use std::path::PathBuf;
use tokmd_format::{
compute_diff_rows, compute_diff_totals, normalize_scan_input, render_diff_md, scan_args,
};
use tokmd_settings::{ChildIncludeMode, ChildrenMode, ScanOptions};
use tokmd_types::{
ConfigMode, ExportArgs, ExportArgsMeta, ExportData, ExportFormat, FileKind, FileRow,
LangArgsMeta, LangReport, LangRow, ModuleArgsMeta, ModuleReport, ModuleRow, RedactMode,
ScanArgs, Totals,
};
#[test]
fn test_scan_args_redact_paths_mode() {
let paths = vec![PathBuf::from("src/lib.rs")];
let global = ScanOptions {
excluded: vec!["target".to_string()],
..Default::default()
};
let args = scan_args(&paths, &global, Some(RedactMode::Paths));
assert!(!args.paths.is_empty());
assert!(args.paths[0].ends_with(".rs"));
assert_ne!(args.paths[0], "src/lib.rs");
assert!(!args.excluded.is_empty());
assert_ne!(args.excluded[0], "target");
assert!(args.excluded_redacted);
}
#[test]
fn test_scan_args_redact_all_mode() {
let paths = vec![PathBuf::from("src/main.rs")];
let global = ScanOptions {
excluded: vec!["node_modules".to_string()],
..Default::default()
};
let args = scan_args(&paths, &global, Some(RedactMode::All));
assert!(args.paths[0].ends_with(".rs"));
assert_ne!(args.paths[0], "src/main.rs");
assert_ne!(args.excluded[0], "node_modules");
assert!(args.excluded_redacted);
}
#[test]
fn test_scan_args_redact_none_mode() {
let paths = vec![PathBuf::from("src/lib.rs")];
let global = ScanOptions {
excluded: vec!["target".to_string()],
..Default::default()
};
let args = scan_args(&paths, &global, Some(RedactMode::None));
assert_eq!(args.paths[0], "src/lib.rs");
assert_eq!(args.excluded[0], "target");
assert!(!args.excluded_redacted);
}
#[test]
fn test_scan_args_no_redact_option() {
let paths = vec![PathBuf::from("src/lib.rs")];
let global = ScanOptions {
excluded: vec!["target".to_string()],
..Default::default()
};
let args = scan_args(&paths, &global, None);
assert_eq!(args.paths[0], "src/lib.rs");
assert_eq!(args.excluded[0], "target");
assert!(!args.excluded_redacted);
}
#[test]
fn test_scan_args_empty_excluded_with_redact() {
let paths = vec![PathBuf::from("src/lib.rs")];
let global = ScanOptions {
excluded: vec![], ..Default::default()
};
let args = scan_args(&paths, &global, Some(RedactMode::Paths));
assert!(args.paths[0].ends_with(".rs"));
assert_ne!(args.paths[0], "src/lib.rs");
assert!(!args.excluded_redacted);
}
#[test]
fn test_scan_args_preserves_global_flags() {
let paths = vec![PathBuf::from(".")];
let global = ScanOptions {
hidden: true,
no_ignore: true,
no_ignore_parent: false,
no_ignore_dot: false,
no_ignore_vcs: false,
treat_doc_strings_as_comments: true,
..Default::default()
};
let args = scan_args(&paths, &global, None);
assert!(args.hidden);
assert!(args.no_ignore);
assert!(args.no_ignore_parent);
assert!(args.no_ignore_dot);
assert!(args.no_ignore_vcs);
assert!(args.treat_doc_strings_as_comments);
}
#[test]
fn test_scan_args_no_ignore_implies_sub_flags() {
let paths = vec![PathBuf::from(".")];
let global = ScanOptions {
no_ignore: true,
no_ignore_parent: false,
no_ignore_dot: false,
no_ignore_vcs: false,
..Default::default()
};
let args = scan_args(&paths, &global, None);
assert!(args.no_ignore_parent);
assert!(args.no_ignore_dot);
assert!(args.no_ignore_vcs);
}
#[test]
fn test_scan_args_individual_sub_flags() {
let paths = vec![PathBuf::from(".")];
let global = ScanOptions {
no_ignore: false,
no_ignore_parent: true,
no_ignore_dot: false,
no_ignore_vcs: true,
..Default::default()
};
let args = scan_args(&paths, &global, None);
assert!(args.no_ignore_parent);
assert!(!args.no_ignore_dot);
assert!(args.no_ignore_vcs);
}
#[test]
fn test_normalize_scan_input_multiple_dot_slash() {
let p = std::path::Path::new("././src/lib.rs");
let normalized = normalize_scan_input(p);
assert_eq!(normalized, "src/lib.rs");
}
#[test]
fn test_normalize_scan_input_empty_after_strip() {
let p = std::path::Path::new("./");
let normalized = normalize_scan_input(p);
assert_eq!(normalized, ".");
}
fn make_lang_row(lang: &str, code: usize, lines: usize, files: usize) -> LangRow {
LangRow {
lang: lang.to_string(),
code,
lines,
files,
bytes: code * 10,
tokens: code / 4,
avg_lines: lines.checked_div(files).unwrap_or(0),
}
}
fn make_totals(code: usize, lines: usize, files: usize) -> Totals {
Totals {
code,
lines,
files,
bytes: code * 10,
tokens: code / 4,
avg_lines: lines.checked_div(files).unwrap_or(0),
}
}
fn make_lang_report(rows: Vec<LangRow>, totals: Totals) -> LangReport {
LangReport {
rows,
total: totals,
with_files: false,
children: ChildrenMode::Collapse,
top: 0,
}
}
#[test]
fn test_compute_diff_rows_delta_calculation() {
let from = make_lang_report(
vec![make_lang_row("Rust", 100, 120, 5)],
make_totals(100, 120, 5),
);
let to = make_lang_report(
vec![make_lang_row("Rust", 150, 180, 7)],
make_totals(150, 180, 7),
);
let rows = compute_diff_rows(&from, &to);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].delta_code, 50); assert_eq!(rows[0].delta_lines, 60); assert_eq!(rows[0].delta_files, 2); }
#[test]
fn test_compute_diff_rows_negative_delta() {
let from = make_lang_report(
vec![make_lang_row("Go", 200, 240, 10)],
make_totals(200, 240, 10),
);
let to = make_lang_report(vec![make_lang_row("Go", 50, 60, 3)], make_totals(50, 60, 3));
let rows = compute_diff_rows(&from, &to);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].delta_code, -150); assert_eq!(rows[0].delta_lines, -180); assert_eq!(rows[0].delta_files, -7); }
#[test]
fn test_compute_diff_totals_accumulation() {
let from = make_lang_report(
vec![
make_lang_row("Rust", 100, 120, 5),
make_lang_row("Go", 200, 240, 10),
],
make_totals(300, 360, 15),
);
let to = make_lang_report(
vec![
make_lang_row("Rust", 150, 180, 7),
make_lang_row("Go", 220, 260, 11),
],
make_totals(370, 440, 18),
);
let rows = compute_diff_rows(&from, &to);
let totals = compute_diff_totals(&rows);
assert_eq!(totals.old_code, 300);
assert_eq!(totals.new_code, 370);
assert_eq!(totals.delta_code, 70);
assert_eq!(totals.old_lines, 360);
assert_eq!(totals.new_lines, 440);
assert_eq!(totals.delta_lines, 80);
}
#[test]
fn test_compute_diff_totals_with_subtraction() {
let from = make_lang_report(
vec![make_lang_row("Python", 500, 600, 20)],
make_totals(500, 600, 20),
);
let to = make_lang_report(
vec![make_lang_row("Python", 600, 700, 25)],
make_totals(600, 700, 25),
);
let rows = compute_diff_rows(&from, &to);
let totals = compute_diff_totals(&rows);
assert_eq!(totals.old_code, 500);
assert_eq!(totals.new_code, 600);
assert_eq!(totals.delta_code, 100);
assert!(totals.delta_code > 0);
}
#[test]
fn test_render_diff_md_non_empty() {
let from = make_lang_report(
vec![make_lang_row("Rust", 100, 120, 5)],
make_totals(100, 120, 5),
);
let to = make_lang_report(
vec![make_lang_row("Rust", 200, 240, 10)],
make_totals(200, 240, 10),
);
let rows = compute_diff_rows(&from, &to);
let totals = compute_diff_totals(&rows);
let md = render_diff_md("baseline", "current", &rows, &totals);
assert!(!md.is_empty());
assert!(md.contains("baseline"));
assert!(md.contains("current"));
assert!(md.contains("Rust"));
assert!(md.contains("|Language|"));
assert!(md.contains("|**Total**|"));
assert!(md.contains("100")); assert!(md.contains("200")); assert!(md.contains("+100")); }
#[test]
fn test_diff_row_old_new_distinct() {
let report = make_lang_report(
vec![make_lang_row("Java", 300, 350, 15)],
make_totals(300, 350, 15),
);
let rows = compute_diff_rows(&report, &report);
assert!(
rows.is_empty(),
"identical reports should have no diff rows"
);
}
#[test]
fn test_diff_row_only_bytes_changed() {
let from = make_lang_report(
vec![LangRow {
lang: "C".to_string(),
code: 100,
lines: 120,
files: 5,
bytes: 1000, tokens: 25,
avg_lines: 24,
}],
make_totals(100, 120, 5),
);
let to = make_lang_report(
vec![LangRow {
lang: "C".to_string(),
code: 100,
lines: 120,
files: 5,
bytes: 2000, tokens: 25,
avg_lines: 24,
}],
make_totals(100, 120, 5),
);
let rows = compute_diff_rows(&from, &to);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].delta_bytes, 1000);
}
#[test]
fn test_diff_row_only_tokens_changed() {
let from = make_lang_report(
vec![LangRow {
lang: "Rust".to_string(),
code: 100,
lines: 120,
files: 5,
bytes: 1000,
tokens: 100, avg_lines: 24,
}],
make_totals(100, 120, 5),
);
let to = make_lang_report(
vec![LangRow {
lang: "Rust".to_string(),
code: 100,
lines: 120,
files: 5,
bytes: 1000,
tokens: 200, avg_lines: 24,
}],
make_totals(100, 120, 5),
);
let rows = compute_diff_rows(&from, &to);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].delta_tokens, 100);
}
#[test]
fn test_write_export_csv_format() {
use std::io::Cursor;
let export = ExportData {
rows: vec![
FileRow {
path: "src/lib.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 100,
comments: 20,
blanks: 10,
lines: 130,
bytes: 1000,
tokens: 250,
},
FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Child,
code: 50,
comments: 5,
blanks: 5,
lines: 60,
bytes: 500,
tokens: 125,
},
],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let _global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Csv,
output: None,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::None,
strip_prefix: None,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_csv_to(&mut buffer, &export, &args).expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
assert!(output.contains("path,module,lang,kind,code,comments,blanks,lines,bytes,tokens"));
assert!(output.contains("src/lib.rs"));
assert!(output.contains("parent"));
assert!(output.contains("child"));
}
#[test]
fn test_write_export_jsonl_with_meta() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "test.rs".to_string(),
module: "".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 2,
blanks: 1,
lines: 13,
bytes: 100,
tokens: 25,
}],
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Jsonl,
output: None,
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::None,
strip_prefix: None,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_jsonl_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let lines: Vec<&str> = output.lines().collect();
assert!(lines.len() >= 2);
assert!(lines[0].contains("\"type\":\"meta\""));
assert!(lines[1].contains("\"type\":\"row\""));
assert!(lines[1].contains("test.rs"));
}
#[test]
fn test_write_export_jsonl_without_meta() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "test.rs".to_string(),
module: "".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 2,
blanks: 1,
lines: 13,
bytes: 100,
tokens: 25,
}],
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Jsonl,
output: None,
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: false, redact: RedactMode::None,
strip_prefix: None,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_jsonl_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let lines: Vec<&str> = output.lines().collect();
assert_eq!(lines.len(), 1);
assert!(lines[0].contains("\"type\":\"row\""));
assert!(!output.contains("\"type\":\"meta\""));
}
#[test]
fn test_write_export_json_with_redaction() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/secret.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 2,
blanks: 1,
lines: 13,
bytes: 100,
tokens: 25,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Json,
output: None,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::All, strip_prefix: None,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_json_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
assert!(!output.contains("src/secret.rs"));
assert!(output.contains(".rs"));
assert!(!output.contains("\"module\":\"src\""));
}
#[test]
fn test_write_export_cyclonedx_structure() {
use std::io::Cursor;
let export = ExportData {
rows: vec![
FileRow {
path: "src/lib.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 100,
comments: 20,
blanks: 10,
lines: 130,
bytes: 1000,
tokens: 250,
},
FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Child,
code: 50,
comments: 5,
blanks: 5,
lines: 60,
bytes: 500,
tokens: 125,
},
],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_cyclonedx_to(&mut buffer, &export, RedactMode::None)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
assert!(output.contains("\"bomFormat\": \"CycloneDX\""));
assert!(output.contains("\"specVersion\": \"1.6\""));
assert!(output.contains("\"components\""));
assert!(output.contains("\"tokmd:lang\""));
assert!(output.contains("\"tokmd:code\""));
assert!(output.contains("\"tokmd:kind\""));
assert!(output.contains("\"child\""));
}
#[test]
fn test_cyclonedx_empty_module_no_group() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "standalone.rs".to_string(),
module: "".to_string(), lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 2,
blanks: 1,
lines: 13,
bytes: 100,
tokens: 25,
}],
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_cyclonedx_to(&mut buffer, &export, RedactMode::None)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
assert!(output.contains("standalone.rs"));
let json: serde_json::Value = serde_json::from_str(&output).expect("must parse valid JSON");
let components = json["components"].as_array().expect("must be a JSON array");
assert_eq!(components.len(), 1);
assert!(components[0].get("group").is_none() || components[0]["group"].is_null());
}
#[test]
fn test_jsonl_generated_at_ms_is_reasonable() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "test.rs".to_string(),
module: "".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 2,
blanks: 1,
lines: 13,
bytes: 100,
tokens: 25,
}],
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Jsonl,
output: None,
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::None,
strip_prefix: None,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_jsonl_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let meta_line = output
.lines()
.next()
.expect("output must have at least one line");
let meta: serde_json::Value = serde_json::from_str(meta_line).expect("operation must succeed");
let generated_at_ms = meta["generated_at_ms"]
.as_u64()
.expect("must be a JSON integer");
const JAN_1_2020_MS: u64 = 1_577_836_800_000;
assert!(
generated_at_ms > JAN_1_2020_MS,
"generated_at_ms ({}) should be greater than Jan 1 2020 ({})",
generated_at_ms,
JAN_1_2020_MS
);
}
#[test]
fn test_json_generated_at_ms_is_reasonable() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "test.rs".to_string(),
module: "".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 2,
blanks: 1,
lines: 13,
bytes: 100,
tokens: 25,
}],
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Json,
output: None,
module_roots: vec![],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::None,
strip_prefix: None,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_json_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let json: serde_json::Value = serde_json::from_str(&output).expect("must parse valid JSON");
let generated_at_ms = json["generated_at_ms"]
.as_u64()
.expect("must be a JSON integer");
const JAN_1_2020_MS: u64 = 1_577_836_800_000;
assert!(
generated_at_ms > JAN_1_2020_MS,
"generated_at_ms ({}) should be greater than Jan 1 2020 ({})",
generated_at_ms,
JAN_1_2020_MS
);
}
#[test]
fn test_diff_receipt_generated_at_ms_is_reasonable() {
use tokmd_format::create_diff_receipt;
use tokmd_types::DiffTotals;
let receipt = create_diff_receipt("from", "to", vec![], DiffTotals::default());
const JAN_1_2020_MS: u128 = 1_577_836_800_000;
assert!(
receipt.generated_at_ms > JAN_1_2020_MS,
"generated_at_ms ({}) should be greater than Jan 1 2020 ({})",
receipt.generated_at_ms,
JAN_1_2020_MS
);
}
#[test]
fn test_write_export_writes_to_file() {
use tokmd_format::write_export;
let export = ExportData {
rows: vec![FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 100,
comments: 10,
blanks: 5,
lines: 115,
bytes: 1000,
tokens: 250,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let temp_file = tempfile::NamedTempFile::new().expect("create temp file");
let temp_path = temp_file.path().to_path_buf();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Jsonl,
output: Some(temp_path.clone()),
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::None,
strip_prefix: None,
};
write_export(&export, &global, &args).expect("write_export should succeed");
let content = std::fs::read_to_string(&temp_path).expect("read temp file");
assert!(
!content.trim().is_empty(),
"exported file must not be empty"
);
let lines: Vec<&str> = content.lines().collect();
assert!(lines.len() >= 2, "should have meta and row lines");
assert!(lines[0].contains("\"type\":\"meta\""));
assert!(lines[1].contains("\"type\":\"row\""));
}
#[test]
fn test_jsonl_strip_prefix_redacted_with_paths_mode() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 1,
blanks: 1,
lines: 12,
bytes: 100,
tokens: 20,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Jsonl,
output: None,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::Paths, strip_prefix: Some(PathBuf::from("src")),
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_jsonl_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let meta_line = output
.lines()
.next()
.expect("output must have at least one line");
let meta: serde_json::Value = serde_json::from_str(meta_line).expect("operation must succeed");
assert_eq!(
meta["args"]["strip_prefix_redacted"], true,
"strip_prefix_redacted must be true when redact=Paths and strip_prefix is set"
);
let sp = meta["args"]["strip_prefix"]
.as_str()
.expect("must be a JSON string");
assert_ne!(sp, "src", "strip_prefix should be redacted, not literal");
assert_eq!(sp.len(), 16, "redacted strip_prefix should be 16 chars");
}
#[test]
fn test_jsonl_strip_prefix_redacted_with_all_mode() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 1,
blanks: 1,
lines: 12,
bytes: 100,
tokens: 20,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Jsonl,
output: None,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::All, strip_prefix: Some(PathBuf::from("prefix")),
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_jsonl_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let meta_line = output
.lines()
.next()
.expect("output must have at least one line");
let meta: serde_json::Value = serde_json::from_str(meta_line).expect("operation must succeed");
assert_eq!(
meta["args"]["strip_prefix_redacted"], true,
"strip_prefix_redacted must be true when redact=All and strip_prefix is set"
);
let sp = meta["args"]["strip_prefix"]
.as_str()
.expect("must be a JSON string");
assert_ne!(sp, "prefix", "strip_prefix should be redacted");
assert_eq!(sp.len(), 16, "redacted strip_prefix should be 16 chars");
}
#[test]
fn test_jsonl_strip_prefix_redacted_false_when_no_strip_prefix() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 1,
blanks: 1,
lines: 12,
bytes: 100,
tokens: 20,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Jsonl,
output: None,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::Paths, strip_prefix: None, };
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_jsonl_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let meta_line = output
.lines()
.next()
.expect("output must have at least one line");
let meta: serde_json::Value = serde_json::from_str(meta_line).expect("operation must succeed");
assert!(
meta["args"]["strip_prefix_redacted"].is_null()
|| meta["args"]["strip_prefix_redacted"] == false,
"strip_prefix_redacted must be false/omitted when strip_prefix is None; got {:?}",
meta["args"]["strip_prefix_redacted"]
);
assert!(
meta["args"]["strip_prefix"].is_null(),
"strip_prefix should be null when not set"
);
}
#[test]
fn test_jsonl_no_redaction_with_none_mode() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 1,
blanks: 1,
lines: 12,
bytes: 100,
tokens: 20,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Jsonl,
output: None,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::None, strip_prefix: Some(PathBuf::from("src")),
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_jsonl_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let meta_line = output
.lines()
.next()
.expect("output must have at least one line");
let meta: serde_json::Value = serde_json::from_str(meta_line).expect("operation must succeed");
assert!(
meta["args"]["strip_prefix_redacted"].is_null()
|| meta["args"]["strip_prefix_redacted"] == false,
"strip_prefix_redacted must be false/omitted when redact=None; got {:?}",
meta["args"]["strip_prefix_redacted"]
);
assert_eq!(
meta["args"]["strip_prefix"]
.as_str()
.expect("must be a JSON string"),
"src",
"strip_prefix should be literal when redact=None"
);
}
#[test]
fn test_json_strip_prefix_redacted_with_paths_mode() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 1,
blanks: 1,
lines: 12,
bytes: 100,
tokens: 20,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Json,
output: None,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::Paths,
strip_prefix: Some(PathBuf::from("src")),
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_json_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let json: serde_json::Value =
serde_json::from_str(output.trim()).expect("must parse valid JSON");
assert_eq!(
json["args"]["strip_prefix_redacted"], true,
"strip_prefix_redacted must be true when redact=Paths and strip_prefix is set"
);
let sp = json["args"]["strip_prefix"]
.as_str()
.expect("must be a JSON string");
assert_ne!(sp, "src");
assert_eq!(sp.len(), 16);
}
#[test]
fn test_json_no_redaction_with_none_mode() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 10,
comments: 1,
blanks: 1,
lines: 12,
bytes: 100,
tokens: 20,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let global = ScanOptions::default();
let args = ExportArgs {
paths: vec![PathBuf::from(".")],
format: ExportFormat::Json,
output: None,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
meta: true,
redact: RedactMode::None,
strip_prefix: Some(PathBuf::from("myprefix")),
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_json_to(&mut buffer, &export, &global, &args)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let json: serde_json::Value =
serde_json::from_str(output.trim()).expect("must parse valid JSON");
assert!(
json["args"]["strip_prefix_redacted"].is_null()
|| json["args"]["strip_prefix_redacted"] == false,
"strip_prefix_redacted must be false/omitted when redact=None; got {:?}",
json["args"]["strip_prefix_redacted"]
);
assert_eq!(
json["args"]["strip_prefix"]
.as_str()
.expect("must be a JSON string"),
"myprefix",
"strip_prefix should be literal when redact=None"
);
}
#[test]
fn test_write_lang_json_to_file_writes_valid_json() {
use tokmd_format::write_lang_json_to_file;
let report = LangReport {
rows: vec![LangRow {
lang: "Rust".to_string(),
code: 100,
lines: 120,
files: 5,
bytes: 5000,
tokens: 250,
avg_lines: 24,
}],
total: Totals {
code: 100,
lines: 120,
files: 5,
bytes: 5000,
tokens: 250,
avg_lines: 24,
},
with_files: true,
children: ChildrenMode::Collapse,
top: 0,
};
let scan = ScanArgs {
paths: vec![".".to_string()],
excluded: vec![],
excluded_redacted: false,
config: ConfigMode::Auto,
hidden: false,
no_ignore: false,
no_ignore_parent: false,
no_ignore_dot: false,
no_ignore_vcs: false,
treat_doc_strings_as_comments: false,
};
let args_meta = LangArgsMeta {
format: "json".to_string(),
top: 0,
with_files: true,
children: ChildrenMode::Collapse,
};
let temp_dir = tempfile::tempdir().expect("create temp dir");
let file_path = temp_dir.path().join("lang.json");
write_lang_json_to_file(&file_path, &report, &scan, &args_meta)
.expect("write_lang_json_to_file should succeed");
let content = std::fs::read_to_string(&file_path).expect("read file");
assert!(!content.trim().is_empty(), "file must not be empty");
let json: serde_json::Value = serde_json::from_str(&content).expect("valid JSON");
assert_eq!(json["mode"], "lang");
assert!(json["rows"].is_array());
assert_eq!(json["rows"][0]["lang"], "Rust");
assert_eq!(json["rows"][0]["code"], 100);
}
#[test]
fn test_write_module_json_to_file_writes_valid_json() {
use tokmd_format::write_module_json_to_file;
let report = ModuleReport {
rows: vec![ModuleRow {
module: "src".to_string(),
code: 200,
lines: 240,
files: 10,
bytes: 10000,
tokens: 500,
avg_lines: 24,
}],
total: Totals {
code: 200,
lines: 240,
files: 10,
bytes: 10000,
tokens: 500,
avg_lines: 24,
},
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
top: 0,
};
let scan = ScanArgs {
paths: vec![".".to_string()],
excluded: vec![],
excluded_redacted: false,
config: ConfigMode::Auto,
hidden: false,
no_ignore: false,
no_ignore_parent: false,
no_ignore_dot: false,
no_ignore_vcs: false,
treat_doc_strings_as_comments: false,
};
let args_meta = ModuleArgsMeta {
format: "json".to_string(),
top: 0,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let temp_dir = tempfile::tempdir().expect("create temp dir");
let file_path = temp_dir.path().join("module.json");
write_module_json_to_file(&file_path, &report, &scan, &args_meta, RedactMode::None)
.expect("write_module_json_to_file should succeed");
let content = std::fs::read_to_string(&file_path).expect("read file");
assert!(!content.trim().is_empty(), "file must not be empty");
let json: serde_json::Value = serde_json::from_str(&content).expect("valid JSON");
assert_eq!(json["mode"], "module");
assert!(json["rows"].is_array());
assert_eq!(json["rows"][0]["module"], "src");
assert_eq!(json["rows"][0]["code"], 200);
}
#[test]
fn test_write_export_jsonl_to_file_writes_valid_jsonl() {
use tokmd_format::write_export_jsonl_to_file;
let export = ExportData {
rows: vec![FileRow {
path: "src/main.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 50,
comments: 10,
blanks: 5,
lines: 65,
bytes: 500,
tokens: 125,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let scan = ScanArgs {
paths: vec![".".to_string()],
excluded: vec![],
excluded_redacted: false,
config: ConfigMode::Auto,
hidden: false,
no_ignore: false,
no_ignore_parent: false,
no_ignore_dot: false,
no_ignore_vcs: false,
treat_doc_strings_as_comments: false,
};
let args_meta = ExportArgsMeta {
format: ExportFormat::Jsonl,
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
min_code: 0,
max_rows: 0,
redact: RedactMode::None,
strip_prefix: None,
strip_prefix_redacted: false,
};
let temp_dir = tempfile::tempdir().expect("create temp dir");
let file_path = temp_dir.path().join("export.jsonl");
write_export_jsonl_to_file(&file_path, &export, &scan, &args_meta)
.expect("write_export_jsonl_to_file should succeed");
let content = std::fs::read_to_string(&file_path).expect("read file");
assert!(!content.trim().is_empty(), "file must not be empty");
let lines: Vec<&str> = content.lines().collect();
assert!(lines.len() >= 2, "should have meta and row lines");
let meta: serde_json::Value = serde_json::from_str(lines[0]).expect("meta is valid JSON");
assert_eq!(meta["type"], "meta");
assert_eq!(meta["mode"], "export");
let row: serde_json::Value = serde_json::from_str(lines[1]).expect("row is valid JSON");
assert_eq!(row["type"], "row");
assert_eq!(row["path"], "src/main.rs");
assert_eq!(row["code"], 50);
}
#[test]
fn test_cyclonedx_parent_has_no_kind_property() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/lib.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent, code: 100,
comments: 20,
blanks: 10,
lines: 130,
bytes: 1000,
tokens: 250,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_cyclonedx_to(&mut buffer, &export, RedactMode::None)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let json: serde_json::Value = serde_json::from_str(&output).expect("must parse valid JSON");
let components = json["components"].as_array().expect("must be a JSON array");
assert_eq!(components.len(), 1);
let properties = components[0]["properties"]
.as_array()
.expect("must be a JSON array");
let has_kind = properties
.iter()
.any(|p| p["name"].as_str() == Some("tokmd:kind"));
assert!(
!has_kind,
"Parent files should NOT have tokmd:kind property"
);
}
#[test]
fn test_cyclonedx_child_has_kind_property() {
use std::io::Cursor;
let export = ExportData {
rows: vec![FileRow {
path: "src/lib.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Child, code: 100,
comments: 20,
blanks: 10,
lines: 130,
bytes: 1000,
tokens: 250,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let mut buffer = Cursor::new(Vec::new());
tokmd_format::write_export_cyclonedx_to(&mut buffer, &export, RedactMode::None)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let json: serde_json::Value = serde_json::from_str(&output).expect("must parse valid JSON");
let components = json["components"].as_array().expect("must be a JSON array");
assert_eq!(components.len(), 1);
let properties = components[0]["properties"]
.as_array()
.expect("must be a JSON array");
let kind_prop = properties
.iter()
.find(|p| p["name"].as_str() == Some("tokmd:kind"));
assert!(
kind_prop.is_some(),
"Child files should have tokmd:kind property"
);
assert_eq!(kind_prop.expect("operation must succeed")["value"], "child");
}
#[test]
fn test_compute_diff_totals_nonzero_deltas() {
use tokmd_types::DiffRow;
let rows = vec![
DiffRow {
lang: "Rust".to_string(),
old_code: 100,
new_code: 160,
delta_code: 60, old_lines: 200,
new_lines: 260,
delta_lines: 60, old_files: 5,
new_files: 8,
delta_files: 3, old_bytes: 1000,
new_bytes: 1800,
delta_bytes: 800, old_tokens: 250,
new_tokens: 400,
delta_tokens: 150, },
DiffRow {
lang: "Go".to_string(),
old_code: 200,
new_code: 180,
delta_code: -20, old_lines: 300,
new_lines: 310,
delta_lines: 10, old_files: 10,
new_files: 9,
delta_files: -1, old_bytes: 2000,
new_bytes: 2100,
delta_bytes: 100, old_tokens: 500,
new_tokens: 520,
delta_tokens: 20, },
];
let totals = compute_diff_totals(&rows);
assert_eq!(totals.old_code, 300);
assert_eq!(totals.new_code, 340);
assert_eq!(totals.delta_code, 40);
assert_eq!(totals.old_lines, 500);
assert_eq!(totals.new_lines, 570);
assert_eq!(totals.delta_lines, 70);
assert_eq!(totals.old_files, 15);
assert_eq!(totals.new_files, 17);
assert_eq!(totals.delta_files, 2);
assert_eq!(totals.old_bytes, 3000);
assert_eq!(totals.new_bytes, 3900);
assert_eq!(totals.delta_bytes, 900);
assert_eq!(totals.old_tokens, 750);
assert_eq!(totals.new_tokens, 920);
assert_eq!(totals.delta_tokens, 170);
}
#[test]
fn test_cyclonedx_snapshot_deterministic() {
use std::io::Cursor;
use tokmd_types::{ChildIncludeMode, FileKind, FileRow};
let export = tokmd_types::ExportData {
rows: vec![FileRow {
path: "src/lib.rs".to_string(),
module: "src".to_string(),
lang: "Rust".to_string(),
kind: FileKind::Parent,
code: 100,
comments: 20,
blanks: 10,
lines: 130,
bytes: 1000,
tokens: 250,
}],
module_roots: vec!["src".to_string()],
module_depth: 1,
children: ChildIncludeMode::Separate,
};
let mut buffer = Cursor::new(Vec::new());
let serial = Some("urn:uuid:00000000-0000-0000-0000-000000000000".to_string());
let timestamp = Some("1970-01-01T00:00:00Z".to_string());
tokmd_format::write_export_cyclonedx_with_options(
&mut buffer,
&export,
tokmd_types::RedactMode::None,
serial,
timestamp,
)
.expect("operation must succeed");
let output = String::from_utf8(buffer.into_inner()).expect("output must be valid UTF-8");
let json: serde_json::Value = serde_json::from_str(&output).expect("must parse valid JSON");
let pretty = serde_json::to_string_pretty(&json).expect("operation must succeed");
insta::assert_snapshot!(pretty);
}