1use std::collections::{BTreeMap, BTreeSet};
24use std::fs;
25use std::io;
26use std::path::{Path, PathBuf};
27
28use thiserror::Error;
29
30use crate::manifest::{canonical_json, sha256_hex, FileEntry, Manifest};
31
32pub(crate) const MANIFEST: &str = "manifest.json";
34
35pub(crate) const FORBIDDEN: [&str; 5] = ["user", "name", "hostname", "node", "email"];
38
39#[derive(Debug, Clone, PartialEq, Eq)]
44pub struct FileSpec {
45 pub path: PathBuf,
47 pub keep_columns: Vec<String>,
50}
51
52#[derive(Debug, Error)]
54pub enum BundleError {
55 #[error("{file}: the column {column} identifies a person or a machine, so nothing was copied")]
58 ForbiddenColumn {
59 file: PathBuf,
61 column: String,
63 },
64 #[error("{file}: a table in an input directory that was not listed as an input, so nothing was copied")]
67 UnlistedFile {
68 file: PathBuf,
70 },
71 #[error("{name}: two inputs would be written under this name")]
73 DuplicateFile {
74 name: String,
76 },
77 #[error("{file}: line {line} has {found} cells, the header has {listed}")]
79 RowWidth {
80 file: PathBuf,
82 line: usize,
84 listed: usize,
86 found: usize,
88 },
89 #[error("{file}: the file has no header row")]
91 NoHeader {
92 file: PathBuf,
94 },
95 #[error("{file}: no column in the header belongs to the allow-list")]
97 NoAllowedColumn {
98 file: PathBuf,
100 },
101 #[error("{file}: {source}")]
103 Io {
104 file: PathBuf,
106 source: io::Error,
108 },
109 #[error("the manifest could not be written: {source}")]
112 Encode {
113 source: serde_json::Error,
115 },
116}
117
118pub fn bundle(
137 inputs: &[FileSpec],
138 out: &Path,
139 meta: BTreeMap<String, String>,
140 created_at: &str,
141 extra_forbidden: &[String],
142) -> Result<Manifest, BundleError> {
143 refuse_unlisted_tables(&inputs_by_directory(inputs))?;
144 refuse_duplicate_names(inputs)?;
145
146 let mut files = Vec::new();
147 let mut dropped = BTreeMap::new();
148 for spec in inputs {
149 let bundled = bundle_one(spec, out, extra_forbidden)?;
150 if !bundled.dropped.is_empty() {
151 dropped.insert(bundled.entry.name.clone(), bundled.dropped.clone());
152 }
153 files.push(bundled.entry);
154 }
155
156 let manifest = Manifest {
157 files,
158 created_at: created_at.to_owned(),
159 tool_version: env!("CARGO_PKG_VERSION").to_owned(),
160 meta,
161 dropped,
162 };
163 let text = canonical_json(
164 &serde_json::to_value(&manifest).map_err(|source| BundleError::Encode { source })?,
165 );
166 write(&out.join(MANIFEST), &text)?;
167
168 Ok(manifest)
169}
170
171fn inputs_by_directory(inputs: &[FileSpec]) -> BTreeMap<PathBuf, Vec<&FileSpec>> {
173 inputs.iter().fold(BTreeMap::new(), |mut groups, spec| {
174 let directory = spec
175 .path
176 .parent()
177 .filter(|parent| !parent.as_os_str().is_empty())
178 .unwrap_or(Path::new("."))
179 .to_owned();
180 groups.entry(directory).or_default().push(spec);
181 groups
182 })
183}
184
185fn refuse_unlisted_tables(groups: &BTreeMap<PathBuf, Vec<&FileSpec>>) -> Result<(), BundleError> {
188 for (directory, specs) in groups {
189 let listed: BTreeSet<&str> = specs
190 .iter()
191 .filter_map(|spec| file_name(&spec.path))
192 .collect();
193 for path in tables_in(directory)? {
194 let name = match file_name(&path) {
195 Some(name) => name.to_owned(),
196 None => continue,
197 };
198 if !listed.contains(name.as_str()) {
199 return Err(BundleError::UnlistedFile { file: path });
200 }
201 }
202 }
203 Ok(())
204}
205
206fn refuse_duplicate_names(inputs: &[FileSpec]) -> Result<(), BundleError> {
208 let mut seen = BTreeSet::new();
209 for spec in inputs {
210 if let Some(name) = file_name(&spec.path) {
211 if !seen.insert(name.to_owned()) {
212 return Err(BundleError::DuplicateFile {
213 name: name.to_owned(),
214 });
215 }
216 }
217 }
218 Ok(())
219}
220
221struct Bundled {
223 entry: FileEntry,
224 dropped: Vec<String>,
225}
226
227fn bundle_one(
229 spec: &FileSpec,
230 out: &Path,
231 extra_forbidden: &[String],
232) -> Result<Bundled, BundleError> {
233 let source = read(&spec.path)?;
234 let filtered = filtered(&source, spec, extra_forbidden)?;
235 let name = file_name(&spec.path).unwrap_or_default().to_owned();
236 write(&out.join(&name), &filtered.text)?;
237 Ok(Bundled {
238 entry: FileEntry {
239 name,
240 sha256: sha256_hex(filtered.text.as_bytes()),
241 rows: filtered.rows,
242 dropped: filtered.dropped.clone(),
243 },
244 dropped: filtered.dropped,
245 })
246}
247
248#[derive(Debug, PartialEq, Eq)]
250struct Filtered {
251 text: String,
252 dropped: Vec<String>,
253 rows: usize,
254}
255
256fn filtered(
268 text: &str,
269 spec: &FileSpec,
270 extra_forbidden: &[String],
271) -> Result<Filtered, BundleError> {
272 let mut lines = text.lines();
273 let header = lines.next().ok_or_else(|| BundleError::NoHeader {
274 file: spec.path.clone(),
275 })?;
276 let columns: Vec<&str> = header.split(',').map(str::trim).collect();
277
278 for column in &columns {
279 if forbidden(column, extra_forbidden) {
280 return Err(BundleError::ForbiddenColumn {
281 file: spec.path.clone(),
282 column: (*column).to_owned(),
283 });
284 }
285 }
286
287 let kept: Vec<usize> = columns
288 .iter()
289 .enumerate()
290 .filter(|(_, column)| spec.keep_columns.iter().any(|keep| keep == *column))
291 .map(|(at, _)| at)
292 .collect();
293 if kept.is_empty() {
294 return Err(BundleError::NoAllowedColumn {
295 file: spec.path.clone(),
296 });
297 }
298
299 let rows: Vec<String> = lines
300 .enumerate()
301 .filter(|(_, line)| !line.trim().is_empty())
302 .map(|(offset, line)| {
303 kept_row(line, columns.len(), &kept).map_err(|found| BundleError::RowWidth {
304 file: spec.path.clone(),
305 line: offset + 2,
306 listed: columns.len(),
307 found,
308 })
309 })
310 .collect::<Result<_, _>>()?;
311
312 let counted = rows.len();
313 let body = std::iter::once(pick(&columns, &kept))
314 .chain(rows)
315 .collect::<Vec<_>>()
316 .join("\n");
317 Ok(Filtered {
318 text: format!("{body}\n"),
319 dropped: columns
320 .iter()
321 .enumerate()
322 .filter(|(at, _)| !kept.contains(at))
323 .map(|(_, column)| (*column).to_owned())
324 .collect(),
325 rows: counted,
326 })
327}
328
329fn forbidden(column: &str, extra: &[String]) -> bool {
334 let lowered = column.to_ascii_lowercase();
335 FORBIDDEN
336 .iter()
337 .copied()
338 .chain(extra.iter().map(String::as_str))
339 .any(|token| lowered.contains(token))
340}
341
342fn kept_row(line: &str, width: usize, kept: &[usize]) -> Result<String, usize> {
345 let cells: Vec<&str> = line.split(',').map(str::trim).collect();
346 if cells.len() == width {
347 Ok(pick(&cells, kept))
348 } else {
349 Err(cells.len())
350 }
351}
352
353fn pick(cells: &[&str], kept: &[usize]) -> String {
355 kept.iter()
356 .filter_map(|at| cells.get(*at))
357 .copied()
358 .collect::<Vec<_>>()
359 .join(",")
360}
361
362fn tables_in(directory: &Path) -> Result<Vec<PathBuf>, BundleError> {
364 let mut found = Vec::new();
365 let entries = fs::read_dir(directory).map_err(|source| BundleError::Io {
366 file: directory.to_owned(),
367 source,
368 })?;
369 for entry in entries {
370 let path = entry
371 .map_err(|source| BundleError::Io {
372 file: directory.to_owned(),
373 source,
374 })?
375 .path();
376 if is_table(&path) {
377 found.push(path);
378 }
379 }
380 found.sort();
381 Ok(found)
382}
383
384fn is_table(path: &Path) -> bool {
386 path.extension().is_some_and(|extension| {
387 extension.eq_ignore_ascii_case("csv") || extension.eq_ignore_ascii_case("tsv")
388 })
389}
390
391fn file_name(path: &Path) -> Option<&str> {
393 path.file_name().and_then(|name| name.to_str())
394}
395
396fn read(path: &Path) -> Result<String, BundleError> {
398 fs::read_to_string(path).map_err(|source| BundleError::Io {
399 file: path.to_owned(),
400 source,
401 })
402}
403
404pub(crate) fn write(path: &Path, text: &str) -> Result<(), BundleError> {
406 if let Some(parent) = path.parent() {
407 fs::create_dir_all(parent).map_err(|source| BundleError::Io {
408 file: parent.to_owned(),
409 source,
410 })?;
411 }
412 fs::write(path, text).map_err(|source| BundleError::Io {
413 file: path.to_owned(),
414 source,
415 })
416}
417
418#[cfg(test)]
420pub(crate) fn unique_directory() -> PathBuf {
421 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
422 let at = NEXT.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
423 std::env::temp_dir().join(format!("export-bundle-test-{}-{at}", std::process::id()))
424}
425
426#[cfg(test)]
427mod tests {
428 use super::*;
429
430 fn spec(path: &str, keep: &[&str]) -> FileSpec {
431 FileSpec {
432 path: PathBuf::from(path),
433 keep_columns: keep.iter().map(|column| (*column).to_owned()).collect(),
434 }
435 }
436
437 fn filtered_of(text: &str, keep: &[&str]) -> Filtered {
438 let spec = spec("meters.csv", keep);
439 filtered(text, &spec, &[]).unwrap_or_else(|error| panic!("the fixture filters: {error}"))
440 }
441
442 #[test]
443 fn a_column_the_allow_list_declares_survives_and_one_it_does_not_is_dropped_and_reported() {
444 let filtered = filtered_of("id,watts,note\n1,60,bench\n", &["id", "watts"]);
445
446 assert_eq!(filtered.text, "id,watts\n1,60\n");
447 assert_eq!(filtered.dropped, vec!["note".to_owned()]);
448 assert_eq!(filtered.rows, 1);
449 }
450
451 #[test]
452 fn a_crlf_file_is_rewritten_as_lf_so_a_hash_does_not_depend_on_the_platform() {
453 let filtered = filtered_of("id,watts\r\n1,60\r\n", &["id", "watts"]);
454
455 assert!(!filtered.text.contains('\r'));
456 assert!(filtered.text.ends_with("1,60\n"));
457 }
458
459 #[test]
460 fn a_trailing_blank_line_is_not_a_row() {
461 let filtered = filtered_of("id,watts\n1,60\n\n", &["id", "watts"]);
462
463 assert_eq!(filtered.rows, 1);
464 assert!(filtered.text.ends_with("1,60\n"));
465 }
466
467 #[test]
468 fn a_row_with_the_wrong_number_of_cells_is_named_by_line_rather_than_re_columned() {
469 let error = filtered(
470 "id,watts\n1,60\n2\n",
471 &spec("meters.csv", &["id", "watts"]),
472 &[],
473 )
474 .expect_err("that row is not on the allow-list");
475
476 assert!(
477 matches!(error, BundleError::RowWidth { line: 3, .. }),
478 "got {error}"
479 );
480 }
481
482 #[test]
483 fn a_file_with_no_allowed_column_at_all_is_refused() {
484 let error = filtered(
485 "note,colour\nbench,red\n",
486 &spec("meters.csv", &["id"]),
487 &[],
488 )
489 .expect_err("nothing on the allow-list");
490
491 assert!(matches!(error, BundleError::NoAllowedColumn { .. }));
492 }
493
494 #[test]
495 fn an_empty_file_is_refused_rather_than_bundled_as_nothing() {
496 let error = filtered("", &spec("meters.csv", &["id"]), &[])
497 .expect_err("an empty file has no header");
498
499 assert!(matches!(error, BundleError::NoHeader { .. }));
500 }
501
502 #[test]
503 fn a_header_column_that_names_a_person_or_a_machine_refuses_the_call() {
504 for header in [
505 "id,user",
506 "id,User",
507 "id,email",
508 "id,hostname",
509 "id,node_list",
510 "id,full_name",
511 ] {
512 let error = filtered(
513 &format!("{header}\n1,x\n"),
514 &spec("meters.csv", &["id"]),
515 &[],
516 )
517 .expect_err("the header names a person or a machine");
518
519 assert!(
520 matches!(error, BundleError::ForbiddenColumn { .. }),
521 "{header} was not refused: {error}"
522 );
523 }
524 }
525
526 #[test]
527 fn the_refusal_names_the_file_and_the_column() {
528 let error = filtered("id,user\n1,nobody\n", &spec("meters.csv", &["id"]), &[])
529 .expect_err("the header names a person");
530
531 assert_eq!(
532 error.to_string(),
533 "meters.csv: the column user identifies a person or a machine, so nothing was copied"
534 );
535 }
536
537 #[test]
538 fn an_extra_forbidden_token_from_the_caller_is_unioned_with_the_defaults() {
539 let spec = spec("meters.csv", &["id"]);
540 let error = filtered("id,region\n1,north\n", &spec, &[String::from("region")])
541 .expect_err("the caller added a forbidden token");
542
543 assert!(
544 matches!(error, BundleError::ForbiddenColumn { ref column, .. } if column == "region"),
545 "got {error}"
546 );
547 }
548
549 #[test]
550 fn the_default_tokens_apply_whenever_the_caller_adds_none() {
551 let spec = spec("meters.csv", &["id"]);
552 let error = filtered("id,user\n1,nobody\n", &spec, &[]).expect_err("default token");
553
554 assert!(matches!(error, BundleError::ForbiddenColumn { .. }));
555 }
556
557 #[test]
558 fn only_a_table_is_inspected_for_an_unlisted_file() {
559 assert!(is_table(Path::new("users.csv")));
560 assert!(is_table(Path::new("export.TSV")));
561 assert!(!is_table(Path::new("notes.json")));
562 assert!(!is_table(Path::new("notes")));
563 }
564
565 #[test]
566 fn bundle_refuses_a_table_in_the_input_directory_that_was_not_listed() {
567 let home = unique_directory();
568 fs::create_dir_all(&home).unwrap();
569 fs::write(home.join("meters.csv"), "id,watts\n1,60\n").unwrap();
570 fs::write(home.join("users.csv"), "user\nnobody\n").unwrap();
571 let out = home.join("bundle");
572
573 let error = bundle(
574 &[spec(
575 home.join("meters.csv").to_str().unwrap(),
576 &["id", "watts"],
577 )],
578 &out,
579 BTreeMap::new(),
580 "2026-01-01T00:00:00Z",
581 &[],
582 )
583 .expect_err("users.csv sits in the same directory and is not listed");
584
585 assert!(
586 error.to_string().contains("users.csv"),
587 "the refusal names the file: {error}"
588 );
589 assert!(!out.join("meters.csv").exists(), "nothing was copied");
590 }
591
592 #[test]
593 fn bundle_writes_the_filtered_files_and_a_manifest_that_verifies() {
594 let home = unique_directory();
595 fs::create_dir_all(&home).unwrap();
596 fs::write(home.join("meters.csv"), "id,watts,note\n1,60,bench\n").unwrap();
597 let out = home.join("bundle");
598
599 let manifest = bundle(
600 &[spec(
601 home.join("meters.csv").to_str().unwrap(),
602 &["id", "watts"],
603 )],
604 &out,
605 BTreeMap::from([("window".to_owned(), "24h".to_owned())]),
606 "2026-01-01T00:00:00Z",
607 &[],
608 )
609 .unwrap_or_else(|error| panic!("the bundle writes: {error}"));
610
611 assert_eq!(manifest.files.len(), 1);
612 assert_eq!(manifest.files[0].name, "meters.csv");
613 assert_eq!(manifest.files[0].rows, 1);
614 assert_eq!(manifest.files[0].dropped, vec!["note".to_owned()]);
615 assert_eq!(manifest.dropped["meters.csv"], vec!["note".to_owned()]);
616 assert_eq!(manifest.meta["window"], "24h");
617 crate::verify(&out).unwrap_or_else(|error| panic!("the bundle verifies: {error}"));
618 }
619
620 #[test]
621 fn two_bundles_of_the_same_bytes_are_equal_whenever_they_were_made() {
622 let home = unique_directory();
623 fs::create_dir_all(&home).unwrap();
624 fs::write(home.join("meters.csv"), "id,watts\n1,60\n").unwrap();
625 let spec = spec(home.join("meters.csv").to_str().unwrap(), &["id", "watts"]);
626
627 let morning = bundle(
628 std::slice::from_ref(&spec),
629 &home.join("morning"),
630 BTreeMap::new(),
631 "2026-01-01T00:00:00Z",
632 &[],
633 )
634 .unwrap();
635 let a_month_later = bundle(
636 &[spec],
637 &home.join("later"),
638 BTreeMap::new(),
639 "2026-02-01T23:59:59Z",
640 &[],
641 )
642 .unwrap();
643
644 assert_eq!(morning, a_month_later);
645 }
646
647 #[test]
648 fn a_bundle_whose_file_changed_is_not_equal_to_one_made_before_the_change() {
649 let home = unique_directory();
650 fs::create_dir_all(&home).unwrap();
651 fs::write(home.join("meters.csv"), "id,watts\n1,60\n").unwrap();
652 let spec = spec(home.join("meters.csv").to_str().unwrap(), &["id", "watts"]);
653
654 let before = bundle(
655 std::slice::from_ref(&spec),
656 &home.join("before"),
657 BTreeMap::new(),
658 "x",
659 &[],
660 )
661 .unwrap();
662 fs::write(home.join("meters.csv"), "id,watts\n1,61\n").unwrap();
663 let after = bundle(&[spec], &home.join("after"), BTreeMap::new(), "x", &[]).unwrap();
664
665 assert_ne!(before, after);
666 }
667
668 #[test]
669 fn two_inputs_from_one_directory_are_both_copied() {
670 let home = unique_directory();
671 fs::create_dir_all(&home).unwrap();
672 fs::write(home.join("meters.csv"), "id,watts\n1,60\n").unwrap();
673 fs::write(home.join("sites.csv"), "site,kw\nroof,4\n").unwrap();
674 let out = home.join("bundle");
675
676 let manifest = bundle(
677 &[
678 spec(home.join("meters.csv").to_str().unwrap(), &["id"]),
679 spec(home.join("sites.csv").to_str().unwrap(), &["site"]),
680 ],
681 &out,
682 BTreeMap::new(),
683 "2026-01-01T00:00:00Z",
684 &[],
685 )
686 .unwrap();
687
688 let names: Vec<&str> = manifest
689 .files
690 .iter()
691 .map(|entry| entry.name.as_str())
692 .collect();
693 assert_eq!(names, vec!["meters.csv", "sites.csv"]);
694 }
695
696 #[test]
697 fn two_inputs_that_would_land_under_one_name_are_refused() {
698 let home = unique_directory();
699 fs::create_dir_all(home.join("one")).unwrap();
700 fs::create_dir_all(home.join("two")).unwrap();
701 fs::write(home.join("one/meters.csv"), "id\n1\n").unwrap();
702 fs::write(home.join("two/meters.csv"), "id\n2\n").unwrap();
703 let out = home.join("bundle");
704
705 let error = bundle(
706 &[
707 spec(home.join("one/meters.csv").to_str().unwrap(), &["id"]),
708 spec(home.join("two/meters.csv").to_str().unwrap(), &["id"]),
709 ],
710 &out,
711 BTreeMap::new(),
712 "2026-01-01T00:00:00Z",
713 &[],
714 )
715 .expect_err("two files would share a name");
716
717 assert!(
718 matches!(error, BundleError::DuplicateFile { ref name } if name == "meters.csv"),
719 "got {error}"
720 );
721 }
722}