1use std::path::{Path, PathBuf};
12
13use sha2::{Digest, Sha256};
14
15use crate::checksum::migration_file_checksum;
16use crate::error::{MigrationError, Result};
17use crate::spec::{MigrationGraph, MigrationSpec};
18
19pub fn load_sidecar(py_path: &Path) -> Result<Option<MigrationSpec>> {
34 let json_path = py_path.with_extension("json");
35 if !json_path.exists() {
36 return Ok(None);
37 }
38 let content = std::fs::read_to_string(&json_path).map_err(|err| MigrationError::Loader {
39 message: format!("failed to read sidecar {}: {err}", json_path.display()),
40 })?;
41 let spec: MigrationSpec =
42 serde_json::from_str(&content).map_err(|err| MigrationError::Loader {
43 message: format!("failed to parse sidecar {}: {err}", json_path.display()),
44 })?;
45 Ok(Some(spec))
46}
47
48pub fn load_dir(dir: &Path) -> Result<MigrationGraph> {
66 let read_dir = std::fs::read_dir(dir).map_err(|err| MigrationError::Loader {
67 message: format!("failed to read migrations dir {}: {err}", dir.display()),
68 })?;
69
70 let mut entries: Vec<(String, PathBuf)> = Vec::new();
71 for entry in read_dir {
72 let entry = entry.map_err(|err| MigrationError::Loader {
73 message: format!("failed to iterate migrations dir {}: {err}", dir.display()),
74 })?;
75 let path = entry.path();
76
77 let file_name = path.file_name().and_then(|name| name.to_str());
78 if file_name.is_some_and(|name| name.ends_with(".adoption.json")) {
80 continue;
81 }
82 if path.extension().and_then(|e| e.to_str()) != Some("json") {
84 continue;
85 }
86
87 let stem = match path.file_stem().and_then(|s| s.to_str()) {
89 Some(s) => s.to_owned(),
90 None => continue,
91 };
92
93 if !is_migration_stem(&stem) {
94 continue;
95 }
96
97 entries.push((stem, path));
98 }
99
100 entries.sort_by(|(a, _), (b, _)| a.cmp(b));
102
103 let mut migrations = Vec::with_capacity(entries.len());
104 for (stem, path) in entries {
105 let content = std::fs::read_to_string(&path).map_err(|err| MigrationError::Loader {
106 message: format!("failed to read sidecar {}: {err}", path.display()),
107 })?;
108 let spec: MigrationSpec =
109 serde_json::from_str(&content).map_err(|err| MigrationError::Loader {
110 message: format!(
111 "failed to parse sidecar {} (stem={stem}): {err}",
112 path.display()
113 ),
114 })?;
115 migrations.push(spec);
116 }
117
118 Ok(MigrationGraph { migrations })
119}
120
121pub fn load_dir_checked(dir: &Path) -> Result<MigrationGraph> {
146 let read_dir = std::fs::read_dir(dir).map_err(|err| MigrationError::Loader {
147 message: format!("failed to read migrations dir {}: {err}", dir.display()),
148 })?;
149
150 let mut entries: Vec<(String, PathBuf)> = Vec::new();
151 let mut python_stems = std::collections::BTreeSet::new();
152 for entry in read_dir {
153 let entry = entry.map_err(|err| MigrationError::Loader {
154 message: format!("failed to iterate migrations dir {}: {err}", dir.display()),
155 })?;
156 let path = entry.path();
157
158 let file_name = path.file_name().and_then(|name| name.to_str());
159 if file_name.is_some_and(|name| name.ends_with(".adoption.json")) {
160 continue;
161 }
162 let extension = path.extension().and_then(|e| e.to_str());
163 let stem = match path.file_stem().and_then(|s| s.to_str()) {
164 Some(s) => s.to_owned(),
165 None => continue,
166 };
167 if !is_migration_stem(&stem) {
168 continue;
169 }
170 match extension {
171 Some("json") => entries.push((stem, path)),
172 Some("py") => {
173 python_stems.insert(stem);
174 }
175 _ => {}
176 }
177 }
178
179 entries.sort_by(|(a, _), (b, _)| a.cmp(b));
180
181 let sidecar_stems: std::collections::BTreeSet<&str> =
187 entries.iter().map(|(stem, _)| stem.as_str()).collect();
188 let orphans: Vec<String> = python_stems
189 .iter()
190 .filter(|stem| !sidecar_stems.contains(stem.as_str()))
191 .cloned()
192 .collect();
193 if !orphans.is_empty() {
194 return Err(MigrationError::Loader {
195 message: format!(
196 "Python-only migrations without JSON sidecars in {}: {}. \
197 The native loader cannot execute a dynamically imported \
198 migration; each listed file needs a JSON sidecar recording \
199 its checked execution spec before the native migration \
200 path can use this history. Generate the sidecars with \
201 `python -m type_bridge.migration.sidecar <migrations-dir>`.",
202 dir.display(),
203 orphans.join(", ")
204 ),
205 });
206 }
207
208 let mut migrations = Vec::with_capacity(entries.len());
209 for (stem, json_path) in entries {
210 let content =
211 std::fs::read_to_string(&json_path).map_err(|err| MigrationError::Loader {
212 message: format!("failed to read sidecar {}: {err}", json_path.display()),
213 })?;
214 let spec: MigrationSpec =
215 serde_json::from_str(&content).map_err(|err| MigrationError::Loader {
216 message: format!(
217 "failed to parse sidecar {} (stem={stem}): {err}",
218 json_path.display()
219 ),
220 })?;
221
222 let py_path = json_path.with_extension("py");
223 let has_python_source = python_stems.contains(&stem);
224
225 if let Some(sidecar_sha256) = &spec.source_sha256 {
230 if !is_lower_hex_sha256(sidecar_sha256) {
231 return Err(MigrationError::Loader {
232 message: format!(
233 "sidecar for {stem} carries a malformed source_sha256; \
234 expected exactly 64 lowercase hexadecimal characters"
235 ),
236 });
237 }
238 if has_python_source {
239 let raw = read_python_bytes(&py_path)?;
240 let computed = format!("{:x}", Sha256::digest(&raw));
241 if computed != *sidecar_sha256 {
242 return Err(MigrationError::Loader {
243 message: format!(
244 "sidecar drift detected for {stem}: the raw .py file has been \
245 modified since the sidecar was generated \
246 (sidecar source_sha256={sidecar_sha256}, \
247 current .py source_sha256={computed}). \
248 Regenerate the migration to sync the sidecar."
249 ),
250 });
251 }
252 }
253 } else if let Some(sidecar_checksum) = &spec.checksum {
254 if has_python_source {
257 let py_text = read_python_text(&py_path)?;
258 let computed = migration_file_checksum(&py_text);
259 if computed != *sidecar_checksum {
260 return Err(MigrationError::Loader {
261 message: format!(
262 "sidecar drift detected for {stem}: the .py file has been \
263 modified since the sidecar was generated \
264 (sidecar checksum={sidecar_checksum}, \
265 current .py checksum={computed}). \
266 Regenerate the migration to sync the sidecar."
267 ),
268 });
269 }
270 }
271 }
272
273 migrations.push(spec);
274 }
275
276 Ok(MigrationGraph { migrations })
277}
278
279fn read_python_text(py_path: &Path) -> Result<String> {
284 let raw = std::fs::read_to_string(py_path).map_err(|err| MigrationError::Loader {
285 message: format!(
286 "failed to read .py for drift check {}: {err}",
287 py_path.display()
288 ),
289 })?;
290 if !raw.contains('\r') {
291 return Ok(raw);
292 }
293 Ok(raw.replace("\r\n", "\n").replace('\r', "\n"))
294}
295
296fn read_python_bytes(py_path: &Path) -> Result<Vec<u8>> {
297 std::fs::read(py_path).map_err(|err| MigrationError::Loader {
298 message: format!(
299 "failed to read .py for raw-source drift check {}: {err}",
300 py_path.display()
301 ),
302 })
303}
304
305fn is_lower_hex_sha256(value: &str) -> bool {
306 value.len() == 64
307 && value
308 .bytes()
309 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
310}
311
312pub(crate) fn open_regular_readonly_nofollow(
318 directory: &cap_std::fs::Dir,
319 name: &std::ffi::OsStr,
320) -> std::io::Result<cap_std::fs::File> {
321 use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt as _};
322
323 let mut options = cap_std::fs::OpenOptions::new();
324 options.read(true).follow(FollowSymlinks::No);
325 let file = directory.open_with(name, &options)?;
326
327 if !file.metadata()?.is_file() {
328 return Err(std::io::Error::new(
329 std::io::ErrorKind::InvalidInput,
330 "migration artifact is not a regular file",
331 ));
332 }
333 Ok(file)
334}
335
336fn is_migration_stem(stem: &str) -> bool {
339 let bytes = stem.as_bytes();
340 if bytes.len() < 5 {
341 return false;
342 }
343 bytes[..4].iter().all(|b| b.is_ascii_digit()) && bytes[4] == b'_'
344}
345
346#[cfg(test)]
347mod tests {
348 use super::*;
349 use crate::spec::{MigrationSpec, OperationSpec};
350
351 fn make_spec(name: &str) -> MigrationSpec {
353 MigrationSpec {
354 app_label: "test_app".to_string(),
355 name: name.to_string(),
356 dependencies: vec![],
357 operations: vec![OperationSpec::RunTypeql {
358 forward: format!("define attribute {name}, value string;"),
359 reverse: None,
360 }],
361 checksum: Some("abc123".to_string()),
362 source_sha256: None,
363 reversible: false,
364 }
365 }
366
367 fn write_sidecar(dir: &Path, stem: &str, spec: &MigrationSpec) {
369 let json = serde_json::to_string(spec).unwrap();
370 std::fs::write(dir.join(format!("{stem}.json")), json).unwrap();
371 }
372
373 fn write_py(dir: &Path, stem: &str) {
375 std::fs::write(dir.join(format!("{stem}.py")), b"class Migration: pass\n").unwrap();
376 }
377
378 #[test]
381 fn load_sidecar_returns_some_for_valid_json() {
382 let tmp = tempfile::tempdir().unwrap();
383 let spec = make_spec("0001_initial");
384 write_sidecar(tmp.path(), "0001_initial", &spec);
385 write_py(tmp.path(), "0001_initial");
386
387 let py_path = tmp.path().join("0001_initial.py");
388 let result = load_sidecar(&py_path).unwrap();
389
390 assert_eq!(result, Some(spec));
391 }
392
393 #[test]
394 fn load_sidecar_returns_none_when_no_json_sibling() {
395 let tmp = tempfile::tempdir().unwrap();
396 write_py(tmp.path(), "0001_initial");
397
398 let py_path = tmp.path().join("0001_initial.py");
399 let result = load_sidecar(&py_path).unwrap();
400
401 assert_eq!(result, None);
402 }
403
404 #[test]
405 fn load_sidecar_returns_error_on_malformed_json() {
406 let tmp = tempfile::tempdir().unwrap();
407 std::fs::write(tmp.path().join("0001_initial.json"), b"{ not valid json }").unwrap();
408 write_py(tmp.path(), "0001_initial");
409
410 let py_path = tmp.path().join("0001_initial.py");
411 let result = load_sidecar(&py_path);
412
413 assert!(result.is_err(), "expected Err on malformed JSON");
414 let err = result.unwrap_err();
415 assert!(
416 matches!(err, MigrationError::Loader { .. }),
417 "expected MigrationError::Loader, got: {err:?}"
418 );
419 }
420
421 #[test]
424 fn load_dir_loads_sidecars_and_skips_bare_py() {
425 let tmp = tempfile::tempdir().unwrap();
426
427 let spec1 = make_spec("0001_initial");
429 let spec2 = make_spec("0002_add_attr");
430 write_sidecar(tmp.path(), "0001_initial", &spec1);
431 write_py(tmp.path(), "0001_initial");
432 write_sidecar(tmp.path(), "0002_add_attr", &spec2);
433 write_py(tmp.path(), "0002_add_attr");
434
435 write_py(tmp.path(), "0003_legacy");
437
438 let graph = load_dir(tmp.path()).unwrap();
439
440 assert_eq!(
441 graph.migrations.len(),
442 2,
443 "expected exactly two specs from the two sidecars"
444 );
445 assert_eq!(
446 graph.migrations[0], spec1,
447 "first spec should be 0001_initial"
448 );
449 assert_eq!(
450 graph.migrations[1], spec2,
451 "second spec should be 0002_add_attr"
452 );
453 }
454
455 #[test]
456 fn load_dir_sorts_by_stem() {
457 let tmp = tempfile::tempdir().unwrap();
458
459 let spec2 = make_spec("0002_b");
461 let spec1 = make_spec("0001_a");
462 write_sidecar(tmp.path(), "0002_b", &spec2);
463 write_sidecar(tmp.path(), "0001_a", &spec1);
464
465 let graph = load_dir(tmp.path()).unwrap();
466
467 assert_eq!(graph.migrations.len(), 2);
468 assert_eq!(graph.migrations[0].name, "0001_a");
469 assert_eq!(graph.migrations[1].name, "0002_b");
470 }
471
472 #[test]
473 fn sidecar_loaders_recognize_minimal_released_migration_stem() {
474 let tmp = tempfile::tempdir().unwrap();
475 let mut spec = make_spec("0001_");
476 spec.checksum = None;
477 write_sidecar(tmp.path(), "0001_", &spec);
478 write_py(tmp.path(), "0001_");
479
480 assert_eq!(load_dir(tmp.path()).unwrap().migrations, vec![spec.clone()]);
481 assert_eq!(load_dir_checked(tmp.path()).unwrap().migrations, vec![spec]);
482 }
483
484 #[test]
485 fn load_dir_integration_smoke_sidecar_and_no_sidecar() {
486 let tmp = tempfile::tempdir().unwrap();
491
492 let spec = make_spec("0001_initial");
493 write_sidecar(tmp.path(), "0001_initial", &spec);
494 write_py(tmp.path(), "0001_initial");
495
496 write_py(tmp.path(), "0002_legacy");
498
499 let graph = load_dir(tmp.path()).unwrap();
500
501 assert_eq!(
502 graph.migrations.len(),
503 1,
504 "load_dir must load only JSON sidecars; the bare .py must not appear"
505 );
506 assert_eq!(graph.migrations[0], spec);
507 }
508
509 fn make_spec_with_real_checksum(name: &str, py_text: &str) -> MigrationSpec {
514 use crate::checksum::migration_file_checksum;
515 MigrationSpec {
516 app_label: "test_app".to_string(),
517 name: name.to_string(),
518 dependencies: vec![],
519 operations: vec![OperationSpec::RunTypeql {
520 forward: format!("define attribute {name}, value string;"),
521 reverse: None,
522 }],
523 checksum: Some(migration_file_checksum(py_text)),
524 source_sha256: None,
525 reversible: false,
526 }
527 }
528
529 #[test]
530 fn load_dir_checked_accepts_matching_checksum() {
531 let tmp = tempfile::tempdir().unwrap();
534 let py_text = "class Migration: pass\n";
535
536 let spec = make_spec_with_real_checksum("0001_initial", py_text);
537 write_sidecar(tmp.path(), "0001_initial", &spec);
538 std::fs::write(tmp.path().join("0001_initial.py"), py_text.as_bytes()).unwrap();
539
540 let graph = load_dir_checked(tmp.path()).unwrap();
541 assert_eq!(graph.migrations.len(), 1);
542 assert_eq!(graph.migrations[0].name, "0001_initial");
543 }
544
545 #[test]
546 fn load_dir_checked_accepts_crlf_checked_out_python() {
547 let tmp = tempfile::tempdir().unwrap();
552 let lf_text = "class Migration: pass\n# checked out with CRLF\n";
553 let crlf_text = lf_text.replace('\n', "\r\n");
554
555 let spec = make_spec_with_real_checksum("0001_initial", lf_text);
556 write_sidecar(tmp.path(), "0001_initial", &spec);
557 std::fs::write(tmp.path().join("0001_initial.py"), crlf_text.as_bytes()).unwrap();
558
559 let graph = load_dir_checked(tmp.path()).unwrap();
560 assert_eq!(graph.migrations.len(), 1);
561 }
562
563 #[test]
564 fn load_dir_checked_raw_digest_supersedes_legacy_text_checksum() {
565 let tmp = tempfile::tempdir().unwrap();
566 let py_bytes = b"# -*- coding: cp1252 -*-\nlabel = '\xe9'\n";
567 let mut spec = make_spec("0001_encoded");
568 spec.checksum = Some("0000000000000000".to_string());
571 spec.source_sha256 = Some(format!("{:x}", Sha256::digest(py_bytes)));
572 write_sidecar(tmp.path(), "0001_encoded", &spec);
573 std::fs::write(tmp.path().join("0001_encoded.py"), py_bytes).unwrap();
574
575 let graph = load_dir_checked(tmp.path()).unwrap();
576 assert_eq!(graph.migrations, vec![spec]);
577 }
578
579 #[test]
580 fn load_dir_checked_rejects_raw_source_drift_before_legacy_checksum() {
581 let tmp = tempfile::tempdir().unwrap();
582 let original = b"class Migration: pass\n";
583 let current = b"class Migration: pass\n# changed\n";
584 let mut spec =
585 make_spec_with_real_checksum("0001_raw_drift", std::str::from_utf8(current).unwrap());
586 spec.source_sha256 = Some(format!("{:x}", Sha256::digest(original)));
587 write_sidecar(tmp.path(), "0001_raw_drift", &spec);
588 std::fs::write(tmp.path().join("0001_raw_drift.py"), current).unwrap();
589
590 let error = load_dir_checked(tmp.path()).unwrap_err().to_string();
591 assert!(error.contains("source_sha256"), "unexpected error: {error}");
592 }
593
594 #[test]
595 fn load_dir_checked_rejects_malformed_raw_source_digest() {
596 let tmp = tempfile::tempdir().unwrap();
597 let mut spec = make_spec("0001_malformed");
598 spec.source_sha256 = Some("ABC".to_string());
599 write_sidecar(tmp.path(), "0001_malformed", &spec);
600
601 let error = load_dir_checked(tmp.path()).unwrap_err().to_string();
602 assert!(
603 error.contains("malformed source_sha256"),
604 "unexpected error: {error}"
605 );
606 }
607
608 #[test]
609 fn load_dir_checked_rejects_python_only_migrations() {
610 let tmp = tempfile::tempdir().unwrap();
614 let py_text = "class Migration: pass\n";
615 let spec = make_spec_with_real_checksum("0001_initial", py_text);
616 write_sidecar(tmp.path(), "0001_initial", &spec);
617 std::fs::write(tmp.path().join("0001_initial.py"), py_text.as_bytes()).unwrap();
618 write_py(tmp.path(), "0002_custom");
619
620 let err = load_dir_checked(tmp.path()).unwrap_err();
621 let message = err.to_string();
622 assert!(
623 message.contains("0002_custom") && message.contains("sidecar"),
624 "error must name the orphan and the sidecar requirement; got: {message}"
625 );
626 }
627
628 #[test]
629 fn load_dir_checked_rejects_stale_sidecar() {
630 let tmp = tempfile::tempdir().unwrap();
634 let original_py_text = "class Migration: pass\n";
635 let mutated_py_text = "class Migration: pass\n# hand-edited after sidecar generation\n";
636
637 let spec = make_spec_with_real_checksum("0001_initial", original_py_text);
639 write_sidecar(tmp.path(), "0001_initial", &spec);
640
641 std::fs::write(
643 tmp.path().join("0001_initial.py"),
644 mutated_py_text.as_bytes(),
645 )
646 .unwrap();
647
648 let result = load_dir_checked(tmp.path());
649 assert!(
650 result.is_err(),
651 "load_dir_checked must reject a stale sidecar"
652 );
653 let err = result.unwrap_err();
654 assert!(
655 matches!(err, MigrationError::Loader { .. }),
656 "expected MigrationError::Loader, got {err:?}"
657 );
658 let msg = err.to_string();
660 assert!(
661 msg.contains("sidecar drift") || msg.contains("regenerate"),
662 "error message should mention sidecar drift or regenerate; got: {msg}"
663 );
664 }
665
666 #[test]
667 fn load_dir_checked_skips_drift_check_when_no_py_file() {
668 let tmp = tempfile::tempdir().unwrap();
671
672 let spec = make_spec("0001_initial");
674 write_sidecar(tmp.path(), "0001_initial", &spec);
675 let graph = load_dir_checked(tmp.path()).unwrap();
678 assert_eq!(
679 graph.migrations.len(),
680 1,
681 "sidecar without .py companion must still be loaded"
682 );
683 }
684
685 #[test]
686 fn load_dir_checked_skips_drift_check_when_no_checksum_in_sidecar() {
687 let tmp = tempfile::tempdir().unwrap();
690 let py_text = "class Migration: pass\n";
691
692 let mut spec = make_spec("0001_initial");
693 spec.checksum = None; write_sidecar(tmp.path(), "0001_initial", &spec);
695 std::fs::write(tmp.path().join("0001_initial.py"), py_text.as_bytes()).unwrap();
696
697 let graph = load_dir_checked(tmp.path()).unwrap();
698 assert_eq!(graph.migrations.len(), 1);
699 }
700
701 #[test]
702 fn executable_loader_ignores_non_executable_adoption_archives() {
703 let tmp = tempfile::tempdir().unwrap();
704 std::fs::write(
705 tmp.path().join("0001_backfill.adoption.json"),
706 br#"{"format":"typebridge.migration-adoption-metadata/v1"}"#,
707 )
708 .unwrap();
709 assert!(load_dir(tmp.path()).unwrap().migrations.is_empty());
710 }
711
712 #[test]
713 fn released_sidecar_loaders_accept_valid_artifact_larger_than_16_mib() {
714 let tmp = tempfile::tempdir().unwrap();
715 let mut spec = make_spec("0001_large");
716 spec.checksum = None;
717 let mut bytes = serde_json::to_vec(&spec).unwrap();
718 bytes.resize(16 * 1024 * 1024 + 1, b' ');
719 std::fs::write(tmp.path().join("0001_large.json"), bytes).unwrap();
720 write_py(tmp.path(), "0001_large");
721
722 assert_eq!(
723 load_sidecar(&tmp.path().join("0001_large.py")).unwrap(),
724 Some(spec.clone())
725 );
726 assert_eq!(load_dir(tmp.path()).unwrap().migrations, vec![spec.clone()]);
727 assert_eq!(load_dir_checked(tmp.path()).unwrap().migrations, vec![spec]);
728 }
729
730 #[test]
731 fn released_sidecar_loaders_ignore_more_than_65536_unrelated_entries() {
732 let tmp = tempfile::tempdir().unwrap();
733 let seeds = (0..128)
736 .map(|index| tmp.path().join(format!("unrelated-seed-{index:03}.txt")))
737 .collect::<Vec<_>>();
738 for seed in &seeds {
739 std::fs::write(seed, b"not a migration").unwrap();
740 }
741 for index in 0..65_537 {
742 std::fs::hard_link(
743 &seeds[index % seeds.len()],
744 tmp.path().join(format!("unrelated-{index:05}.txt")),
745 )
746 .unwrap();
747 }
748 let mut spec = make_spec("0001_initial");
749 spec.checksum = None;
750 write_sidecar(tmp.path(), "0001_initial", &spec);
751 write_py(tmp.path(), "0001_initial");
752
753 assert_eq!(load_dir(tmp.path()).unwrap().migrations, vec![spec.clone()]);
754 assert_eq!(load_dir_checked(tmp.path()).unwrap().migrations, vec![spec]);
755 }
756
757 #[cfg(unix)]
758 #[test]
759 fn released_sidecar_loaders_follow_regular_file_symlinks() {
760 use std::os::unix::fs::symlink;
761
762 let tmp = tempfile::tempdir().unwrap();
763 let py_text = "class Migration: pass\n";
764 let spec = make_spec_with_real_checksum("0001_linked", py_text);
765 std::fs::write(tmp.path().join("source.py"), py_text).unwrap();
766 std::fs::write(
767 tmp.path().join("source.json"),
768 serde_json::to_vec(&spec).unwrap(),
769 )
770 .unwrap();
771 symlink("source.py", tmp.path().join("0001_linked.py")).unwrap();
772 symlink("source.json", tmp.path().join("0001_linked.json")).unwrap();
773
774 assert_eq!(
775 load_sidecar(&tmp.path().join("0001_linked.py")).unwrap(),
776 Some(spec.clone())
777 );
778 assert_eq!(load_dir(tmp.path()).unwrap().migrations, vec![spec.clone()]);
779 assert_eq!(load_dir_checked(tmp.path()).unwrap().migrations, vec![spec]);
780 }
781}