1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
//! `delete-branch`, `protect`, `unprotect` subcommands.
//!
//! Each operation is anchored at `<prefix>/refs/heads/<branch>/`, the same
//! key space the protocol REPL writes bundles into. When the URL has no
//! repository prefix (root-of-bucket repos, `<prefix>` is empty), keys
//! collapse to `refs/heads/<branch>/...` with no leading slash.
//!
//! All operator-visible output goes through a `Write`-bound writer (the
//! `*_into` entry points) so tests can capture and assert on the
//! messages. The public `delete()`, `protect()`, `unprotect()` methods
//! wrap their `*_into` siblings with `std::io::stdout()` for the
//! management CLI.
use std::collections::HashSet;
use std::io::Write;
use std::sync::Arc;
use bytes::Bytes;
use time::OffsetDateTime;
use tracing::{info, warn};
use super::{ManageError, Prompter};
use crate::git::RefName;
use crate::keys;
use crate::object_store::{ObjectMeta, ObjectStore, ObjectStoreError, PutOpts};
use crate::packchain::gc::try_write_baseline_tombstone;
use crate::protocol::push::{
LockGuard, acquire_lock, lock_key, lock_ttl_from_env, release_lock,
verify_no_orphan_protected_after_delete,
};
/// Operations on a single branch within a repository.
pub struct ManageBranch<'a> {
store: Arc<dyn ObjectStore>,
prefix: String,
branch: String,
prompter: &'a dyn Prompter,
}
impl<'a> ManageBranch<'a> {
/// Open a branch handle, verifying it exists by listing
/// `<prefix>/refs/heads/<branch>/` (or `refs/heads/<branch>/` when
/// `prefix` is empty).
///
/// # Errors
///
/// Returns [`ManageError::InvalidBranch`] if `branch` fails
/// `gix-validate`'s strict ref-name check. Returns
/// [`ManageError::BranchNotFound`] when no objects exist under the
/// branch prefix. Returns [`ManageError::Store`] for object-store
/// failures.
pub async fn open(
store: Arc<dyn ObjectStore>,
prefix: impl Into<String>,
branch: impl Into<String>,
prompter: &'a dyn Prompter,
) -> Result<Self, ManageError> {
let branch = branch.into();
// Reject branch names that git itself would reject. S3 / Azure
// are case-sensitive byte stores with no path semantics, so a
// value like `foo/../bar` would be stored verbatim and produce
// unrecoverable junk under `<prefix>/refs/heads/`. The strict
// `RefName::is_valid` (delegating to `gix_validate::reference::name`)
// rejects empties, `..`, control characters, and the rest of
// git's invalid-ref alphabet. Use the borrow-only predicate
// here so we don't allocate a wrapped `RefName` we'd discard.
if !RefName::is_valid(&format!("refs/heads/{branch}")) {
return Err(ManageError::InvalidBranch(branch));
}
let mb = Self {
store,
prefix: prefix.into(),
branch,
prompter,
};
if mb.store.list(&mb.branch_prefix()).await?.is_empty() {
return Err(ManageError::BranchNotFound(mb.branch));
}
Ok(mb)
}
fn branch_prefix(&self) -> String {
keys::ref_listing_prefix(Some(&self.prefix), &format!("refs/heads/{}", self.branch))
}
fn protected_key(&self) -> String {
keys::join(
Some(&self.prefix),
&format!(
"refs/heads/{}/{}",
self.branch,
keys::PROTECTED_MARKER_SEGMENT,
),
)
}
/// Delete every object under the branch's prefix after a `yes/no`
/// confirmation. Aborts (returns `Ok(())`) if the user answers no;
/// the `Cancelled` variant is reserved for prompt I/O failures.
///
/// Refuses outright when a `PROTECTED#` marker is present under the
/// branch prefix — the operator must run `unprotect` first. This
/// mirrors the refusal the helper-protocol delete path
/// (`delete_remote_ref_under_lock`) emits, so a `git push :branch`
/// against a protected ref and a management-CLI `delete-branch` of
/// the same ref fail the same way.
///
/// # Per-ref lock (#158)
///
/// After the operator confirms the prompt, `delete-branch` acquires
/// the same `<prefix>/<ref>/LOCK#.lock` the helper-protocol push
/// and delete paths take. The lock is held across the fresh re-list,
/// the baseline tombstone write (#143), and the synchronous sweep.
/// Without it a concurrent `git push` could land a new bundle after
/// the post-prompt re-list, the sweep would delete only the stale
/// snapshot, and the ref would survive with the just-pushed bundle
/// even though delete-branch reported success.
///
/// Lock acquisition runs AFTER the prompt — the prompt is
/// interactive and could block indefinitely, and holding the lock
/// across user input would make every other writer wait on the
/// operator's keyboard. If the lock is contended at acquisition
/// time the function returns [`ManageError::LockContended`] and
/// makes no changes. Release failures are downgraded to a `warn!`
/// because the lock's TTL guarantees a stale lock is recovered by
/// the next acquirer; matches the protocol-push pattern.
///
/// The prompt-display and protection-marker check use a first listing
/// for accuracy of the displayed object count, then a **second
/// listing is taken under the lock immediately before the deletion
/// loop**. The fresh listing drives the sweep so that any concurrent
/// push landing under the branch prefix during the prompt window —
/// before the lock window opens — is caught and deleted rather than
/// left as a zombie object (#139). The protection-marker check is
/// re-evaluated on the fresh listing so a `protect` racing with the
/// prompt is honoured (#131) — the post-prompt re-check is what
/// closes the TOCTOU window between the initial marker check and
/// the deletion loop. If the fresh listing is empty (a concurrent
/// delete won the race) the function reports it and returns
/// `Ok(())` rather than silently claiming success.
///
/// `NotFound` errors observed during the sweep are tolerated — they
/// mean a concurrent deleter swept the key first, which still
/// satisfies the operator's intent. Other per-key delete errors
/// (Network, `AccessDenied`, ...) are collected: the loop does NOT
/// short-circuit, every remaining key is still attempted, and the
/// function returns [`ManageError::PartialDelete`] with the exact
/// list of keys that survived so a retry can converge (#122). A
/// list-call failure still propagates immediately because there is
/// nothing to recover — without a listing the sweep cannot proceed.
///
/// Packchain refs with a parseable `chain.json` skip immediate
/// deletion of the baseline bundle (`<full_at>.bundle`): a
/// baseline tombstone is written first and the bundle is left for
/// `gc sweep` to reclaim after the grace window (#143). The
/// synchronous sweep still removes `chain.json`,
/// `path-index.json`, and any other residue. The deferral protects
/// an in-flight fetcher that already read the prior `chain.json`
/// from a `BaselineMissing` range-GET failure; a fresh reader
/// sees the missing chain and the ref is gone from its
/// perspective. Bundle-engine refs, refs with an unparseable
/// chain, and any tombstone PUT failure fall through to immediate
/// bundle deletion so the operator's "ref is gone" intent is
/// never blocked on the tombstone path.
///
/// # Errors
///
/// Returns [`ManageError::Protected`] if the branch carries a
/// `PROTECTED#` marker (checked on both listings),
/// [`ManageError::LockContended`] if another writer holds the
/// per-ref lock at acquisition time,
/// [`ManageError::Cancelled`] if the user cancels the prompt,
/// [`ManageError::Io`] for prompt or write I/O failures,
/// [`ManageError::Store`] if a list operation fails, or
/// [`ManageError::PartialDelete`] when one or more per-key deletes
/// fail with a non-`NotFound` error after every key in the fresh
/// listing has been attempted.
pub async fn delete(&self) -> Result<(), ManageError> {
self.delete_into(&mut std::io::stdout()).await
}
/// Same contract as [`delete`](Self::delete) but writes human-readable
/// output to `out`. Tests use this to capture the operator messages
/// (e.g. the "already gone" race notice from #139) so a regression
/// that drops the message — silently turning a concurrent race into
/// an apparent success — is caught.
///
/// # Errors
///
/// Same as [`delete`](Self::delete), plus [`ManageError::Io`] if a
/// write to `out` fails.
pub(crate) async fn delete_into<W: Write>(&self, out: &mut W) -> Result<(), ManageError> {
let listing_prefix = self.branch_prefix();
let initial = self.store.list(&listing_prefix).await?;
if keys::entries_have_protected_marker(&initial) {
return Err(ManageError::Protected(self.branch.clone()));
}
let prompt = format!("Delete branch {} ({} objects)?", self.branch, initial.len());
if !self.prompter.confirm(&prompt)? {
writeln!(out, "Aborted")?;
return Ok(());
}
// Acquire the per-ref lock AFTER the prompt and BEFORE the
// fresh re-list / tombstone / sweep. Holding the lock across
// the prompt would block every concurrent writer on the
// operator's keyboard; the lock window starts only once the
// operator has confirmed the intent (#158). The protocol push
// / delete paths and `packchain::compact` use the same lock
// key, so a concurrent `git push` or `compact` racing this
// delete is mutually excluded.
let ref_name = self.validated_ref_name()?;
let (lock_key, guard) = self.acquire_ref_lock("delete-branch").await?;
let work = self
.delete_under_lock(out, &listing_prefix, &lock_key, &initial, &ref_name)
.await;
self.release_or_warn(guard, &lock_key, "delete-branch")
.await;
work
}
/// The lock-held body of [`Self::delete_into`]: fresh re-list,
/// protection re-check, tombstone write, sweep. Extracted so the
/// caller's `release_lock` runs unconditionally on every exit
/// path. The lock key is filtered from the fresh listing so the
/// sweep does not delete the very lock we hold.
async fn delete_under_lock<W: Write>(
&self,
out: &mut W,
listing_prefix: &str,
lock: &str,
initial: &[ObjectMeta],
ref_name: &RefName,
) -> Result<(), ManageError> {
// Re-list under the lock so concurrent pushes that landed
// during the prompt window — before the lock window opened —
// are included in the deletion set. With the lock now held,
// no further writes can sneak in between this listing and the
// sweep. Filter out the lock key itself: we hold it and the
// release tail removes it; sweeping it mid-critical-section
// would let another acquirer take the lock under us.
let fresh: Vec<ObjectMeta> = self
.store
.list(listing_prefix)
.await?
.into_iter()
.filter(|m| m.key != lock)
.collect();
if fresh.is_empty() {
writeln!(
out,
"Branch {} is already gone (concurrent delete during prompt); nothing to do",
self.branch,
)?;
info!(
branch = %self.branch,
"branch already deleted by concurrent operation",
);
return Ok(());
}
if keys::entries_have_protected_marker(&fresh) {
return Err(ManageError::Protected(self.branch.clone()));
}
let initial_keys: HashSet<&str> = initial.iter().map(|m| m.key.as_str()).collect();
let concurrent_adds = fresh
.iter()
.filter(|m| !initial_keys.contains(m.key.as_str()))
.count();
if concurrent_adds > 0 {
warn!(
branch = %self.branch,
added = concurrent_adds,
"concurrent activity detected during prompt; sweeping fresh listing",
);
}
// Issue #143: if the ref is a packchain ref with a parseable
// `chain.json`, write a baseline tombstone naming the current
// `full_at` bundle BEFORE the synchronous sweep, then exclude
// that bundle key from the delete loop. A concurrent fetcher
// that read the prior `chain.json` (t₀) and is mid-range-GET
// on `<full_at>.bundle` then completes against the still-live
// bundle; `gc sweep` reclaims it after the grace window. The
// synchronous sweep still removes `chain.json`,
// `path-index.json`, and every other key — from a fresh
// reader's perspective the ref is gone the moment those
// commit. Bundle-engine refs (no `chain.json`) and refs with
// an unparseable chain fall through to the immediate-delete
// path: there is nothing for `sweep` to reconcile against, so
// deferral would just orphan the bundle.
//
// The tombstone write runs UNDER the lock (#158): a concurrent
// push that landed between the tombstone and the chain.json
// delete would otherwise leave the bucket with a tombstone
// referencing a SHA no longer in the chain, and `gc sweep`
// would reclaim a live bundle.
let deferred_bundle_key = self.try_tombstone_baseline(&fresh).await;
if let Some(ref key) = deferred_bundle_key {
info!(
branch = %self.branch,
key = %key,
"delete-branch: deferred baseline bundle delete via tombstone",
);
}
// Collect, don't short-circuit: a transient failure on key #2
// of a 4-key listing must not leave #3 and #4 standing with no
// inventory of what survived. NotFound continues to be tolerated
// (the key is gone — operator intent satisfied). Every other
// per-key error is logged and recorded; at the end we either
// declare full success or return PartialDelete naming every
// surviving key so a retry can converge (#122).
let mut undeleted: Vec<String> = Vec::new();
for object in &fresh {
// The baseline bundle (if any) is left for `gc sweep` —
// see the tombstone block above. Other keys (chain.json,
// path-index.json, PROTECTED# is already refused earlier)
// are deleted synchronously. The lock key was filtered
// from `fresh` above, so it is not in the iteration.
if deferred_bundle_key.as_deref() == Some(object.key.as_str()) {
continue;
}
match self.store.delete(&object.key).await {
Ok(()) | Err(ObjectStoreError::NotFound(_)) => {}
Err(err) => {
warn!(
branch = %self.branch,
key = %object.key,
error = %err,
"delete-branch: per-key delete failed; continuing sweep",
);
undeleted.push(object.key.clone());
}
}
}
// attempted excludes the deferred bundle (if any): that key was
// intentionally skipped via tombstone, not "attempted and missing"
// — the operator-facing count must reflect what was swept, not
// what was listed.
let attempted = fresh.len() - usize::from(deferred_bundle_key.is_some());
if !undeleted.is_empty() {
return Err(ManageError::PartialDelete {
branch: self.branch.clone(),
undeleted,
attempted,
});
}
// Issue #151 defence-in-depth: post-sweep, with the lock still
// held, confirm no `PROTECTED#` marker is present for this ref.
// The primary defence is the per-ref lock — `protect` /
// `unprotect` both acquire the same `<prefix>/<ref>/LOCK#.lock`
// per #159, so a marker cannot land between the under-lock
// listing and the sweep. This `head` probe is belt-and-suspenders
// surveillance: an orphan marker observed here would indicate a
// contract violation (lock bypass, bucket inconsistency, or
// misbehaving sibling tool). The helper logs at `error!` and
// does NOT change the delete's success outcome — the branch's
// bundle artefacts are gone, so the operator's intent stands.
verify_no_orphan_protected_after_delete(self.store.as_ref(), self.prefix_opt(), ref_name)
.await;
writeln!(out, "Branch {} has been deleted", self.branch)?;
info!(branch = %self.branch, count = attempted, "branch deleted");
Ok(())
}
/// Build a validated `RefName` for `refs/heads/<branch>`. `open`
/// already accepted this value, so the parse is effectively
/// infallible — but we surface a parse failure as
/// [`ManageError::InvalidBranch`] rather than panicking so a
/// future loosening of `open`'s validator cannot turn delete-branch
/// into a panic surface.
fn validated_ref_name(&self) -> Result<RefName, ManageError> {
RefName::new(format!("refs/heads/{}", self.branch))
.map_err(|_| ManageError::InvalidBranch(self.branch.clone()))
}
/// Returns `Some(&prefix)` when a non-empty bucket prefix is
/// configured, `None` for root-prefixed buckets. Centralises the
/// `(!self.prefix.is_empty()).then_some(self.prefix.as_str())`
/// pattern previously duplicated across delete and tombstone paths.
fn prefix_opt(&self) -> Option<&str> {
(!self.prefix.is_empty()).then_some(self.prefix.as_str())
}
/// Attempt to tombstone the baseline bundle for a packchain ref so
/// the synchronous delete loop can skip it (issue #143). Returns
/// the bundle key that was deferred, or `None` if no deferral is
/// possible. Thin caller-side wrapper that resolves `&self`'s
/// prefix / ref-name and delegates to the shared
/// [`try_write_baseline_tombstone`] helper for the actual
/// load-chain / listing-check / tombstone-write logic (#221).
async fn try_tombstone_baseline(
&self,
fresh: &[crate::object_store::ObjectMeta],
) -> Option<String> {
// `RefName::new` re-runs the same `gix-validate` check `open`
// already accepted, so this is effectively infallible. Surface
// a parse failure as "no tombstone" rather than panicking — a
// future loosening of `open`'s validator must not make
// delete-branch unsafe.
let ref_name = RefName::new(format!("refs/heads/{}", self.branch)).ok()?;
try_write_baseline_tombstone(
self.store.as_ref(),
self.prefix_opt(),
&ref_name,
fresh,
"delete-branch",
)
.await
}
/// Mark the branch as protected by writing the `PROTECTED#` sentinel.
/// Idempotent — overwrites any existing marker.
///
/// # Per-ref lock (#159)
///
/// `protect` acquires the same `<prefix>/<ref>/LOCK#.lock` the
/// helper-protocol push, helper-protocol delete, and `delete-branch`
/// take. Pre-#159, the push path's pre-bundle `is_protected` check
/// could race a concurrent `protect`: a force-push that observed no
/// marker would still overwrite the bundle even if `protect` landed
/// between the under-lock `is_protected` and the bundle upload —
/// because `protect` was a lockless `put_bytes`. Taking the same
/// lock serialises protection state changes against the writers
/// that consult it, closing the entire write window rather than
/// narrowing it to a second sample.
///
/// If the lock is contended (a push, delete, or compact holds it),
/// `protect` returns [`ManageError::LockContended`] and makes no
/// changes. Operators can retry. Stale-lock recovery is inherited
/// from `acquire_lock` (a previous holder that crashed without
/// releasing).
///
/// Re-lists the branch prefix under the lock so a concurrent
/// `delete-branch` (or last-bundle removal) that landed between
/// [`ManageBranch::open`] and the lock window is caught and the
/// marker is NOT written for a non-existent branch (#137). Without
/// this re-check the orphaned `PROTECTED#` would persist with no
/// automated cleanup and would silently block a future recreation
/// of the same branch from being force-pushed or deleted. The
/// re-listing filters out stale lock keys and any pre-existing
/// `PROTECTED#` marker so a branch whose only residue is operational
/// metadata is treated as gone.
///
/// # Errors
///
/// Returns [`ManageError::BranchNotFound`] if the under-lock listing
/// shows the branch was deleted concurrently. Returns
/// [`ManageError::LockContended`] if another writer holds the
/// per-ref lock at acquisition time. Returns [`ManageError::Store`]
/// if a list or put operation fails.
pub async fn protect(&self) -> Result<(), ManageError> {
self.protect_into(&mut std::io::stdout()).await
}
/// Writer-injecting variant of [`Self::protect`] so tests can
/// capture the "now protected" operator message. Mirrors the
/// pattern established by [`Self::delete_into`] (#145) and the
/// management CLI's other writer-aware entry points.
///
/// # Errors
///
/// Same as [`Self::protect`], plus [`ManageError::Io`] if a write
/// to `out` fails.
pub(crate) async fn protect_into<W: Write>(&self, out: &mut W) -> Result<(), ManageError> {
let (lock_key, guard) = self.acquire_ref_lock("protect").await?;
let work = self.protect_under_lock(out).await;
self.release_or_warn(guard, &lock_key, "protect").await;
work
}
/// Lock-held body of [`Self::protect_into`]: re-list under the
/// lock, reject if the branch has been deleted concurrently,
/// otherwise write the `PROTECTED#` sentinel. Extracted so the
/// acquire/release tail in `protect_into` runs unconditionally on
/// every exit path — including the `BranchNotFound` early return.
async fn protect_under_lock<W: Write>(&self, out: &mut W) -> Result<(), ManageError> {
let fresh = self.store.list(&self.branch_prefix()).await?;
if !super::has_branch_data(&fresh) {
warn!(
branch = %self.branch,
"branch was deleted concurrently between open and protect; refusing to write orphaned marker",
);
return Err(ManageError::BranchNotFound(self.branch.clone()));
}
self.store
.put_bytes(&self.protected_key(), Bytes::new(), PutOpts::default())
.await?;
writeln!(out, "Branch {} is now protected", self.branch)?;
Ok(())
}
/// Remove the `PROTECTED#` sentinel. A missing marker is treated as
/// already-unprotected rather than an error.
///
/// # Per-ref lock (#159)
///
/// `unprotect` acquires the same per-ref lock as [`Self::protect`]
/// so ALL protection state changes serialise against pushes,
/// deletes, and compactions. Without taking the lock here a
/// concurrent push observing `is_protected() == true` could
/// otherwise commit to the protected refusal path just as
/// `unprotect` landed, leaving the writer's behaviour out of step
/// with operator intent. Symmetry with `protect` keeps the lock the
/// single point of serialisation for protection state.
///
/// # Errors
///
/// Returns [`ManageError::LockContended`] if another writer holds
/// the per-ref lock at acquisition time. Returns
/// [`ManageError::Store`] for object-store failures other than
/// `NotFound`.
pub async fn unprotect(&self) -> Result<(), ManageError> {
self.unprotect_into(&mut std::io::stdout()).await
}
/// Writer-injecting variant of [`Self::unprotect`] so tests can
/// capture the "now unprotected" operator message.
///
/// # Errors
///
/// Same as [`Self::unprotect`], plus [`ManageError::Io`] if a
/// write to `out` fails.
pub(crate) async fn unprotect_into<W: Write>(&self, out: &mut W) -> Result<(), ManageError> {
let (lock_key, guard) = self.acquire_ref_lock("unprotect").await?;
let work = self.unprotect_under_lock(out).await;
self.release_or_warn(guard, &lock_key, "unprotect").await;
work
}
/// Lock-held body of [`Self::unprotect_into`]: delete the
/// `PROTECTED#` marker, treating `NotFound` as
/// already-unprotected. The lock scope is mechanical (no listing
/// or recovery work needed); we still hold it so a concurrent
/// `protect` cannot land between here and the delete and leave
/// the operator's "unprotect" intent silently overridden.
async fn unprotect_under_lock<W: Write>(&self, out: &mut W) -> Result<(), ManageError> {
match self.store.delete(&self.protected_key()).await {
Ok(()) | Err(ObjectStoreError::NotFound(_)) => {
writeln!(out, "Branch {} is now unprotected", self.branch)?;
Ok(())
}
Err(other) => Err(other.into()),
}
}
/// Acquire the per-ref lock for `op` (`delete-branch`, `protect`,
/// `unprotect`, or any future ref-mutating caller). Returns the
/// resolved lock object-store key alongside the guard so the
/// matching `release_or_warn` tail can name the key in its log
/// line without re-deriving it.
///
/// Contention surfaces as [`ManageError::LockContended`] with the
/// branch name, lock key, and current TTL — matching the wording
/// `delete-branch` (#158) uses so operators see one shape of error
/// across the management surface.
async fn acquire_ref_lock(&self, op: &'static str) -> Result<(String, LockGuard), ManageError> {
let ref_name = self.validated_ref_name()?;
let prefix_opt = self.prefix_opt();
let lock_key = lock_key(prefix_opt, &ref_name);
let ttl = lock_ttl_from_env();
let now = OffsetDateTime::now_utc();
let Some(guard) = acquire_lock(Arc::clone(&self.store), &lock_key, ttl, now).await? else {
warn!(
branch = %self.branch,
op = op,
key = %lock_key,
"{op}: per-ref lock is held by another writer; refusing to race",
);
return Err(ManageError::LockContended {
branch: self.branch.clone(),
lock: lock_key,
ttl_seconds: ttl.whole_seconds(),
});
};
Ok((lock_key, guard))
}
/// Release a previously acquired lock, downgrading release failures
/// to a `warn!` so the caller's primary error (or success) is what
/// surfaces. The lock's TTL recovers a leaked key on the next
/// acquirer (#150), so the worst case is a delayed retry rather
/// than a permanently stuck ref.
async fn release_or_warn(&self, guard: LockGuard, lock_key: &str, op: &'static str) {
if let Err(e) = release_lock(guard).await {
warn!(
branch = %self.branch,
op = op,
key = %lock_key,
error = %e,
"{op}: failed to release per-ref lock; will age out by TTL",
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::manage::{Prompter, ScriptedPrompter, scripted::Answer};
use crate::object_store::mock::MockStore;
use crate::packchain::gc::baseline_tombstone_listing_prefix;
use bytes::Bytes;
fn seed_with_branch(branch: &str) -> MockStore {
let mock = MockStore::new();
mock.insert(
format!("myrepo/refs/heads/{branch}/abc.bundle"),
Bytes::from("body"),
);
mock
}
#[tokio::test]
async fn open_returns_branch_not_found_when_empty() {
let mock = MockStore::new();
let store: Arc<dyn ObjectStore> = Arc::new(mock);
let prompter = ScriptedPrompter::new([]);
match ManageBranch::open(store, "myrepo", "missing", &prompter).await {
Err(ManageError::BranchNotFound(name)) => assert_eq!(name, "missing"),
Err(other) => panic!("expected BranchNotFound, got {other:?}"),
Ok(_) => panic!("expected open to fail"),
}
}
#[tokio::test]
async fn delete_removes_every_key_when_confirmed() {
// No PROTECTED# marker — only the bundle. A confirmed delete
// must clear it AND release the per-ref lock it acquires
// (#158), leaving the bucket empty.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete().await.expect("delete");
assert!(
mock.keys().is_empty(),
"all keys removed (including the LOCK#.lock that delete-branch acquired and released): {:?}",
mock.keys()
);
assert_eq!(prompter.remaining(), 0);
}
#[tokio::test]
async fn delete_refuses_when_protected_marker_present() {
// `protect` then `delete-branch` must refuse — same wording the
// helper-protocol delete path emits. The prompt is never reached,
// so the script queues no answer; the marker and bundle survive.
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/PROTECTED#", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.delete()
.await
.expect_err("delete must refuse when PROTECTED# is present");
match &err {
ManageError::Protected(name) => assert_eq!(name, "main"),
other => panic!("expected ManageError::Protected, got {other:?}"),
}
assert!(
err.to_string()
.contains("git-remote-object-store unprotect"),
"error message must point at unprotect, got: {err}",
);
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
assert!(mock.contains("myrepo/refs/heads/main/abc.bundle"));
// Prompt must not have been consumed.
assert_eq!(prompter.remaining(), 0);
}
#[tokio::test]
async fn delete_succeeds_after_unprotect_clears_marker() {
// Protect, then unprotect, then delete — the canonical recovery
// path. The final delete must remove every remaining key.
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/PROTECTED#", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.unprotect().await.expect("unprotect");
mb.delete().await.expect("delete after unprotect");
assert!(
mock.keys().is_empty(),
"all keys removed after unprotect+delete: {:?}",
mock.keys()
);
}
#[tokio::test]
async fn delete_no_keeps_keys() {
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(false)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete().await.expect("delete (aborted)");
assert_eq!(mock.keys().len(), 1, "branch still present");
}
#[tokio::test]
async fn protect_creates_marker_idempotent() {
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.protect().await.expect("protect");
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
// Second call overwrites without error.
mb.protect().await.expect("protect again");
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
}
#[tokio::test]
async fn protect_refuses_when_branch_deleted_between_open_and_protect() {
// Issue #137: TOCTOU between `open` (which lists to verify the
// branch exists) and `protect` (which writes the marker). A
// concurrent `delete-branch` or last-bundle removal lands
// between the two calls. Pre-fix, `protect` wrote a marker for
// a non-existent branch — an orphaned `PROTECTED#` that never
// gets cleaned up and silently blocks a future recreation of
// the same branch. The fix re-lists immediately before the put
// and refuses with BranchNotFound if the branch is gone.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
// Simulate a concurrent delete sweeping every key under the
// branch prefix after `open` returned but before `protect` runs.
for key in mock.keys() {
if key.starts_with("myrepo/refs/heads/main/") {
let _ = mock.remove_key(&key);
}
}
let err = mb
.protect()
.await
.expect_err("protect must refuse against a concurrently-deleted branch");
match &err {
ManageError::BranchNotFound(name) => assert_eq!(name, "main"),
other => panic!("expected BranchNotFound, got {other:?}"),
}
// The orphaned marker must NOT have been written — that is the
// exact regression #137 fixes.
assert!(
!mock.contains("myrepo/refs/heads/main/PROTECTED#"),
"orphaned PROTECTED# must not be written when branch is gone",
);
assert!(
mock.keys().is_empty(),
"store remains empty: {:?}",
mock.keys()
);
}
#[tokio::test]
async fn protect_refuses_when_only_stale_lock_key_remains() {
// A `LOCK#.lock` key is operational metadata, not branch data.
// Treating a lock-only listing as "branch exists" would let a
// `protect` write a marker for a branch that has no bundles —
// the same orphan-marker pathology #137 describes, just with a
// lock as the misleading residue instead of an empty listing.
//
// The lock is seeded stale (older than TTL) so #159's lock
// acquisition recovers it rather than reporting contention —
// otherwise we would assert the wrong error. The data-presence
// re-check then runs and refuses the orphan write.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/abc.bundle", Bytes::from("body"));
let stale = OffsetDateTime::now_utc() - time::Duration::days(1);
mock.insert_with(
"myrepo/refs/heads/main/LOCK#.lock",
Bytes::new(),
stale,
PutOpts::default(),
);
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
// Concurrent push-then-delete leaves only the lock behind.
let _ = mock.remove_key("myrepo/refs/heads/main/abc.bundle");
let err = mb
.protect()
.await
.expect_err("protect must refuse when only a lock key remains");
assert!(
matches!(err, ManageError::BranchNotFound(ref name) if name == "main"),
"expected BranchNotFound, got {err:?}",
);
assert!(!mock.contains("myrepo/refs/heads/main/PROTECTED#"));
// protect must recover the stale lock AND release the fresh one
// it acquired. A regression that leaked the lock would still
// pass the BranchNotFound assertion above.
assert!(
!mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"stale lock must be recovered and the acquired lock released",
);
}
#[tokio::test]
async fn protect_remains_idempotent_when_marker_already_present() {
// The pre-existing marker plus a real bundle means the branch
// is alive. `protect` must still succeed (idempotent overwrite)
// — the data-presence check must not regress to "any marker
// means orphan" and refuse a legitimate re-protect.
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/PROTECTED#", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.protect()
.await
.expect("protect must remain idempotent over an existing marker");
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
assert!(mock.contains("myrepo/refs/heads/main/abc.bundle"));
}
#[tokio::test]
async fn protect_into_writes_operator_message_to_writer() {
// Mirror the delete_into pattern (#145): the writer-injecting
// variant must emit the operator-visible message through `out`,
// not via stdout. A regression that dropped the message — or
// emitted it on stdout instead of the writer — would slip past
// any test calling `protect()` because that wraps stdout.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let mut out: Vec<u8> = Vec::new();
mb.protect_into(&mut out).await.expect("protect_into");
let captured = String::from_utf8(out).expect("utf8");
assert!(
captured.contains("Branch main is now protected"),
"protect_into must emit the operator message; got: {captured:?}",
);
}
#[tokio::test]
async fn unprotect_into_writes_operator_message_to_writer() {
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/PROTECTED#", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let mut out: Vec<u8> = Vec::new();
mb.unprotect_into(&mut out).await.expect("unprotect_into");
let captured = String::from_utf8(out).expect("utf8");
assert!(
captured.contains("Branch main is now unprotected"),
"unprotect_into must emit the operator message; got: {captured:?}",
);
}
#[tokio::test]
async fn unprotect_deletes_marker_when_present() {
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/PROTECTED#", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.unprotect().await.expect("unprotect");
assert!(!mock.contains("myrepo/refs/heads/main/PROTECTED#"));
}
#[tokio::test]
async fn unprotect_idempotent_when_marker_absent() {
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock);
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.unprotect()
.await
.expect("unprotect should be idempotent");
}
#[tokio::test]
async fn open_rejects_invalid_branch_name() {
// Attempting `delete-branch foo/../bar` would otherwise build
// literal `<prefix>/refs/heads/foo/../bar/...` keys on S3.
let mock = MockStore::new();
let store: Arc<dyn ObjectStore> = Arc::new(mock);
let prompter = ScriptedPrompter::new([]);
match ManageBranch::open(store, "myrepo", "foo/../bar", &prompter).await {
Err(ManageError::InvalidBranch(name)) => assert_eq!(name, "foo/../bar"),
Err(other) => panic!("expected InvalidBranch, got {other:?}"),
Ok(_) => panic!("expected open to reject `foo/../bar`"),
}
}
#[tokio::test]
async fn open_rejects_branch_with_control_char() {
let mock = MockStore::new();
let store: Arc<dyn ObjectStore> = Arc::new(mock);
let prompter = ScriptedPrompter::new([]);
match ManageBranch::open(store, "myrepo", "main\nrefs/heads/other", &prompter).await {
Err(ManageError::InvalidBranch(_)) => {}
Err(other) => panic!("expected InvalidBranch, got {other:?}"),
Ok(_) => panic!("expected open to reject control-char branch"),
}
}
#[tokio::test]
async fn delete_partial_failure_continues_and_returns_structured_error() {
// Issue #122: pre-fix, `delete` short-circuited on the first
// per-key error, leaving the later keys untouched and the
// operator with no inventory of what survived. The fix is to
// collect failures, continue past each, and return a structured
// `PartialDelete` naming exactly the keys that remain.
//
// `MockStore::list` returns keys in lexicographic (BTreeMap)
// order. The loop deletes aaa, attempts bbb (armed to fail
// transiently), and must still attempt ccc. Post-fix: aaa and
// ccc are gone, bbb remains, the error names bbb explicitly.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/aaa.bundle", Bytes::from("a"));
mock.insert("myrepo/refs/heads/main/bbb.bundle", Bytes::from("b"));
mock.insert("myrepo/refs/heads/main/ccc.bundle", Bytes::from("c"));
mock.arm(crate::object_store::mock::Fault::NetworkOnDelete {
key: "myrepo/refs/heads/main/bbb.bundle".to_owned(),
});
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(
Arc::clone(&store),
"myrepo",
"main",
&prompter as &dyn Prompter,
)
.await
.expect("open");
let err = mb
.delete()
.await
.expect_err("partial delete must surface PartialDelete");
match &err {
ManageError::PartialDelete {
branch,
undeleted,
attempted,
} => {
assert_eq!(branch, "main");
assert_eq!(*attempted, 3);
assert_eq!(
undeleted.as_slice(),
["myrepo/refs/heads/main/bbb.bundle"],
"undeleted list must name exactly the failed key",
);
}
other => panic!("expected PartialDelete, got {other:?}"),
}
// The error message must name the failed key so a copy-paste
// retry tool (or human) can act on it.
let rendered = err.to_string();
assert!(
rendered.contains("myrepo/refs/heads/main/bbb.bundle"),
"error message must name surviving key, got: {rendered}",
);
assert!(
rendered.contains("retry to converge"),
"error message must point at the retry path, got: {rendered}",
);
assert!(
rendered.contains("1 of 3"),
"render should pin the count framing, got: {rendered}",
);
// The loop did NOT short-circuit on bbb — aaa AND ccc are
// both gone, and only bbb survives.
assert!(!mock.contains("myrepo/refs/heads/main/aaa.bundle"));
assert!(mock.contains("myrepo/refs/heads/main/bbb.bundle"));
assert!(!mock.contains("myrepo/refs/heads/main/ccc.bundle"));
assert_eq!(mock.pending_faults(), 0);
// Retry-converges: clear nothing extra (the fault is already
// consumed) and run delete again. The fresh listing inside
// `delete` will only show bbb; the loop deletes it; the branch
// is now fully gone.
let prompter2 = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb2 = ManageBranch::open(store, "myrepo", "main", &prompter2 as &dyn Prompter)
.await
.expect("re-open after partial delete");
mb2.delete().await.expect("retry must converge to Ok");
assert!(
mock.keys().is_empty(),
"retry must remove the surviving key: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn delete_partial_failure_attempts_every_key_in_listing() {
// Issue #122 explicit four-key case: a transient failure on
// key #2 of a 4-key listing must not stop the loop from
// attempting keys #3 and #4. Pre-fix, this seeded with key
// names a-d, fault on bbb, would leave bbb/ccc/ddd standing.
// Post-fix, only bbb survives (the named failure).
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/aaa.bundle", Bytes::from("a"));
mock.insert("myrepo/refs/heads/main/bbb.bundle", Bytes::from("b"));
mock.insert("myrepo/refs/heads/main/ccc.bundle", Bytes::from("c"));
mock.insert("myrepo/refs/heads/main/ddd.bundle", Bytes::from("d"));
mock.arm(crate::object_store::mock::Fault::NetworkOnDelete {
key: "myrepo/refs/heads/main/bbb.bundle".to_owned(),
});
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb.delete().await.expect_err("partial delete expected");
match err {
ManageError::PartialDelete {
undeleted,
attempted,
..
} => {
assert_eq!(attempted, 4, "loop must visit every listed key");
assert_eq!(undeleted.as_slice(), ["myrepo/refs/heads/main/bbb.bundle"]);
}
other => panic!("expected PartialDelete, got {other:?}"),
}
// Keys #1, #3, #4 were all attempted and succeeded; only the
// named failure key survives.
assert!(!mock.contains("myrepo/refs/heads/main/aaa.bundle"));
assert!(mock.contains("myrepo/refs/heads/main/bbb.bundle"));
assert!(!mock.contains("myrepo/refs/heads/main/ccc.bundle"));
assert!(!mock.contains("myrepo/refs/heads/main/ddd.bundle"));
}
#[tokio::test]
async fn delete_all_keys_fail_returns_full_inventory() {
// Two faults arm against two of the three keys, plus a third
// standalone failure. We assert that PartialDelete lists every
// surviving key in lexicographic order so an operator (or
// tooling that reads the structured field) gets a complete
// inventory rather than just the first failure.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/aaa.bundle", Bytes::from("a"));
mock.insert("myrepo/refs/heads/main/bbb.bundle", Bytes::from("b"));
mock.insert("myrepo/refs/heads/main/ccc.bundle", Bytes::from("c"));
for key in [
"myrepo/refs/heads/main/aaa.bundle",
"myrepo/refs/heads/main/bbb.bundle",
"myrepo/refs/heads/main/ccc.bundle",
] {
mock.arm(crate::object_store::mock::Fault::NetworkOnDelete {
key: key.to_owned(),
});
}
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb.delete().await.expect_err("all-fail must surface error");
match err {
ManageError::PartialDelete {
undeleted,
attempted,
..
} => {
assert_eq!(attempted, 3);
assert_eq!(
undeleted,
vec![
"myrepo/refs/heads/main/aaa.bundle".to_owned(),
"myrepo/refs/heads/main/bbb.bundle".to_owned(),
"myrepo/refs/heads/main/ccc.bundle".to_owned(),
],
"every surviving key must be reported, in listing order",
);
}
other => panic!("expected PartialDelete, got {other:?}"),
}
// All three originals survive — nothing was deleted.
assert_eq!(mock.keys().len(), 3);
}
#[tokio::test]
async fn delete_mixed_notfound_and_failure_only_lists_real_failures() {
// NotFound mid-sweep is tolerated (#139). The PartialDelete
// inventory must NOT include keys that the listing showed but
// that a concurrent sweeper had already removed — those are
// success from the operator's POV. Only the genuine network
// failure on bbb should be in `undeleted`.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/aaa.bundle", Bytes::from("a"));
mock.insert("myrepo/refs/heads/main/bbb.bundle", Bytes::from("b"));
mock.insert("myrepo/refs/heads/main/ccc.bundle", Bytes::from("c"));
// aaa raced and is gone; bbb is a genuine network failure; ccc
// succeeds normally.
mock.arm(crate::object_store::mock::Fault::NotFoundOnDelete {
key: "myrepo/refs/heads/main/aaa.bundle".to_owned(),
});
mock.arm(crate::object_store::mock::Fault::NetworkOnDelete {
key: "myrepo/refs/heads/main/bbb.bundle".to_owned(),
});
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb.delete().await.expect_err("bbb failure must surface");
match err {
ManageError::PartialDelete {
undeleted,
attempted,
..
} => {
assert_eq!(attempted, 3);
assert_eq!(
undeleted.as_slice(),
["myrepo/refs/heads/main/bbb.bundle"],
"only the genuine non-NotFound failure must appear",
);
}
other => panic!("expected PartialDelete, got {other:?}"),
}
// ccc was deleted by the loop. bbb survives. aaa's NotFound
// fault short-circuited its delete BEFORE the actual removal,
// so the body is still in the mock — same observable as the
// pre-existing `delete_tolerates_notfound_mid_sweep` test.
assert!(!mock.contains("myrepo/refs/heads/main/ccc.bundle"));
assert!(mock.contains("myrepo/refs/heads/main/bbb.bundle"));
}
/// Prompter that performs a side effect against a [`MockStore`]
/// before replying to `confirm`, simulating a concurrent operation
/// landing during the user's prompt window. Each call consumes one
/// queued `(action, answer)` pair; running dry returns
/// [`ManageError::Cancelled`] so an under-armed script fails loudly.
struct ConcurrentPrompter {
store: MockStore,
actions: std::sync::Mutex<std::collections::VecDeque<(ConcurrentAction, bool)>>,
}
enum ConcurrentAction {
/// Insert `(key, body)` into the store.
Insert(String, Bytes),
/// Insert multiple `(key, body)` pairs in one prompt window —
/// used to model an interleaved `git push` + `protect` race
/// against a single user prompt (#131).
InsertMany(Vec<(String, Bytes)>),
/// Delete every key currently under `prefix` (simulates a
/// concurrent `delete-branch` winning the race).
DeleteAllUnder(String),
}
impl ConcurrentPrompter {
fn new(
store: MockStore,
actions: impl IntoIterator<Item = (ConcurrentAction, bool)>,
) -> Self {
Self {
store,
actions: std::sync::Mutex::new(actions.into_iter().collect()),
}
}
}
impl Prompter for ConcurrentPrompter {
fn select(&self, _prompt: &str, _options: &[String]) -> Result<usize, ManageError> {
panic!("ConcurrentPrompter does not expect select");
}
fn confirm(&self, _prompt: &str) -> Result<bool, ManageError> {
let (action, answer) = self
.actions
.lock()
.expect("concurrent mutex poisoned")
.pop_front()
.ok_or(ManageError::Cancelled)?;
match action {
ConcurrentAction::Insert(key, body) => self.store.insert(key, body),
ConcurrentAction::InsertMany(pairs) => {
for (key, body) in pairs {
self.store.insert(key, body);
}
}
ConcurrentAction::DeleteAllUnder(prefix) => {
for key in self.store.keys() {
if key.starts_with(&prefix) {
let _ = self.store.remove_key(&key);
}
}
}
}
Ok(answer)
}
}
#[tokio::test]
async fn delete_sweeps_objects_added_during_prompt() {
// Issue #139: a concurrent push lands a new bundle key between
// the initial LIST and the deletion loop. Pre-fix, that key was
// not in the captured listing and survived the "successful"
// delete. The fix re-lists after the prompt, so the new key is
// included in the sweep.
let mock = seed_with_branch("main");
let new_key = "myrepo/refs/heads/main/concurrent.bundle".to_owned();
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ConcurrentPrompter::new(
mock.clone(),
[(
ConcurrentAction::Insert(new_key.clone(), Bytes::from("racing body")),
true,
)],
);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete()
.await
.expect("delete must include concurrently-added key");
assert!(
mock.keys().is_empty(),
"fresh listing must drive sweep; zombie keys remaining: {:?}",
mock.keys(),
);
assert!(
!mock.contains(&new_key),
"concurrently-added bundle must be deleted, not left as a zombie",
);
}
#[tokio::test]
async fn delete_refuses_when_marker_lands_during_prompt() {
// Initial listing has no PROTECTED# marker, so the protection
// check passes and the prompt fires. A concurrent `protect`
// lands during the prompt, then the user answers "yes". The
// fresh-listing protection check must catch the marker and
// refuse — otherwise the operator silently bulldozes a ref that
// was just protected.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ConcurrentPrompter::new(
mock.clone(),
[(
ConcurrentAction::Insert(
"myrepo/refs/heads/main/PROTECTED#".to_owned(),
Bytes::new(),
),
true,
)],
);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.delete()
.await
.expect_err("delete must refuse marker that landed during prompt");
assert!(
matches!(err, ManageError::Protected(ref name) if name == "main"),
"expected Protected, got {err:?}",
);
// Both the marker and the original bundle survive.
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
assert!(mock.contains("myrepo/refs/heads/main/abc.bundle"));
}
#[tokio::test]
async fn issue_131_protect_during_prompt_blocks_delete_even_with_concurrent_push() {
// Issue #131 regression: TOCTOU between the initial protection
// check and the deletion loop. This pins the specific scenario
// where a `protect` lands DURING the user prompt — distinct from
// #139's pure-push race. The combined push+protect interleaving
// here proves two things about the post-prompt re-check:
//
// 1. The marker check fires on the FRESH listing, not the
// stale initial listing (otherwise the marker is missed
// because it didn't exist when `delete` started).
// 2. The marker check takes precedence over the sweep even
// when other concurrent activity (a racing push) would
// otherwise look "successful" — the operator must not
// silently bulldoze a freshly-protected ref just because
// the listing also grew.
//
// Pre-#139 the marker check was only on the initial listing, so
// both concurrent writes were ignored and the original bundle
// was deleted. The fix re-lists after the prompt and re-checks
// for the marker, refusing the delete entirely.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ConcurrentPrompter::new(
mock.clone(),
[(
ConcurrentAction::InsertMany(vec![
("myrepo/refs/heads/main/PROTECTED#".to_owned(), Bytes::new()),
(
"myrepo/refs/heads/main/racing-push.bundle".to_owned(),
Bytes::from("pushed during prompt"),
),
]),
true,
)],
);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.delete()
.await
.expect_err("delete must refuse marker even when push also raced");
assert!(
matches!(err, ManageError::Protected(ref name) if name == "main"),
"expected Protected (post-prompt re-check), got {err:?}",
);
// The marker, the racing push, and the original bundle all
// survive — refusal is total, not partial.
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
assert!(mock.contains("myrepo/refs/heads/main/racing-push.bundle"));
assert!(mock.contains("myrepo/refs/heads/main/abc.bundle"));
}
#[tokio::test]
async fn delete_handles_empty_initial_listing_when_branch_swept_between_open_and_delete() {
// Distinct from the prompt-window race
// (`delete_reports_already_gone_on_concurrent_delete_race`):
// here the branch is swept BETWEEN `open()` succeeding (data
// existed at open time) and the FIRST listing inside
// `delete()`. The function must handle the empty-initial-list
// path without panicking, without spuriously claiming success,
// and without surfacing an unexpected error variant. The fresh
// re-listing inside `delete()` is also empty, so the
// "already gone" branch fires and the function returns Ok(()).
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
// One confirm answer queued: the current implementation does
// NOT short-circuit on an empty INITIAL listing — it falls
// through to the prompt (the operator may want to confirm a
// "0 objects" delete) and only the empty FRESH listing path
// (post-prompt) returns Ok(()). Queuing a single `true`
// exercises that exact path.
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open succeeds while branch data is still present");
// Sweep every key under the branch between open() and delete().
// Mirrors a concurrent `delete-branch` or last-bundle removal
// that ran while the caller was still holding the open handle.
for key in mock.keys() {
if key.starts_with("myrepo/refs/heads/main/") {
let _ = mock.remove_key(&key);
}
}
assert!(
mock.keys().is_empty(),
"pre-condition: branch must be fully swept before delete()",
);
mb.delete()
.await
.expect("delete() must handle an empty initial listing without error");
assert!(
mock.keys().is_empty(),
"delete() against an already-empty branch must not resurrect any key",
);
}
#[tokio::test]
async fn delete_reports_already_gone_on_concurrent_delete_race() {
// A concurrent `delete-branch` (or last-bundle removal) clears
// every object under the branch prefix during the prompt
// window. The fresh listing is empty; the function must report
// the race and return Ok(()), not claim success without doing
// anything.
//
// The store-state asserts here are intentionally weak: the
// ConcurrentPrompter side effect already cleared the store
// before `delete()` resumed from the prompt, so `keys()` is
// empty regardless of which production branch was taken
// (#145). The load-bearing assert is the captured-stdout
// substring — without the "already gone" notice the operator
// cannot tell a successful delete from a no-op race-loss.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ConcurrentPrompter::new(
mock.clone(),
[(
ConcurrentAction::DeleteAllUnder("myrepo/refs/heads/main/".to_owned()),
true,
)],
);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let mut out: Vec<u8> = Vec::new();
mb.delete_into(&mut out)
.await
.expect("empty fresh listing must return Ok, not silent success");
let captured = String::from_utf8(out).expect("captured output must be UTF-8");
assert!(
captured.contains("is already gone"),
"operator message must announce the concurrent race; got: {captured:?}",
);
assert!(
!captured.contains("has been deleted"),
"must not claim a successful delete when nothing was swept; got: {captured:?}",
);
assert!(mock.keys().is_empty(), "store remains empty");
}
#[tokio::test]
async fn delete_tolerates_notfound_mid_sweep() {
// A concurrent sweeper races between our fresh listing and a
// per-key delete: the listing still reports `bbb`, but by the
// time `delete(bbb)` fires the key is gone. Pre-fix, the
// ObjectStoreError::NotFound surfaced as ManageError::Store and
// aborted the sweep mid-flight. The fix tolerates NotFound in
// the loop so a partial concurrent delete doesn't leave the
// rest of the branch standing.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/aaa.bundle", Bytes::from("a"));
mock.insert("myrepo/refs/heads/main/bbb.bundle", Bytes::from("b"));
mock.insert("myrepo/refs/heads/main/ccc.bundle", Bytes::from("c"));
mock.arm(crate::object_store::mock::Fault::NotFoundOnDelete {
key: "myrepo/refs/heads/main/bbb.bundle".to_owned(),
});
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete()
.await
.expect("NotFound mid-sweep must not abort the loop");
// aaa and ccc are deleted; the NotFound fault on bbb is
// tolerated and the fault is consumed (the body remains because
// the fault fired BEFORE the actual removal).
assert!(!mock.contains("myrepo/refs/heads/main/aaa.bundle"));
assert!(!mock.contains("myrepo/refs/heads/main/ccc.bundle"));
// bbb's body is still present because the fault short-circuited
// the delete with NotFound before removal. In production the
// analogous case is a concurrent sweeper that ALREADY removed
// it — same observable: key gone or not, the loop continues.
assert_eq!(mock.pending_faults(), 0);
}
// --- Root-of-bucket (empty prefix) coverage --------------------------
#[tokio::test]
async fn root_prefix_delete_removes_keys_without_leading_slash() {
// Repo lives at the bucket root: keys have no `<prefix>/`
// segment. A leading-slash regression here would surface as
// `BranchNotFound` (the list of `/refs/heads/main/` returns
// nothing) or as a delete that fails to match the real keys.
// No PROTECTED# marker is seeded — protected-ref refusal is
// covered separately by
// `root_prefix_delete_refuses_when_protected_marker_present`.
// The `LOCK#.lock` is created and removed by `delete`'s own
// acquire/release tail (#158) — pre-seeding a fresh lock here
// would (correctly) surface as `LockContended`.
let mock = MockStore::new();
mock.insert("refs/heads/main/abc.bundle", Bytes::from("body"));
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "", "main", &prompter as &dyn Prompter)
.await
.expect("open at root");
mb.delete().await.expect("delete at root");
assert!(mock.keys().is_empty(), "all root keys removed");
}
#[tokio::test]
async fn root_prefix_delete_refuses_when_protected_marker_present() {
// Root-of-bucket layout (no `<prefix>/` segment) must use the
// same final-segment match the helper-protocol delete path uses;
// a substring-only check could miss the unprefixed marker key.
let mock = MockStore::new();
mock.insert("refs/heads/main/abc.bundle", Bytes::from("body"));
mock.insert("refs/heads/main/PROTECTED#", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "", "main", &prompter as &dyn Prompter)
.await
.expect("open at root");
let err = mb
.delete()
.await
.expect_err("delete at root must refuse PROTECTED#");
assert!(
matches!(err, ManageError::Protected(ref name) if name == "main"),
"expected ManageError::Protected, got {err:?}",
);
assert!(mock.contains("refs/heads/main/PROTECTED#"));
assert!(mock.contains("refs/heads/main/abc.bundle"));
}
#[tokio::test]
async fn root_prefix_protect_writes_marker_at_root_layout() {
let mock = MockStore::new();
mock.insert("refs/heads/main/abc.bundle", Bytes::from("body"));
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "", "main", &prompter as &dyn Prompter)
.await
.expect("open at root");
mb.protect().await.expect("protect at root");
// Root-of-bucket layout: no leading slash, no synthetic prefix.
assert!(mock.contains("refs/heads/main/PROTECTED#"));
assert!(!mock.contains("/refs/heads/main/PROTECTED#"));
}
#[tokio::test]
async fn root_prefix_unprotect_removes_marker_at_root_layout() {
let mock = MockStore::new();
mock.insert("refs/heads/main/abc.bundle", Bytes::from("body"));
mock.insert("refs/heads/main/PROTECTED#", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "", "main", &prompter as &dyn Prompter)
.await
.expect("open at root");
mb.unprotect().await.expect("unprotect at root");
assert!(!mock.contains("refs/heads/main/PROTECTED#"));
// The bundle alongside the marker must survive — `unprotect` is
// a marker-only delete and a regression that broadened the
// delete scope would leave the bundle missing.
assert!(mock.contains("refs/heads/main/abc.bundle"));
}
#[tokio::test]
async fn root_prefix_open_reports_branch_not_found_for_missing_branch() {
let mock = MockStore::new();
let store: Arc<dyn ObjectStore> = Arc::new(mock);
let prompter = ScriptedPrompter::new([]);
match ManageBranch::open(store, "", "missing", &prompter).await {
Err(ManageError::BranchNotFound(name)) => assert_eq!(name, "missing"),
Err(other) => panic!("expected BranchNotFound, got {other:?}"),
Ok(_) => panic!("expected open at root to fail with BranchNotFound"),
}
}
// --- Baseline-bundle tombstone on delete-branch (#143) ---------------
/// SHA used as `<full_at>` in the seeded `chain.json` for the
/// tombstone tests below. The exact value is irrelevant — the
/// tests assert that whatever SHA the chain names is the SHA the
/// tombstone references and the SHA whose `<sha>.bundle` survives
/// the synchronous sweep.
const TOMBSTONE_TEST_FULL_AT: &str = "0123456789abcdef0123456789abcdef01234567";
/// Seed a packchain-style branch at `<prefix>/refs/heads/<branch>/`
/// with a baseline bundle, a `chain.json` naming that bundle as
/// `full_at`, and a `path-index.json`. Returns the bundle's
/// full key so tests can pin survival/deletion against the
/// exact byte string.
async fn seed_packchain_branch(
store: &crate::object_store::mock::MockStore,
prefix: &str,
branch: &str,
) -> String {
use crate::packchain::manifest::write_chain;
use crate::packchain::schema::{ChainManifest, ChainSegment, Sha40};
let ref_name = RefName::new(format!("refs/heads/{branch}")).unwrap();
let prefix_opt = (!prefix.is_empty()).then_some(prefix);
let full_at = Sha40::try_new(TOMBSTONE_TEST_FULL_AT).unwrap();
let chain = ChainManifest {
v: 1,
tip: full_at.clone(),
full_at: full_at.clone(),
segments: vec![ChainSegment {
sha: full_at.clone(),
parent_sha: None,
pack: format!("packs/{TOMBSTONE_TEST_FULL_AT}.pack"),
bytes: 1_024,
}],
};
write_chain(store, prefix_opt, &ref_name, &chain)
.await
.unwrap();
// path-index.json — written verbatim so we can assert the
// synchronous sweep removes it alongside chain.json.
let path_index_key = crate::packchain::keys::path_index_key(prefix_opt, &ref_name);
store.insert(path_index_key, Bytes::from_static(b"{\"v\":1,\"root\":{}}"));
let bundle_key = keys::bundle_key(prefix_opt, ref_name.as_str(), full_at.as_str());
store.insert(bundle_key.clone(), Bytes::from_static(b"PACKBUNDLE"));
bundle_key
}
#[tokio::test]
async fn delete_writes_baseline_tombstone_and_defers_bundle() {
// Issue #143: a packchain delete-branch must write a
// `<prefix>/gc/baseline-tomb-*.json` tombstone naming the
// current `full_at` SHA, and the synchronous sweep must
// leave that bundle in place. chain.json and path-index.json
// ARE removed synchronously — from a fresh reader's
// perspective the ref is gone the moment the chain commits
// its deletion; the bundle stays only so an in-flight
// fetcher that already loaded the prior chain can finish.
let mock = crate::object_store::mock::MockStore::new();
let bundle_key = seed_packchain_branch(&mock, "repo", "main").await;
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "repo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete().await.expect("delete");
// chain.json and path-index.json are deleted synchronously.
assert!(
!mock.contains("repo/refs/heads/main/chain.json"),
"chain.json must be removed synchronously: {:?}",
mock.keys(),
);
assert!(
!mock.contains("repo/refs/heads/main/path-index.json"),
"path-index.json must be removed synchronously: {:?}",
mock.keys(),
);
// The baseline bundle is NOT removed — it is left for `gc sweep`.
assert!(
mock.contains(&bundle_key),
"baseline bundle must survive synchronous delete: {:?}",
mock.keys(),
);
// Exactly one baseline tombstone is written under
// `<prefix>/gc/baseline-tomb-*.json`, and it names the
// bundle's SHA. The shape (UUID-named JSON body) belongs to
// `BaselineTombstone`; this test pins only the listing
// prefix and the SHA inside, since the UUID is intentionally
// non-deterministic.
let tomb_keys: Vec<String> = mock
.keys()
.into_iter()
.filter(|k| k.starts_with(&baseline_tombstone_listing_prefix(Some("repo"))))
.collect();
assert_eq!(
tomb_keys.len(),
1,
"exactly one baseline tombstone must exist: {tomb_keys:?}",
);
let body = mock
.get_bytes(&tomb_keys[0])
.await
.expect("tombstone body present");
let parsed: serde_json::Value =
serde_json::from_slice(&body).expect("tombstone is valid JSON");
assert_eq!(parsed["v"], 1);
assert_eq!(parsed["sha"], TOMBSTONE_TEST_FULL_AT);
assert_eq!(parsed["ref_name"], "refs/heads/main");
}
#[tokio::test]
async fn gc_sweep_after_grace_window_reclaims_deferred_bundle() {
// Round-trip the #143 contract: write a tombstone via
// delete-branch, then run `gc sweep --force` (skips the
// grace window). The bundle must now be gone. This proves
// the tombstone's body is shaped exactly the way the
// existing sweep code expects — a regression in the
// delete-branch tombstone shape would surface as a deferred
// sweep step rather than a reclaim.
let mock = crate::object_store::mock::MockStore::new();
let bundle_key = seed_packchain_branch(&mock, "repo", "main").await;
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(
Arc::clone(&store),
"repo",
"main",
&prompter as &dyn Prompter,
)
.await
.expect("open");
mb.delete().await.expect("delete");
// Pre-condition: bundle still present, tombstone written.
assert!(mock.contains(&bundle_key));
// `--force` skips the grace window. The sweep finds a
// chain.json-less ref (delete-branch removed it) and so
// proceeds with the bundle delete.
let outcome = crate::packchain::gc::sweep(
store.as_ref(),
"repo",
crate::packchain::gc::SweepOpts {
grace_hours: 0,
force: true,
},
)
.await
.expect("sweep");
assert_eq!(
outcome.swept_tombstones, 1,
"sweep must reclaim exactly the tombstone delete-branch wrote",
);
assert!(
!mock.contains(&bundle_key),
"baseline bundle must be deleted by sweep: surviving keys = {:?}",
mock.keys(),
);
// The tombstone itself is also gone after a successful sweep.
let surviving_tombs: Vec<String> = mock
.keys()
.into_iter()
.filter(|k| k.starts_with(&baseline_tombstone_listing_prefix(Some("repo"))))
.collect();
assert!(
surviving_tombs.is_empty(),
"tombstone must be deleted by sweep: {surviving_tombs:?}",
);
}
#[tokio::test]
async fn delete_bundle_engine_ref_with_no_chain_uses_immediate_delete() {
// Bundle-engine refs (no `chain.json`) have no baseline to
// tombstone. The function must fall through to the existing
// immediate-delete path — no tombstone written, every key
// swept synchronously. This guards against a regression that
// would write a spurious tombstone naming a non-existent
// SHA, or that would leave a bundle-engine ref's `.bundle`
// standing.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete().await.expect("delete");
assert!(
mock.keys().is_empty(),
"bundle-engine ref must be fully swept synchronously: {:?}",
mock.keys(),
);
// No baseline tombstone written.
let tomb_keys: Vec<String> = mock
.keys()
.into_iter()
.filter(|k| k.contains(crate::packchain::gc::BASELINE_TOMBSTONE_KEY_FRAGMENT))
.collect();
assert!(
tomb_keys.is_empty(),
"no tombstone must be written for a chain-less ref: {tomb_keys:?}",
);
}
#[tokio::test]
async fn delete_unparseable_chain_falls_back_to_synchronous_bundle_delete() {
// A malformed `chain.json` (truncated, wrong schema version,
// etc.) means `load_chain` fails. The delete path must NOT
// block on this — the operator already confirmed the delete;
// the ref is going away. The fallback is the existing
// immediate-delete behaviour: sweep every key including any
// bundles. Without the fallback an operator could be stuck
// unable to delete a corrupted ref.
let mock = crate::object_store::mock::MockStore::new();
// Hand-craft an unparseable `chain.json` body (not JSON).
mock.insert(
"repo/refs/heads/main/chain.json",
Bytes::from_static(b"not a json"),
);
// Seed a bundle whose name matches what a chain.json MIGHT
// have pointed at — must still be swept synchronously since
// we have no tombstone protection.
let bundle_key = format!("repo/refs/heads/main/{TOMBSTONE_TEST_FULL_AT}.bundle");
mock.insert(bundle_key.clone(), Bytes::from_static(b"BUNDLE"));
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "repo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete().await.expect("delete");
assert!(
mock.keys().is_empty(),
"unparseable chain must fall back to immediate sweep: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn delete_chain_pointing_at_missing_bundle_sweeps_remaining_keys() {
// Pathological case: `chain.json` parses and names a
// `full_at`, but the corresponding `<sha>.bundle` is NOT in
// the fresh listing (already deleted, or never written).
// `try_tombstone_baseline` returns None on this branch —
// there is nothing to defer. The synchronous sweep must
// still remove chain.json and any other residue.
use crate::packchain::manifest::write_chain;
use crate::packchain::schema::{ChainManifest, ChainSegment, Sha40};
let mock = crate::object_store::mock::MockStore::new();
// Seed chain.json + path-index.json but NOT the bundle.
let ref_name = RefName::new("refs/heads/main").unwrap();
let full_at = Sha40::try_new(TOMBSTONE_TEST_FULL_AT).unwrap();
let chain = ChainManifest {
v: 1,
tip: full_at.clone(),
full_at: full_at.clone(),
segments: vec![ChainSegment {
sha: full_at.clone(),
parent_sha: None,
pack: format!("packs/{TOMBSTONE_TEST_FULL_AT}.pack"),
bytes: 1_024,
}],
};
write_chain(&mock, Some("repo"), &ref_name, &chain)
.await
.unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "repo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete().await.expect("delete");
// chain.json removed; no bundle was ever there; no
// tombstone written (deferring nothing is pointless).
assert!(
!mock.contains("repo/refs/heads/main/chain.json"),
"chain.json must be removed: {:?}",
mock.keys(),
);
let tomb_keys: Vec<String> = mock
.keys()
.into_iter()
.filter(|k| k.starts_with(&baseline_tombstone_listing_prefix(Some("repo"))))
.collect();
assert!(
tomb_keys.is_empty(),
"no tombstone for a chain whose bundle is already absent: {tomb_keys:?}",
);
}
// --- Per-ref lock acquisition / release (#158) ----------------------
#[tokio::test]
async fn delete_refuses_when_per_ref_lock_is_held_by_another_writer() {
// Issue #158: pre-fix, `delete-branch` performed a fresh
// listing + sweep without taking the per-ref `LOCK#.lock`. A
// concurrent `git push` that acquired the lock and started
// uploading a new bundle after the listing would land that
// bundle AFTER the delete sweep, leaving the ref alive while
// delete-branch reported success.
//
// The fix takes the same lock the helper-protocol push and
// delete paths take. This test seeds a FRESH lock (matching a
// concurrent push holding it) and asserts that delete-branch
// returns `LockContended` and makes NO changes — the bundle
// and the lock both survive verbatim.
let mock = seed_with_branch("main");
// Fresh `last_modified` = now → `acquire_lock` sees
// `age <= ttl` and reports contention (Ok(None)).
mock.insert("myrepo/refs/heads/main/LOCK#.lock", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.delete()
.await
.expect_err("delete must refuse to race a fresh lock holder");
match &err {
ManageError::LockContended {
branch,
lock,
ttl_seconds,
} => {
assert_eq!(branch, "main");
assert_eq!(lock, "myrepo/refs/heads/main/LOCK#.lock");
assert!(
*ttl_seconds > 0,
"ttl_seconds must be positive, got {ttl_seconds}",
);
}
other => panic!("expected LockContended, got {other:?}"),
}
// The operator-facing wording must name the lock key (so a
// doctor invocation can copy it) and surface the TTL.
let rendered = err.to_string();
assert!(
rendered.contains("myrepo/refs/heads/main/LOCK#.lock"),
"error must name the lock key, got: {rendered}",
);
assert!(
rendered.contains("doctor"),
"error must point operators at doctor, got: {rendered}",
);
// NOTHING was deleted: the bundle, the lock, and the prompt's
// confirmation are all preserved. Pre-#158 this exact race
// produced a "success" with the bundle missing.
assert!(
mock.contains("myrepo/refs/heads/main/abc.bundle"),
"bundle must survive a contended-lock refusal",
);
assert!(
mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"the racing writer's lock must NOT be deleted",
);
}
#[tokio::test]
async fn delete_recovers_stale_lock_and_proceeds() {
// A `LOCK#.lock` older than the TTL means a previous writer
// crashed before releasing it. `acquire_lock` recovers it by
// deleting and re-acquiring. The delete must then complete
// normally — refusing on a stale lock would let a crashed
// writer block the bucket forever.
//
// This pins the staleness boundary by seeding a lock dated
// well in the past (sufficient for any reasonable TTL up to
// hours) and asserting both the sweep and the final release
// ran.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/abc.bundle", Bytes::from("body"));
let stale = OffsetDateTime::now_utc() - time::Duration::days(1);
mock.insert_with(
"myrepo/refs/heads/main/LOCK#.lock",
Bytes::new(),
stale,
PutOpts::default(),
);
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete().await.expect("stale lock must be recovered");
assert!(
mock.keys().is_empty(),
"bundle and lock both gone after stale-lock recovery + release: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn delete_releases_lock_after_successful_sweep() {
// A successful delete must clean up the LOCK#.lock it
// acquired. Without release, a subsequent operation would
// see a fresh (just-released) lock and report contention
// until the TTL elapses — defeating the point of explicit
// release.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(
Arc::clone(&store),
"myrepo",
"main",
&prompter as &dyn Prompter,
)
.await
.expect("open");
mb.delete().await.expect("delete");
assert!(
!mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"lock must be released (deleted) after a successful sweep: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn delete_releases_lock_even_when_sweep_returns_partial_delete() {
// The lock must be released regardless of how the lock-held
// body returned. A `PartialDelete` error means one per-key
// delete failed; the lock is still released so a retry isn't
// gated on TTL recovery.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/aaa.bundle", Bytes::from("a"));
mock.insert("myrepo/refs/heads/main/bbb.bundle", Bytes::from("b"));
mock.arm(crate::object_store::mock::Fault::NetworkOnDelete {
key: "myrepo/refs/heads/main/bbb.bundle".to_owned(),
});
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.delete()
.await
.expect_err("partial delete must still surface its error");
assert!(matches!(err, ManageError::PartialDelete { .. }));
assert!(
!mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"lock must be released even when sweep returns PartialDelete: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn delete_does_not_iterate_over_its_own_lock_key() {
// Mirrors `protocol::push::delete_remote_ref_under_lock`'s
// lock-key filter (#133): the fresh under-lock listing must
// exclude the lock we hold so the sweep does not delete our
// own coordination key mid-critical-section. The expression
// of this guarantee in the test: a stale lock that the
// acquire path recovered is replaced by OUR fresh lock; the
// sweep must NOT report `attempted = 2` (the lock + the
// bundle) but `attempted = 1` (the bundle only).
//
// Indirect proof: we arm a fault on the lock key. If the
// sweep iterates over the lock, the fault fires and the
// delete returns `PartialDelete { undeleted: [lock] }`. The
// test asserts the fault is NOT consumed — the sweep skipped
// the lock as expected.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/abc.bundle", Bytes::from("body"));
// Arm a fault on the lock key; if `delete` iterates over the
// lock the fault fires.
mock.arm(crate::object_store::mock::Fault::NetworkOnDelete {
key: "myrepo/refs/heads/main/LOCK#.lock".to_owned(),
});
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
// Note: `release_lock`'s delete also goes through the mock
// and will trip the fault. We accept either outcome here —
// the load-bearing assertion is that the sweep loop did not
// iterate over the lock (which would have produced a
// `PartialDelete`). The release-delete simply returns a
// warn-logged error and is swallowed; the sweep success path
// surfaces as `Ok(())`.
let result = mb.delete().await;
assert!(
!matches!(result, Err(ManageError::PartialDelete { .. })),
"sweep must not iterate over the lock key (no PartialDelete on the lock): {result:?}",
);
// The bundle was deleted by the sweep.
assert!(
!mock.contains("myrepo/refs/heads/main/abc.bundle"),
"bundle must still be swept: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn delete_release_failure_does_not_mask_sweep_success() {
// Issue #158: release failures are downgraded to `warn!` —
// the operator's "ref is gone" intent is satisfied as soon
// as the sweep succeeds, and an orphan lock will age out via
// the next acquirer's TTL recovery. A regression that
// propagated the release error would surface a spurious
// failure for a delete that actually succeeded.
//
// Arm a fault on the lock-key delete (which is exactly what
// `release_lock` calls). The sweep is unaffected (bundle is
// a different key); the delete must return `Ok(())`.
let mock = MockStore::new();
mock.insert("myrepo/refs/heads/main/abc.bundle", Bytes::from("body"));
mock.arm(crate::object_store::mock::Fault::NetworkOnDelete {
key: "myrepo/refs/heads/main/LOCK#.lock".to_owned(),
});
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete()
.await
.expect("release failure must not mask sweep success");
// The bundle was deleted; only the now-orphan lock survives
// (the release fault consumed the lock's delete).
assert!(!mock.contains("myrepo/refs/heads/main/abc.bundle"));
}
// -----------------------------------------------------------------
// Issue #159 — protect / unprotect must acquire the per-ref lock so
// a concurrent push-in-progress cannot land a force-push between
// the under-lock `is_protected` sample and the bundle upload.
// -----------------------------------------------------------------
#[tokio::test]
async fn protect_refuses_when_per_ref_lock_is_held_by_another_writer() {
// Issue #159: pre-fix, `protect` was a lockless `put_bytes`. A
// concurrent `git push` that had taken the per-ref lock and
// already passed its under-lock `is_protected()` check could
// overwrite the bundle even if `protect` landed between that
// check and the bundle upload. The fix takes the same lock the
// push path takes; this test seeds a fresh lock (matching a
// push holding it) and asserts `protect` returns
// `LockContended` and writes NO marker.
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/LOCK#.lock", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.protect()
.await
.expect_err("protect must refuse to race a fresh lock holder");
match &err {
ManageError::LockContended {
branch,
lock,
ttl_seconds,
} => {
assert_eq!(branch, "main");
assert_eq!(lock, "myrepo/refs/heads/main/LOCK#.lock");
assert!(*ttl_seconds > 0);
}
other => panic!("expected LockContended, got {other:?}"),
}
// The marker must NOT be written and the racing writer's lock
// must NOT be deleted. Pre-#159 this exact race let `protect`
// land its marker AFTER the push's `is_protected` check, with
// the push completing the force-push anyway.
assert!(
!mock.contains("myrepo/refs/heads/main/PROTECTED#"),
"no marker may be written under a contended lock",
);
assert!(
mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"the racing writer's lock must survive a contention refusal",
);
assert!(mock.contains("myrepo/refs/heads/main/abc.bundle"));
}
#[tokio::test]
async fn unprotect_refuses_when_per_ref_lock_is_held_by_another_writer() {
// Symmetry with the protect contention test: `unprotect` must
// also block on a held lock so protection state changes are
// serialised against every other writer. Pre-#159, `unprotect`
// was a lockless `delete`; a concurrent push observing
// `is_protected() == true` and a racing `unprotect` could land
// with the push still on the protected-refusal path.
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/PROTECTED#", Bytes::new());
mock.insert("myrepo/refs/heads/main/LOCK#.lock", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.unprotect()
.await
.expect_err("unprotect must refuse to race a fresh lock holder");
assert!(
matches!(err, ManageError::LockContended { ref branch, .. } if branch == "main"),
"expected LockContended, got {err:?}",
);
// The marker must remain — `unprotect` did not get to remove it.
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
assert!(mock.contains("myrepo/refs/heads/main/LOCK#.lock"));
}
#[tokio::test]
async fn protect_releases_lock_after_successful_write() {
// A successful protect must release the LOCK#.lock it acquired.
// Without release, a subsequent push or unprotect would see a
// fresh lock and report contention until TTL — defeating the
// point of an explicit release.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.protect().await.expect("protect");
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
assert!(
!mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"lock must be released after a successful protect: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn unprotect_releases_lock_after_successful_delete() {
// Mirror of `protect_releases_lock_after_successful_write`: the
// unprotect path must release its lock on the success branch.
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/PROTECTED#", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.unprotect().await.expect("unprotect");
assert!(!mock.contains("myrepo/refs/heads/main/PROTECTED#"));
assert!(
!mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"lock must be released after a successful unprotect: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn protect_recovers_stale_lock_and_proceeds() {
// A `LOCK#.lock` older than the TTL means a previous writer
// crashed before releasing it. `acquire_lock` recovers it by
// deleting and re-acquiring. The protect must then complete
// normally — refusing on a stale lock would let a crashed
// writer block protection state changes forever.
let mock = seed_with_branch("main");
let stale = OffsetDateTime::now_utc() - time::Duration::days(1);
mock.insert_with(
"myrepo/refs/heads/main/LOCK#.lock",
Bytes::new(),
stale,
PutOpts::default(),
);
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.protect().await.expect("stale lock must be recovered");
assert!(mock.contains("myrepo/refs/heads/main/PROTECTED#"));
assert!(
!mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"stale lock recovered and our fresh lock released: {:?}",
mock.keys(),
);
}
#[tokio::test]
async fn protect_release_failure_does_not_mask_marker_write_success() {
// Issue #159 / #158 symmetry: release failures are downgraded
// to `warn!`. A regression that propagated the release error
// would lie to the operator about a `protect` that actually
// succeeded — the marker is on the bucket; the orphan lock
// ages out via the next acquirer's TTL recovery.
let mock = seed_with_branch("main");
mock.arm(crate::object_store::mock::Fault::NetworkOnDelete {
key: "myrepo/refs/heads/main/LOCK#.lock".to_owned(),
});
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.protect()
.await
.expect("release failure must not mask marker-write success");
assert!(
mock.contains("myrepo/refs/heads/main/PROTECTED#"),
"marker must be written even when lock release fails",
);
}
#[tokio::test]
async fn issue_159_protect_cannot_land_during_active_push() {
// The headline regression test for #159. Models the documented
// race verbatim:
//
// 1. push acquires LOCK#.lock
// 2. push reads is_protected -> NotFound
// 3. operator runs `protect`, which (pre-#159) put_bytes the
// marker without taking the lock — succeeds
// 4. push uploads the new bundle, force-overwriting a now
// "protected" ref
//
// The fix makes step 3 fail with `LockContended`. With the
// lock still on the bucket from step 1, `protect` cannot run
// until the push releases — at which point the push has
// already either committed or refused on its under-lock
// `is_protected` check, with no half-state in between.
//
// The test seeds the lock directly (representing step 1's
// holder) and asserts step 3 fails. The push's actual upload
// is not exercised here because it is covered by the
// helper-protocol push tests; the load-bearing claim is "no
// mid-push protect can sneak in".
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/LOCK#.lock", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.protect()
.await
.expect_err("protect must not land during an active push");
assert!(
matches!(err, ManageError::LockContended { .. }),
"expected LockContended, got {err:?}",
);
// Marker NOT written: the under-lock push (when it eventually
// releases) will see no marker, take whichever branch
// is_protected dictates, and operator intent never crosses
// streams with the writer's snapshot.
assert!(!mock.contains("myrepo/refs/heads/main/PROTECTED#"));
// The racing writer's lock must survive the contention refusal —
// protect must not have touched LOCK#.lock owned by another
// operation. Pinning this directly makes the test self-sufficient.
assert!(
mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"the writer's LOCK#.lock must survive a contended protect attempt",
);
}
// -----------------------------------------------------------------
// Issue #151 — delete paths must not miss a `PROTECTED#` marker
// written after the under-lock listing. Closed mechanically by the
// per-ref lock (#158 for delete-branch, #159 for protect/unprotect):
// `protect` blocks on the same key the delete acquired, so a marker
// cannot land between the under-lock listing and the sweep. These
// tests pin the lock-contract guarantee and the post-sweep
// defensive verification that surfaces a contract violation if one
// ever arises.
// -----------------------------------------------------------------
#[tokio::test]
async fn issue_151_protect_cannot_inject_marker_during_active_delete() {
// The headline regression test for #151. Models the documented
// race verbatim:
//
// 1. delete-branch acquires LOCK#.lock and does its
// under-lock listing (no marker).
// 2. operator runs `protect`, which (pre-#159) put_bytes the
// marker without taking the lock — would succeed.
// 3. delete-branch sweeps the listing it took at step 1,
// missing the marker entirely; delete reports success
// while the marker is orphaned.
//
// The fix (#159) makes step 2 fail with `LockContended` because
// `protect` now serialises through the same per-ref lock
// delete-branch holds. The test seeds the lock directly
// (representing the delete-branch holder at step 1) and asserts
// step 2 fails — proving the race window is mechanically closed.
let mock = seed_with_branch("main");
mock.insert("myrepo/refs/heads/main/LOCK#.lock", Bytes::new());
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
let err = mb
.protect()
.await
.expect_err("protect must not land during an active delete-branch");
assert!(
matches!(err, ManageError::LockContended { .. }),
"expected LockContended (lock held by delete-branch), got {err:?}",
);
// No marker landed — the delete's sweep will not encounter a
// mid-flow PROTECTED# the listing did not see.
assert!(
!mock.contains("myrepo/refs/heads/main/PROTECTED#"),
"no marker may be written while a delete holds the lock",
);
// The delete-branch holder's lock survives the contention
// refusal verbatim.
assert!(
mock.contains("myrepo/refs/heads/main/LOCK#.lock"),
"the delete-branch holder's lock must survive contention",
);
}
#[tokio::test]
async fn issue_151_post_sweep_verification_passes_on_clean_delete() {
// The post-sweep `verify_no_orphan_protected_after_delete`
// probe is belt-and-suspenders telemetry: with the lock
// contract in place there is no way for a marker to appear
// post-sweep. The probe must be silent on the happy path so an
// operator reading logs is not chasing phantoms.
//
// This test exercises the success path end-to-end: seed only
// the bundle, confirm, sweep. The delete must return Ok and
// the bucket must be empty (including the lock the release
// step removed). A regression that flipped the post-sweep
// probe into a hard error (rather than telemetry) would surface
// here as an unexpected `Err`.
let mock = seed_with_branch("main");
let store: Arc<dyn ObjectStore> = Arc::new(mock.clone());
let prompter = ScriptedPrompter::new([Answer::Confirm(true)]);
let mb = ManageBranch::open(store, "myrepo", "main", &prompter as &dyn Prompter)
.await
.expect("open");
mb.delete()
.await
.expect("clean delete must pass the post-sweep probe silently");
assert!(
mock.keys().is_empty(),
"bundle + lock both gone after a clean delete-and-release: {:?}",
mock.keys(),
);
}
}