1use std::collections::HashSet;
26
27use mkit_core::hash::{Hash, to_hex};
28use mkit_core::object::{Object, ObjectType};
29use mkit_core::ops::graph::{ClosureMode, MAX_REACHABLE, children, reachable_closure_checked};
30use mkit_core::ops::merge::is_ancestor;
31use mkit_core::pack;
32use mkit_core::protocol::UploadLimits;
33use mkit_core::refs::RefWriteCondition;
34use mkit_core::store::{ObjectStore, StoreError};
35use mkit_core::transfer::{self, PackPlan};
36
37use super::{
38 DispatchError, PackSink, build_and_upload_packs, effective_payload_cap,
39 encode_delta_candidates_batch, estimate_pack_sizes,
40};
41
42pub const MAX_SPLIT_STEPS: usize = 1_000;
44
45pub const MAX_CHAIN_COMMITS: usize = 1_000_000;
47
48pub trait StepAuthority: Send + Sync {
51 fn authorize(
58 &self,
59 branch: &str,
60 first: RefWriteCondition,
61 later_steps: bool,
62 ) -> Result<(), String>;
63}
64
65pub struct PushControl<'a> {
67 pub authority: Option<&'a dyn StepAuthority>,
69 pub max_steps: usize,
71 pub max_chain: usize,
73 pub shallow: HashSet<Hash>,
76}
77
78impl std::fmt::Debug for PushControl<'_> {
79 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
80 f.debug_struct("PushControl")
81 .field("checks_grants", &self.authority.is_some())
82 .field("max_steps", &self.max_steps)
83 .field("max_chain", &self.max_chain)
84 .field("shallow", &self.shallow.len())
85 .finish()
86 }
87}
88
89impl Default for PushControl<'_> {
90 fn default() -> Self {
91 Self {
92 authority: None,
93 max_steps: MAX_SPLIT_STEPS,
94 max_chain: MAX_CHAIN_COMMITS,
95 shallow: HashSet::new(),
96 }
97 }
98}
99
100pub(super) fn data_pack_budget(limits: UploadLimits) -> usize {
103 limits
104 .tickets_per_advance
105 .map_or(6, |tickets| tickets.min(7).saturating_sub(1))
106}
107
108struct Chain {
110 commits: Vec<Hash>,
112 seed: HashSet<Hash>,
116}
117
118fn first_parent(obj: &Object) -> Option<Hash> {
119 match obj {
120 Object::Commit(c) => c.parents.first().copied(),
121 Object::Remix(r) => r.parents.first().copied(),
122 _ => None,
123 }
124}
125
126fn held_closure(store: &ObjectStore, root: Option<Hash>) -> Result<HashSet<Hash>, DispatchError> {
131 let Some(root) = root.filter(|root| store.contains(root)) else {
132 return Ok(HashSet::new());
133 };
134 match reachable_closure_checked(store, [&root]) {
135 Ok((set, false)) => Ok(set.into_iter().collect()),
136 Ok((_, true)) => Err(too_large_to_split()),
137 Err(StoreError::ObjectNotFound(_)) => Ok(HashSet::new()),
138 Err(e) => Err(e.into()),
139 }
140}
141
142fn too_large_to_split() -> DispatchError {
143 DispatchError::PushSplitLimit(format!(
144 "the history is too large to split ({MAX_REACHABLE} objects or more); no advance of this branch was published"
145 ))
146}
147
148fn walk_chain(
151 store: &ObjectStore,
152 tip: Hash,
153 remote_tip: Option<Hash>,
154 ctl: &PushControl<'_>,
155) -> Result<Chain, DispatchError> {
156 let held = held_closure(store, remote_tip)?;
157 let mut commits = Vec::new();
158 let mut base = None;
159 let mut cursor = tip;
160 loop {
161 if held.contains(&cursor) {
162 base = Some(cursor);
163 break;
164 }
165 if commits.len() >= ctl.max_chain {
166 return Err(DispatchError::PushSplitLimit(format!(
167 "the first-parent history between the remote and the local tip is longer than {} commits, too long to split; no advance of this branch was published",
168 ctl.max_chain
169 )));
170 }
171 if crate::signal::is_shutdown() {
172 return Err(DispatchError::Interrupted);
173 }
174 commits.push(cursor);
175 if ctl.shallow.contains(&cursor) {
176 break;
177 }
178 match first_parent(&store.read_object(&cursor)?) {
179 Some(parent) => cursor = parent,
180 None => break,
181 }
182 }
183 commits.reverse();
184 let seed = match base {
185 Some(base) if Some(base) == remote_tip => held,
186 base => held_closure(store, base)?,
187 };
188 Ok(Chain { commits, seed })
189}
190
191fn cumulative_weights(
196 store: &ObjectStore,
197 chain: &Chain,
198 ctl: &PushControl<'_>,
199) -> Result<Vec<u64>, DispatchError> {
200 let mut seen = chain.seed.clone();
201 let mut cumulative = Vec::with_capacity(chain.commits.len() + 1);
202 let mut total = 0_u64;
203 cumulative.push(0);
204 for commit in &chain.commits {
205 if crate::signal::is_shutdown() {
206 return Err(DispatchError::Interrupted);
207 }
208 let mut stack = vec![*commit];
209 while let Some(hash) = stack.pop() {
210 if !seen.insert(hash) {
211 continue;
212 }
213 if seen.len() > MAX_REACHABLE {
214 return Err(too_large_to_split());
215 }
216 total = total
217 .saturating_add(store.object_metadata(&hash)?.len())
218 .saturating_add(pack::ENTRY_FRAME_LEN as u64);
219 if matches!(
220 store.object_type(&hash)?,
221 ObjectType::Blob | ObjectType::Delta
222 ) {
223 continue;
224 }
225 let mode = if ctl.shallow.contains(&hash) {
227 ClosureMode::Snapshot
228 } else {
229 ClosureMode::History
230 };
231 stack.extend(children(&store.read_object(&hash)?, mode));
232 }
233 cumulative.push(total);
234 }
235 Ok(cumulative)
236}
237
238fn pack_weight_cap(payload_cap: u64, max_pack_bytes: Option<u64>) -> u64 {
242 let serialized = max_pack_bytes.map_or(u64::MAX, |limit| {
243 limit.saturating_sub((pack::HEADER_LEN + pack::TRAILER_LEN) as u64)
244 });
245 payload_cap.min(serialized).max(1)
246}
247
248struct Cutter<'a> {
249 store: &'a ObjectStore,
250 chain: &'a Chain,
251 weights: &'a [u64],
252 remote_tip: Option<Hash>,
253 limits: UploadLimits,
254 cap: u64,
255 budget: usize,
256 must_reach: usize,
258}
259
260impl Cutter<'_> {
261 fn probe(&self, from: usize, to: usize) -> Result<(PackPlan, usize), DispatchError> {
264 if crate::signal::is_shutdown() {
265 return Err(DispatchError::Interrupted);
266 }
267 let base = match from {
268 0 => self.remote_tip,
269 from => Some(self.chain.commits[from - 1]),
270 };
271 let plan = transfer::plan_pack_with(
272 self.store,
273 self.chain.commits[to - 1],
274 base,
275 encode_delta_candidates_batch,
276 )?;
277 let packs =
278 estimate_pack_sizes(self.store, &plan, self.cap, self.limits.max_pack_bytes)?.len();
279 Ok((plan, packs))
280 }
281
282 fn furthest(&self, from: usize, limit: u64) -> usize {
285 let bound = self.weights[from].saturating_add(limit);
286 let count = self.weights.partition_point(|&weight| weight <= bound);
287 count
288 .saturating_sub(1)
289 .max(from + 1)
290 .min(self.weights.len() - 1)
291 }
292
293 fn cut(&self, from: usize) -> Result<usize, DispatchError> {
295 let wcap = pack_weight_cap(self.cap, self.limits.max_pack_bytes);
296 let floor_weight = wcap.saturating_mul((self.budget / 2) as u64);
299 let natural = self.furthest(from, floor_weight);
300 let floor = if from == 0 {
304 natural.max(self.must_reach)
305 } else {
306 natural
307 };
308 let ceiling = self
309 .furthest(from, wcap.saturating_mul(self.budget as u64))
310 .max(floor);
311 let (mut fits, mut over) = (floor, ceiling);
312 while fits < over {
313 let mid = fits + (over - fits).div_ceil(2);
314 if self.probe(from, mid)?.1 <= self.budget {
315 fits = mid;
316 } else {
317 over = mid - 1;
318 }
319 }
320 let heavy = self.weights[fits] - self.weights[from] > floor_weight;
324 if fits == floor && heavy {
325 self.verify_forced(from, fits, from == 0 && floor > natural)?;
326 }
327 Ok(fits)
328 }
329
330 fn verify_forced(
335 &self,
336 from: usize,
337 to: usize,
338 holds_remote_head: bool,
339 ) -> Result<(), DispatchError> {
340 let (plan, packs) = self.probe(from, to)?;
341 if packs <= self.budget {
342 return Ok(());
343 }
344 build_and_upload_packs(PackSink::Count, self.store, plan, self.cap, self.limits)
345 .map(drop)
346 .map_err(|error| match error {
347 DispatchError::PushTooLarge { packs, limit, .. } => DispatchError::PushTooLarge {
348 packs,
349 limit,
350 commit: Some(self.describe(self.chain.commits[to - 1])),
351 holds_remote_head,
352 },
353 other => other,
354 })
355 }
356
357 fn describe(&self, commit: Hash) -> String {
358 match self.store.read_object(&commit) {
359 Ok(Object::Commit(c)) if c.parents.len() > 1 => format!(
360 "merge commit {} (second parent {})",
361 to_hex(&commit),
362 to_hex(&c.parents[1])
363 ),
364 _ => format!("commit {}", to_hex(&commit)),
365 }
366 }
367}
368
369fn positions_to_reach(
374 store: &ObjectStore,
375 chain: &Chain,
376 remote_tip: Option<Hash>,
377) -> Result<usize, DispatchError> {
378 let Some(remote) = remote_tip.filter(|remote| store.contains(remote)) else {
379 return Ok(0);
380 };
381 let (mut low, mut high) = (1, chain.commits.len());
382 if !is_ancestor(store, remote, chain.commits[high - 1])? {
383 return Ok(0);
384 }
385 while low < high {
386 let mid = low + (high - low) / 2;
387 if is_ancestor(store, remote, chain.commits[mid - 1])? {
388 high = mid;
389 } else {
390 low = mid + 1;
391 }
392 }
393 Ok(low)
394}
395
396#[allow(clippy::too_many_arguments)]
407pub fn plan_push_steps(
408 store: &ObjectStore,
409 tip: Hash,
410 remote_tip: Option<Hash>,
411 limits: UploadLimits,
412 pack_payload_cap: u64,
413 ctl: &PushControl<'_>,
414 condition: RefWriteCondition,
415 branch: &str,
416) -> Result<Vec<Hash>, DispatchError> {
417 let cap = effective_payload_cap(pack_payload_cap, limits.max_pack_bytes)?;
418 let chain = walk_chain(store, tip, remote_tip, ctl)?;
419 if chain.commits.is_empty() {
420 return Ok(vec![tip]);
421 }
422 if let Some(authority) = ctl.authority {
423 authority
424 .authorize(branch, condition, chain.commits.len() > 1)
425 .map_err(DispatchError::PushNotAuthorized)?;
426 }
427 let weights = cumulative_weights(store, &chain, ctl)?;
428 let cutter = Cutter {
429 store,
430 chain: &chain,
431 weights: &weights,
432 remote_tip,
433 limits,
434 cap,
435 budget: data_pack_budget(limits),
436 must_reach: positions_to_reach(store, &chain, remote_tip)?,
437 };
438 let mut steps = Vec::new();
439 let mut from = 0;
440 while from < chain.commits.len() {
441 if steps.len() >= ctl.max_steps {
442 return Err(DispatchError::PushSplitLimit(format!(
443 "this push would need more than {} advances; no advance of this branch was published. Ask the operator to raise max_pack_bytes, or push an ancestor commit first",
444 ctl.max_steps
445 )));
446 }
447 from = cutter.cut(from)?;
448 steps.push(chain.commits[from - 1]);
449 }
450 Ok(steps)
451}
452
453#[cfg(test)]
454mod tests {
455 use super::*;
456 use mkit_core::layout::RepoLayout;
457 use mkit_core::object::{Blob, Commit, EntryMode, Identity, Tree, TreeEntry};
458 use mkit_core::ops::graph::reachable_objects;
459 use mkit_core::serialize;
460 use proptest::prelude::*;
461 use tempfile::TempDir;
462
463 fn store() -> (TempDir, ObjectStore) {
464 let dir = TempDir::new().unwrap();
465 let store = ObjectStore::init(&RepoLayout::single(dir.path())).unwrap();
466 (dir, store)
467 }
468
469 fn put(store: &ObjectStore, obj: &Object) -> Hash {
470 store.write(&serialize::serialize(obj).unwrap()).unwrap()
471 }
472
473 fn bytes(seed: u64, len: usize, compressible: bool) -> Vec<u8> {
475 if compressible {
476 return vec![seed.to_le_bytes()[0]; len];
477 }
478 let mut out = vec![0; len];
479 let mut state = seed | 1;
480 for chunk in out.chunks_mut(8) {
481 state = state
482 .wrapping_mul(6_364_136_223_846_793_005)
483 .wrapping_add(1_442_695_040_888_963_407);
484 chunk.copy_from_slice(&state.to_le_bytes()[..chunk.len()]);
485 }
486 out
487 }
488
489 fn commit(store: &ObjectStore, files: &[Hash], parents: Vec<Hash>, seed: u8) -> Hash {
491 let mut entries: Vec<TreeEntry> = files
492 .iter()
493 .enumerate()
494 .map(|(i, hash)| TreeEntry {
495 name: format!("f{i:04}").into_bytes(),
496 mode: EntryMode::Blob,
497 object_hash: *hash,
498 })
499 .collect();
500 entries.sort_by(|a, b| a.name.cmp(&b.name));
501 let tree = put(store, &Object::Tree(Tree { entries }));
502 put(
503 store,
504 &Object::Commit(Commit::new_unannotated(
505 tree,
506 parents,
507 Identity::ed25519([7; 32]),
508 [0; 32],
509 vec![seed],
510 u64::from(seed),
511 [0; 64],
512 )),
513 )
514 }
515
516 fn blob(store: &ObjectStore, seed: u64, len: usize) -> Hash {
517 put(
518 store,
519 &Object::Blob(Blob {
520 data: bytes(seed, len, false),
521 }),
522 )
523 }
524
525 fn line(store: &ObjectStore, n: u8) -> Vec<Hash> {
527 let mut files = Vec::new();
528 let mut commits: Vec<Hash> = Vec::new();
529 for i in 0..n {
530 files.push(blob(store, u64::from(i) + 1, 100));
531 let parents = commits.last().copied().into_iter().collect();
532 commits.push(commit(store, &files, parents, i));
533 }
534 commits
535 }
536
537 #[test]
538 fn the_walk_stops_at_the_commit_the_remote_holds() {
539 let (_dir, store) = store();
540 let c = line(&store, 6);
541 let chain = walk_chain(&store, c[5], Some(c[2]), &PushControl::default()).unwrap();
542 assert_eq!(chain.commits, [c[3], c[4], c[5]]);
543 assert_eq!(chain.seed, held_closure(&store, Some(c[2])).unwrap());
544
545 let chain = walk_chain(&store, c[5], None, &PushControl::default()).unwrap();
547 assert_eq!(chain.commits, c);
548 assert!(chain.seed.is_empty());
549 }
550
551 #[test]
552 fn a_diverged_remote_seeds_the_shared_ancestor_not_its_own_tip() {
553 let (_dir, store) = store();
554 let c = line(&store, 6);
555 let other = blob(&store, 900, 100);
556 let theirs = commit(&store, &[other], vec![c[1]], 90);
557 let chain = walk_chain(&store, c[5], Some(theirs), &PushControl::default()).unwrap();
558 assert_eq!(chain.commits, [c[2], c[3], c[4], c[5]]);
559 assert_eq!(chain.seed, held_closure(&store, Some(c[1])).unwrap());
560 }
561
562 #[test]
563 fn the_walk_never_enters_a_second_parent_side() {
564 let (_dir, store) = store();
565 let c = line(&store, 4);
566 let side_file = blob(&store, 500, 100);
567 let side = commit(&store, &[side_file], vec![c[0]], 50);
568 let merge = commit(&store, &[side_file], vec![c[3], side], 60);
569 let chain = walk_chain(&store, merge, None, &PushControl::default()).unwrap();
570 assert_eq!(chain.commits, [c[0], c[1], c[2], c[3], merge]);
571 let weights = cumulative_weights(&store, &chain, &PushControl::default()).unwrap();
573 let brought = weights[5] - weights[4];
574 let lone = weights[4] - weights[3];
575 assert!(brought > lone, "{weights:?}");
576 }
577
578 #[test]
579 fn a_shallow_boundary_ends_the_walk_and_its_parents_are_not_followed() {
580 let (_dir, store) = store();
581 let c = line(&store, 6);
582 let control = PushControl {
583 shallow: std::iter::once(c[3]).collect(),
584 ..PushControl::default()
585 };
586 let chain = walk_chain(&store, c[5], None, &control).unwrap();
587 assert_eq!(chain.commits, [c[3], c[4], c[5]]);
588 let weights = cumulative_weights(&store, &chain, &control).unwrap();
590 let everything = cumulative_weights(&store, &chain, &PushControl::default()).unwrap();
591 assert!(weights[1] < everything[1]);
592 }
593
594 #[test]
595 fn a_too_long_walk_is_refused() {
596 let (_dir, store) = store();
597 let c = line(&store, 6);
598 let control = PushControl {
599 max_chain: 4,
600 ..PushControl::default()
601 };
602 let error = walk_chain(&store, c[5], None, &control).err().unwrap();
603 assert!(
604 matches!(error, DispatchError::PushSplitLimit(_)),
605 "{error:?}"
606 );
607 }
608
609 #[test]
610 fn cumulative_weights_add_up_to_the_new_objects() {
611 let (_dir, store) = store();
612 let c = line(&store, 6);
613 let chain = walk_chain(&store, c[5], Some(c[1]), &PushControl::default()).unwrap();
614 let weights = cumulative_weights(&store, &chain, &PushControl::default()).unwrap();
615 assert_eq!(weights.len(), chain.commits.len() + 1);
616 assert!(weights.windows(2).all(|w| w[0] < w[1]));
617 let held = reachable_objects(&store, &c[1]).unwrap();
618 let expected: u64 = reachable_objects(&store, &c[5])
619 .unwrap()
620 .difference(&held)
621 .map(|h| store.object_metadata(h).unwrap().len() + pack::ENTRY_FRAME_LEN as u64)
622 .sum();
623 assert_eq!(*weights.last().unwrap(), expected);
624 }
625
626 #[test]
629 fn a_step_forced_to_reach_the_remote_head_says_how_to_avoid_it() {
630 let (_dir, store) = store();
631 let a = line(&store, 4);
632 let their_file = blob(&store, 900, 100);
633 let theirs = commit(&store, &[their_file], Vec::new(), 90);
634 let big: Vec<Hash> = (0..20).map(|i| blob(&store, 300 + i, 2048)).collect();
635 let merge = commit(&store, &big, vec![a[3], theirs], 91);
636 let limits = UploadLimits {
637 max_pack_bytes: Some(8192),
638 tickets_per_advance: Some(7),
639 ticket_threshold_bytes: Some(0),
640 };
641 let error = plan_push_steps(
642 &store,
643 merge,
644 Some(theirs),
645 limits,
646 4096,
647 &PushControl::default(),
648 RefWriteCondition::Match(theirs),
649 "main",
650 )
651 .unwrap_err();
652 assert!(
653 matches!(
654 &error,
655 DispatchError::PushTooLarge {
656 holds_remote_head: true,
657 ..
658 }
659 ),
660 "{error:?}"
661 );
662 assert!(
663 error.to_string().contains("rebase onto the remote head"),
664 "{error}"
665 );
666 }
667
668 proptest! {
669 #![proptest_config(ProptestConfig::with_cases(64))]
670
671 #[test]
677 fn estimate_bounds_the_real_pack_count_and_three_caps_always_fit(
678 sizes in prop::collection::vec((16_usize..1500, any::<bool>()), 1..12),
679 payload_cap in 512_u64..16_384,
680 max_pack in prop::option::of(2048_u64..12_000),
681 with_deltas in any::<bool>(),
682 ) {
683 let (_dir, store) = store();
684 let files: Vec<Hash> = sizes
685 .iter()
686 .enumerate()
687 .map(|(i, (len, compressible))| {
688 put(&store, &Object::Blob(Blob { data: bytes(i as u64 + 1, *len, *compressible) }))
689 })
690 .collect();
691 let (tip, base, seed) = if with_deltas {
692 let base = commit(&store, &files, Vec::new(), 1);
694 let edited: Vec<Hash> = sizes
695 .iter()
696 .enumerate()
697 .map(|(i, (len, compressible))| {
698 let mut data = bytes(i as u64 + 1, *len, *compressible);
699 let middle = data.len() / 2;
700 data[middle] ^= 0xff;
701 put(&store, &Object::Blob(Blob { data }))
702 })
703 .collect();
704 let tip = commit(&store, &edited, vec![base], 2);
705 (tip, Some(base), held_closure(&store, Some(base)).unwrap())
706 } else {
707 (commit(&store, &files, Vec::new(), 1), None, HashSet::new())
708 };
709 let plan = if with_deltas {
710 transfer::plan_pack_with(&store, tip, base, encode_delta_candidates_batch).unwrap()
711 } else {
712 transfer::plan_pack_with(&store, tip, None, |_, candidates| {
713 Ok(vec![None; candidates.len()])
714 })
715 .unwrap()
716 };
717 let limits = UploadLimits {
719 max_pack_bytes: max_pack,
720 tickets_per_advance: None,
721 ticket_threshold_bytes: None,
722 };
723 let cap = effective_payload_cap(payload_cap, max_pack).unwrap();
724 let estimate = estimate_pack_sizes(&store, &plan, cap, max_pack).unwrap().len();
725 let keys = build_and_upload_packs(PackSink::Count, &store, plan, cap, limits)
726 .map_err(|e| TestCaseError::fail(format!("{e:?}")))?;
727 prop_assert!(keys.len() <= estimate, "{} > {estimate}", keys.len());
728 let chain = Chain { commits: vec![tip], seed };
729 let weight = *cumulative_weights(&store, &chain, &PushControl::default())
730 .unwrap()
731 .last()
732 .unwrap();
733 if weight <= 3 * pack_weight_cap(cap, max_pack) {
734 prop_assert!(estimate <= 6, "{estimate} packs for weight {weight}");
735 }
736 }
737 }
738}