1use std::{
2 collections::{BTreeMap, HashMap},
3 io::{self, Seek, SeekFrom, Write},
4 sync::{
5 Arc, Mutex,
6 atomic::{AtomicUsize, Ordering},
7 },
8 time::{Duration, Instant},
9};
10
11use grpc::heddle::v1::{
12 GetBlobRequest, GitCheckpointTransfer, GitLaneTransfer, GitPackTransfer,
13 GitRefKind as GrpcGitRefKind, GitRefUpdateTransfer, ListRefsRequest, ObjectAvailabilityStatus,
14 ObjectDescriptor, PackChunk, PackStreamKind, PartialFetchStatus, PullMessage, PullRequest,
15 PushMessage, PushRequest, RedactionTransfer, StateVisibilityTransfer, ThreadConfidenceSummary,
16 ThreadIntegrationPolicy, ThreadMetadata, ThreadVerificationSummary, TransportMode,
17 UpdateRefRequest, WantObjects, git_lane_transfer, pull_message, push_message,
18};
19use objects::{
20 Progress,
21 object::{ChangeId, ContentHash, MarkerName, ThreadName},
22 store::{AnyStore, ObjectStore, PackObjectId},
23};
24use repo::{
25 GitRefKind as ClassifiedGitRefKind, GitRefName, Repository, RepositoryCapability,
26 RevisionAddress, SyncedThreadMetadata, ThreadManager,
27};
28use sley::{
29 ObjectId as GitObjectId, RefPrecondition, ReferenceTarget, Repository as SleyRepository,
30};
31use tempfile::NamedTempFile;
32use tokio::sync::mpsc;
33use tokio_stream::wrappers::ReceiverStream;
34use tonic::Request;
35use wire::{
36 GitLaneTransferIntent, ObjectInfo, ObjectType, ObjectTypeBucket, PlannedObject, ProtocolError,
37 PullComplete, PushComplete, RefEntry, RefUpdated, RepositoryTransferPlan,
38};
39
40use super::{
41 HostedGrpcClient, PullMaterialization,
42 helpers::{
43 descriptor_id, descriptor_id_from_info, object_descriptor_with_status, object_type_name,
44 parse_descriptor_to_info, status_to_protocol_error, to_proto_object_info,
45 transport_mode_name,
46 },
47};
48
49#[derive(Clone, Copy)]
50struct PullOptions<'a> {
51 local_thread: Option<&'a str>,
52 depth: Option<u32>,
53 target_state: Option<ChangeId>,
54 materialization: PullMaterialization,
55}
56
57struct PullWantPlan {
58 wants: Vec<ObjectDescriptor>,
59 transfer_plan: RepositoryTransferPlan<ObjectInfo>,
60 wanted_types: WantedTypes,
61 want_full_closure: bool,
62}
63
64type WantedTypes = HashMap<PackObjectId, Vec<ObjectType>>;
65
66struct GitLanePushPlan {
67 local_revision_address: String,
68 pack: Option<GitPackPushPlan>,
75 ref_updates: Vec<PushMessage>,
81}
82
83#[derive(Clone)]
84struct GitPackPushPlan {
85 transfer_id: String,
86 pack_id: Vec<u8>,
87 pack_size: u64,
88 #[cfg_attr(not(test), allow(dead_code))]
93 roots: Vec<GitObjectId>,
94 pack_file: Arc<Mutex<NamedTempFile>>,
97}
98
99const PUSH_FULL_DESCRIPTOR_OBJECT_THRESHOLD: usize = 512;
100const PULL_PACK_SPOOL_OBJECT_THRESHOLD: usize = 512;
101const NATIVE_PACK_DRAIN_OBJECT_INTERVAL: usize = 32;
102const NATIVE_PACK_OBJECT_PREFETCH_LIMIT: usize = 32;
103const NATIVE_PACK_OBJECT_LOAD_WORKER_LIMIT: usize = 8;
104
105#[derive(Debug, Clone, Default)]
106pub struct PullObjectMix {
107 pub blobs: usize,
108 pub trees: usize,
109 pub states: usize,
110 pub actions: usize,
111 pub redactions: usize,
112 pub state_visibilities: usize,
113}
114
115#[derive(Debug, Clone)]
116pub struct HostedRefEntry {
117 pub name: String,
118 pub change_id: ChangeId,
119 pub is_thread: bool,
120 pub revision_address: String,
121}
122
123impl PullObjectMix {
124 fn record(&mut self, obj_type: ObjectType) {
125 match obj_type.bucket() {
126 ObjectTypeBucket::Blob => self.blobs += 1,
127 ObjectTypeBucket::Tree => self.trees += 1,
128 ObjectTypeBucket::State => self.states += 1,
129 ObjectTypeBucket::Action => self.actions += 1,
130 ObjectTypeBucket::Redaction => self.redactions += 1,
131 ObjectTypeBucket::StateVisibility => self.state_visibilities += 1,
132 }
133 }
134
135 pub fn total(&self) -> usize {
136 self.blobs
137 + self.trees
138 + self.states
139 + self.actions
140 + self.redactions
141 + self.state_visibilities
142 }
143}
144
145#[derive(Debug, Clone, Default)]
146pub struct PullProfile {
147 pub ready_wait: Duration,
148 pub receive_and_apply: Duration,
149 pub decode: Duration,
150 pub store_receive_object: Duration,
151 pub metadata_sync: Duration,
152 pub pack_decode_apply: Duration,
153 pub raw_decode_apply: Duration,
154 pub pack_decode: Duration,
155 pub raw_decode: Duration,
156 pub bytes_received: usize,
157 pub pack_bytes_received: usize,
158 pub raw_bytes_received: usize,
159 pub objects_received: usize,
160 pub object_mix: PullObjectMix,
161}
162
163impl HostedGrpcClient {
164 pub async fn list_refs(&mut self, repo_path: &str) -> Result<Vec<RefEntry>, ProtocolError> {
165 Ok(self
166 .list_refs_with_revision_addresses(repo_path)
167 .await?
168 .into_iter()
169 .map(|entry| RefEntry {
170 name: entry.name,
171 change_id: entry.change_id,
172 is_thread: entry.is_thread,
173 })
174 .collect())
175 }
176
177 pub async fn list_refs_with_revision_addresses(
178 &mut self,
179 repo_path: &str,
180 ) -> Result<Vec<HostedRefEntry>, ProtocolError> {
181 let mut request = Request::new(ListRefsRequest {
182 repo_path: repo_path.to_string(),
183 });
184 self.apply_signed_auth(&mut request, "/heddle.v1.RepoSyncService/ListRefs")?;
185 let response = self
186 .inner
187 .list_refs(request)
188 .await
189 .map_err(status_to_protocol_error)?
190 .into_inner();
191 response
192 .refs
193 .into_iter()
194 .map(|entry| {
195 Ok(HostedRefEntry {
196 name: entry.name,
197 change_id: ChangeId::try_from_slice(&entry.change_id)
198 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?,
199 is_thread: entry.is_thread,
200 revision_address: entry.revision_address,
201 })
202 })
203 .collect()
204 }
205
206 #[allow(clippy::too_many_arguments)]
207 pub async fn update_ref(
208 &mut self,
209 repo_path: &str,
210 name: &str,
211 is_thread: bool,
212 old_value: Option<ChangeId>,
213 new_value: ChangeId,
214 force: bool,
215 thread_metadata: Option<&SyncedThreadMetadata>,
216 ) -> Result<RefUpdated, ProtocolError> {
217 let mut request = Request::new(UpdateRefRequest {
218 repo_path: repo_path.to_string(),
219 name: name.to_string(),
220 is_thread,
221 force,
222 old_value: old_value
223 .map(|value| value.to_string_full())
224 .unwrap_or_default(),
225 new_value: new_value.to_string_full(),
226 thread_metadata: thread_metadata.map(to_proto_thread_metadata),
227 old_revision_address: old_value
228 .map(|value| RevisionAddress::heddle(value).to_string())
229 .unwrap_or_default(),
230 new_revision_address: RevisionAddress::heddle(new_value).to_string(),
231 client_operation_id: String::new(),
232 });
233 self.apply_signed_auth(&mut request, "/heddle.v1.RepoSyncService/UpdateRef")?;
234 let response = self
235 .inner
236 .update_ref(request)
237 .await
238 .map_err(status_to_protocol_error)?
239 .into_inner();
240 Ok(RefUpdated {
241 success: response.success,
242 old_value: if response.old_value.is_empty() {
243 None
244 } else {
245 Some(
246 ChangeId::parse(&response.old_value)
247 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?,
248 )
249 },
250 error: (!response.error.is_empty()).then_some(response.error),
251 })
252 }
253
254 pub async fn push(
255 &mut self,
256 repo: &Repository,
257 repo_path: &str,
258 local_state: ChangeId,
259 target_thread: &str,
260 force: bool,
261 ) -> Result<PushComplete, ProtocolError> {
262 self.push_with_revision(
263 repo,
264 repo_path,
265 local_state,
266 target_thread,
267 force,
268 RevisionAddress::heddle(local_state).to_string(),
269 None,
270 &Progress::null(),
271 )
272 .await
273 }
274
275 pub async fn push_git_overlay_mirror(
290 &mut self,
291 repo: &Repository,
292 repo_path: &str,
293 local_state: ChangeId,
294 target_thread: &str,
295 force: bool,
296 progress: &Progress,
297 ) -> Result<PushComplete, ProtocolError> {
298 progress.set_phase("packing refs");
299 let remote_ref_expectations = self.git_mirror_ref_expectations(repo_path).await?;
300 let git_lane = build_git_mirror_push_plan(
301 repo,
302 self.transport.chunk_size.max(1),
303 &remote_ref_expectations,
304 )?;
305 let local_revision_address = git_lane.local_revision_address.clone();
306 self.push_with_revision(
307 repo,
308 repo_path,
309 local_state,
310 target_thread,
311 force,
312 local_revision_address,
313 Some(git_lane),
314 progress,
315 )
316 .await
317 }
318
319 async fn git_mirror_ref_expectations(
323 &mut self,
324 repo_path: &str,
325 ) -> Result<HashMap<String, GitRefRemoteExpectation>, ProtocolError> {
326 let remote_refs = self.list_refs_with_revision_addresses(repo_path).await?;
327 let mut expectations = HashMap::with_capacity(remote_refs.len());
328 for entry in remote_refs {
329 let expectation = parse_git_ref_expectation(&entry.revision_address)?;
330 expectations.insert(entry.name, expectation);
331 }
332 Ok(expectations)
333 }
334
335 #[allow(clippy::too_many_arguments)]
336 async fn push_with_revision(
337 &mut self,
338 repo: &Repository,
339 repo_path: &str,
340 local_state: ChangeId,
341 target_thread: &str,
342 force: bool,
343 local_revision_address: String,
344 git_lane: Option<GitLanePushPlan>,
345 progress: &Progress,
346 ) -> Result<PushComplete, ProtocolError> {
347 let _ = self.transport.chunk_size;
348 let _ = self.transport.resume_attempts;
349 let git_lane_intent = if git_lane.is_some() {
354 GitLaneTransferIntent::ExistingImplementation
355 } else {
356 GitLaneTransferIntent::HeddleObjectsOnly
357 };
358 let closure = wire::enumerate_state_closure_transfer_with_options(
359 repo.store(),
360 local_state,
361 wire::StateClosureOptions::default(),
362 PUSH_FULL_DESCRIPTOR_OBJECT_THRESHOLD,
363 )?;
364 let object_plan =
365 RepositoryTransferPlan::from_planned_objects(closure.planned_objects, git_lane_intent);
366 let full_objects = closure.full_objects;
367 let object_count = full_objects
368 .as_ref()
369 .map_or(object_plan.stats.total_objects, std::vec::Vec::len);
370 let transfer_id = push_transfer_id(repo_path, local_state, target_thread);
371 let transport_mode = preferred_transport_mode(&self.transport, object_count);
372 let thread_metadata = load_thread_metadata(repo, target_thread, local_state)?;
373 let request_message = PushMessage {
374 body: Some(push_message::Body::Request(PushRequest {
375 repo_path: repo_path.to_string(),
376 local_state: local_state.to_string_full(),
377 target_thread: target_thread.to_string(),
378 create_thread: true,
379 force,
380 objects: full_objects.as_ref().map_or_else(
381 || {
382 object_plan
383 .partitions
384 .iter()
385 .map(to_proto_planned_object)
386 .collect()
387 },
388 |objects| objects.iter().map(to_proto_object_info).collect(),
389 ),
390 transfer: Some(self.transport.transfer_checkpoint_with_mode(
391 transfer_id.clone(),
392 transport_mode,
393 0,
394 0,
395 false,
396 )),
397 partial_fetch_status: partial_fetch_status_for_repo(repo),
398 allow_partial_fetch: true,
399 thread_metadata: thread_metadata
400 .map(|metadata| to_proto_thread_metadata(&metadata)),
401 local_revision_address,
402 client_operation_id: String::new(),
403 })),
404 };
405
406 let (tx, rx) = mpsc::channel(self.transport.max_inflight_objects.max(4));
407 tx.send(request_message).await.map_err(|_| {
408 ProtocolError::InvalidState("failed to initialize push stream".to_string())
409 })?;
410 let mut request = Request::new(ReceiverStream::new(rx));
411 self.apply_auth(&mut request)?;
412 let mut response = self
413 .inner
414 .push(request)
415 .await
416 .map_err(status_to_protocol_error)?
417 .into_inner();
418
419 let ready = match response.message().await.map_err(status_to_protocol_error)? {
420 Some(PushMessage {
421 body: Some(push_message::Body::Ready(ready)),
422 }) => ready,
423 _ => {
424 return Err(ProtocolError::InvalidState(
425 "expected PushReady from gRPC server".to_string(),
426 ));
427 }
428 };
429 let object_index = match full_objects {
430 Some(objects) => objects
431 .into_iter()
432 .map(|info| (descriptor_id_from_info(&info), info))
433 .collect::<HashMap<_, _>>(),
434 None => object_plan
435 .partitions
436 .iter()
437 .map(|object| {
438 (
439 descriptor_id_from_plan(object),
440 object_info_from_plan(object),
441 )
442 })
443 .collect::<HashMap<_, _>>(),
444 };
445
446 let ready_transport_mode = ready
447 .transfer
448 .as_ref()
449 .and_then(|transfer| TransportMode::try_from(transfer.transport_mode).ok())
450 .unwrap_or(transport_mode);
451 let wanted_infos = ready
452 .want_objects
453 .into_iter()
454 .map(|want| {
455 object_index
456 .get(&descriptor_id(&want))
457 .cloned()
458 .ok_or_else(|| {
459 ProtocolError::InvalidState("server requested unknown object".to_string())
460 })
461 })
462 .collect::<Result<Vec<_>, _>>()?;
463
464 let wanted_plan = RepositoryTransferPlan::from_object_infos(wanted_infos, git_lane_intent);
465
466 if !wanted_plan.partitions.packable_objects.is_empty() {
467 send_native_pack_streaming_messages(
468 &tx,
469 repo,
470 &wanted_plan.partitions.packable_objects,
471 &transfer_id,
472 self.transport.chunk_size.max(1),
473 &self.transport,
474 ready_transport_mode,
475 )
476 .await?;
477 }
478
479 for info in wanted_plan.partitions.sidecar_objects {
480 let message = sidecar_push_message(repo, info)?;
481 tx.send(message).await.map_err(|_| {
482 ProtocolError::InvalidState("push stream closed unexpectedly".to_string())
483 })?;
484 }
485
486 if let Some(git_lane) = git_lane {
487 if let Some(pack) = git_lane.pack.as_ref() {
493 send_git_pack_streaming_messages(
494 &tx,
495 pack,
496 self.transport.chunk_size.max(1),
497 progress,
498 )
499 .await?;
500 } else {
501 progress.set_phase("no new objects to pack");
502 }
503 progress.set_phase(format!("writing {} refs", git_lane.ref_updates.len()));
504 for ref_update in git_lane.ref_updates {
505 tx.send(ref_update).await.map_err(|_| {
506 ProtocolError::InvalidState("push stream closed unexpectedly".to_string())
507 })?;
508 }
509 }
510 drop(tx);
511
512 let result = match response.message().await.map_err(status_to_protocol_error)? {
513 Some(PushMessage {
514 body: Some(push_message::Body::Complete(complete)),
515 }) => PushComplete {
516 success: complete.success,
517 new_state: if complete.new_state.is_empty() {
518 None
519 } else {
520 Some(
521 ChangeId::parse(&complete.new_state)
522 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?,
523 )
524 },
525 error: (!complete.error.is_empty()).then_some(complete.error),
526 transfer_id: complete
527 .transfer
528 .as_ref()
529 .map(|transfer| transfer.transfer_id.clone())
530 .unwrap_or_default(),
531 transport_mode: complete
532 .transfer
533 .as_ref()
534 .map(|transfer| transport_mode_name(transfer.transport_mode))
535 .unwrap_or("raw-objects")
536 .to_string(),
537 resume_offset: complete
538 .transfer
539 .as_ref()
540 .map(|transfer| transfer.resume_offset)
541 .unwrap_or_default(),
542 chunk_index: complete
543 .transfer
544 .as_ref()
545 .map(|transfer| transfer.chunk_index)
546 .unwrap_or_default(),
547 checkpoint: complete
548 .transfer
549 .as_ref()
550 .map(|transfer| transfer.checkpoint.clone())
551 .unwrap_or_default(),
552 is_complete: complete
553 .transfer
554 .as_ref()
555 .map(|transfer| transfer.is_complete)
556 .unwrap_or(false),
557 },
558 _ => {
559 return Err(ProtocolError::InvalidState(
560 "expected PushComplete from gRPC server".to_string(),
561 ));
562 }
563 };
564
565 if result.success {
566 self.sync_remote_markers(repo, repo_path, local_state)
567 .await?;
568 }
569 Ok(result)
570 }
571
572 pub async fn pull(
573 &mut self,
574 repo: &Repository,
575 repo_path: &str,
576 remote_thread: &str,
577 local_thread: Option<&str>,
578 ) -> Result<PullComplete, ProtocolError> {
579 self.pull_with_options(
580 repo,
581 repo_path,
582 remote_thread,
583 PullOptions {
584 local_thread,
585 depth: None,
586 target_state: None,
587 materialization: PullMaterialization::Full,
588 },
589 )
590 .await
591 }
592
593 pub async fn pull_profiled(
594 &mut self,
595 repo: &Repository,
596 repo_path: &str,
597 remote_thread: &str,
598 local_thread: Option<&str>,
599 ) -> Result<(PullComplete, PullProfile), ProtocolError> {
600 self.pull_exchange(
601 repo,
602 repo_path,
603 remote_thread,
604 PullOptions {
605 local_thread,
606 depth: None,
607 target_state: None,
608 materialization: PullMaterialization::Full,
609 },
610 )
611 .await
612 .map(|exchange| (exchange.result, exchange.profile))
613 }
614
615 pub async fn pull_partial(
616 &mut self,
617 repo: &Repository,
618 repo_path: &str,
619 remote_thread: &str,
620 local_thread: Option<&str>,
621 ) -> Result<PullComplete, ProtocolError> {
622 self.pull_with_options(
623 repo,
624 repo_path,
625 remote_thread,
626 PullOptions {
627 local_thread,
628 depth: None,
629 target_state: None,
630 materialization: PullMaterialization::Lazy,
631 },
632 )
633 .await
634 }
635
636 pub async fn pull_with_depth_and_materialization(
637 &mut self,
638 repo: &Repository,
639 repo_path: &str,
640 remote_thread: &str,
641 local_thread: Option<&str>,
642 depth: Option<u32>,
643 materialization: PullMaterialization,
644 ) -> Result<PullComplete, ProtocolError> {
645 self.pull_with_options(
646 repo,
647 repo_path,
648 remote_thread,
649 PullOptions {
650 local_thread,
651 depth,
652 target_state: None,
653 materialization,
654 },
655 )
656 .await
657 }
658
659 pub async fn pull_with_depth(
660 &mut self,
661 repo: &Repository,
662 repo_path: &str,
663 remote_thread: &str,
664 local_thread: Option<&str>,
665 depth: Option<u32>,
666 ) -> Result<PullComplete, ProtocolError> {
667 self.pull_with_depth_and_materialization(
668 repo,
669 repo_path,
670 remote_thread,
671 local_thread,
672 depth,
673 PullMaterialization::Full,
674 )
675 .await
676 }
677
678 pub async fn fetch_state(
679 &mut self,
680 repo: &Repository,
681 repo_path: &str,
682 remote_thread: &str,
683 target_state: ChangeId,
684 ) -> Result<usize, ProtocolError> {
685 self.pull_exchange(
686 repo,
687 repo_path,
688 remote_thread,
689 PullOptions {
690 local_thread: None,
691 depth: None,
692 target_state: Some(target_state),
693 materialization: PullMaterialization::Full,
694 },
695 )
696 .await
697 .map(|exchange| exchange.object_count)
698 }
699
700 pub async fn fetch_state_partial(
701 &mut self,
702 repo: &Repository,
703 repo_path: &str,
704 remote_thread: &str,
705 target_state: ChangeId,
706 ) -> Result<usize, ProtocolError> {
707 self.pull_exchange(
708 repo,
709 repo_path,
710 remote_thread,
711 PullOptions {
712 local_thread: None,
713 depth: None,
714 target_state: Some(target_state),
715 materialization: PullMaterialization::Lazy,
716 },
717 )
718 .await
719 .map(|exchange| exchange.object_count)
720 }
721
722 pub async fn hydrate_blob_at_path(
723 &mut self,
724 repo: &Repository,
725 repo_path: &str,
726 reference: &str,
727 path: &str,
728 ) -> Result<objects::object::Blob, ProtocolError> {
729 let mut request = Request::new(GetBlobRequest {
730 repo_path: repo_path.to_string(),
731 r#ref: reference.to_string(),
732 path: path.to_string(),
733 });
734 self.apply_signed_auth(&mut request, "/heddle.v1.ContentService/GetBlob")?;
735 let response = self
736 .content
737 .get_blob(request)
738 .await
739 .map_err(status_to_protocol_error)?
740 .into_inner();
741
742 let content = super::helpers::decode_blob_content(response.content, response.is_binary)?;
743 let blob = objects::object::Blob::new(content);
744 repo.store().put_blob(&blob)?;
745 repo.clear_missing_blob(&blob.hash())?;
746 Ok(blob)
747 }
748
749 pub async fn hydrate_missing_blobs_for_state(
750 &mut self,
751 repo: &Repository,
752 repo_path: &str,
753 remote_thread: &str,
754 target_state: ChangeId,
755 ) -> Result<usize, ProtocolError> {
756 let exchange = self
757 .pull_exchange(
758 repo,
759 repo_path,
760 remote_thread,
761 PullOptions {
762 local_thread: None,
763 depth: None,
764 target_state: Some(target_state),
765 materialization: PullMaterialization::Full,
766 },
767 )
768 .await?;
769 clear_missing_blobs_for_state(repo, target_state)?;
770 Ok(exchange.object_count)
771 }
772
773 async fn pull_with_options(
774 &mut self,
775 repo: &Repository,
776 repo_path: &str,
777 remote_thread: &str,
778 options: PullOptions<'_>,
779 ) -> Result<PullComplete, ProtocolError> {
780 self.pull_exchange(repo, repo_path, remote_thread, options)
781 .await
782 .map(|exchange| exchange.result)
783 }
784
785 async fn pull_exchange(
786 &mut self,
787 repo: &Repository,
788 repo_path: &str,
789 remote_thread: &str,
790 options: PullOptions<'_>,
791 ) -> Result<PullExchange, ProtocolError> {
792 let exchange_start = Instant::now();
793 let mut exclude_states = Vec::new();
794 let advertised_head = if let Some(local_thread) = options.local_thread {
805 locally_complete_local_thread_head(repo, local_thread, options.target_state)?
806 } else {
807 locally_complete_pull_head(repo, remote_thread, options.target_state)?
808 };
809 if let Some(head) = advertised_head {
810 exclude_states.push(head);
811 }
812 let allow_partial_fetch = options.materialization.allows_partial_fetch();
813 let fresh_full_pull =
814 supports_compact_full_pull(repo, allow_partial_fetch, &exclude_states)?;
815
816 let transfer_id = pull_transfer_id(
817 repo_path,
818 remote_thread,
819 options.local_thread,
820 options.depth,
821 options.target_state,
822 );
823 let request_message = PullMessage {
824 body: Some(pull_message::Body::Request(PullRequest {
825 repo_path: repo_path.to_string(),
826 remote_thread: remote_thread.to_string(),
827 local_thread: options.local_thread.unwrap_or_default().to_string(),
828 target_state: options
829 .target_state
830 .map(|value| value.to_string_full())
831 .unwrap_or_default(),
832 depth: options.depth.unwrap_or_default(),
833 exclude_states: exclude_states
834 .iter()
835 .map(ChangeId::to_string_full)
836 .collect(),
837 transfer: Some(self.transport.transfer_checkpoint_with_mode(
838 transfer_id.clone(),
839 TransportMode::NativePack,
840 0,
841 0,
842 false,
843 )),
844 partial_fetch_status: partial_fetch_status_for_repo(repo),
845 allow_partial_fetch,
846 fresh_full_pull,
847 target_revision_address: options
848 .target_state
849 .map(|state| RevisionAddress::heddle(state).to_string())
850 .unwrap_or_default(),
851 client_operation_id: String::new(),
852 })),
853 };
854
855 let (tx, rx) = mpsc::channel(self.transport.max_inflight_objects.max(4));
856 tx.send(request_message).await.map_err(|_| {
857 ProtocolError::InvalidState("failed to initialize pull stream".to_string())
858 })?;
859 let mut request = Request::new(ReceiverStream::new(rx));
860 self.apply_auth(&mut request)?;
861 let mut response = self
862 .inner
863 .pull(request)
864 .await
865 .map_err(status_to_protocol_error)?
866 .into_inner();
867
868 let ready = match response.message().await.map_err(status_to_protocol_error)? {
869 Some(PullMessage {
870 body: Some(pull_message::Body::Ready(ready)),
871 }) => ready,
872 _ => {
873 return Err(ProtocolError::InvalidState(
874 "expected PullReady from gRPC server".to_string(),
875 ));
876 }
877 };
878 let mut profile = PullProfile {
879 ready_wait: exchange_start.elapsed(),
880 ..PullProfile::default()
881 };
882 let remote_state = ChangeId::parse(&ready.remote_state)
883 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
884 let advertised_object_count = ready.objects_to_fetch.len();
885 let PullWantPlan {
886 wants,
887 transfer_plan,
888 wanted_types,
889 want_full_closure,
890 } = plan_pull_wants(
891 repo,
892 &remote_state,
893 ready.full_closure_available,
894 ready.objects_to_fetch,
895 allow_partial_fetch,
896 )?;
897 let native_pack_required = native_pack_required_for_pull(want_full_closure, &transfer_plan);
898
899 tx.send(PullMessage {
900 body: Some(pull_message::Body::Want(WantObjects {
901 objects: wants.clone(),
902 want_full_closure,
903 transfer: Some(self.transport.transfer_checkpoint_with_mode(
904 transfer_id.clone(),
905 TransportMode::NativePack,
906 0,
907 0,
908 false,
909 )),
910 })),
911 })
912 .await
913 .map_err(|_| ProtocolError::InvalidState("pull stream closed unexpectedly".to_string()))?;
914 drop(tx);
915
916 let receive_start = Instant::now();
917 let use_pack_spool = advertised_object_count > PULL_PACK_SPOOL_OBJECT_THRESHOLD;
918 let mut pack_state = wire::PackChunkState::default();
919 let mut pack_spool = if use_pack_spool {
920 Some(wire::PackChunkSpool::new_in(repo.heddle_dir())?)
921 } else {
922 None
923 };
924 let mut git_lane_repo = None;
925 let mut git_pack_state = GitPackPullInstallState::default();
926 let mut received = 0usize;
927 while let Some(message) = response.message().await.map_err(status_to_protocol_error)? {
928 match message.body {
929 Some(pull_message::Body::Pack(chunk)) => {
930 profile.bytes_received =
931 profile.bytes_received.saturating_add(chunk.data.len());
932 profile.pack_bytes_received =
933 profile.pack_bytes_received.saturating_add(chunk.data.len());
934 let transfer = chunk.transfer.as_ref().ok_or_else(|| {
935 ProtocolError::InvalidState(
936 "native pack chunk missing transfer checkpoint".to_string(),
937 )
938 })?;
939 let stream_kind = PackStreamKind::try_from(chunk.stream_kind)
940 .unwrap_or(PackStreamKind::Unspecified);
941 if stream_kind == PackStreamKind::Unspecified {
942 return Err(ProtocolError::InvalidState(
943 "native pack chunk missing stream kind".to_string(),
944 ));
945 }
946 let decode_start = Instant::now();
947 if let Some(pack_spool) = pack_spool.as_mut() {
948 pack_spool.receive_chunk(
949 stream_kind == PackStreamKind::Index,
950 transfer.resume_offset,
951 transfer.chunk_index,
952 transfer.is_complete,
953 &chunk.data,
954 chunk.is_final_chunk,
955 )?;
956 } else {
957 wire::receive_pack_chunk(
958 &mut pack_state,
959 stream_kind == PackStreamKind::Index,
960 transfer.resume_offset,
961 transfer.chunk_index,
962 transfer.is_complete,
963 &chunk.data,
964 chunk.is_final_chunk,
965 )?;
966 }
967 let decode_elapsed = decode_start.elapsed();
968 profile.pack_decode += decode_elapsed;
969 profile.pack_decode_apply += decode_elapsed;
970 profile.decode += decode_elapsed;
971 }
972 Some(pull_message::Body::Redaction(transfer)) => {
973 wire::check_received_transfer_blob_size(
979 transfer.redactions_blob.len(),
980 wire::MAX_RECEIVED_REDACTIONS_BLOB_SIZE,
981 "redactions",
982 )?;
983 profile.bytes_received = profile
984 .bytes_received
985 .saturating_add(transfer.redactions_blob.len());
986 profile.object_mix.record(ObjectType::Redaction);
987 let blob = ContentHash::from_hex(&transfer.blob_hash).map_err(|err| {
988 ProtocolError::InvalidState(format!(
989 "RedactionTransfer.blob_hash is not a valid content hash: {err}"
990 ))
991 })?;
992 let decode_start = Instant::now();
993 repo.accept_wire_redactions(blob, &transfer.redactions_blob)
994 .map_err(|err| {
995 ProtocolError::InvalidState(format!(
996 "accept_wire_redactions for blob {}: {err}",
997 transfer.blob_hash
998 ))
999 })?;
1000 let decode_elapsed = decode_start.elapsed();
1001 profile.store_receive_object += decode_elapsed;
1002 }
1003 Some(pull_message::Body::StateVisibility(transfer)) => {
1004 wire::check_received_transfer_blob_size(
1005 transfer.state_visibility_blob.len(),
1006 wire::MAX_RECEIVED_STATE_VISIBILITY_BLOB_SIZE,
1007 "state-visibility",
1008 )?;
1009 profile.bytes_received = profile
1010 .bytes_received
1011 .saturating_add(transfer.state_visibility_blob.len());
1012 profile.object_mix.record(ObjectType::StateVisibility);
1013 let state = ChangeId::parse(&transfer.state_id).map_err(|err| {
1014 ProtocolError::InvalidState(format!(
1015 "StateVisibilityTransfer.state_id is not a valid ChangeId: {err}"
1016 ))
1017 })?;
1018 let decode_start = Instant::now();
1019 repo.accept_wire_state_visibility(state, &transfer.state_visibility_blob)
1020 .map_err(|err| {
1021 ProtocolError::InvalidState(format!(
1022 "accept_wire_state_visibility for state {}: {err}",
1023 transfer.state_id
1024 ))
1025 })?;
1026 let decode_elapsed = decode_start.elapsed();
1027 profile.store_receive_object += decode_elapsed;
1028 }
1029 Some(pull_message::Body::GitLane(transfer)) => {
1030 let decode_start = Instant::now();
1031 profile.bytes_received = profile
1032 .bytes_received
1033 .saturating_add(git_lane_transfer_size(&transfer));
1034 accept_git_lane_pull_transfer(
1035 repo,
1036 &mut git_lane_repo,
1037 &mut git_pack_state,
1038 transfer,
1039 )?;
1040 let decode_elapsed = decode_start.elapsed();
1041 profile.store_receive_object += decode_elapsed;
1042 }
1043 Some(pull_message::Body::Complete(complete)) => {
1044 profile.receive_and_apply = receive_start.elapsed();
1045 git_pack_state.ensure_idle()?;
1046 let final_state = if complete.new_state.is_empty() {
1047 None
1048 } else {
1049 Some(
1050 ChangeId::parse(&complete.new_state)
1051 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?,
1052 )
1053 };
1054
1055 if complete.success {
1056 if native_pack_required {
1057 let store_start = Instant::now();
1058 let installed_ids = if let Some(pack_spool) = pack_spool.as_mut() {
1059 if !pack_spool.is_complete() {
1060 return Err(ProtocolError::InvalidState(
1061 "pull completed before native pack stream finished"
1062 .to_string(),
1063 ));
1064 }
1065 pack_spool.install_into(repo.store())?
1066 } else {
1067 if !pack_state.is_complete() {
1068 return Err(ProtocolError::InvalidState(
1069 "pull completed before native pack stream finished"
1070 .to_string(),
1071 ));
1072 }
1073 wire::install_received_pack(
1074 repo.store(),
1075 &pack_state.pack_data,
1076 &pack_state.index_data,
1077 )?
1078 };
1079 profile.store_receive_object += store_start.elapsed();
1080 received = installed_ids.len();
1081 for id in installed_ids {
1082 match (id, wanted_packable_type(&wanted_types, &id)) {
1083 (PackObjectId::Hash(hash), Some(ObjectType::Blob)) => {
1084 profile.object_mix.record(ObjectType::Blob);
1085 repo.clear_missing_blob(&hash)?;
1086 }
1087 (_, Some(obj_type)) => {
1088 profile.object_mix.record(obj_type);
1089 }
1090 (PackObjectId::ChangeId(_), None) => {
1091 profile.object_mix.record(ObjectType::State);
1092 }
1093 (PackObjectId::Hash(hash), None) => {
1094 let inferred =
1095 infer_installed_hash_object_type(repo, &hash)?;
1096 profile.object_mix.record(inferred);
1097 }
1098 }
1099 }
1100 }
1101
1102 let metadata_start = Instant::now();
1103 if let Some(local_thread) = options.local_thread
1104 && let Some(state) = final_state
1105 {
1106 repo.refs()
1107 .set_thread(&ThreadName::from(local_thread), &state)?;
1108 }
1109 if let Some(state) = final_state
1110 && allow_partial_fetch
1111 {
1112 mark_missing_blobs_for_state(repo, state)?;
1113 } else if final_state.is_some() {
1114 let _ = repo.clear_all_missing_blobs()?;
1115 }
1116 let synced_markers = complete
1117 .transfer
1118 .as_ref()
1119 .map(|transfer| apply_marker_snapshot(repo, &transfer.checkpoint))
1120 .transpose()?
1121 .unwrap_or(false);
1122 if !synced_markers {
1123 self.sync_local_markers(repo, repo_path).await?;
1124 }
1125 profile.metadata_sync = metadata_start.elapsed();
1126 profile.objects_received = received;
1127 return Ok(PullExchange {
1128 result: PullComplete {
1129 success: true,
1130 final_state,
1131 error: None,
1132 transfer_id: complete
1133 .transfer
1134 .as_ref()
1135 .map(|transfer| transfer.transfer_id.clone())
1136 .unwrap_or_default(),
1137 transport_mode: complete
1138 .transfer
1139 .as_ref()
1140 .map(|transfer| transport_mode_name(transfer.transport_mode))
1141 .unwrap_or("native-pack")
1142 .to_string(),
1143 resume_offset: complete
1144 .transfer
1145 .as_ref()
1146 .map(|transfer| transfer.resume_offset)
1147 .unwrap_or_default(),
1148 chunk_index: complete
1149 .transfer
1150 .as_ref()
1151 .map(|transfer| transfer.chunk_index)
1152 .unwrap_or_default(),
1153 checkpoint: complete
1154 .transfer
1155 .as_ref()
1156 .map(|transfer| transfer.checkpoint.clone())
1157 .unwrap_or_default(),
1158 is_complete: complete
1159 .transfer
1160 .as_ref()
1161 .map(|transfer| transfer.is_complete)
1162 .unwrap_or(false),
1163 },
1164 object_count: received,
1165 profile,
1166 });
1167 }
1168
1169 profile.objects_received = received;
1170 return Ok(PullExchange {
1171 result: PullComplete {
1172 success: false,
1173 final_state,
1174 error: (!complete.error.is_empty()).then_some(complete.error),
1175 transfer_id: complete
1176 .transfer
1177 .as_ref()
1178 .map(|transfer| transfer.transfer_id.clone())
1179 .unwrap_or_default(),
1180 transport_mode: complete
1181 .transfer
1182 .as_ref()
1183 .map(|transfer| transport_mode_name(transfer.transport_mode))
1184 .unwrap_or("native-pack")
1185 .to_string(),
1186 resume_offset: complete
1187 .transfer
1188 .as_ref()
1189 .map(|transfer| transfer.resume_offset)
1190 .unwrap_or_default(),
1191 chunk_index: complete
1192 .transfer
1193 .as_ref()
1194 .map(|transfer| transfer.chunk_index)
1195 .unwrap_or_default(),
1196 checkpoint: complete
1197 .transfer
1198 .as_ref()
1199 .map(|transfer| transfer.checkpoint.clone())
1200 .unwrap_or_default(),
1201 is_complete: complete
1202 .transfer
1203 .as_ref()
1204 .map(|transfer| transfer.is_complete)
1205 .unwrap_or(false),
1206 },
1207 object_count: received,
1208 profile,
1209 });
1210 }
1211 _ => {}
1212 }
1213 }
1214
1215 Err(ProtocolError::InvalidState(format!(
1216 "pull stream ended unexpectedly after receiving {received} packed objects"
1217 )))
1218 }
1219}
1220
1221fn redaction_push_message(
1222 repo: &Repository,
1223 info: wire::ObjectInfo,
1224) -> Result<PushMessage, ProtocolError> {
1225 let wire::ObjectId::Hash(blob) = info.id else {
1226 return Err(ProtocolError::InvalidState(
1227 "wanted Redaction must be keyed by ObjectId::Hash(content_hash)".to_string(),
1228 ));
1229 };
1230 let hex = blob.to_hex();
1231 let bytes = repo
1236 .store()
1237 .get_redactions_bytes_for_blob(&blob)
1238 .map_err(|err| {
1239 ProtocolError::InvalidState(format!("load redactions sidecar for {}: {err}", hex))
1240 })?
1241 .ok_or_else(|| {
1242 ProtocolError::InvalidState(format!(
1243 "server wants redaction for blob {} but sender has no sidecar",
1244 hex
1245 ))
1246 })?;
1247 Ok(PushMessage {
1248 body: Some(push_message::Body::Redaction(RedactionTransfer {
1249 blob_hash: hex,
1250 redactions_blob: bytes.into(),
1251 })),
1252 })
1253}
1254
1255fn native_pack_required_for_pull(
1256 want_full_closure: bool,
1257 transfer_plan: &RepositoryTransferPlan<ObjectInfo>,
1258) -> bool {
1259 transfer_plan.requires_native_pack(want_full_closure)
1260}
1261
1262fn object_info_from_plan(object: &PlannedObject) -> ObjectInfo {
1263 ObjectInfo {
1264 id: object.id.clone(),
1265 obj_type: object.obj_type,
1266 size: 0,
1267 delta_base: None,
1268 }
1269}
1270
1271fn to_proto_planned_object(object: &PlannedObject) -> ObjectDescriptor {
1272 object_descriptor_with_status(
1273 &object_info_from_plan(object),
1274 ObjectAvailabilityStatus::Present,
1275 "",
1276 )
1277}
1278
1279fn descriptor_id_from_plan(object: &PlannedObject) -> (String, String) {
1280 let id = match &object.id {
1281 wire::ObjectId::Hash(hash) => hash.to_hex(),
1282 wire::ObjectId::ChangeId(change_id) => change_id.to_string_full(),
1283 };
1284 (id, object_type_name(object.obj_type).to_string())
1285}
1286
1287fn record_wanted_type(wanted_types: &mut WantedTypes, pack_id: PackObjectId, obj_type: ObjectType) {
1288 let types = wanted_types.entry(pack_id).or_default();
1289 if !types.contains(&obj_type) {
1290 types.push(obj_type);
1291 }
1292}
1293
1294fn wanted_packable_type(wanted_types: &WantedTypes, pack_id: &PackObjectId) -> Option<ObjectType> {
1295 wanted_types
1296 .get(pack_id)
1297 .and_then(|types| types.iter().copied().find(|obj_type| obj_type.packable()))
1298}
1299
1300fn sidecar_push_message(
1301 repo: &Repository,
1302 info: wire::ObjectInfo,
1303) -> Result<PushMessage, ProtocolError> {
1304 match info.obj_type {
1305 ObjectType::Redaction => redaction_push_message(repo, info),
1306 ObjectType::StateVisibility => state_visibility_push_message(repo, info),
1307 obj_type => Err(ProtocolError::InvalidState(format!(
1308 "{obj_type:?} is not an out-of-pack sidecar object"
1309 ))),
1310 }
1311}
1312
1313fn state_visibility_push_message(
1314 repo: &Repository,
1315 info: wire::ObjectInfo,
1316) -> Result<PushMessage, ProtocolError> {
1317 let wire::ObjectId::ChangeId(state) = info.id else {
1318 return Err(ProtocolError::InvalidState(
1319 "wanted StateVisibility must be keyed by ObjectId::ChangeId(state)".to_string(),
1320 ));
1321 };
1322 let state_id = state.to_string_full();
1323 let bytes = repo
1324 .get_state_visibility_bytes_for_state(&state)
1325 .map_err(|err| {
1326 ProtocolError::InvalidState(format!(
1327 "load state-visibility sidecar for {}: {err}",
1328 state_id
1329 ))
1330 })?
1331 .ok_or_else(|| {
1332 ProtocolError::InvalidState(format!(
1333 "server wants state visibility for state {} but sender has no sidecar",
1334 state_id
1335 ))
1336 })?;
1337 Ok(PushMessage {
1338 body: Some(push_message::Body::StateVisibility(
1339 StateVisibilityTransfer {
1340 state_id,
1341 state_visibility_blob: bytes.into(),
1342 },
1343 )),
1344 })
1345}
1346
1347fn load_thread_metadata(
1348 repo: &Repository,
1349 target_thread: &str,
1350 local_state: ChangeId,
1351) -> Result<Option<SyncedThreadMetadata>, ProtocolError> {
1352 let thread_manager = ThreadManager::new(repo.heddle_dir());
1353 Ok(thread_manager.find_synced_record_by_thread(repo, target_thread, Some(local_state))?)
1354}
1355
1356fn plan_pull_wants(
1357 repo: &Repository,
1358 remote_state: &ChangeId,
1359 full_closure_available: bool,
1360 objects_to_fetch: Vec<ObjectDescriptor>,
1361 allow_partial_fetch: bool,
1362) -> Result<PullWantPlan, ProtocolError> {
1363 if full_closure_available {
1364 return Ok(PullWantPlan {
1365 wants: Vec::new(),
1366 transfer_plan: RepositoryTransferPlan::from_object_infos(
1367 Vec::<ObjectInfo>::new(),
1368 GitLaneTransferIntent::HeddleObjectsOnly,
1369 ),
1370 wanted_types: HashMap::new(),
1371 want_full_closure: true,
1372 });
1373 }
1374 let request_full_closure =
1375 should_request_full_closure(repo, remote_state, allow_partial_fetch)?;
1376 let mut wants = Vec::with_capacity(objects_to_fetch.len());
1377 let mut wanted_infos = Vec::with_capacity(objects_to_fetch.len());
1378 let mut wanted_types = HashMap::with_capacity(objects_to_fetch.len());
1379
1380 for descriptor in objects_to_fetch {
1381 let info = parse_descriptor_to_info(descriptor)?;
1382 let pack_id = match &info.id {
1383 wire::ObjectId::Hash(hash) => PackObjectId::Hash(*hash),
1384 wire::ObjectId::ChangeId(change_id) => PackObjectId::ChangeId(*change_id),
1385 };
1386 let include = if request_full_closure {
1387 true
1388 } else {
1389 let has = wire::has_object(repo.store(), &info)?;
1390 !(has || (allow_partial_fetch && matches!(info.obj_type, ObjectType::Blob)))
1391 };
1392
1393 if include {
1394 record_wanted_type(&mut wanted_types, pack_id, info.obj_type);
1395 wants.push(object_descriptor_with_status(
1396 &info,
1397 ObjectAvailabilityStatus::Missing,
1398 "requested by client",
1399 ));
1400 wanted_infos.push(info);
1401 }
1402 }
1403
1404 Ok(PullWantPlan {
1405 wants,
1406 transfer_plan: RepositoryTransferPlan::from_object_infos(
1407 wanted_infos,
1408 GitLaneTransferIntent::HeddleObjectsOnly,
1409 ),
1410 wanted_types,
1411 want_full_closure: false,
1412 })
1413}
1414
1415fn supports_compact_full_pull(
1416 repo: &Repository,
1417 allow_partial_fetch: bool,
1418 exclude_states: &[ChangeId],
1419) -> Result<bool, ProtocolError> {
1420 if allow_partial_fetch || !exclude_states.is_empty() {
1421 return Ok(false);
1422 }
1423 repo_looks_fresh(repo)
1424}
1425
1426fn locally_complete_pull_head(
1449 repo: &Repository,
1450 remote_thread: &str,
1451 target_state: Option<ChangeId>,
1452) -> Result<Option<ChangeId>, ProtocolError> {
1453 locally_complete_thread_head(repo, remote_thread, target_state)
1454}
1455
1456fn locally_complete_local_thread_head(
1465 repo: &Repository,
1466 local_thread: &str,
1467 target_state: Option<ChangeId>,
1468) -> Result<Option<ChangeId>, ProtocolError> {
1469 locally_complete_thread_head(repo, local_thread, target_state)
1470}
1471
1472fn locally_complete_thread_head(
1492 repo: &Repository,
1493 thread: &str,
1494 target_state: Option<ChangeId>,
1495) -> Result<Option<ChangeId>, ProtocolError> {
1496 if target_state.is_some() {
1499 return Ok(None);
1500 }
1501 if !repo.missing_blobs()?.is_empty() {
1504 return Ok(None);
1505 }
1506 let Some(head) = repo.refs().get_thread(&ThreadName::from(thread))? else {
1507 return Ok(None);
1510 };
1511 match wire::enumerate_state_closure(repo.store(), head) {
1516 Ok(_) => Ok(Some(head)),
1517 Err(ProtocolError::ObjectNotFound(_)) => Ok(None),
1518 Err(err) => Err(err),
1519 }
1520}
1521
1522fn should_request_full_closure(
1523 repo: &Repository,
1524 remote_state: &ChangeId,
1525 allow_partial_fetch: bool,
1526) -> Result<bool, ProtocolError> {
1527 if allow_partial_fetch || repo.store().has_state(remote_state)? {
1528 return Ok(false);
1529 }
1530 repo_looks_fresh(repo)
1531}
1532
1533fn repo_looks_fresh(repo: &Repository) -> Result<bool, ProtocolError> {
1534 if repo.head()?.is_some() {
1535 return Ok(false);
1536 }
1537 if !repo.refs().list_threads()?.is_empty() || !repo.refs().list_markers()?.is_empty() {
1538 return Ok(false);
1539 }
1540 Ok(repo.missing_blobs()?.is_empty())
1541}
1542
1543fn infer_installed_hash_object_type(
1544 repo: &Repository,
1545 hash: &ContentHash,
1546) -> Result<ObjectType, ProtocolError> {
1547 let store = repo.store();
1548 if store.get_tree(hash)?.is_some() {
1549 return Ok(ObjectType::Tree);
1550 }
1551 if store
1552 .get_action(&objects::object::ActionId::from_hash(*hash))?
1553 .is_some()
1554 {
1555 return Ok(ObjectType::Action);
1556 }
1557 Ok(ObjectType::Blob)
1558}
1559
1560fn apply_marker_snapshot(repo: &Repository, checkpoint: &[u8]) -> Result<bool, ProtocolError> {
1561 const HEADER: &str = "heddle-markers-v1\n";
1562 if checkpoint.is_empty() {
1563 return Ok(false);
1564 }
1565 let payload = std::str::from_utf8(checkpoint)
1566 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
1567 let Some(lines) = payload.strip_prefix(HEADER) else {
1568 return Ok(false);
1569 };
1570
1571 for line in lines.lines() {
1572 if line.is_empty() {
1573 continue;
1574 }
1575 let Some((name, change_id)) = line.split_once('\t') else {
1576 return Err(ProtocolError::InvalidState(
1577 "invalid marker snapshot line".to_string(),
1578 ));
1579 };
1580 let change_id = ChangeId::parse(change_id)
1581 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
1582 if !repo.store().has_state(&change_id)? {
1583 continue;
1584 }
1585 let name = MarkerName::from(name);
1586 match repo.refs().get_marker(&name)? {
1587 Some(existing) if existing == change_id => {}
1588 Some(existing) => repo.refs().set_marker_cas(
1589 &name,
1590 refs::RefExpectation::Value(existing),
1591 &change_id,
1592 )?,
1593 None => repo.refs().create_marker(&name, &change_id)?,
1594 }
1595 }
1596
1597 Ok(true)
1598}
1599
1600fn change_id_string_to_bytes(s: &str) -> Vec<u8> {
1601 if s.is_empty() {
1602 return Vec::new();
1603 }
1604 objects::object::ChangeId::parse(s)
1605 .map(|id| id.as_bytes().to_vec())
1606 .unwrap_or_default()
1607}
1608
1609fn to_proto_thread_metadata(metadata: &SyncedThreadMetadata) -> ThreadMetadata {
1610 ThreadMetadata {
1611 name: metadata.thread.clone(),
1612 target_thread: metadata.target_thread.clone(),
1613 parent_thread: metadata.parent_thread.clone(),
1614 task: metadata.task.clone(),
1615 thread_mode: metadata.mode.to_string(),
1616 thread_state: metadata.state.to_string(),
1617 freshness: metadata.freshness.to_string(),
1618 base_state: change_id_string_to_bytes(&metadata.base_state),
1619 base_root: change_id_string_to_bytes(&metadata.base_root),
1620 current_state: metadata
1621 .current_state
1622 .as_deref()
1623 .map(change_id_string_to_bytes),
1624 merged_state: metadata
1625 .merged_state
1626 .as_deref()
1627 .map(change_id_string_to_bytes),
1628 changed_paths: metadata.changed_paths.clone(),
1629 impact_categories: metadata
1630 .impact_categories
1631 .iter()
1632 .map(ToString::to_string)
1633 .collect(),
1634 heavy_impact_paths: metadata.heavy_impact_paths.clone(),
1635 promotion_suggested: metadata.promotion_suggested,
1636 verification_summary: Some(ThreadVerificationSummary {
1637 tests_passed: metadata.verification_summary.tests_passed,
1638 tests_failed: metadata
1639 .verification_summary
1640 .tests_failed
1641 .unwrap_or_default(),
1642 coverage_pct: metadata.verification_summary.coverage_pct,
1643 lint_warnings: metadata.verification_summary.lint_warnings,
1644 }),
1645 confidence_summary: Some(ThreadConfidenceSummary {
1646 value: metadata.confidence_summary.value,
1647 band: metadata
1648 .confidence_summary
1649 .band
1650 .as_ref()
1651 .map(ToString::to_string),
1652 }),
1653 integration_policy_result: Some(ThreadIntegrationPolicy {
1654 status: metadata
1655 .integration_policy_result
1656 .status
1657 .clone()
1658 .unwrap_or_default(),
1659 reason: metadata
1660 .integration_policy_result
1661 .reason
1662 .clone()
1663 .unwrap_or_default(),
1664 }),
1665 created_at: Some(prost_types::Timestamp {
1666 seconds: metadata.created_at.timestamp(),
1667 nanos: metadata.created_at.timestamp_subsec_nanos() as i32,
1668 }),
1669 updated_at: Some(prost_types::Timestamp {
1670 seconds: metadata.updated_at.timestamp(),
1671 nanos: metadata.updated_at.timestamp_subsec_nanos() as i32,
1672 }),
1673 }
1674}
1675
1676struct PullExchange {
1677 result: PullComplete,
1678 object_count: usize,
1679 profile: PullProfile,
1680}
1681
1682fn mark_missing_blobs_for_state(
1683 repo: &Repository,
1684 state_id: ChangeId,
1685) -> Result<(), ProtocolError> {
1686 let state = repo
1687 .store()
1688 .get_state(&state_id)?
1689 .ok_or_else(|| ProtocolError::ObjectNotFound(state_id.to_string_full()))?;
1690 let mut missing = wire::missing_blobs_in_tree(repo.store(), state.tree)?;
1691 if let Some(context_root) = state.context.as_ref() {
1692 missing.extend(wire::missing_blobs_in_tree(repo.store(), *context_root)?);
1693 }
1694 if let Some(discussions_blob) = state.discussions.as_ref()
1695 && !repo.store().has_blob(discussions_blob)?
1696 {
1697 missing.push(*discussions_blob);
1698 }
1699 missing
1700 .into_iter()
1701 .try_for_each(|hash| repo.record_missing_blob(hash).map_err(ProtocolError::from))
1702}
1703
1704fn clear_missing_blobs_for_state(
1705 repo: &Repository,
1706 state_id: ChangeId,
1707) -> Result<(), ProtocolError> {
1708 let state = repo
1709 .store()
1710 .get_state(&state_id)?
1711 .ok_or_else(|| ProtocolError::ObjectNotFound(state_id.to_string_full()))?;
1712 let mut missing = wire::missing_blobs_in_tree(repo.store(), state.tree)?;
1713 if let Some(context_root) = state.context.as_ref() {
1714 missing.extend(wire::missing_blobs_in_tree(repo.store(), *context_root)?);
1715 }
1716 if let Some(discussions_blob) = state.discussions.as_ref() {
1717 missing.push(*discussions_blob);
1718 }
1719 missing
1720 .into_iter()
1721 .try_for_each(|hash| repo.clear_missing_blob(&hash).map_err(ProtocolError::from))
1722}
1723
1724fn partial_fetch_status_for_repo(repo: &Repository) -> i32 {
1725 match repo.missing_blobs() {
1726 Ok(missing) if !missing.is_empty() => PartialFetchStatus::Enabled as i32,
1727 Ok(_) => PartialFetchStatus::Disabled as i32,
1728 Err(_) => PartialFetchStatus::Unspecified as i32,
1729 }
1730}
1731
1732fn pull_transfer_id(
1733 repo_path: &str,
1734 remote_thread: &str,
1735 local_thread: Option<&str>,
1736 depth: Option<u32>,
1737 target_state: Option<ChangeId>,
1738) -> String {
1739 format!(
1740 "pull:{repo_path}:{remote_thread}:{}:{depth:?}:{}",
1741 local_thread.unwrap_or_default(),
1742 target_state
1743 .map(|value| value.to_string_full())
1744 .unwrap_or_default()
1745 )
1746}
1747
1748fn push_transfer_id(repo_path: &str, local_state: ChangeId, target_thread: &str) -> String {
1749 format!(
1750 "push:{repo_path}:{}:{target_thread}",
1751 local_state.to_string_full()
1752 )
1753}
1754
1755fn build_git_mirror_push_plan(
1769 repo: &Repository,
1770 chunk_size: usize,
1771 remote_ref_expectations: &HashMap<String, GitRefRemoteExpectation>,
1772) -> Result<GitLanePushPlan, ProtocolError> {
1773 if repo.capability() != RepositoryCapability::GitOverlay {
1774 return Err(ProtocolError::InvalidState(
1775 "Git remote mirror pushes require a git-overlay repository".to_string(),
1776 ));
1777 }
1778 let git_repo = repo
1779 .git_overlay_sley_repository()
1780 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
1781 .ok_or_else(|| {
1782 ProtocolError::InvalidState("git-overlay repository has no Git store".to_string())
1783 })?;
1784 build_git_mirror_plan_from_sley(&git_repo, chunk_size, remote_ref_expectations)
1785}
1786
1787fn build_git_mirror_plan_from_sley(
1790 git_repo: &SleyRepository,
1791 chunk_size: usize,
1792 remote_ref_expectations: &HashMap<String, GitRefRemoteExpectation>,
1793) -> Result<GitLanePushPlan, ProtocolError> {
1794 let refs = git_repo
1795 .references()
1796 .list_refs()
1797 .map_err(|err| ProtocolError::InvalidState(format!("list git-overlay refs: {err}")))?;
1798
1799 let mut roots: Vec<GitObjectId> = Vec::new();
1800 let mut ref_updates: Vec<PushMessage> = Vec::new();
1801 let mut newest_root: Option<GitObjectId> = None;
1802
1803 for reference in refs {
1804 if GitRefName::new(&reference.name).is_local_only() {
1818 continue;
1819 }
1820
1821 let target_oid = match reference.target {
1826 ReferenceTarget::Direct(oid) => oid,
1827 ReferenceTarget::Symbolic(_) => continue,
1828 };
1829
1830 let object = git_repo.read_object(&target_oid).map_err(|err| {
1834 ProtocolError::InvalidState(format!(
1835 "git-overlay ref {} target {} is not readable: {err}",
1836 reference.name,
1837 target_oid.to_hex()
1838 ))
1839 })?;
1840
1841 let peeled_oid = if object.object_type == sley::GitObjectType::Tag {
1842 let peeled = git_repo.peel_to_object_oid(target_oid).map_err(|err| {
1843 ProtocolError::InvalidState(format!(
1844 "peel git-overlay tag ref {}: {err}",
1845 reference.name
1846 ))
1847 })?;
1848 Some(peeled)
1849 } else {
1850 None
1851 };
1852
1853 roots.push(target_oid);
1857 newest_root.get_or_insert(target_oid);
1858
1859 let expectation = remote_ref_expectations
1860 .get(&reference.name)
1861 .cloned()
1862 .unwrap_or(GitRefRemoteExpectation::Missing);
1863
1864 let kind = grpc_git_ref_kind(GitRefName::new(&reference.name).wire_kind());
1865 let mut message =
1866 git_ref_update_message(&reference.name, kind, target_oid, peeled_oid, None);
1867 apply_git_ref_expectation_value(&mut message, &expectation)?;
1868 ref_updates.push(message);
1869 }
1870
1871 if roots.is_empty() {
1872 return Err(ProtocolError::InvalidState(
1873 "git-overlay repository has no direct refs to mirror".to_string(),
1874 ));
1875 }
1876
1877 let format = git_repo.object_format();
1888 let mut have_boundary: Vec<GitObjectId> = Vec::new();
1889 for expectation in remote_ref_expectations.values() {
1890 if let GitRefRemoteExpectation::Value(oid_bytes) = expectation
1891 && let Ok(oid) = GitObjectId::from_raw(format, oid_bytes)
1892 {
1893 have_boundary.push(oid);
1897 }
1898 }
1899
1900 let pack = build_git_lane_multi_root_pack_plan(git_repo, roots, have_boundary, chunk_size)?;
1901
1902 let local_revision_address = newest_root
1905 .map(|oid| RevisionAddress::git_commit(oid.to_hex()).to_string())
1906 .unwrap_or_default();
1907
1908 Ok(GitLanePushPlan {
1909 local_revision_address,
1910 pack,
1911 ref_updates,
1912 })
1913}
1914
1915fn grpc_git_ref_kind(kind: ClassifiedGitRefKind) -> GrpcGitRefKind {
1916 match kind {
1917 ClassifiedGitRefKind::Branch => GrpcGitRefKind::Branch,
1918 ClassifiedGitRefKind::Tag => GrpcGitRefKind::Tag,
1919 ClassifiedGitRefKind::Note => GrpcGitRefKind::Note,
1920 ClassifiedGitRefKind::Other => GrpcGitRefKind::Other,
1921 }
1922}
1923
1924fn build_git_lane_multi_root_pack_plan(
1930 git_repo: &SleyRepository,
1931 roots: Vec<GitObjectId>,
1932 excluded: Vec<GitObjectId>,
1933 chunk_size: usize,
1934) -> Result<Option<GitPackPushPlan>, ProtocolError> {
1935 if roots.is_empty() {
1936 return Err(ProtocolError::InvalidState(
1937 "cannot plan a Git pack with no roots".to_string(),
1938 ));
1939 }
1940 let Some(plan) = git_repo
1941 .reachable_pack_plan()
1942 .roots(roots.iter().copied())
1943 .exclusions(excluded)
1944 .build()
1945 .map_err(|err| {
1946 ProtocolError::InvalidState(format!("plan reachable Git pack stream: {err}"))
1947 })?
1948 else {
1949 return Ok(None);
1952 };
1953 let pack_file = NamedTempFile::new()
1954 .map_err(|err| ProtocolError::InvalidState(format!("create Git pack tempfile: {err}")))?;
1955 let prepared = plan.prepare_to_file(pack_file.path()).map_err(|err| {
1956 ProtocolError::InvalidState(format!("write reachable Git pack tempfile: {err}"))
1957 })?;
1958 let pack_size = prepared.summary.pack_size;
1959 let checksum = prepared.summary.checksum;
1960 if pack_size > wire::MAX_RECEIVED_GIT_PACK_SIZE {
1961 return Err(ProtocolError::InvalidState(format!(
1962 "Git pack exceeds maximum transfer size of {} bytes; multi-pack split for repos over this size is a follow-up (plan §B.2)",
1963 wire::MAX_RECEIVED_GIT_PACK_SIZE
1964 )));
1965 }
1966 let chunk_size = chunk_size.max(1) as u64;
1967 let chunk_count = pack_size.div_ceil(chunk_size);
1968 if chunk_count > u32::MAX as u64 {
1969 return Err(ProtocolError::InvalidState(
1970 "Git pack chunk count exceeds u32".to_string(),
1971 ));
1972 }
1973 let transfer_id = format!("git-pack:{}", checksum.to_hex());
1974 Ok(Some(GitPackPushPlan {
1975 transfer_id,
1976 pack_id: checksum.as_bytes().to_vec(),
1977 pack_size,
1978 roots,
1979 pack_file: Arc::new(Mutex::new(pack_file)),
1980 }))
1981}
1982
1983async fn send_git_pack_streaming_messages(
1984 tx: &mpsc::Sender<PushMessage>,
1985 pack: &GitPackPushPlan,
1986 chunk_size: usize,
1987 progress: &Progress,
1988) -> Result<(), ProtocolError> {
1989 let tx = tx.clone();
1990 let pack = pack.clone();
1991 let progress = progress.clone();
1992 tokio::task::spawn_blocking(move || {
1993 stream_git_pack_messages_blocking(tx, pack, chunk_size, progress)
1994 })
1995 .await
1996 .map_err(|err| ProtocolError::InvalidState(format!("Git pack streaming task failed: {err}")))?
1997}
1998
1999fn stream_git_pack_messages_blocking(
2000 tx: mpsc::Sender<PushMessage>,
2001 pack: GitPackPushPlan,
2002 chunk_size: usize,
2003 progress: Progress,
2004) -> Result<(), ProtocolError> {
2005 let mut writer = GitPackPushMessageWriter::new(
2006 tx,
2007 pack.transfer_id.clone(),
2008 pack.pack_id.clone(),
2009 pack.pack_size,
2010 chunk_size,
2011 progress,
2012 );
2013 let mut pack_file = pack
2014 .pack_file
2015 .lock()
2016 .map_err(|err| ProtocolError::InvalidState(format!("lock Git pack tempfile: {err}")))?;
2017 pack_file
2018 .seek(SeekFrom::Start(0))
2019 .map_err(|err| ProtocolError::InvalidState(format!("rewind Git pack tempfile: {err}")))?;
2020 let streamed = io::copy(&mut pack_file.as_file_mut(), &mut writer)
2021 .map_err(|err| ProtocolError::InvalidState(format!("stream Git pack tempfile: {err}")))?;
2022 if streamed != pack.pack_size {
2023 return Err(ProtocolError::InvalidState(format!(
2024 "Git pack stream changed while sending; expected {} bytes/{}, streamed {} bytes",
2025 pack.pack_size,
2026 hex::encode(&pack.pack_id),
2027 streamed
2028 )));
2029 }
2030 writer.finish()?;
2031 Ok(())
2032}
2033
2034struct GitPackPushMessageWriter {
2035 tx: mpsc::Sender<PushMessage>,
2036 transfer_id: String,
2037 pack_id: Vec<u8>,
2038 pack_size: u64,
2039 chunk_size: usize,
2040 buffer: Vec<u8>,
2041 offset: u64,
2042 chunk_index: u32,
2043 progress: Progress,
2046 last_progress_pct: u64,
2050}
2051
2052impl GitPackPushMessageWriter {
2053 fn new(
2054 tx: mpsc::Sender<PushMessage>,
2055 transfer_id: String,
2056 pack_id: Vec<u8>,
2057 pack_size: u64,
2058 chunk_size: usize,
2059 progress: Progress,
2060 ) -> Self {
2061 let chunk_size = chunk_size.max(1);
2062 Self {
2063 tx,
2064 transfer_id,
2065 pack_id,
2066 pack_size,
2067 chunk_size,
2068 buffer: Vec::with_capacity(chunk_size),
2069 offset: 0,
2070 chunk_index: 0,
2071 progress,
2072 last_progress_pct: u64::MAX,
2073 }
2074 }
2075
2076 fn send_buffer(&mut self) -> io::Result<()> {
2077 if self.buffer.is_empty() {
2078 return Ok(());
2079 }
2080 let chunk = std::mem::take(&mut self.buffer);
2081 let next_offset = self.offset.checked_add(chunk.len() as u64).ok_or_else(|| {
2082 io::Error::new(io::ErrorKind::InvalidData, "Git pack offset overflow")
2083 })?;
2084 if next_offset > self.pack_size {
2085 return Err(io::Error::new(
2086 io::ErrorKind::InvalidData,
2087 format!(
2088 "Git pack stream exceeded planned size {}; got at least {}",
2089 self.pack_size, next_offset
2090 ),
2091 ));
2092 }
2093 let is_final_chunk = next_offset == self.pack_size;
2094 self.tx
2095 .blocking_send(git_lane_push_message(git_lane_transfer::Body::Pack(
2096 GitPackTransfer {
2097 transfer_id: self.transfer_id.clone(),
2098 offset: self.offset,
2099 chunk_index: self.chunk_index,
2100 is_final_chunk,
2101 pack_size: self.pack_size,
2102 pack_chunk: chunk.into(),
2103 pack_id: self.pack_id.clone().into(),
2104 },
2105 )))
2106 .map_err(|_| {
2107 io::Error::new(io::ErrorKind::BrokenPipe, "push stream closed unexpectedly")
2108 })?;
2109 self.offset = next_offset;
2110 self.chunk_index = self.chunk_index.checked_add(1).ok_or_else(|| {
2111 io::Error::new(io::ErrorKind::InvalidData, "Git pack chunk index overflow")
2112 })?;
2113 self.report_upload_progress();
2114 Ok(())
2115 }
2116
2117 fn report_upload_progress(&mut self) {
2122 if !self.progress.is_active() {
2123 return;
2124 }
2125 let pct = self.offset.saturating_mul(100) / self.pack_size.max(1);
2126 if pct == self.last_progress_pct {
2127 return;
2128 }
2129 self.last_progress_pct = pct;
2130 self.progress.set_phase(format!(
2131 "uploading {}/{} bytes ({pct}%)",
2132 self.offset, self.pack_size
2133 ));
2134 }
2135
2136 fn finish(mut self) -> Result<(), ProtocolError> {
2137 self.send_buffer().map_err(|err| {
2138 ProtocolError::InvalidState(format!("send final Git pack chunk: {err}"))
2139 })?;
2140 if self.offset != self.pack_size {
2141 return Err(ProtocolError::InvalidState(format!(
2142 "Git pack stream length mismatch: expected {}, got {}",
2143 self.pack_size, self.offset
2144 )));
2145 }
2146 Ok(())
2147 }
2148}
2149
2150impl Write for GitPackPushMessageWriter {
2151 fn write(&mut self, mut buf: &[u8]) -> io::Result<usize> {
2152 let written = buf.len();
2153 while !buf.is_empty() {
2154 if self.offset == self.pack_size {
2155 return Err(io::Error::new(
2156 io::ErrorKind::InvalidData,
2157 format!("Git pack stream wrote past planned size {}", self.pack_size),
2158 ));
2159 }
2160 let capacity = self
2161 .chunk_size
2162 .checked_sub(self.buffer.len())
2163 .ok_or_else(|| {
2164 io::Error::new(io::ErrorKind::InvalidData, "Git pack chunk buffer overflow")
2165 })?;
2166 let take = capacity.min(buf.len());
2167 self.buffer.extend_from_slice(&buf[..take]);
2168 buf = &buf[take..];
2169 if self.buffer.len() == self.chunk_size {
2170 self.send_buffer()?;
2171 }
2172 }
2173 Ok(written)
2174 }
2175
2176 fn flush(&mut self) -> io::Result<()> {
2177 Ok(())
2178 }
2179}
2180
2181fn git_ref_update_message(
2188 name: &str,
2189 kind: GrpcGitRefKind,
2190 target_oid: GitObjectId,
2191 peeled_oid: Option<GitObjectId>,
2192 checkpoint: Option<GitCheckpointTransfer>,
2193) -> PushMessage {
2194 git_lane_push_message(git_lane_transfer::Body::RefUpdate(GitRefUpdateTransfer {
2195 name: name.to_string(),
2196 kind: kind as i32,
2197 target_oid: target_oid.as_bytes().to_vec().into(),
2198 peeled_oid: peeled_oid
2199 .map(|oid| oid.as_bytes().to_vec())
2200 .unwrap_or_default()
2201 .into(),
2202 expected_missing: false,
2203 expected_target_oid: Vec::new().into(),
2204 checkpoint,
2205 }))
2206}
2207
2208fn apply_git_ref_expectation_value(
2209 message: &mut PushMessage,
2210 expectation: &GitRefRemoteExpectation,
2211) -> Result<(), ProtocolError> {
2212 let Some(push_message::Body::GitLane(GitLaneTransfer {
2213 body: Some(git_lane_transfer::Body::RefUpdate(update)),
2214 })) = message.body.as_mut()
2215 else {
2216 return Err(ProtocolError::InvalidState(
2217 "Git lane push plan missing ref update message".to_string(),
2218 ));
2219 };
2220 match expectation {
2221 GitRefRemoteExpectation::Missing => {
2222 update.expected_missing = true;
2223 update.expected_target_oid = Vec::new().into();
2224 }
2225 GitRefRemoteExpectation::Value(oid) => {
2226 update.expected_missing = false;
2227 update.expected_target_oid = oid.clone().into();
2228 }
2229 }
2230 Ok(())
2231}
2232
2233#[derive(Clone, Debug, PartialEq, Eq)]
2234enum GitRefRemoteExpectation {
2235 Missing,
2236 Value(Vec<u8>),
2237}
2238
2239fn parse_git_ref_expectation(
2240 remote_revision_address: &str,
2241) -> Result<GitRefRemoteExpectation, ProtocolError> {
2242 if remote_revision_address.is_empty() {
2243 return Ok(GitRefRemoteExpectation::Missing);
2244 }
2245
2246 match remote_revision_address.parse::<RevisionAddress>() {
2247 Ok(RevisionAddress::Heddle(_)) => Ok(GitRefRemoteExpectation::Missing),
2248 Ok(RevisionAddress::GitCommit(oid)) => hex::decode(&oid)
2249 .map(GitRefRemoteExpectation::Value)
2250 .map_err(|err| {
2251 ProtocolError::InvalidState(format!(
2252 "server returned invalid Git remote_revision_address: {err}"
2253 ))
2254 }),
2255 Err(err) => Err(ProtocolError::InvalidState(format!(
2256 "server returned invalid remote_revision_address {remote_revision_address:?}: {err}"
2257 ))),
2258 }
2259}
2260
2261fn git_lane_push_message(body: git_lane_transfer::Body) -> PushMessage {
2262 PushMessage {
2263 body: Some(push_message::Body::GitLane(GitLaneTransfer {
2264 body: Some(body),
2265 })),
2266 }
2267}
2268
2269fn git_lane_transfer_size(transfer: &GitLaneTransfer) -> usize {
2270 match transfer.body.as_ref() {
2271 Some(git_lane_transfer::Body::Pack(pack)) => pack.pack_chunk.len(),
2272 Some(git_lane_transfer::Body::RefUpdate(update)) => update
2273 .target_oid
2274 .len()
2275 .saturating_add(update.peeled_oid.len())
2276 .saturating_add(update.expected_target_oid.len())
2277 .saturating_add(
2278 update
2279 .checkpoint
2280 .as_ref()
2281 .map(git_checkpoint_transfer_size)
2282 .unwrap_or_default(),
2283 ),
2284 Some(git_lane_transfer::Body::Checkpoint(checkpoint)) => {
2285 git_checkpoint_transfer_size(checkpoint)
2286 }
2287 None => 0,
2288 }
2289}
2290
2291fn git_checkpoint_transfer_size(checkpoint: &GitCheckpointTransfer) -> usize {
2292 checkpoint
2293 .heddle_change_id
2294 .len()
2295 .saturating_add(checkpoint.git_commit_oid.len())
2296 .saturating_add(checkpoint.thread.len())
2297 .saturating_add(checkpoint.metadata_json.len())
2298}
2299
2300#[derive(Default)]
2301struct GitPackPullInstallState {
2302 active: Option<GitPackPullInstall>,
2303}
2304
2305impl GitPackPullInstallState {
2306 fn ensure_idle(&self) -> Result<(), ProtocolError> {
2307 if self.active.is_none() {
2308 Ok(())
2309 } else {
2310 Err(ProtocolError::InvalidState(
2311 "Git pack transfer ended before final chunk".to_string(),
2312 ))
2313 }
2314 }
2315
2316 fn receive_chunk(
2317 &mut self,
2318 git_repo: &SleyRepository,
2319 pack: GitPackTransfer,
2320 ) -> Result<(), ProtocolError> {
2321 if pack.transfer_id.is_empty() {
2322 return Err(ProtocolError::InvalidState(
2323 "Git pack transfer_id is required".to_string(),
2324 ));
2325 }
2326 if pack.pack_size > wire::MAX_RECEIVED_GIT_PACK_SIZE {
2327 return Err(ProtocolError::InvalidState(format!(
2328 "Git pack exceeds maximum transfer size of {} bytes",
2329 wire::MAX_RECEIVED_GIT_PACK_SIZE
2330 )));
2331 }
2332 if pack.pack_chunk.is_empty() {
2333 return Err(ProtocolError::InvalidState(
2334 "Git pack chunk must not be empty".to_string(),
2335 ));
2336 }
2337 let pack_id =
2338 GitObjectId::from_raw(git_repo.object_format(), &pack.pack_id).map_err(|err| {
2339 ProtocolError::InvalidState(format!("GitPackTransfer.pack_id: {err}"))
2340 })?;
2341 if self.active.is_none() {
2342 if pack.offset != 0 {
2343 return Err(ProtocolError::InvalidState(format!(
2344 "Git pack offset mismatch: expected 0, got {}",
2345 pack.offset
2346 )));
2347 }
2348 if pack.chunk_index != 0 {
2349 return Err(ProtocolError::InvalidState(format!(
2350 "Git pack chunk index mismatch: expected 0, got {}",
2351 pack.chunk_index
2352 )));
2353 }
2354 let writer = git_repo
2355 .objects()
2356 .begin_raw_pack_install(pack_id, pack.pack_size)
2357 .map_err(|err| {
2358 ProtocolError::InvalidState(format!("begin Git pack install: {err}"))
2359 })?;
2360 self.active = Some(GitPackPullInstall {
2361 transfer_id: pack.transfer_id.clone(),
2362 pack_id,
2363 pack_size: pack.pack_size,
2364 next_offset: 0,
2365 next_chunk_index: 0,
2366 writer,
2367 });
2368 }
2369
2370 let active = self.active.as_mut().ok_or_else(|| {
2371 ProtocolError::InvalidState("Git pack install not active".to_string())
2372 })?;
2373 active.receive_chunk(&pack, pack_id)?;
2374 if pack.is_final_chunk {
2375 let active = self.active.take().ok_or_else(|| {
2376 ProtocolError::InvalidState("Git pack install not active".to_string())
2377 })?;
2378 active.finish()?;
2379 git_repo.refresh_objects();
2380 }
2381 Ok(())
2382 }
2383}
2384
2385struct GitPackPullInstall {
2386 transfer_id: String,
2387 pack_id: GitObjectId,
2388 pack_size: u64,
2389 next_offset: u64,
2390 next_chunk_index: u32,
2391 writer: sley::plumbing::sley_odb::RawPackStreamingInstall,
2392}
2393
2394impl GitPackPullInstall {
2395 fn receive_chunk(
2396 &mut self,
2397 pack: &GitPackTransfer,
2398 pack_id: GitObjectId,
2399 ) -> Result<(), ProtocolError> {
2400 if self.transfer_id != pack.transfer_id {
2401 return Err(ProtocolError::InvalidState(format!(
2402 "Git pack transfer id changed from {:?} to {:?}",
2403 self.transfer_id, pack.transfer_id
2404 )));
2405 }
2406 if self.pack_id != pack_id {
2407 return Err(ProtocolError::InvalidState(
2408 "Git pack id changed during transfer".to_string(),
2409 ));
2410 }
2411 if self.pack_size != pack.pack_size {
2412 return Err(ProtocolError::InvalidState(
2413 "Git pack size changed during transfer".to_string(),
2414 ));
2415 }
2416 if pack.offset != self.next_offset {
2417 return Err(ProtocolError::InvalidState(format!(
2418 "Git pack offset mismatch: expected {}, got {}",
2419 self.next_offset, pack.offset
2420 )));
2421 }
2422 if pack.chunk_index != self.next_chunk_index {
2423 return Err(ProtocolError::InvalidState(format!(
2424 "Git pack chunk index mismatch: expected {}, got {}",
2425 self.next_chunk_index, pack.chunk_index
2426 )));
2427 }
2428 let chunk_len = u64::try_from(pack.pack_chunk.len()).map_err(|_| {
2429 ProtocolError::InvalidState("Git pack chunk length exceeds u64".to_string())
2430 })?;
2431 let next_offset = self
2432 .next_offset
2433 .checked_add(chunk_len)
2434 .ok_or_else(|| ProtocolError::InvalidState("Git pack offset overflow".to_string()))?;
2435 if next_offset > self.pack_size {
2436 return Err(ProtocolError::InvalidState(
2437 "Git pack chunk exceeds declared pack size".to_string(),
2438 ));
2439 }
2440 self.writer.write_all(&pack.pack_chunk).map_err(|err| {
2441 ProtocolError::InvalidState(format!("write streamed Git pack chunk: {err}"))
2442 })?;
2443 self.next_offset = next_offset;
2444 self.next_chunk_index = self.next_chunk_index.checked_add(1).ok_or_else(|| {
2445 ProtocolError::InvalidState("Git pack chunk index overflow".to_string())
2446 })?;
2447 if pack.is_final_chunk {
2448 if self.next_offset != self.pack_size {
2449 return Err(ProtocolError::InvalidState(format!(
2450 "Git pack final size mismatch: declared {}, received {}",
2451 self.pack_size, self.next_offset
2452 )));
2453 }
2454 } else if self.next_offset == self.pack_size {
2455 return Err(ProtocolError::InvalidState(
2456 "Git pack reached declared size without final chunk marker".to_string(),
2457 ));
2458 }
2459 Ok(())
2460 }
2461
2462 fn finish(self) -> Result<(), ProtocolError> {
2463 self.writer
2464 .finish()
2465 .map_err(|err| ProtocolError::InvalidState(format!("install Git pack: {err}")))?;
2466 Ok(())
2467 }
2468}
2469
2470fn accept_git_lane_pull_transfer(
2471 repo: &Repository,
2472 git_repo: &mut Option<SleyRepository>,
2473 git_pack_state: &mut GitPackPullInstallState,
2474 transfer: GitLaneTransfer,
2475) -> Result<(), ProtocolError> {
2476 if repo.capability() != RepositoryCapability::GitOverlay {
2477 return Err(ProtocolError::InvalidState(format!(
2478 "received git-lane pull transfer for non-GitOverlay repository (capability {:?})",
2479 repo.capability()
2480 )));
2481 }
2482 match transfer.body {
2483 Some(git_lane_transfer::Body::Pack(pack)) => {
2484 accept_git_lane_pack(repo, git_repo, git_pack_state, pack)
2485 }
2486 Some(git_lane_transfer::Body::RefUpdate(update)) => {
2487 accept_git_lane_ref_update(repo, git_repo, update)
2488 }
2489 Some(git_lane_transfer::Body::Checkpoint(checkpoint)) => {
2490 record_git_lane_checkpoint(repo, git_lane_sley_repository(repo, git_repo)?, checkpoint)
2491 }
2492 None => Err(ProtocolError::InvalidState(
2493 "GitLaneTransfer body is required".to_string(),
2494 )),
2495 }
2496}
2497
2498fn accept_git_lane_pack(
2499 repo: &Repository,
2500 git_repo: &mut Option<SleyRepository>,
2501 git_pack_state: &mut GitPackPullInstallState,
2502 pack: GitPackTransfer,
2503) -> Result<(), ProtocolError> {
2504 let git_repo = git_lane_sley_repository(repo, git_repo)?;
2505 git_pack_state.receive_chunk(git_repo, pack)?;
2506 Ok(())
2507}
2508
2509fn accept_git_lane_ref_update(
2519 repo: &Repository,
2520 git_repo: &mut Option<SleyRepository>,
2521 update: GitRefUpdateTransfer,
2522) -> Result<(), ProtocolError> {
2523 let git_repo = git_lane_sley_repository(repo, git_repo)?;
2524 let target = git_oid_from_bytes(
2525 git_repo,
2526 "GitRefUpdateTransfer.target_oid",
2527 &update.target_oid,
2528 )?;
2529 git_repo.read_commit(&target).map_err(|err| {
2530 ProtocolError::InvalidState(format!(
2531 "Git ref {} target commit {} is not present after pack receive: {err}",
2532 update.name,
2533 target.to_hex()
2534 ))
2535 })?;
2536 let refs = git_repo.references();
2537 let mut tx = refs.transaction();
2538 tx.update_to(
2539 update.name.clone(),
2540 ReferenceTarget::Direct(target),
2541 RefPrecondition::Any,
2542 None,
2543 );
2544 tx.commit().map_err(|err| {
2545 ProtocolError::InvalidState(format!("update Git ref {}: {err}", update.name))
2546 })?;
2547 if let Some(checkpoint) = update.checkpoint {
2548 record_git_lane_checkpoint(repo, git_repo, checkpoint)?;
2549 }
2550 Ok(())
2551}
2552
2553fn record_git_lane_checkpoint(
2554 repo: &Repository,
2555 git_repo: &SleyRepository,
2556 checkpoint: GitCheckpointTransfer,
2557) -> Result<(), ProtocolError> {
2558 let state = ChangeId::try_from_slice(&checkpoint.heddle_change_id)
2559 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
2560 let commit_oid = git_oid_from_bytes(
2561 git_repo,
2562 "GitCheckpointTransfer.git_commit_oid",
2563 &checkpoint.git_commit_oid,
2564 )?;
2565 let commit_hex = commit_oid.to_hex();
2566 if repo
2567 .latest_git_checkpoint_for_change(&state)
2568 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
2569 .is_some_and(|record| record.git_commit == commit_hex)
2570 {
2571 return Ok(());
2572 }
2573 let summary = serde_json::from_str::<serde_json::Value>(&checkpoint.metadata_json)
2574 .ok()
2575 .and_then(|metadata| {
2576 metadata
2577 .get("summary")
2578 .and_then(serde_json::Value::as_str)
2579 .map(str::to_string)
2580 })
2581 .unwrap_or_else(|| "pulled Git checkpoint".to_string());
2582 repo.record_git_checkpoint(&state, commit_hex, summary)
2583 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
2584 Ok(())
2585}
2586
2587fn git_lane_sley_repository<'a>(
2588 repo: &Repository,
2589 git_repo: &'a mut Option<SleyRepository>,
2590) -> Result<&'a SleyRepository, ProtocolError> {
2591 if git_repo.is_none() {
2592 *git_repo = Some(
2593 repo.git_overlay_sley_repository()
2594 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
2595 .ok_or_else(|| {
2596 ProtocolError::InvalidState(
2597 "git-overlay repository has no Git store".to_string(),
2598 )
2599 })?,
2600 );
2601 }
2602 match git_repo.as_ref() {
2603 Some(git_repo) => Ok(git_repo),
2604 None => Err(ProtocolError::InvalidState(
2605 "git-overlay repository has no Git store".to_string(),
2606 )),
2607 }
2608}
2609
2610fn git_oid_from_bytes(
2611 git_repo: &SleyRepository,
2612 field: &str,
2613 bytes: &[u8],
2614) -> Result<GitObjectId, ProtocolError> {
2615 GitObjectId::from_raw(git_repo.object_format(), bytes)
2616 .map_err(|err| ProtocolError::InvalidState(format!("{field}: {err}")))
2617}
2618
2619async fn send_native_pack_streaming_messages(
2620 tx: &mpsc::Sender<PushMessage>,
2621 repo: &Repository,
2622 objects: &[ObjectInfo],
2623 transfer_id: &str,
2624 chunk_size: usize,
2625 transport: &super::helpers::HostedTransportPolicy,
2626 transport_mode: TransportMode,
2627) -> Result<(), ProtocolError> {
2628 let object_count = u64::try_from(objects.len()).map_err(|_| {
2629 ProtocolError::InvalidState("native pack object count exceeds u64".to_string())
2630 })?;
2631 let mut writer = wire::NativePackStreamingWriter::new_in(repo.heddle_dir(), object_count)?;
2632 let mut pack_reader = wire::GrowingPackChunkReader::open(writer.pack_path(), chunk_size)?;
2633 let (loaded_tx, mut loaded_rx) = mpsc::channel::<(
2634 usize,
2635 Result<wire::ObjectData, ProtocolError>,
2636 )>(NATIVE_PACK_OBJECT_PREFETCH_LIMIT);
2637 let store = repo.store().clone();
2638 let object_plan = objects.to_vec();
2639 let loader = tokio::task::spawn_blocking(move || {
2640 load_native_pack_objects_parallel(store, object_plan, loaded_tx);
2641 });
2642
2643 let mut next_index = 0usize;
2644 let mut pending = BTreeMap::new();
2645 while next_index < objects.len() {
2646 let (index, object) = loaded_rx.recv().await.ok_or_else(|| {
2647 ProtocolError::InvalidState(
2648 "native pack object loader stopped before sending all objects".to_string(),
2649 )
2650 })?;
2651 pending.insert(index, object);
2652
2653 while let Some(object) = pending.remove(&next_index) {
2654 let object = object?;
2655 let should_drain = object.data.len() >= chunk_size
2656 || (next_index + 1).is_multiple_of(NATIVE_PACK_DRAIN_OBJECT_INTERVAL);
2657 writer.add_object_data(object)?;
2658 if should_drain {
2659 writer.flush_pack()?;
2660 drain_growing_native_pack_stream(
2661 tx,
2662 &mut pack_reader,
2663 false,
2664 PackStreamKind::Pack,
2665 transfer_id,
2666 transport,
2667 transport_mode,
2668 )
2669 .await?;
2670 }
2671 next_index += 1;
2672 }
2673 }
2674 loader.await.map_err(|err| {
2675 ProtocolError::InvalidState(format!("native pack object loader task failed: {err}"))
2676 })?;
2677
2678 let bundle = writer.finish()?;
2679 drain_growing_native_pack_stream(
2680 tx,
2681 &mut pack_reader,
2682 true,
2683 PackStreamKind::Pack,
2684 transfer_id,
2685 transport,
2686 transport_mode,
2687 )
2688 .await?;
2689 send_native_pack_file_stream(
2690 tx,
2691 &bundle.index_path,
2692 PackStreamKind::Index,
2693 transfer_id,
2694 chunk_size,
2695 transport,
2696 transport_mode,
2697 )
2698 .await
2699}
2700
2701fn load_native_pack_objects_parallel(
2702 store: AnyStore,
2703 objects: Vec<ObjectInfo>,
2704 tx: mpsc::Sender<(usize, Result<wire::ObjectData, ProtocolError>)>,
2705) {
2706 if objects.is_empty() {
2707 return;
2708 }
2709 let worker_count = native_pack_object_load_worker_count(objects.len());
2710 let objects = Arc::new(objects);
2711 let next_index = Arc::new(AtomicUsize::new(0));
2712
2713 std::thread::scope(|scope| {
2714 for _ in 0..worker_count {
2715 let store = store.clone();
2716 let objects = Arc::clone(&objects);
2717 let next_index = Arc::clone(&next_index);
2718 let tx = tx.clone();
2719 scope.spawn(move || {
2720 loop {
2721 let index = next_index.fetch_add(1, Ordering::Relaxed);
2722 let Some(info) = objects.get(index) else {
2723 break;
2724 };
2725 let object = wire::load_object_data(&store, &info.id, info.obj_type);
2726 if tx.blocking_send((index, object)).is_err() {
2727 break;
2728 }
2729 }
2730 });
2731 }
2732 });
2733}
2734
2735fn native_pack_object_load_worker_count(object_count: usize) -> usize {
2736 let available = std::thread::available_parallelism()
2737 .map(usize::from)
2738 .unwrap_or(1);
2739 object_count
2740 .min(available)
2741 .clamp(1, NATIVE_PACK_OBJECT_LOAD_WORKER_LIMIT)
2742}
2743
2744async fn drain_growing_native_pack_stream(
2745 tx: &mpsc::Sender<PushMessage>,
2746 reader: &mut wire::GrowingPackChunkReader,
2747 final_stream: bool,
2748 stream_kind: PackStreamKind,
2749 transfer_id: &str,
2750 transport: &super::helpers::HostedTransportPolicy,
2751 transport_mode: TransportMode,
2752) -> Result<(), ProtocolError> {
2753 while let Some((offset, chunk_index, data, is_final_chunk)) =
2754 reader.next_available_chunk(final_stream)?
2755 {
2756 send_pack_chunk(
2757 tx,
2758 stream_kind,
2759 data,
2760 transfer_id,
2761 transport,
2762 transport_mode,
2763 chunk_index,
2764 offset,
2765 is_final_chunk,
2766 )
2767 .await?;
2768 }
2769 Ok(())
2770}
2771
2772async fn send_native_pack_file_stream(
2773 tx: &mpsc::Sender<PushMessage>,
2774 path: &std::path::Path,
2775 stream_kind: PackStreamKind,
2776 transfer_id: &str,
2777 chunk_size: usize,
2778 transport: &super::helpers::HostedTransportPolicy,
2779 transport_mode: TransportMode,
2780) -> Result<(), ProtocolError> {
2781 let mut reader = wire::PackFileChunkReader::open(path, chunk_size)?;
2782 while let Some((offset, chunk_index, data, is_final_chunk)) = reader.next_chunk()? {
2783 send_pack_chunk(
2784 tx,
2785 stream_kind,
2786 data,
2787 transfer_id,
2788 transport,
2789 transport_mode,
2790 chunk_index,
2791 offset,
2792 is_final_chunk,
2793 )
2794 .await?;
2795 }
2796 Ok(())
2797}
2798
2799#[allow(clippy::too_many_arguments)]
2800async fn send_pack_chunk(
2801 tx: &mpsc::Sender<PushMessage>,
2802 stream_kind: PackStreamKind,
2803 data: Vec<u8>,
2804 transfer_id: &str,
2805 transport: &super::helpers::HostedTransportPolicy,
2806 transport_mode: TransportMode,
2807 chunk_index: u32,
2808 offset: u64,
2809 is_final_chunk: bool,
2810) -> Result<(), ProtocolError> {
2811 let chunk_length = data.len().min(u32::MAX as usize) as u32;
2812 tx.send(PushMessage {
2813 body: Some(push_message::Body::Pack(PackChunk {
2814 stream_kind: stream_kind as i32,
2815 data: data.into(),
2816 transfer: Some(transport.transfer_checkpoint_with_mode(
2817 transfer_id,
2818 transport_mode,
2819 chunk_index,
2820 offset,
2821 is_final_chunk,
2822 )),
2823 chunk_length,
2824 is_final_chunk,
2825 })),
2826 })
2827 .await
2828 .map_err(|_| ProtocolError::InvalidState("push stream closed unexpectedly".to_string()))
2829}
2830
2831fn preferred_transport_mode(
2832 transport: &super::helpers::HostedTransportPolicy,
2833 object_count: usize,
2834) -> TransportMode {
2835 let _ = transport;
2836 let _ = object_count;
2837 TransportMode::NativePack
2838}
2839
2840#[cfg(test)]
2841mod tests {
2842 use std::collections::HashSet;
2843
2844 use chrono::{TimeZone, Utc};
2845 use cli_shared::ClientConfig;
2846 use grpc::heddle::v1::{
2847 ListRefsRequest, ListRefsResponse, PullComplete as GrpcPullComplete, PullReady,
2848 TransferCheckpoint, UpdateRefRequest, UpdateRefResponse, push_message,
2849 repo_sync_service_server::{RepoSyncService, RepoSyncServiceServer},
2850 };
2851 use objects::{
2852 object::{
2853 Attribution, Blob, ChangeId, ContentHash, Principal, Redaction, State, StateVisibility,
2854 StateVisibilityBlob, Tree, TreeEntry, VisibilityTier,
2855 },
2856 store::ObjectStore,
2857 };
2858 use tempfile::TempDir;
2859 use tonic::{Response, Status, transport::Server};
2860 use wire::{ObjectId, ObjectInfo};
2861
2862 use super::*;
2863 use crate::grpc_hosted::helpers::{descriptor_id_from_info, to_proto_object_info};
2864
2865 fn temp_repo() -> (TempDir, Repository) {
2866 let dir = TempDir::new().expect("tempdir");
2867 let repo = Repository::init_default(dir.path()).expect("init repo");
2868 (dir, repo)
2869 }
2870
2871 fn temp_repo_unseeded() -> (TempDir, Repository) {
2875 let dir = TempDir::new().expect("tempdir");
2876 let repo = Repository::init(dir.path()).expect("init repo");
2877 (dir, repo)
2878 }
2879
2880 fn redaction_info(blob: ContentHash) -> ObjectInfo {
2881 ObjectInfo {
2882 id: ObjectId::Hash(blob),
2883 obj_type: ObjectType::Redaction,
2884 size: 0,
2885 delta_base: None,
2886 }
2887 }
2888
2889 fn state_info(state: ChangeId) -> ObjectInfo {
2890 ObjectInfo {
2891 id: ObjectId::ChangeId(state),
2892 obj_type: ObjectType::State,
2893 size: 0,
2894 delta_base: None,
2895 }
2896 }
2897
2898 fn state_visibility_info(state: ChangeId) -> ObjectInfo {
2899 ObjectInfo {
2900 id: ObjectId::ChangeId(state),
2901 obj_type: ObjectType::StateVisibility,
2902 size: 0,
2903 delta_base: None,
2904 }
2905 }
2906
2907 fn sample_blob() -> ContentHash {
2908 ContentHash::from_bytes([7u8; 32])
2909 }
2910
2911 #[test]
2912 fn descriptor_id_from_info_matches_proto_encode_path() {
2913 let infos = [
2914 redaction_info(sample_blob()),
2915 state_info(ChangeId::from_bytes([3u8; 16])),
2916 state_visibility_info(ChangeId::from_bytes([9u8; 16])),
2917 ];
2918 for info in infos {
2919 assert_eq!(
2920 descriptor_id_from_info(&info),
2921 descriptor_id(&to_proto_object_info(&info)),
2922 "keying path must stay byte-identical to the throwaway-encode path",
2923 );
2924 }
2925 }
2926
2927 #[test]
2928 fn git_lane_messages_pack_reachable_commit_graph_and_attach_checkpoint() {
2929 let dir = TempDir::new().expect("tempdir");
2930 let git = sley::Repository::init(dir.path()).expect("init git");
2931 let blob_oid = git.write_blob(b"hello\n").expect("write blob");
2932 let tree = sley::TreeObject {
2933 entries: vec![sley::plumbing::sley_object::TreeEntry {
2934 mode: 0o100644,
2935 name: sley::BString::from(b"hello.txt"),
2936 oid: blob_oid,
2937 }],
2938 };
2939 let tree_oid = git
2940 .write_raw_object(sley::GitObjectType::Tree, tree.write())
2941 .expect("write tree");
2942 let commit = sley::CommitObject {
2943 tree: tree_oid,
2944 parents: Vec::new(),
2945 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
2946 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
2947 encoding: None,
2948 message: b"checkpoint\n".to_vec(),
2949 };
2950 let commit_oid = git
2951 .write_raw_object(sley::GitObjectType::Commit, commit.write())
2952 .expect("write commit");
2953
2954 let pack =
2955 build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64 * 1024)
2956 .expect("build git lane pack plan")
2957 .expect("non-empty pack plan");
2958 assert_eq!(pack.pack_id.len(), git.object_format().raw_len());
2959 let (tx, mut rx) = mpsc::channel(8);
2960 stream_git_pack_messages_blocking(tx, pack.clone(), 64 * 1024, Progress::null())
2961 .expect("stream git lane pack");
2962 let mut pack_bytes = Vec::new();
2963 let mut chunks = Vec::new();
2964 while let Some(chunk) = rx.blocking_recv() {
2965 let Some(push_message::Body::GitLane(GitLaneTransfer {
2966 body: Some(git_lane_transfer::Body::Pack(pack)),
2967 })) = chunk.body
2968 else {
2969 panic!("expected git pack chunk");
2970 };
2971 pack_bytes.extend_from_slice(&pack.pack_chunk);
2972 chunks.push(pack);
2973 }
2974 assert!(!chunks.is_empty());
2975 assert!(chunks.last().expect("pack chunk").is_final_chunk);
2976 assert_eq!(pack_bytes.len() as u64, pack.pack_size);
2977 let indexed = sley::plumbing::sley_odb::index_raw_pack(&pack_bytes, git.object_format())
2978 .expect("generated pack should index");
2979 assert_eq!(indexed.pack_id.as_bytes(), pack.pack_id.as_slice());
2980 assert_eq!(indexed.objects.len(), 3);
2981
2982 let state = ChangeId::from_bytes([9u8; 16]);
2983 let ref_message = git_ref_update_message(
2984 "refs/heads/main",
2985 GrpcGitRefKind::Branch,
2986 commit_oid,
2987 None,
2988 Some(GitCheckpointTransfer {
2989 heddle_change_id: state.as_bytes().to_vec().into(),
2990 git_commit_oid: commit_oid.as_bytes().to_vec().into(),
2991 thread: "main".to_string(),
2992 metadata_json: String::new(),
2993 }),
2994 );
2995 let Some(push_message::Body::GitLane(GitLaneTransfer {
2996 body: Some(git_lane_transfer::Body::RefUpdate(update)),
2997 })) = ref_message.body
2998 else {
2999 panic!("expected git ref update message");
3000 };
3001 assert_eq!(update.name, "refs/heads/main");
3002 assert_eq!(update.kind, GrpcGitRefKind::Branch as i32);
3003 assert_eq!(update.target_oid.as_ref(), commit_oid.as_bytes());
3004 let checkpoint = update.checkpoint.expect("checkpoint");
3005 assert_eq!(checkpoint.heddle_change_id.as_ref(), state.as_bytes());
3006 assert_eq!(checkpoint.git_commit_oid.as_ref(), commit_oid.as_bytes());
3007 assert_eq!(checkpoint.thread, "main");
3008 }
3009
3010 fn sample_ref_update_message(commit_oid: GitObjectId) -> PushMessage {
3011 git_ref_update_message(
3012 "refs/heads/main",
3013 GrpcGitRefKind::Branch,
3014 commit_oid,
3015 None,
3016 Some(GitCheckpointTransfer {
3017 heddle_change_id: ChangeId::from_bytes([9u8; 16]).as_bytes().to_vec().into(),
3018 git_commit_oid: commit_oid.as_bytes().to_vec().into(),
3019 thread: "main".to_string(),
3020 metadata_json: String::new(),
3021 }),
3022 )
3023 }
3024
3025 fn write_commit(git: &sley::Repository, seed: &str) -> GitObjectId {
3028 let blob_oid = git
3029 .write_blob(format!("content-{seed}\n").as_bytes())
3030 .expect("write blob");
3031 let tree = sley::TreeObject {
3032 entries: vec![sley::plumbing::sley_object::TreeEntry {
3033 mode: 0o100644,
3034 name: sley::BString::from(format!("{seed}.txt").into_bytes()),
3035 oid: blob_oid,
3036 }],
3037 };
3038 let tree_oid = git
3039 .write_raw_object(sley::GitObjectType::Tree, tree.write())
3040 .expect("write tree");
3041 let commit = sley::CommitObject {
3042 tree: tree_oid,
3043 parents: Vec::new(),
3044 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3045 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3046 encoding: None,
3047 message: format!("commit {seed}\n").into_bytes(),
3048 };
3049 git.write_raw_object(sley::GitObjectType::Commit, commit.write())
3050 .expect("write commit")
3051 }
3052
3053 fn set_ref(git: &mut sley::Repository, name: &str, target: GitObjectId) {
3054 let refs = git.references();
3055 let mut tx = refs.transaction();
3056 tx.update_to(
3057 name.to_string(),
3058 ReferenceTarget::Direct(target),
3059 RefPrecondition::Any,
3060 None,
3061 );
3062 tx.commit().expect("commit ref");
3063 }
3064
3065 fn mirror_ref_updates(plan: &GitLanePushPlan) -> Vec<GitRefUpdateTransfer> {
3066 plan.ref_updates
3067 .iter()
3068 .map(git_ref_update_from_message)
3069 .cloned()
3070 .collect()
3071 }
3072
3073 fn write_child_commit(git: &sley::Repository, parent: GitObjectId, seed: &str) -> GitObjectId {
3077 let blob_oid = git
3078 .write_blob(format!("content-{seed}\n").as_bytes())
3079 .expect("write blob");
3080 let tree = sley::TreeObject {
3081 entries: vec![sley::plumbing::sley_object::TreeEntry {
3082 mode: 0o100644,
3083 name: sley::BString::from(format!("{seed}.txt").into_bytes()),
3084 oid: blob_oid,
3085 }],
3086 };
3087 let tree_oid = git
3088 .write_raw_object(sley::GitObjectType::Tree, tree.write())
3089 .expect("write tree");
3090 let commit = sley::CommitObject {
3091 tree: tree_oid,
3092 parents: vec![parent],
3093 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3094 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3095 encoding: None,
3096 message: format!("commit {seed}\n").into_bytes(),
3097 };
3098 git.write_raw_object(sley::GitObjectType::Commit, commit.write())
3099 .expect("write commit")
3100 }
3101
3102 fn value_expectation(oid: GitObjectId) -> GitRefRemoteExpectation {
3103 GitRefRemoteExpectation::Value(oid.as_bytes().to_vec())
3104 }
3105
3106 #[test]
3115 fn git_mirror_plan_packs_only_wanted_objects_against_have_boundary() {
3116 let dir = TempDir::new().expect("tempdir");
3117 let mut git = sley::Repository::init(dir.path()).expect("init git");
3118
3119 let base = write_commit(&git, "base-with-a-reasonably-large-payload");
3122 let mut tip = base;
3123 for i in 0..8 {
3124 tip = write_child_commit(&git, tip, &format!("layer-{i}"));
3125 }
3126 let descendant = write_child_commit(&git, tip, "descendant-adds-one-object");
3127 set_ref(&mut git, "refs/heads/main", descendant);
3128
3129 let full =
3132 build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new()).expect("full plan");
3133 let full_pack = full.pack.as_ref().expect("fresh server produces a pack");
3134 let full_size = full_pack.pack_size;
3135 assert!(full_size > 0, "baseline full pack must be non-empty");
3136
3137 let mut have_tip = HashMap::new();
3141 have_tip.insert("refs/heads/main".to_string(), value_expectation(tip));
3142 let delta =
3143 build_git_mirror_plan_from_sley(&git, 64 * 1024, &have_tip).expect("delta plan");
3144 let delta_pack = delta
3145 .pack
3146 .as_ref()
3147 .expect("descendant still has one new object to pack");
3148 let delta_size = delta_pack.pack_size;
3149 assert!(
3150 delta_size * 4 < full_size,
3151 "want-only packing must be dramatically smaller than a full re-pack \
3152 (delta {delta_size} bytes vs full {full_size} bytes)",
3153 );
3154
3155 let mut have_target = HashMap::new();
3159 have_target.insert("refs/heads/main".to_string(), value_expectation(descendant));
3160 let ref_only =
3161 build_git_mirror_plan_from_sley(&git, 64 * 1024, &have_target).expect("ref-only plan");
3162 assert!(
3163 ref_only.pack.is_none(),
3164 "pushing a ref the server already has must send NO pack",
3165 );
3166 assert_eq!(
3168 ref_only.ref_updates.len(),
3169 1,
3170 "the ref update still ships even with an empty want set",
3171 );
3172 }
3173
3174 #[test]
3177 fn git_lane_reachable_pack_plan_matches_legacy_writer_checksum() {
3178 let dir = TempDir::new().expect("tempdir");
3179 let git = sley::Repository::init(dir.path()).expect("init git");
3180 let commit_oid = write_commit(&git, "byte-identity");
3181
3182 let pack_plan =
3183 build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64 * 1024)
3184 .expect("build pack plan")
3185 .expect("non-empty pack plan");
3186
3187 let mut legacy_pack = Vec::new();
3188 let legacy = sley::plumbing::sley_odb::write_reachable_pack_to_writer(
3189 git.objects().as_ref(),
3190 git.object_format(),
3191 std::iter::once(commit_oid),
3192 &HashSet::new(),
3193 &mut legacy_pack,
3194 )
3195 .expect("legacy reachable pack")
3196 .expect("legacy pack summary");
3197
3198 assert_eq!(pack_plan.pack_id, legacy.checksum.as_bytes().to_vec());
3199 assert_eq!(pack_plan.pack_size, legacy.pack_size);
3200 assert_eq!(pack_plan.pack_size, legacy_pack.len() as u64);
3201
3202 let mut planned_pack = Vec::new();
3203 pack_plan
3204 .pack_file
3205 .lock()
3206 .expect("lock pack tempfile")
3207 .seek(SeekFrom::Start(0))
3208 .expect("rewind pack tempfile");
3209 io::copy(
3210 &mut pack_plan
3211 .pack_file
3212 .lock()
3213 .expect("lock pack tempfile")
3214 .as_file_mut(),
3215 &mut planned_pack,
3216 )
3217 .expect("read planned pack");
3218 assert_eq!(planned_pack, legacy_pack);
3219 }
3220
3221 #[test]
3225 fn git_mirror_plan_builds_one_pack_and_checkpointless_updates_for_all_refs() {
3226 let dir = TempDir::new().expect("tempdir");
3227 let mut git = sley::Repository::init(dir.path()).expect("init git");
3228
3229 let main_commit = write_commit(&git, "main");
3230 let feature_commit = write_commit(&git, "feature");
3231 let tagged_commit = write_commit(&git, "tagged");
3232
3233 set_ref(&mut git, "refs/heads/main", main_commit);
3234 set_ref(&mut git, "refs/heads/feature", feature_commit);
3235
3236 let tag = sley::TagObject {
3238 object: tagged_commit,
3239 object_type: sley::GitObjectType::Commit,
3240 name: b"v1".to_vec(),
3241 tagger: Some(b"Tester <test@example.com> 1700000000 +0000".to_vec()),
3242 message: b"release v1\n".to_vec(),
3243 raw_body: None,
3244 };
3245 let tag_oid = git
3246 .write_raw_object(sley::GitObjectType::Tag, tag.write())
3247 .expect("write tag");
3248 set_ref(&mut git, "refs/tags/v1", tag_oid);
3249
3250 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3251 .expect("build mirror plan");
3252
3253 let updates = mirror_ref_updates(&plan);
3254 assert_eq!(updates.len(), 3, "one ref update per direct ref");
3255
3256 assert!(
3259 updates.iter().all(|u| u.checkpoint.is_none()),
3260 "mirror mode signals via checkpoint: None on every ref update",
3261 );
3262
3263 let by_name: HashMap<&str, &GitRefUpdateTransfer> =
3264 updates.iter().map(|u| (u.name.as_str(), u)).collect();
3265
3266 let main = by_name["refs/heads/main"];
3267 assert_eq!(main.kind, GrpcGitRefKind::Branch as i32);
3268 assert_eq!(main.target_oid.as_ref(), main_commit.as_bytes());
3269 assert!(main.peeled_oid.is_empty(), "commit refs are not peeled");
3270
3271 let feature = by_name["refs/heads/feature"];
3272 assert_eq!(feature.kind, GrpcGitRefKind::Branch as i32);
3273 assert_eq!(feature.target_oid.as_ref(), feature_commit.as_bytes());
3274
3275 let tag_update = by_name["refs/tags/v1"];
3276 assert_eq!(tag_update.kind, GrpcGitRefKind::Tag as i32);
3277 assert_eq!(tag_update.target_oid.as_ref(), tag_oid.as_bytes());
3278 assert_eq!(
3279 tag_update.peeled_oid.as_ref(),
3280 tagged_commit.as_bytes(),
3281 "annotated tag ref is peeled to its underlying commit",
3282 );
3283
3284 let plan_pack = plan.pack.as_ref().expect("full pack for fresh server");
3288 let mut pack_roots: Vec<GitObjectId> = plan_pack.roots.clone();
3289 pack_roots.sort_by_key(|oid| oid.to_hex());
3290 let mut expected_roots = vec![main_commit, feature_commit, tag_oid];
3291 expected_roots.sort_by_key(|oid| oid.to_hex());
3292 assert_eq!(
3293 pack_roots, expected_roots,
3294 "pack roots == resolved ref targets",
3295 );
3296
3297 let (tx, mut rx) = mpsc::channel(64);
3300 stream_git_pack_messages_blocking(tx, plan_pack.clone(), 64 * 1024, Progress::null())
3301 .expect("stream mirror pack");
3302 let mut pack_bytes = Vec::new();
3303 while let Some(message) = rx.blocking_recv() {
3304 if let Some(push_message::Body::GitLane(GitLaneTransfer {
3305 body: Some(git_lane_transfer::Body::Pack(pack)),
3306 })) = message.body
3307 {
3308 pack_bytes.extend_from_slice(&pack.pack_chunk);
3309 }
3310 }
3311 assert_eq!(
3312 pack_bytes.len() as u64,
3313 plan_pack.pack_size,
3314 "streamed pack size must match plan",
3315 );
3316 let indexed = sley::plumbing::sley_odb::index_raw_pack(&pack_bytes, git.object_format())
3317 .expect("mirror pack indexes");
3318 let packed: HashSet<Vec<u8>> = indexed
3319 .objects
3320 .iter()
3321 .map(|obj| obj.oid.as_bytes().to_vec())
3322 .collect();
3323 for oid in [main_commit, feature_commit, tagged_commit, tag_oid] {
3324 assert!(
3325 packed.contains(oid.as_bytes()),
3326 "pack must contain {}",
3327 oid.to_hex()
3328 );
3329 }
3330 }
3331
3332 #[test]
3335 fn git_mirror_plan_skips_symbolic_refs() {
3336 let dir = TempDir::new().expect("tempdir");
3337 let mut git = sley::Repository::init(dir.path()).expect("init git");
3338 let main_commit = write_commit(&git, "main");
3339 set_ref(&mut git, "refs/heads/main", main_commit);
3340 {
3343 let refs = git.references();
3344 let mut tx = refs.transaction();
3345 tx.update_to(
3346 "HEAD".to_string(),
3347 ReferenceTarget::Symbolic("refs/heads/main".to_string()),
3348 RefPrecondition::Any,
3349 None,
3350 );
3351 tx.commit().expect("set HEAD");
3352 }
3353
3354 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3355 .expect("build mirror plan");
3356 let updates = mirror_ref_updates(&plan);
3357 assert_eq!(updates.len(), 1, "only the direct ref is pushed");
3358 assert_eq!(updates[0].name, "refs/heads/main");
3359 }
3360
3361 #[test]
3364 fn git_mirror_plan_applies_per_ref_remote_expectations() {
3365 let dir = TempDir::new().expect("tempdir");
3366 let mut git = sley::Repository::init(dir.path()).expect("init git");
3367 let main_commit = write_commit(&git, "main");
3368 let feature_commit = write_commit(&git, "feature");
3369 set_ref(&mut git, "refs/heads/main", main_commit);
3370 set_ref(&mut git, "refs/heads/feature", feature_commit);
3371
3372 let remote_oid = "89abcdef012345670123456789abcdef01234567";
3373 let mut expectations = HashMap::new();
3374 expectations.insert(
3375 "refs/heads/main".to_string(),
3376 GitRefRemoteExpectation::Value(hex::decode(remote_oid).expect("hex")),
3377 );
3378 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &expectations)
3381 .expect("build mirror plan");
3382 let updates = mirror_ref_updates(&plan);
3383 let by_name: HashMap<&str, &GitRefUpdateTransfer> =
3384 updates.iter().map(|u| (u.name.as_str(), u)).collect();
3385
3386 let main = by_name["refs/heads/main"];
3387 assert!(!main.expected_missing);
3388 assert_eq!(hex::encode(main.expected_target_oid.as_ref()), remote_oid);
3389
3390 let feature = by_name["refs/heads/feature"];
3391 assert!(
3392 feature.expected_missing,
3393 "ref absent from ListRefs is expected-missing (create)",
3394 );
3395 }
3396
3397 #[test]
3401 fn git_mirror_plan_includes_pull_request_refs() {
3402 let dir = TempDir::new().expect("tempdir");
3403 let mut git = sley::Repository::init(dir.path()).expect("init git");
3404 let pr_commit = write_commit(&git, "pr");
3405 set_ref(&mut git, "refs/pull/42/head", pr_commit);
3406
3407 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3408 .expect("build mirror plan");
3409 let updates = mirror_ref_updates(&plan);
3410 let pull = updates
3411 .iter()
3412 .find(|update| update.name == "refs/pull/42/head")
3413 .expect("refs/pull/* ref must be mirrored");
3414 assert_eq!(
3415 pull.kind,
3416 GrpcGitRefKind::Other as i32,
3417 "refs/pull/* classifies as Other",
3418 );
3419 assert_eq!(pull.target_oid.as_ref(), pr_commit.as_bytes());
3420 assert!(
3421 pull.checkpoint.is_none(),
3422 "mirror ref updates are checkpoint-less",
3423 );
3424 assert!(pull.peeled_oid.is_empty(), "a commit ref is not peeled");
3425 }
3426
3427 #[test]
3432 fn git_mirror_plan_excludes_local_only_refs_and_keeps_content() {
3433 let dir = TempDir::new().expect("tempdir");
3434 let mut git = sley::Repository::init(dir.path()).expect("init git");
3435
3436 let main_commit = write_commit(&git, "main");
3437 let notes_commit = write_commit(&git, "notes");
3438 let stash_commit = write_commit(&git, "stash");
3439 let remote_commit = write_commit(&git, "remote");
3440 let original_commit = write_commit(&git, "original");
3441 let replace_commit = write_commit(&git, "replace");
3442
3443 set_ref(&mut git, "refs/heads/main", main_commit);
3445 set_ref(&mut git, "refs/notes/heddle", notes_commit);
3446
3447 set_ref(&mut git, "refs/stash", stash_commit);
3449 set_ref(&mut git, "refs/remotes/origin/main", remote_commit);
3450 set_ref(&mut git, "refs/original/refs/heads/main", original_commit);
3451 set_ref(&mut git, "refs/replace/deadbeef", replace_commit);
3452
3453 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3454 .expect("build mirror plan");
3455 let names: Vec<&str> = plan
3456 .ref_updates
3457 .iter()
3458 .map(|m| git_ref_update_from_message(m).name.as_str())
3459 .collect();
3460
3461 assert!(
3462 names.contains(&"refs/heads/main"),
3463 "content branch is mirrored: {names:?}",
3464 );
3465 assert!(
3466 names.contains(&"refs/notes/heddle"),
3467 "heddle state-note ref is mirrored: {names:?}",
3468 );
3469 for excluded in [
3470 "refs/stash",
3471 "refs/remotes/origin/main",
3472 "refs/original/refs/heads/main",
3473 "refs/replace/deadbeef",
3474 ] {
3475 assert!(
3476 !names.contains(&excluded),
3477 "local-only ref {excluded} must NOT be mirrored: {names:?}",
3478 );
3479 }
3480 assert_eq!(
3481 names.len(),
3482 2,
3483 "exactly the two content refs ship: {names:?}"
3484 );
3485 }
3486
3487 #[test]
3492 fn git_mirror_plan_ignores_dangling_local_only_ref() {
3493 let dir = TempDir::new().expect("tempdir");
3494 let mut git = sley::Repository::init(dir.path()).expect("init git");
3495
3496 let main_commit = write_commit(&git, "main");
3497 set_ref(&mut git, "refs/heads/main", main_commit);
3498
3499 let missing = GitObjectId::from_hex(
3501 sley::ObjectFormat::Sha1,
3502 "0123456789abcdef0123456789abcdef01234567",
3503 )
3504 .expect("oid");
3505 set_ref(&mut git, "refs/original/refs/heads/main", missing);
3506
3507 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3508 .expect("dangling local-only ref must not fail the mirror plan");
3509 let names: Vec<&str> = plan
3510 .ref_updates
3511 .iter()
3512 .map(|m| git_ref_update_from_message(m).name.as_str())
3513 .collect();
3514 assert_eq!(
3515 names,
3516 vec!["refs/heads/main"],
3517 "only content ships: {names:?}"
3518 );
3519 }
3520
3521 #[test]
3522 fn git_ref_expectation_marks_missing_when_remote_has_no_git_revision() {
3523 let commit_oid = GitObjectId::from_hex(
3524 sley::ObjectFormat::Sha1,
3525 "0123456789abcdef0123456789abcdef01234567",
3526 )
3527 .expect("oid");
3528 let mut message = sample_ref_update_message(commit_oid);
3529
3530 let expectation = parse_git_ref_expectation("").expect("missing expectation");
3531 apply_git_ref_expectation_value(&mut message, &expectation).expect("apply expectation");
3532 let update = git_ref_update_from_message(&message);
3533 assert!(update.expected_missing);
3534 assert!(update.expected_target_oid.is_empty());
3535 }
3536
3537 #[test]
3538 fn git_ref_expectation_uses_remote_git_revision_oid() {
3539 let commit_oid = GitObjectId::from_hex(
3540 sley::ObjectFormat::Sha1,
3541 "0123456789abcdef0123456789abcdef01234567",
3542 )
3543 .expect("oid");
3544 let remote_oid = "89abcdef012345670123456789abcdef01234567";
3545 let mut message = sample_ref_update_message(commit_oid);
3546
3547 let expectation =
3548 parse_git_ref_expectation(&format!("git:{remote_oid}")).expect("git expectation");
3549 apply_git_ref_expectation_value(&mut message, &expectation).expect("apply expectation");
3550 let update = git_ref_update_from_message(&message);
3551 assert!(!update.expected_missing);
3552 assert_eq!(hex::encode(update.expected_target_oid.as_ref()), remote_oid);
3553 }
3554
3555 #[test]
3556 fn git_pack_stream_writer_emits_ordered_chunks() {
3557 let (tx, mut rx) = mpsc::channel(4);
3558 let mut writer = GitPackPushMessageWriter::new(
3559 tx,
3560 "git-pack:test".to_string(),
3561 vec![0x42; 20],
3562 10,
3563 4,
3564 Progress::null(),
3565 );
3566 writer.write_all(b"abcdefghij").expect("write pack bytes");
3567 writer.finish().expect("finish pack stream");
3568
3569 let mut chunks = Vec::new();
3570 while let Some(message) = rx.blocking_recv() {
3571 let Some(push_message::Body::GitLane(GitLaneTransfer {
3572 body: Some(git_lane_transfer::Body::Pack(pack)),
3573 })) = message.body
3574 else {
3575 panic!("expected git pack chunk");
3576 };
3577 chunks.push(pack);
3578 }
3579
3580 assert_eq!(chunks.len(), 3);
3581 assert_eq!(chunks[0].offset, 0);
3582 assert_eq!(chunks[0].chunk_index, 0);
3583 assert!(!chunks[0].is_final_chunk);
3584 assert_eq!(chunks[0].pack_chunk.as_ref(), b"abcd");
3585 assert_eq!(chunks[1].offset, 4);
3586 assert_eq!(chunks[1].chunk_index, 1);
3587 assert!(!chunks[1].is_final_chunk);
3588 assert_eq!(chunks[1].pack_chunk.as_ref(), b"efgh");
3589 assert_eq!(chunks[2].offset, 8);
3590 assert_eq!(chunks[2].chunk_index, 2);
3591 assert!(chunks[2].is_final_chunk);
3592 assert_eq!(chunks[2].pack_chunk.as_ref(), b"ij");
3593 assert!(
3594 chunks
3595 .iter()
3596 .all(|chunk| chunk.pack_id.as_ref() == &[0x42; 20][..])
3597 );
3598 }
3599
3600 #[derive(Default)]
3603 struct PhaseCapturingSink {
3604 phases: std::sync::Mutex<Vec<String>>,
3605 }
3606
3607 impl PhaseCapturingSink {
3608 fn phases(&self) -> Vec<String> {
3609 self.phases.lock().unwrap().clone()
3610 }
3611 }
3612
3613 impl objects::Sink for PhaseCapturingSink {
3614 fn render(&self, snap: objects::ProgressSnapshot) {
3615 self.phases.lock().unwrap().push(snap.phase);
3616 }
3617 }
3618
3619 #[test]
3623 fn git_pack_stream_reports_upload_progress() {
3624 let dir = TempDir::new().expect("tempdir");
3625 let git = sley::Repository::init(dir.path()).expect("init git");
3626 let commit_oid = write_commit(&git, "progress");
3627 let pack = build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64)
3629 .expect("build pack plan")
3630 .expect("non-empty pack plan");
3631
3632 let sink = std::sync::Arc::new(PhaseCapturingSink::default());
3633 struct Forward(std::sync::Arc<PhaseCapturingSink>);
3634 impl objects::Sink for Forward {
3635 fn render(&self, snap: objects::ProgressSnapshot) {
3636 self.0.render(snap);
3637 }
3638 }
3639 let progress = Progress::with_sink(Box::new(Forward(std::sync::Arc::clone(&sink))));
3640
3641 let (tx, mut rx) = mpsc::channel(1024);
3642 stream_git_pack_messages_blocking(tx, pack.clone(), 64, progress.clone())
3643 .expect("stream pack");
3644 while rx.blocking_recv().is_some() {}
3645
3646 let phases = sink.phases();
3647 assert!(
3648 phases.iter().any(|phase| phase.contains("uploading")),
3649 "pack streamer must drive an 'uploading' progress phase; saw {phases:?}",
3650 );
3651 let last_uploading = phases
3652 .iter()
3653 .rev()
3654 .find(|phase| phase.contains("uploading"))
3655 .cloned();
3656 assert!(
3657 last_uploading
3658 .as_deref()
3659 .is_some_and(|phase| phase.contains(&pack.pack_size.to_string())),
3660 "final uploading phase must show the full pack size ({} bytes); saw {last_uploading:?}",
3661 pack.pack_size,
3662 );
3663 }
3664
3665 #[test]
3666 fn git_pack_pull_install_state_streams_pack_into_sley_store() {
3667 let source_dir = TempDir::new().expect("source tempdir");
3668 let source = sley::Repository::init(source_dir.path()).expect("init source git");
3669 let blob_oid = source.write_blob(b"streamed pull pack\n").expect("blob");
3670 let pack = sley::plumbing::sley_odb::build_reachable_pack(
3671 source.objects().as_ref(),
3672 source.object_format(),
3673 [blob_oid],
3674 &HashSet::new(),
3675 )
3676 .expect("build pack")
3677 .expect("reachable pack");
3678
3679 let dest_dir = TempDir::new().expect("dest tempdir");
3680 let dest = sley::Repository::init(dest_dir.path()).expect("init dest git");
3681 let mut state = GitPackPullInstallState::default();
3682 let chunk_size = 7usize;
3683 let mut offset = 0u64;
3684 for (chunk_index, chunk) in pack.pack.chunks(chunk_size).enumerate() {
3685 let next_offset = offset + chunk.len() as u64;
3686 state
3687 .receive_chunk(
3688 &dest,
3689 GitPackTransfer {
3690 transfer_id: "git-pack:test".to_string(),
3691 offset,
3692 chunk_index: chunk_index as u32,
3693 is_final_chunk: next_offset == pack.pack.len() as u64,
3694 pack_size: pack.pack.len() as u64,
3695 pack_chunk: chunk.to_vec().into(),
3696 pack_id: pack.checksum.as_bytes().to_vec().into(),
3697 },
3698 )
3699 .expect("receive chunk");
3700 offset = next_offset;
3701 }
3702
3703 state.ensure_idle().expect("stream should finish");
3704 let object = dest.read_object(&blob_oid).expect("read installed object");
3705 assert_eq!(object.body.as_slice(), b"streamed pull pack\n");
3706 }
3707
3708 #[test]
3709 fn accept_git_lane_pull_transfer_errors_for_non_overlay_repo() {
3710 let (_dir, repo) = temp_repo();
3711 assert_ne!(
3712 repo.capability(),
3713 RepositoryCapability::GitOverlay,
3714 "init_default repo must be non-GitOverlay for this guard test",
3715 );
3716 let mut git_repo = None;
3717 let mut state = GitPackPullInstallState::default();
3718 let transfer = GitLaneTransfer {
3719 body: Some(git_lane_transfer::Body::Pack(GitPackTransfer {
3720 transfer_id: "git-pack:test".to_string(),
3721 offset: 0,
3722 chunk_index: 0,
3723 is_final_chunk: true,
3724 pack_size: 0,
3725 pack_chunk: Vec::new().into(),
3726 pack_id: Vec::new().into(),
3727 })),
3728 };
3729 let err = accept_git_lane_pull_transfer(&repo, &mut git_repo, &mut state, transfer)
3730 .expect_err("git-lane pull to a non-GitOverlay repo must fail loud, not silently drop");
3731 assert!(
3732 matches!(err, ProtocolError::InvalidState(_)),
3733 "expected InvalidState protocol error, got {err:?}",
3734 );
3735 }
3736
3737 fn git_ref_update_from_message(message: &PushMessage) -> &GitRefUpdateTransfer {
3738 let Some(push_message::Body::GitLane(GitLaneTransfer {
3739 body: Some(git_lane_transfer::Body::RefUpdate(update)),
3740 })) = message.body.as_ref()
3741 else {
3742 panic!("expected git ref update message");
3743 };
3744 update
3745 }
3746
3747 fn loose_tree_path(repo: &Repository, hash: &ContentHash) -> std::path::PathBuf {
3748 let hex = hash.to_hex();
3749 let (prefix, rest) = hex.split_at(2);
3750 repo.heddle_dir()
3751 .join("objects")
3752 .join("trees")
3753 .join(prefix)
3754 .join(rest)
3755 }
3756
3757 fn sample_redaction(blob: ContentHash) -> Redaction {
3758 Redaction {
3759 redacted_blob: blob,
3760 state: ChangeId::from_bytes([1u8; 16]),
3761 path: "config/secrets.toml".into(),
3762 reason: "leaked credential".into(),
3763 redactor: Principal {
3764 name: "Grace Hopper".into(),
3765 email: "grace@example.com".into(),
3766 },
3767 redacted_at: Utc.with_ymd_and_hms(2026, 5, 10, 14, 33, 0).unwrap(),
3768 signature: None,
3769 purged_at: None,
3770 supersedes: None,
3771 }
3772 }
3773
3774 fn sample_state_visibility(state: ChangeId) -> StateVisibility {
3775 StateVisibility {
3776 state,
3777 tier: VisibilityTier::Restricted {
3778 scope_label: "security-embargo".into(),
3779 },
3780 embargo_until: None,
3781 declarer: Principal {
3782 name: "Grace Hopper".into(),
3783 email: "grace@example.com".into(),
3784 },
3785 declared_at: Utc.with_ymd_and_hms(2026, 6, 1, 12, 0, 0).unwrap(),
3786 signature: None,
3787 supersedes: None,
3788 }
3789 }
3790
3791 #[test]
3792 fn non_packable_object_types_are_in_out_of_pack_transfer_partition() {
3793 for obj_type in wire::native_pack_excluded_object_types() {
3794 assert!(
3795 wire::TransferPartitions::<ObjectInfo>::is_sidecar_object_type(*obj_type),
3796 "{obj_type:?} is excluded from native packs but missing from the out-of-pack transfer partition"
3797 );
3798 }
3799 }
3800
3801 #[test]
3802 fn native_pack_required_tracks_packable_pull_wants() {
3803 let blob = sample_blob();
3804 let state = ChangeId::from_bytes([9u8; 16]);
3805
3806 let sidecar_only = RepositoryTransferPlan::from_object_infos(
3807 vec![state_visibility_info(state)],
3808 GitLaneTransferIntent::HeddleObjectsOnly,
3809 );
3810 assert!(!native_pack_required_for_pull(false, &sidecar_only));
3811
3812 let redaction_only = RepositoryTransferPlan::from_object_infos(
3813 vec![redaction_info(blob)],
3814 GitLaneTransferIntent::HeddleObjectsOnly,
3815 );
3816 assert!(!native_pack_required_for_pull(false, &redaction_only));
3817
3818 let packable = RepositoryTransferPlan::from_object_infos(
3819 vec![ObjectInfo {
3820 id: ObjectId::Hash(blob),
3821 obj_type: ObjectType::Blob,
3822 size: 0,
3823 delta_base: None,
3824 }],
3825 GitLaneTransferIntent::HeddleObjectsOnly,
3826 );
3827 assert!(native_pack_required_for_pull(false, &packable));
3828
3829 let state_with_sidecar = RepositoryTransferPlan::from_object_infos(
3830 vec![state_info(state), state_visibility_info(state)],
3831 GitLaneTransferIntent::HeddleObjectsOnly,
3832 );
3833 assert!(native_pack_required_for_pull(false, &state_with_sidecar));
3834 let empty = RepositoryTransferPlan::from_object_infos(
3835 Vec::<ObjectInfo>::new(),
3836 GitLaneTransferIntent::HeddleObjectsOnly,
3837 );
3838 assert!(native_pack_required_for_pull(true, &empty));
3839 }
3840
3841 #[test]
3842 fn plan_pull_wants_accumulates_state_and_visibility_for_same_change_id() {
3843 let (_dir, repo) = temp_repo();
3844 let state = ChangeId::from_bytes([9u8; 16]);
3845 let plan = plan_pull_wants(
3846 &repo,
3847 &state,
3848 false,
3849 vec![
3850 object_descriptor_with_status(
3851 &state_info(state),
3852 ObjectAvailabilityStatus::Missing,
3853 "missing state",
3854 ),
3855 object_descriptor_with_status(
3856 &state_visibility_info(state),
3857 ObjectAvailabilityStatus::Missing,
3858 "missing state visibility",
3859 ),
3860 ],
3861 false,
3862 )
3863 .expect("plan pull wants");
3864
3865 let wanted = plan
3866 .wanted_types
3867 .get(&PackObjectId::ChangeId(state))
3868 .expect("same ChangeId want entry");
3869 assert_eq!(
3870 wanted.as_slice(),
3871 &[ObjectType::State, ObjectType::StateVisibility]
3872 );
3873 assert!(native_pack_required_for_pull(
3874 plan.want_full_closure,
3875 &plan.transfer_plan
3876 ));
3877 }
3878
3879 #[derive(Clone)]
3880 struct SidecarOnlyPullService {
3881 state: ChangeId,
3882 state_visibility_blob: Vec<u8>,
3883 }
3884
3885 #[tonic::async_trait]
3886 impl RepoSyncService for SidecarOnlyPullService {
3887 async fn list_refs(
3888 &self,
3889 _request: tonic::Request<ListRefsRequest>,
3890 ) -> Result<Response<ListRefsResponse>, Status> {
3891 Ok(Response::new(ListRefsResponse::default()))
3892 }
3893
3894 async fn update_ref(
3895 &self,
3896 _request: tonic::Request<UpdateRefRequest>,
3897 ) -> Result<Response<UpdateRefResponse>, Status> {
3898 Ok(Response::new(UpdateRefResponse::default()))
3899 }
3900
3901 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushMessage, Status>>;
3902
3903 async fn push(
3904 &self,
3905 _request: tonic::Request<tonic::Streaming<PushMessage>>,
3906 ) -> Result<Response<Self::PushStream>, Status> {
3907 let (_tx, rx) = mpsc::channel(1);
3908 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
3909 rx,
3910 )))
3911 }
3912
3913 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullMessage, Status>>;
3914
3915 async fn pull(
3916 &self,
3917 request: tonic::Request<tonic::Streaming<PullMessage>>,
3918 ) -> Result<Response<Self::PullStream>, Status> {
3919 let state = self.state;
3920 let state_visibility_blob = self.state_visibility_blob.clone();
3921 let (tx, rx) = mpsc::channel(4);
3922
3923 tokio::spawn(async move {
3924 let mut inbound = request.into_inner();
3925 match inbound.message().await {
3926 Ok(Some(PullMessage {
3927 body: Some(pull_message::Body::Request(_)),
3928 })) => {}
3929 other => {
3930 let _ = tx
3931 .send(Err(Status::invalid_argument(format!(
3932 "expected pull request, got {other:?}"
3933 ))))
3934 .await;
3935 return;
3936 }
3937 }
3938
3939 let descriptor = object_descriptor_with_status(
3940 &state_visibility_info(state),
3941 ObjectAvailabilityStatus::Missing,
3942 "missing state visibility",
3943 );
3944 let ready = PullMessage {
3945 body: Some(pull_message::Body::Ready(PullReady {
3946 remote_state: state.to_string_full(),
3947 objects_to_fetch: vec![descriptor],
3948 transfer: None,
3949 partial_fetch_status: PartialFetchStatus::Disabled as i32,
3950 missing_objects: Vec::new(),
3951 full_closure_available: false,
3952 object_count: 1,
3953 remote_revision_address: RevisionAddress::heddle(state).to_string(),
3954 })),
3955 };
3956 if tx.send(Ok(ready)).await.is_err() {
3957 return;
3958 }
3959
3960 match inbound.message().await {
3961 Ok(Some(PullMessage {
3962 body: Some(pull_message::Body::Want(want)),
3963 })) if !want.want_full_closure
3964 && want.objects.len() == 1
3965 && want.objects[0].object_type == "state_visibility" => {}
3966 other => {
3967 let _ = tx
3968 .send(Err(Status::invalid_argument(format!(
3969 "expected sidecar-only want, got {other:?}"
3970 ))))
3971 .await;
3972 return;
3973 }
3974 }
3975
3976 let transfer = PullMessage {
3977 body: Some(pull_message::Body::StateVisibility(
3978 StateVisibilityTransfer {
3979 state_id: state.to_string_full(),
3980 state_visibility_blob: state_visibility_blob.into(),
3981 },
3982 )),
3983 };
3984 if tx.send(Ok(transfer)).await.is_err() {
3985 return;
3986 }
3987
3988 let complete = PullMessage {
3989 body: Some(pull_message::Body::Complete(GrpcPullComplete {
3990 success: true,
3991 new_state: state.to_string_full(),
3992 error: String::new(),
3993 transfer: Some(TransferCheckpoint {
3994 transfer_id: "sidecar-only-test".to_string(),
3995 transport_mode: TransportMode::NativePack as i32,
3996 resume_offset: 0,
3997 chunk_index: 0,
3998 checkpoint: b"heddle-markers-v1\n".to_vec(),
3999 is_complete: true,
4000 }),
4001 new_revision_address: RevisionAddress::heddle(state).to_string(),
4002 })),
4003 };
4004 let _ = tx.send(Ok(complete)).await;
4005 });
4006
4007 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4008 rx,
4009 )))
4010 }
4011 }
4012
4013 async fn connect_sidecar_only_service(
4014 service: SidecarOnlyPullService,
4015 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
4016 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
4017 Ok(listener) => listener,
4018 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
4019 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
4020 return None;
4021 }
4022 Err(err) => panic!("bind test server: {err}"),
4023 };
4024 let addr = listener.local_addr().expect("local addr");
4025 let incoming = futures::stream::unfold(listener, |listener| async {
4026 match listener.accept().await {
4027 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
4028 Err(err) => Some((Err(err), listener)),
4029 }
4030 });
4031
4032 let handle = tokio::spawn(async move {
4033 Server::builder()
4034 .add_service(RepoSyncServiceServer::new(service))
4035 .serve_with_incoming(incoming)
4036 .await
4037 .expect("serve sidecar-only test service");
4038 });
4039
4040 let client = HostedGrpcClient::connect(addr, &ClientConfig::default())
4041 .await
4042 .expect("connect client");
4043 Some((client, handle))
4044 }
4045
4046 #[tokio::test]
4047 async fn state_visibility_sidecar_only_pull_completes_without_native_pack() {
4048 let (_dir, repo) = temp_repo();
4049 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4050 let state = State::new_snapshot(
4051 tree_hash,
4052 vec![],
4053 Attribution::human(Principal {
4054 name: "Grace Hopper".into(),
4055 email: "grace@example.com".into(),
4056 }),
4057 );
4058 let state_id = state.change_id;
4059 repo.store().put_state(&state).expect("put state");
4060 assert!(
4061 repo.get_state_visibility_bytes_for_state(&state_id)
4062 .expect("load local sidecar")
4063 .is_none(),
4064 "test starts with state present and StateVisibility sidecar absent"
4065 );
4066
4067 let state_visibility_blob =
4068 StateVisibilityBlob::new(vec![sample_state_visibility(state_id)])
4069 .encode()
4070 .expect("encode state visibility blob");
4071 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4072 state: state_id,
4073 state_visibility_blob,
4074 })
4075 .await
4076 else {
4077 return;
4078 };
4079
4080 let exchange = tokio::time::timeout(
4081 Duration::from_secs(5),
4082 client.pull_exchange(
4083 &repo,
4084 "owner/repo",
4085 "main",
4086 PullOptions {
4087 local_thread: None,
4088 depth: None,
4089 target_state: Some(state_id),
4090 materialization: PullMaterialization::Full,
4091 },
4092 ),
4093 )
4094 .await
4095 .expect("sidecar-only pull must not hang waiting for native pack")
4096 .expect("sidecar-only pull succeeds");
4097 server.abort();
4098
4099 assert!(exchange.result.success);
4100 assert_eq!(exchange.object_count, 0);
4101 assert_eq!(exchange.profile.pack_bytes_received, 0);
4102 assert_eq!(exchange.profile.object_mix.state_visibilities, 1);
4103 assert!(
4104 repo.get_state_visibility_for_state(&state_id)
4105 .expect("load accepted sidecar")
4106 .has_record(),
4107 "pull must accept the out-of-pack StateVisibility sidecar"
4108 );
4109 }
4110
4111 #[tokio::test]
4115 async fn legitimate_large_sidecar_blob_decodes_and_installs() {
4116 let (_dir, repo) = temp_repo();
4117 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4118 let state = State::new_snapshot(
4119 tree_hash,
4120 vec![],
4121 Attribution::human(Principal {
4122 name: "Grace Hopper".into(),
4123 email: "grace@example.com".into(),
4124 }),
4125 );
4126 let state_id = state.change_id;
4127 repo.store().put_state(&state).expect("put state");
4128
4129 let mut record = sample_state_visibility(state_id);
4132 record.tier = VisibilityTier::Restricted {
4133 scope_label: "x".repeat(8 * 1024 * 1024),
4134 };
4135 let state_visibility_blob = StateVisibilityBlob::new(vec![record])
4136 .encode()
4137 .expect("encode large state visibility blob");
4138 assert!(
4139 state_visibility_blob.len() > 4 * 1024 * 1024,
4140 "blob must exceed tonic's 4 MiB default to exercise the raised decode limit"
4141 );
4142 assert!(
4143 (state_visibility_blob.len() as u64) <= wire::MAX_RECEIVED_STATE_VISIBILITY_BLOB_SIZE,
4144 "blob must stay within the legitimate sidecar receive cap"
4145 );
4146
4147 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4148 state: state_id,
4149 state_visibility_blob,
4150 })
4151 .await
4152 else {
4153 return;
4154 };
4155
4156 let exchange = tokio::time::timeout(
4157 Duration::from_secs(30),
4158 client.pull_exchange(
4159 &repo,
4160 "owner/repo",
4161 "main",
4162 PullOptions {
4163 local_thread: None,
4164 depth: None,
4165 target_state: Some(state_id),
4166 materialization: PullMaterialization::Full,
4167 },
4168 ),
4169 )
4170 .await
4171 .expect("large-sidecar pull must not hang")
4172 .expect("large but legitimate sidecar pull succeeds");
4173 server.abort();
4174
4175 assert!(exchange.result.success);
4176 assert!(
4177 repo.get_state_visibility_for_state(&state_id)
4178 .expect("load accepted sidecar")
4179 .has_record(),
4180 "pull must accept a legitimately-large StateVisibility sidecar"
4181 );
4182 }
4183
4184 #[tokio::test]
4188 async fn oversized_sidecar_blob_rejected_at_grpc_decode_boundary() {
4189 let (_dir, repo) = temp_repo();
4190 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4191 let state = State::new_snapshot(
4192 tree_hash,
4193 vec![],
4194 Attribution::human(Principal {
4195 name: "Grace Hopper".into(),
4196 email: "grace@example.com".into(),
4197 }),
4198 );
4199 let state_id = state.change_id;
4200 repo.store().put_state(&state).expect("put state");
4201
4202 let oversized = vec![0u8; wire::MAX_PULL_DECODE_MESSAGE_SIZE + 1];
4205 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4206 state: state_id,
4207 state_visibility_blob: oversized,
4208 })
4209 .await
4210 else {
4211 return;
4212 };
4213
4214 let result = tokio::time::timeout(
4215 Duration::from_secs(30),
4216 client.pull_exchange(
4217 &repo,
4218 "owner/repo",
4219 "main",
4220 PullOptions {
4221 local_thread: None,
4222 depth: None,
4223 target_state: Some(state_id),
4224 materialization: PullMaterialization::Full,
4225 },
4226 ),
4227 )
4228 .await
4229 .expect("oversized-sidecar pull must not hang");
4230 server.abort();
4231
4232 let err = match result {
4233 Err(err) => err,
4234 Ok(_) => panic!("oversized sidecar PullMessage must be rejected at decode"),
4235 };
4236 let message = err.to_string();
4237 assert!(
4238 !message.contains("exceeds receive size limit"),
4239 "rejection must come from the decode-size limit, before the post-decode check: {message}"
4240 );
4241 assert!(
4242 repo.get_state_visibility_for_state(&state_id)
4243 .expect("load sidecar")
4244 .latest()
4245 .expect("resolve visibility")
4246 .is_none(),
4247 "an oversized sidecar must never be installed"
4248 );
4249 }
4250
4251 #[derive(Clone)]
4252 struct StateAndVisibilityPullService {
4253 state: ChangeId,
4254 pack_bundle: wire::NativePackBundle,
4255 state_visibility_blob: Vec<u8>,
4256 }
4257
4258 #[tonic::async_trait]
4259 impl RepoSyncService for StateAndVisibilityPullService {
4260 async fn list_refs(
4261 &self,
4262 _request: tonic::Request<ListRefsRequest>,
4263 ) -> Result<Response<ListRefsResponse>, Status> {
4264 Ok(Response::new(ListRefsResponse::default()))
4265 }
4266
4267 async fn update_ref(
4268 &self,
4269 _request: tonic::Request<UpdateRefRequest>,
4270 ) -> Result<Response<UpdateRefResponse>, Status> {
4271 Ok(Response::new(UpdateRefResponse::default()))
4272 }
4273
4274 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushMessage, Status>>;
4275
4276 async fn push(
4277 &self,
4278 _request: tonic::Request<tonic::Streaming<PushMessage>>,
4279 ) -> Result<Response<Self::PushStream>, Status> {
4280 let (_tx, rx) = mpsc::channel(1);
4281 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4282 rx,
4283 )))
4284 }
4285
4286 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullMessage, Status>>;
4287
4288 async fn pull(
4289 &self,
4290 request: tonic::Request<tonic::Streaming<PullMessage>>,
4291 ) -> Result<Response<Self::PullStream>, Status> {
4292 let state = self.state;
4293 let pack_bundle = self.pack_bundle.clone();
4294 let state_visibility_blob = self.state_visibility_blob.clone();
4295 let (tx, rx) = mpsc::channel(8);
4296
4297 tokio::spawn(async move {
4298 let mut inbound = request.into_inner();
4299 match inbound.message().await {
4300 Ok(Some(PullMessage {
4301 body: Some(pull_message::Body::Request(_)),
4302 })) => {}
4303 other => {
4304 let _ = tx
4305 .send(Err(Status::invalid_argument(format!(
4306 "expected pull request, got {other:?}"
4307 ))))
4308 .await;
4309 return;
4310 }
4311 }
4312
4313 let ready = PullMessage {
4314 body: Some(pull_message::Body::Ready(PullReady {
4315 remote_state: state.to_string_full(),
4316 objects_to_fetch: vec![
4317 object_descriptor_with_status(
4318 &state_info(state),
4319 ObjectAvailabilityStatus::Missing,
4320 "missing state",
4321 ),
4322 object_descriptor_with_status(
4323 &state_visibility_info(state),
4324 ObjectAvailabilityStatus::Missing,
4325 "missing state visibility",
4326 ),
4327 ],
4328 transfer: None,
4329 partial_fetch_status: PartialFetchStatus::Disabled as i32,
4330 missing_objects: Vec::new(),
4331 full_closure_available: false,
4332 object_count: 2,
4333 remote_revision_address: RevisionAddress::heddle(state).to_string(),
4334 })),
4335 };
4336 if tx.send(Ok(ready)).await.is_err() {
4337 return;
4338 }
4339
4340 match inbound.message().await {
4341 Ok(Some(PullMessage {
4342 body: Some(pull_message::Body::Want(want)),
4343 })) if !want.want_full_closure
4344 && want.objects.len() == 2
4345 && want
4346 .objects
4347 .iter()
4348 .any(|object| object.object_type == "state")
4349 && want
4350 .objects
4351 .iter()
4352 .any(|object| object.object_type == "state_visibility") => {}
4353 other => {
4354 let _ = tx
4355 .send(Err(Status::invalid_argument(format!(
4356 "expected state + sidecar wants, got {other:?}"
4357 ))))
4358 .await;
4359 return;
4360 }
4361 }
4362
4363 for message in
4364 encode_pull_native_pack_messages(&pack_bundle, "state-and-visibility-test", 16)
4365 {
4366 if tx.send(Ok(message)).await.is_err() {
4367 return;
4368 }
4369 }
4370
4371 let transfer = PullMessage {
4372 body: Some(pull_message::Body::StateVisibility(
4373 StateVisibilityTransfer {
4374 state_id: state.to_string_full(),
4375 state_visibility_blob: state_visibility_blob.into(),
4376 },
4377 )),
4378 };
4379 if tx.send(Ok(transfer)).await.is_err() {
4380 return;
4381 }
4382
4383 let complete = PullMessage {
4384 body: Some(pull_message::Body::Complete(GrpcPullComplete {
4385 success: true,
4386 new_state: state.to_string_full(),
4387 error: String::new(),
4388 transfer: Some(TransferCheckpoint {
4389 transfer_id: "state-and-visibility-test".to_string(),
4390 transport_mode: TransportMode::NativePack as i32,
4391 resume_offset: 0,
4392 chunk_index: 0,
4393 checkpoint: b"heddle-markers-v1\n".to_vec(),
4394 is_complete: true,
4395 }),
4396 new_revision_address: RevisionAddress::heddle(state).to_string(),
4397 })),
4398 };
4399 let _ = tx.send(Ok(complete)).await;
4400 });
4401
4402 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4403 rx,
4404 )))
4405 }
4406 }
4407
4408 fn encode_pull_native_pack_messages(
4409 bundle: &wire::NativePackBundle,
4410 transfer_id: &str,
4411 chunk_size: usize,
4412 ) -> Vec<PullMessage> {
4413 let mut messages = Vec::new();
4414 let chunk_size = chunk_size.max(1);
4415
4416 let pack_total_chunks = wire::chunk_count(bundle.pack_data.len(), chunk_size);
4417 for chunk_index in 0..pack_total_chunks.max(1) {
4418 let Some((start, len)) =
4419 wire::chunk_bounds(bundle.pack_data.len(), chunk_size, chunk_index)
4420 else {
4421 break;
4422 };
4423 messages.push(PullMessage {
4424 body: Some(pull_message::Body::Pack(PackChunk {
4425 stream_kind: PackStreamKind::Pack as i32,
4426 data: bundle.pack_data[start..start + len].to_vec().into(),
4427 transfer: Some(TransferCheckpoint {
4428 transfer_id: transfer_id.to_string(),
4429 transport_mode: TransportMode::NativePack as i32,
4430 resume_offset: start as u64,
4431 chunk_index: chunk_index as u32,
4432 checkpoint: Vec::new(),
4433 is_complete: chunk_index + 1 == pack_total_chunks,
4434 }),
4435 chunk_length: len as u32,
4436 is_final_chunk: chunk_index + 1 == pack_total_chunks,
4437 })),
4438 });
4439 }
4440
4441 let index_total_chunks = wire::chunk_count(bundle.index_data.len(), chunk_size);
4442 for chunk_index in 0..index_total_chunks.max(1) {
4443 let Some((start, len)) =
4444 wire::chunk_bounds(bundle.index_data.len(), chunk_size, chunk_index)
4445 else {
4446 break;
4447 };
4448 messages.push(PullMessage {
4449 body: Some(pull_message::Body::Pack(PackChunk {
4450 stream_kind: PackStreamKind::Index as i32,
4451 data: bundle.index_data[start..start + len].to_vec().into(),
4452 transfer: Some(TransferCheckpoint {
4453 transfer_id: transfer_id.to_string(),
4454 transport_mode: TransportMode::NativePack as i32,
4455 resume_offset: start as u64,
4456 chunk_index: chunk_index as u32,
4457 checkpoint: Vec::new(),
4458 is_complete: chunk_index + 1 == index_total_chunks,
4459 }),
4460 chunk_length: len as u32,
4461 is_final_chunk: chunk_index + 1 == index_total_chunks,
4462 })),
4463 });
4464 }
4465
4466 messages
4467 }
4468
4469 async fn connect_state_and_visibility_service(
4470 service: StateAndVisibilityPullService,
4471 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
4472 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
4473 Ok(listener) => listener,
4474 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
4475 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
4476 return None;
4477 }
4478 Err(err) => panic!("bind test server: {err}"),
4479 };
4480 let addr = listener.local_addr().expect("local addr");
4481 let incoming = futures::stream::unfold(listener, |listener| async {
4482 match listener.accept().await {
4483 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
4484 Err(err) => Some((Err(err), listener)),
4485 }
4486 });
4487
4488 let handle = tokio::spawn(async move {
4489 Server::builder()
4490 .add_service(RepoSyncServiceServer::new(service))
4491 .serve_with_incoming(incoming)
4492 .await
4493 .expect("serve state-and-visibility test service");
4494 });
4495
4496 let client = HostedGrpcClient::connect(addr, &ClientConfig::default())
4497 .await
4498 .expect("connect client");
4499 Some((client, handle))
4500 }
4501
4502 #[tokio::test]
4503 async fn state_and_visibility_same_change_id_pull_requests_pack_and_sidecar() {
4504 let (_source_dir, source_repo) = temp_repo();
4505 let (_target_dir, target_repo) = temp_repo();
4506 let tree_hash = source_repo
4507 .store()
4508 .put_tree(&Tree::new())
4509 .expect("put tree");
4510 let state = State::new_snapshot(
4511 tree_hash,
4512 vec![],
4513 Attribution::human(Principal {
4514 name: "Grace Hopper".into(),
4515 email: "grace@example.com".into(),
4516 }),
4517 );
4518 let state_id = state.change_id;
4519 source_repo
4520 .store()
4521 .put_state(&state)
4522 .expect("put source state");
4523 let state_visibility_blob =
4524 StateVisibilityBlob::new(vec![sample_state_visibility(state_id)])
4525 .encode()
4526 .expect("encode state visibility blob");
4527 source_repo
4528 .accept_wire_state_visibility(state_id, &state_visibility_blob)
4529 .expect("put source state visibility");
4530 let pack_bundle = wire::build_native_pack(source_repo.store(), &[state_info(state_id)])
4531 .expect("build state pack");
4532
4533 assert!(
4534 target_repo
4535 .store()
4536 .get_state(&state_id)
4537 .expect("load target state")
4538 .is_none(),
4539 "test starts with state absent"
4540 );
4541 assert!(
4542 target_repo
4543 .get_state_visibility_bytes_for_state(&state_id)
4544 .expect("load target sidecar")
4545 .is_none(),
4546 "test starts with StateVisibility sidecar absent"
4547 );
4548
4549 let Some((mut client, server)) =
4550 connect_state_and_visibility_service(StateAndVisibilityPullService {
4551 state: state_id,
4552 pack_bundle,
4553 state_visibility_blob,
4554 })
4555 .await
4556 else {
4557 return;
4558 };
4559
4560 let exchange = tokio::time::timeout(
4561 Duration::from_secs(5),
4562 client.pull_exchange(
4563 &target_repo,
4564 "owner/repo",
4565 "main",
4566 PullOptions {
4567 local_thread: None,
4568 depth: None,
4569 target_state: Some(state_id),
4570 materialization: PullMaterialization::Full,
4571 },
4572 ),
4573 )
4574 .await
4575 .expect("state + sidecar pull must not hang waiting for native pack")
4576 .expect("state + sidecar pull succeeds");
4577 server.abort();
4578
4579 assert!(exchange.result.success);
4580 assert_eq!(exchange.object_count, 1);
4581 assert!(exchange.profile.pack_bytes_received > 0);
4582 assert_eq!(exchange.profile.object_mix.states, 1);
4583 assert_eq!(exchange.profile.object_mix.state_visibilities, 1);
4584 assert!(
4585 target_repo
4586 .store()
4587 .get_state(&state_id)
4588 .expect("load installed state")
4589 .is_some(),
4590 "native pack must install the State"
4591 );
4592 assert!(
4593 target_repo
4594 .get_state_visibility_for_state(&state_id)
4595 .expect("load accepted sidecar")
4596 .has_record(),
4597 "pull must accept the out-of-pack StateVisibility sidecar"
4598 );
4599 }
4600
4601 #[test]
4602 fn missing_blobs_in_tree_treats_absent_tree_as_empty() {
4603 let (_dir, repo) = temp_repo();
4604 let absent_tree = ContentHash::from_bytes([99u8; 32]);
4605
4606 let missing = wire::missing_blobs_in_tree(repo.store(), absent_tree)
4607 .expect("absent tree is not an error");
4608
4609 assert!(missing.is_empty());
4610 }
4611
4612 #[test]
4613 fn missing_blobs_in_tree_reports_only_genuinely_missing_blobs() {
4614 let (_dir, repo) = temp_repo();
4615 let present_blob = Blob::from("already local");
4616 let present_hash = repo.store().put_blob(&present_blob).expect("put blob");
4617 let missing_hash = ContentHash::from_bytes([42u8; 32]);
4618 let tree = Tree::from_entries(vec![
4619 TreeEntry::file("local.txt", present_hash, false).expect("present entry"),
4620 TreeEntry::file("remote.txt", missing_hash, false).expect("missing entry"),
4621 ]);
4622 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
4623
4624 let missing =
4625 wire::missing_blobs_in_tree(repo.store(), tree_hash).expect("collect missing blobs");
4626
4627 assert_eq!(missing, vec![missing_hash]);
4628 }
4629
4630 #[test]
4631 fn missing_blobs_in_tree_reports_corrupt_tree_read() {
4632 let (_dir, repo) = temp_repo();
4633 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4634 std::fs::write(loose_tree_path(&repo, &tree_hash), [0xc1]).expect("corrupt tree");
4635 repo.store().clear_recent_caches();
4636
4637 let err = wire::missing_blobs_in_tree(repo.store(), tree_hash)
4638 .expect_err("corrupt tree must fail");
4639
4640 assert!(matches!(err, ProtocolError::InvalidState(_)));
4641 assert!(
4642 err.to_string().contains(&format!(
4643 "load tree {} while collecting lazy hydration missing blobs",
4644 tree_hash.to_hex()
4645 )),
4646 "unexpected error: {err}"
4647 );
4648 }
4649
4650 #[test]
4651 fn redaction_push_message_uses_hex_keyed_sidecar_payload() {
4652 let (_dir, repo) = temp_repo();
4653 let blob = sample_blob();
4654 repo.put_redaction(sample_redaction(blob))
4655 .expect("put redaction");
4656 let expected_bytes = repo
4657 .store()
4658 .get_redactions_bytes_for_blob(&blob)
4659 .expect("load sidecar")
4660 .expect("sidecar exists");
4661
4662 let message = redaction_push_message(&repo, redaction_info(blob)).expect("message");
4663
4664 let Some(push_message::Body::Redaction(transfer)) = message.body else {
4665 panic!("expected redaction transfer");
4666 };
4667 assert_eq!(transfer.blob_hash, blob.to_hex());
4668 assert_eq!(transfer.redactions_blob, expected_bytes);
4669 }
4670
4671 #[test]
4672 fn redaction_push_message_reports_missing_sidecar_with_blob_hex() {
4673 let (_dir, repo) = temp_repo();
4674 let blob = sample_blob();
4675
4676 let err = redaction_push_message(&repo, redaction_info(blob)).expect_err("missing sidecar");
4677
4678 assert!(matches!(err, ProtocolError::InvalidState(_)));
4679 assert!(
4680 err.to_string().contains(&format!(
4681 "server wants redaction for blob {} but sender has no sidecar",
4682 blob.to_hex()
4683 )),
4684 "unexpected error: {err}"
4685 );
4686 }
4687
4688 #[test]
4689 fn redaction_push_message_reports_sidecar_load_error_with_blob_hex() {
4690 let (_dir, repo) = temp_repo();
4691 let blob = sample_blob();
4692 let redaction_path = repo
4693 .heddle_dir()
4694 .join("redactions")
4695 .join(format!("{}.bin", blob.to_hex()));
4696 std::fs::create_dir_all(&redaction_path).expect("directory at redaction path");
4697
4698 let err = redaction_push_message(&repo, redaction_info(blob)).expect_err("load error");
4699
4700 assert!(matches!(err, ProtocolError::InvalidState(_)));
4701 assert!(
4702 err.to_string()
4703 .contains(&format!("load redactions sidecar for {}:", blob.to_hex())),
4704 "unexpected error: {err}"
4705 );
4706 }
4707
4708 #[test]
4709 fn state_visibility_push_message_uses_state_keyed_sidecar_payload() {
4710 let (_dir, repo) = temp_repo();
4711 let state = ChangeId::from_bytes([17u8; 16]);
4712 repo.put_state_visibility(sample_state_visibility(state))
4713 .expect("put state visibility");
4714 let expected_bytes = repo
4715 .get_state_visibility_bytes_for_state(&state)
4716 .expect("load sidecar")
4717 .expect("sidecar exists");
4718
4719 let message =
4720 state_visibility_push_message(&repo, state_visibility_info(state)).expect("message");
4721
4722 let Some(push_message::Body::StateVisibility(transfer)) = message.body else {
4723 panic!("expected state visibility transfer");
4724 };
4725 assert_eq!(transfer.state_id, state.to_string_full());
4726 assert_eq!(transfer.state_visibility_blob, expected_bytes);
4727 }
4728
4729 fn sample_attribution() -> Attribution {
4732 Attribution::human(Principal {
4733 name: "Grace Hopper".into(),
4734 email: "grace@example.com".into(),
4735 })
4736 }
4737
4738 fn put_state_with_blob(repo: &Repository, contents: &str, parents: Vec<ChangeId>) -> ChangeId {
4741 let blob = Blob::from(contents);
4742 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
4743 let tree = Tree::from_entries(vec![
4744 TreeEntry::file(format!("{contents}.txt"), blob_hash, false).expect("tree entry"),
4745 ]);
4746 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
4747 let state = State::new_snapshot(tree_hash, parents, sample_attribution());
4748 let state_id = state.change_id;
4749 repo.store().put_state(&state).expect("put state");
4750 state_id
4751 }
4752
4753 #[test]
4754 fn locally_complete_pull_head_advertises_fully_present_thread_head() {
4755 let (_dir, repo) = temp_repo();
4756 let head = put_state_with_blob(&repo, "alpha", vec![]);
4757 repo.refs()
4758 .set_thread(&ThreadName::from("main"), &head)
4759 .expect("set thread");
4760
4761 let advertised =
4762 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
4763 assert_eq!(
4764 advertised,
4765 Some(head),
4766 "a thread head whose full closure is present locally must be advertised"
4767 );
4768 }
4769
4770 #[test]
4771 fn locally_complete_pull_head_skips_unknown_thread() {
4772 let (_dir, repo) = temp_repo_unseeded();
4776 let advertised =
4777 locally_complete_pull_head(&repo, "nonexistent", None).expect("completeness check");
4778 assert_eq!(advertised, None);
4779 }
4780
4781 #[test]
4782 fn locally_complete_pull_head_skips_when_target_state_override_in_play() {
4783 let (_dir, repo) = temp_repo();
4784 let head = put_state_with_blob(&repo, "alpha", vec![]);
4785 repo.refs()
4786 .set_thread(&ThreadName::from("main"), &head)
4787 .expect("set thread");
4788
4789 let advertised =
4792 locally_complete_pull_head(&repo, "main", Some(head)).expect("completeness check");
4793 assert_eq!(advertised, None);
4794 }
4795
4796 #[test]
4797 fn locally_complete_pull_head_skips_repo_with_missing_blobs() {
4798 let (_dir, repo) = temp_repo();
4803 let head = put_state_with_blob(&repo, "alpha", vec![]);
4804 repo.refs()
4805 .set_thread(&ThreadName::from("main"), &head)
4806 .expect("set thread");
4807 repo.record_missing_blob(ContentHash::from_bytes([88u8; 32]))
4808 .expect("record missing blob");
4809
4810 let advertised =
4811 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
4812 assert_eq!(
4813 advertised, None,
4814 "a repo carrying missing blobs must never advertise a head"
4815 );
4816 }
4817
4818 #[test]
4819 fn locally_complete_pull_head_skips_head_with_incomplete_closure() {
4820 let (_dir, repo) = temp_repo();
4825 let absent_parent = ChangeId::generate();
4826 let blob = Blob::from("orphan");
4827 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
4828 let tree = Tree::from_entries(vec![
4829 TreeEntry::file("orphan.txt", blob_hash, false).expect("tree entry"),
4830 ]);
4831 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
4832 let child = State::new_snapshot(tree_hash, vec![absent_parent], sample_attribution());
4834 let child_id = child.change_id;
4835 repo.store().put_state(&child).expect("put child state");
4836 repo.refs()
4837 .set_thread(&ThreadName::from("main"), &child_id)
4838 .expect("set thread");
4839
4840 let advertised =
4841 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
4842 assert_eq!(
4843 advertised, None,
4844 "a head whose closure has an absent parent state must not be advertised"
4845 );
4846 }
4847
4848 #[test]
4849 fn locally_complete_local_thread_head_advertises_fully_present_thread_head() {
4850 let (_dir, repo) = temp_repo();
4853 let head = put_state_with_blob(&repo, "alpha", vec![]);
4854 repo.refs()
4855 .set_thread(&ThreadName::from("feature"), &head)
4856 .expect("set thread");
4857
4858 let advertised =
4859 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
4860 assert_eq!(
4861 advertised,
4862 Some(head),
4863 "an explicit local-thread head whose full closure is present must be advertised"
4864 );
4865 }
4866
4867 #[test]
4868 fn locally_complete_local_thread_head_skips_unknown_thread() {
4869 let (_dir, repo) = temp_repo_unseeded();
4872 let advertised = locally_complete_local_thread_head(&repo, "nonexistent", None)
4873 .expect("completeness check");
4874 assert_eq!(advertised, None);
4875 }
4876
4877 #[test]
4878 fn locally_complete_local_thread_head_skips_when_target_state_override_in_play() {
4879 let (_dir, repo) = temp_repo();
4880 let head = put_state_with_blob(&repo, "alpha", vec![]);
4881 repo.refs()
4882 .set_thread(&ThreadName::from("feature"), &head)
4883 .expect("set thread");
4884
4885 let advertised = locally_complete_local_thread_head(&repo, "feature", Some(head))
4888 .expect("completeness check");
4889 assert_eq!(advertised, None);
4890 }
4891
4892 #[test]
4893 fn locally_complete_local_thread_head_skips_repo_with_missing_blobs() {
4894 let (_dir, repo) = temp_repo();
4897 let head = put_state_with_blob(&repo, "alpha", vec![]);
4898 repo.refs()
4899 .set_thread(&ThreadName::from("feature"), &head)
4900 .expect("set thread");
4901 repo.record_missing_blob(ContentHash::from_bytes([88u8; 32]))
4902 .expect("record missing blob");
4903
4904 let advertised =
4905 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
4906 assert_eq!(
4907 advertised, None,
4908 "a repo carrying missing blobs must never advertise an explicit local-thread head"
4909 );
4910 }
4911
4912 #[test]
4913 fn locally_complete_local_thread_head_skips_head_with_incomplete_closure() {
4914 let (_dir, repo) = temp_repo();
4919 let absent_parent = ChangeId::generate();
4920 let blob = Blob::from("orphan");
4921 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
4922 let tree = Tree::from_entries(vec![
4923 TreeEntry::file("orphan.txt", blob_hash, false).expect("tree entry"),
4924 ]);
4925 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
4926 let child = State::new_snapshot(tree_hash, vec![absent_parent], sample_attribution());
4927 let child_id = child.change_id;
4928 repo.store().put_state(&child).expect("put child state");
4929 repo.refs()
4930 .set_thread(&ThreadName::from("feature"), &child_id)
4931 .expect("set thread");
4932
4933 let advertised =
4934 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
4935 assert_eq!(
4936 advertised, None,
4937 "an explicit local-thread head whose closure has an absent parent must not be advertised"
4938 );
4939 }
4940
4941 #[derive(Clone)]
4947 struct DeltaAwarePullService {
4948 remote_state: ChangeId,
4949 full_closure: Vec<ObjectInfo>,
4955 delta_objects: Vec<ObjectInfo>,
4956 known_parent: ChangeId,
4957 full_pack: wire::NativePackBundle,
4958 delta_pack: wire::NativePackBundle,
4959 captured_exclude: std::sync::Arc<std::sync::Mutex<Option<Vec<String>>>>,
4960 }
4961
4962 #[tonic::async_trait]
4963 impl RepoSyncService for DeltaAwarePullService {
4964 async fn list_refs(
4965 &self,
4966 _request: tonic::Request<ListRefsRequest>,
4967 ) -> Result<Response<ListRefsResponse>, Status> {
4968 Ok(Response::new(ListRefsResponse::default()))
4969 }
4970
4971 async fn update_ref(
4972 &self,
4973 _request: tonic::Request<UpdateRefRequest>,
4974 ) -> Result<Response<UpdateRefResponse>, Status> {
4975 Ok(Response::new(UpdateRefResponse::default()))
4976 }
4977
4978 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushMessage, Status>>;
4979
4980 async fn push(
4981 &self,
4982 _request: tonic::Request<tonic::Streaming<PushMessage>>,
4983 ) -> Result<Response<Self::PushStream>, Status> {
4984 let (_tx, rx) = mpsc::channel(1);
4985 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4986 rx,
4987 )))
4988 }
4989
4990 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullMessage, Status>>;
4991
4992 async fn pull(
4993 &self,
4994 request: tonic::Request<tonic::Streaming<PullMessage>>,
4995 ) -> Result<Response<Self::PullStream>, Status> {
4996 let svc = self.clone();
4997 let (tx, rx) = mpsc::channel(16);
4998
4999 tokio::spawn(async move {
5000 let mut inbound = request.into_inner();
5001 let exclude = match inbound.message().await {
5002 Ok(Some(PullMessage {
5003 body: Some(pull_message::Body::Request(req)),
5004 })) => req.exclude_states,
5005 other => {
5006 let _ = tx
5007 .send(Err(Status::invalid_argument(format!(
5008 "expected pull request, got {other:?}"
5009 ))))
5010 .await;
5011 return;
5012 }
5013 };
5014 *svc.captured_exclude.lock().unwrap() = Some(exclude.clone());
5015
5016 let remote_full = svc.remote_state.to_string_full();
5017 let parent_full = svc.known_parent.to_string_full();
5018
5019 let (objects, pack, short_circuit) = if exclude.contains(&remote_full) {
5021 (Vec::new(), None, true)
5023 } else if exclude.contains(&parent_full) {
5024 (
5026 svc.delta_objects.clone(),
5027 Some(svc.delta_pack.clone()),
5028 false,
5029 )
5030 } else {
5031 (svc.full_closure.clone(), Some(svc.full_pack.clone()), false)
5033 };
5034
5035 let descriptors: Vec<_> = objects
5036 .iter()
5037 .map(|info| {
5038 object_descriptor_with_status(
5039 info,
5040 ObjectAvailabilityStatus::Missing,
5041 "requested by client",
5042 )
5043 })
5044 .collect();
5045
5046 let ready = PullMessage {
5047 body: Some(pull_message::Body::Ready(PullReady {
5048 remote_state: remote_full.clone(),
5049 objects_to_fetch: descriptors,
5050 transfer: None,
5051 partial_fetch_status: PartialFetchStatus::Disabled as i32,
5052 missing_objects: Vec::new(),
5053 full_closure_available: false,
5054 object_count: objects.len() as u32,
5055 remote_revision_address: RevisionAddress::heddle(svc.remote_state)
5056 .to_string(),
5057 })),
5058 };
5059 if tx.send(Ok(ready)).await.is_err() {
5060 return;
5061 }
5062
5063 match inbound.message().await {
5065 Ok(Some(PullMessage {
5066 body: Some(pull_message::Body::Want(_)),
5067 })) => {}
5068 other => {
5069 let _ = tx
5070 .send(Err(Status::invalid_argument(format!(
5071 "expected want, got {other:?}"
5072 ))))
5073 .await;
5074 return;
5075 }
5076 }
5077
5078 if let Some(pack) = pack
5079 && !short_circuit
5080 {
5081 for message in encode_pull_native_pack_messages(&pack, "delta-aware-test", 64) {
5082 if tx.send(Ok(message)).await.is_err() {
5083 return;
5084 }
5085 }
5086 }
5087
5088 let complete = PullMessage {
5089 body: Some(pull_message::Body::Complete(GrpcPullComplete {
5090 success: true,
5091 new_state: remote_full,
5092 error: String::new(),
5093 transfer: Some(TransferCheckpoint {
5094 transfer_id: "delta-aware-test".to_string(),
5095 transport_mode: TransportMode::NativePack as i32,
5096 resume_offset: 0,
5097 chunk_index: 0,
5098 checkpoint: b"heddle-markers-v1\n".to_vec(),
5099 is_complete: true,
5100 }),
5101 new_revision_address: RevisionAddress::heddle(svc.remote_state).to_string(),
5102 })),
5103 };
5104 let _ = tx.send(Ok(complete)).await;
5105 });
5106
5107 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
5108 rx,
5109 )))
5110 }
5111 }
5112
5113 async fn connect_delta_aware_service(
5114 service: DeltaAwarePullService,
5115 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
5116 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
5117 Ok(listener) => listener,
5118 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
5119 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
5120 return None;
5121 }
5122 Err(err) => panic!("bind test server: {err}"),
5123 };
5124 let addr = listener.local_addr().expect("local addr");
5125 let incoming = futures::stream::unfold(listener, |listener| async {
5126 match listener.accept().await {
5127 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
5128 Err(err) => Some((Err(err), listener)),
5129 }
5130 });
5131 let handle = tokio::spawn(async move {
5132 Server::builder()
5133 .add_service(RepoSyncServiceServer::new(service))
5134 .serve_with_incoming(incoming)
5135 .await
5136 .expect("serve delta-aware test service");
5137 });
5138 let client = HostedGrpcClient::connect(addr, &ClientConfig::default())
5139 .await
5140 .expect("connect client");
5141 Some((client, handle))
5142 }
5143
5144 #[tokio::test]
5145 async fn warm_bare_pull_advertises_head_and_fires_zero_delta_short_circuit() {
5146 let (_dir, repo) = temp_repo();
5150 let head = put_state_with_blob(&repo, "tip", vec![]);
5151 repo.refs()
5152 .set_thread(&ThreadName::from("main"), &head)
5153 .expect("set thread");
5154
5155 let full_closure =
5156 wire::enumerate_state_closure(repo.store(), head).expect("enumerate closure");
5157 let full_pack =
5158 wire::build_native_pack(repo.store(), &full_closure).expect("build full pack");
5159 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5160
5161 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5162 remote_state: head,
5163 full_closure,
5164 delta_objects: Vec::new(),
5165 known_parent: ChangeId::generate(),
5166 full_pack: full_pack.clone(),
5167 delta_pack: full_pack,
5168 captured_exclude: captured.clone(),
5169 })
5170 .await
5171 else {
5172 return;
5173 };
5174
5175 let exchange = tokio::time::timeout(
5176 Duration::from_secs(5),
5177 client.pull_exchange(
5178 &repo,
5179 "owner/repo",
5180 "main",
5181 PullOptions {
5182 local_thread: None,
5183 depth: None,
5184 target_state: None,
5185 materialization: PullMaterialization::Full,
5186 },
5187 ),
5188 )
5189 .await
5190 .expect("warm bare pull must not hang")
5191 .expect("warm bare pull succeeds");
5192 server.abort();
5193
5194 let advertised = captured.lock().unwrap().clone().expect("request captured");
5195 assert_eq!(
5196 advertised,
5197 vec![head.to_string_full()],
5198 "a bare pull at the tip must advertise that tip in exclude_states"
5199 );
5200 assert!(exchange.result.success);
5201 assert_eq!(
5202 exchange.object_count, 0,
5203 "the zero-delta short-circuit must transfer no objects, not the full closure"
5204 );
5205 }
5206
5207 #[tokio::test]
5208 async fn behind_client_bare_pull_receives_exactly_the_missing_delta() {
5209 let (_src_dir, src_repo) = temp_repo_unseeded();
5216 let parent = put_state_with_blob(&src_repo, "base", vec![]);
5217 let child = put_state_with_blob(&src_repo, "feature", vec![parent]);
5218 let child_closure =
5219 wire::enumerate_state_closure(src_repo.store(), child).expect("child closure");
5220
5221 let (_dir, repo) = temp_repo_unseeded();
5225 let parent_closure =
5226 wire::enumerate_state_closure(src_repo.store(), parent).expect("parent closure");
5227 let parent_pack =
5228 wire::build_native_pack(src_repo.store(), &parent_closure).expect("parent pack");
5229 wire::install_received_pack(
5230 repo.store(),
5231 &parent_pack.pack_data,
5232 &parent_pack.index_data,
5233 )
5234 .expect("install parent closure into client");
5235 repo.refs()
5236 .set_thread(&ThreadName::from("main"), &parent)
5237 .expect("set thread to parent");
5238 wire::enumerate_state_closure(repo.store(), parent).expect("client holds parent closure");
5240 assert!(
5242 repo.store()
5243 .get_state(&child)
5244 .expect("probe child")
5245 .is_none(),
5246 "client must start without the child state"
5247 );
5248 let parent_clone = parent;
5249 let full_pack =
5250 wire::build_native_pack(src_repo.store(), &child_closure).expect("full pack");
5251 let delta_objects = wire::enumerate_state_closure_with_options(
5254 src_repo.store(),
5255 child,
5256 wire::StateClosureOptions {
5257 depth: None,
5258 exclude_states: vec![parent_clone],
5259 },
5260 )
5261 .expect("delta closure");
5262 assert!(
5263 !delta_objects.is_empty() && delta_objects.len() < child_closure.len(),
5264 "delta must be a strict, non-empty subset of the full closure"
5265 );
5266 let delta_pack =
5267 wire::build_native_pack(src_repo.store(), &delta_objects).expect("delta pack");
5268
5269 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5270 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5271 remote_state: child,
5272 full_closure: child_closure.clone(),
5273 delta_objects: delta_objects.clone(),
5274 known_parent: parent_clone,
5275 full_pack,
5276 delta_pack,
5277 captured_exclude: captured.clone(),
5278 })
5279 .await
5280 else {
5281 return;
5282 };
5283
5284 let exchange = tokio::time::timeout(
5285 Duration::from_secs(5),
5286 client.pull_exchange(
5287 &repo,
5288 "owner/repo",
5289 "main",
5290 PullOptions {
5291 local_thread: None,
5292 depth: None,
5293 target_state: None,
5294 materialization: PullMaterialization::Full,
5295 },
5296 ),
5297 )
5298 .await
5299 .expect("behind-client pull must not hang")
5300 .expect("behind-client pull succeeds");
5301 server.abort();
5302
5303 let advertised = captured.lock().unwrap().clone().expect("request captured");
5304 assert_eq!(
5305 advertised,
5306 vec![parent.to_string_full()],
5307 "a behind client must advertise the parent head it holds"
5308 );
5309 assert!(exchange.result.success);
5310 let reassembled = wire::enumerate_state_closure(repo.store(), child)
5314 .expect("the client must hold the complete child closure after a delta pull");
5315 assert_eq!(
5316 reassembled.len(),
5317 child_closure.len(),
5318 "delta pull must leave the client with the full child closure, no gaps"
5319 );
5320 }
5321
5322 #[tokio::test]
5323 async fn fresh_bare_pull_advertises_nothing_and_gets_full_closure() {
5324 let (_dir, repo) = temp_repo_unseeded();
5328 let (_src_dir, src_repo) = temp_repo_unseeded();
5329 let remote = put_state_with_blob(&src_repo, "seed", vec![]);
5330 let full_closure =
5331 wire::enumerate_state_closure(src_repo.store(), remote).expect("closure");
5332 let full_pack =
5333 wire::build_native_pack(src_repo.store(), &full_closure).expect("full pack");
5334
5335 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5336 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5337 remote_state: remote,
5338 full_closure: full_closure.clone(),
5339 delta_objects: Vec::new(),
5340 known_parent: ChangeId::generate(),
5341 full_pack: full_pack.clone(),
5342 delta_pack: full_pack,
5343 captured_exclude: captured.clone(),
5344 })
5345 .await
5346 else {
5347 return;
5348 };
5349
5350 let exchange = tokio::time::timeout(
5351 Duration::from_secs(5),
5352 client.pull_exchange(
5353 &repo,
5354 "owner/repo",
5355 "main",
5356 PullOptions {
5357 local_thread: None,
5358 depth: None,
5359 target_state: None,
5360 materialization: PullMaterialization::Full,
5361 },
5362 ),
5363 )
5364 .await
5365 .expect("fresh bare pull must not hang")
5366 .expect("fresh bare pull succeeds");
5367 server.abort();
5368
5369 let advertised = captured.lock().unwrap().clone().expect("request captured");
5370 assert!(
5371 advertised.is_empty(),
5372 "a fresh repo must advertise no exclude_states"
5373 );
5374 assert!(exchange.result.success);
5375 assert_eq!(
5376 exchange.object_count,
5377 full_closure.len(),
5378 "a fresh pull must receive the full closure, nothing wrongly excluded"
5379 );
5380 assert!(
5381 wire::enumerate_state_closure(repo.store(), remote).is_ok(),
5382 "fresh pull must install the complete closure"
5383 );
5384 }
5385
5386 #[tokio::test]
5387 async fn explicit_local_thread_incomplete_closure_does_not_advertise_and_repairs() {
5388 let (_src_dir, src_repo) = temp_repo_unseeded();
5398 let parent = put_state_with_blob(&src_repo, "base", vec![]);
5399 let child = put_state_with_blob(&src_repo, "feature", vec![parent]);
5400 let child_closure =
5401 wire::enumerate_state_closure(src_repo.store(), child).expect("child closure");
5402 let full_pack =
5403 wire::build_native_pack(src_repo.store(), &child_closure).expect("full pack");
5404
5405 let (_dir, repo) = temp_repo_unseeded();
5410 let child_only = wire::enumerate_state_closure_with_options(
5411 src_repo.store(),
5412 child,
5413 wire::StateClosureOptions {
5414 depth: None,
5415 exclude_states: vec![parent],
5416 },
5417 )
5418 .expect("child-only objects");
5419 let child_only_pack =
5420 wire::build_native_pack(src_repo.store(), &child_only).expect("child-only pack");
5421 wire::install_received_pack(
5422 repo.store(),
5423 &child_only_pack.pack_data,
5424 &child_only_pack.index_data,
5425 )
5426 .expect("install child-only objects");
5427 repo.refs()
5428 .set_thread(&ThreadName::from("feature"), &child)
5429 .expect("set local thread to child");
5430 assert!(
5433 matches!(
5434 wire::enumerate_state_closure(repo.store(), child),
5435 Err(ProtocolError::ObjectNotFound(_))
5436 ),
5437 "the explicit thread head's closure must start incomplete"
5438 );
5439
5440 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5441 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5442 remote_state: child,
5443 full_closure: child_closure.clone(),
5444 delta_objects: Vec::new(),
5445 known_parent: parent,
5446 full_pack: full_pack.clone(),
5447 delta_pack: full_pack,
5448 captured_exclude: captured.clone(),
5449 })
5450 .await
5451 else {
5452 return;
5453 };
5454
5455 let exchange = tokio::time::timeout(
5456 Duration::from_secs(5),
5457 client.pull_exchange(
5458 &repo,
5459 "owner/repo",
5460 "main",
5461 PullOptions {
5462 local_thread: Some("feature"),
5463 depth: None,
5464 target_state: None,
5465 materialization: PullMaterialization::Full,
5466 },
5467 ),
5468 )
5469 .await
5470 .expect("explicit-thread pull must not hang")
5471 .expect("explicit-thread pull succeeds");
5472 server.abort();
5473
5474 let advertised = captured.lock().unwrap().clone().expect("request captured");
5475 assert!(
5476 advertised.is_empty(),
5477 "an explicit --local-thread with an incomplete closure must advertise nothing, \
5478 falling back to a full pull (got {advertised:?})"
5479 );
5480 assert!(exchange.result.success);
5481 let reassembled = wire::enumerate_state_closure(repo.store(), child)
5483 .expect("the client must hold the complete child closure after the fallback full pull");
5484 assert_eq!(
5485 reassembled.len(),
5486 child_closure.len(),
5487 "the full-pull fallback must leave the client with the complete closure, no gaps"
5488 );
5489 }
5490
5491 #[tokio::test]
5492 async fn explicit_local_thread_complete_closure_advertises_and_fires_short_circuit() {
5493 let (_dir, repo) = temp_repo();
5498 let head = put_state_with_blob(&repo, "tip", vec![]);
5499 repo.refs()
5500 .set_thread(&ThreadName::from("feature"), &head)
5501 .expect("set local thread");
5502 wire::enumerate_state_closure(repo.store(), head)
5504 .expect("client holds the complete thread closure");
5505
5506 let full_closure =
5507 wire::enumerate_state_closure(repo.store(), head).expect("enumerate closure");
5508 let full_pack =
5509 wire::build_native_pack(repo.store(), &full_closure).expect("build full pack");
5510 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5511
5512 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5513 remote_state: head,
5514 full_closure,
5515 delta_objects: Vec::new(),
5516 known_parent: ChangeId::generate(),
5517 full_pack: full_pack.clone(),
5518 delta_pack: full_pack,
5519 captured_exclude: captured.clone(),
5520 })
5521 .await
5522 else {
5523 return;
5524 };
5525
5526 let exchange = tokio::time::timeout(
5527 Duration::from_secs(5),
5528 client.pull_exchange(
5529 &repo,
5530 "owner/repo",
5531 "main",
5532 PullOptions {
5533 local_thread: Some("feature"),
5534 depth: None,
5535 target_state: None,
5536 materialization: PullMaterialization::Full,
5537 },
5538 ),
5539 )
5540 .await
5541 .expect("explicit-thread pull must not hang")
5542 .expect("explicit-thread pull succeeds");
5543 server.abort();
5544
5545 let advertised = captured.lock().unwrap().clone().expect("request captured");
5546 assert_eq!(
5547 advertised,
5548 vec![head.to_string_full()],
5549 "an explicit --local-thread with a complete closure must advertise its head"
5550 );
5551 assert!(exchange.result.success);
5552 assert_eq!(
5553 exchange.object_count, 0,
5554 "advertising the complete head must fire the zero-delta short-circuit, not a full pull"
5555 );
5556 }
5557}