1use std::{
2 collections::{BTreeMap, HashMap},
3 io::{self, Seek, SeekFrom, Write},
4 sync::{
5 Arc, Mutex,
6 atomic::{AtomicUsize, Ordering},
7 },
8 time::{Duration, Instant},
9};
10
11use grpc::heddle::api::v1alpha1::{
12 GetBlobRequest, GitCheckpointTransfer, GitLaneTransfer, GitObjectAlgorithm,
13 GitObjectId as ProtoGitObjectId, GitPackTransfer, GitRefKind as GrpcGitRefKind,
14 GitRefUpdateTransfer, ListRefsRequest, ObjectAvailabilityStatus, ObjectDescriptor, PackChunk,
15 PackStreamKind, PartialFetchStatus, PullClientFrame, PullRequest, PullServerFrame,
16 PushClientFrame, PushRequest, PushServerFrame, RedactionTransfer, StateVisibilityTransfer,
17 StreamOpeningProof, ThreadConfidenceSummary, ThreadIntegrationPolicy, ThreadMetadata,
18 ThreadVerificationSummary, TransportMode, UpdateRefRequest, WantObjects, git_lane_transfer,
19 pull_client_frame, pull_server_frame, push_client_frame, push_server_frame,
20};
21use objects::{
22 Progress,
23 object::{ContentHash, MarkerName, StateId, ThreadName},
24 store::{AnyStore, ObjectStore, PackObjectId},
25};
26use repo::{
27 GitRefKind as ClassifiedGitRefKind, GitRefName, Repository, RepositoryCapability,
28 RevisionAddress, SyncedThreadMetadata, ThreadManager,
29};
30use sley::{
31 ObjectId as GitObjectId, RefPrecondition, ReferenceTarget, Repository as SleyRepository,
32};
33use tempfile::NamedTempFile;
34use tokio::sync::mpsc;
35use tokio_stream::wrappers::ReceiverStream;
36use tonic::Request;
37use wire::{
38 GitLaneTransferIntent, ObjectInfo, ObjectType, ObjectTypeBucket, PlannedObject, ProtocolError,
39 PullComplete, PushComplete, RefEntry, RefUpdated, RepositoryTransferPlan,
40};
41
42use super::{
43 HostedGrpcClient, PullMaterialization,
44 helpers::{
45 descriptor_id, descriptor_id_from_info, object_descriptor_with_status, object_type_name,
46 parse_descriptor_to_info, status_to_protocol_error, to_proto_object_info,
47 transport_mode_name,
48 },
49 operation_id::ClientOperationId,
50};
51
52#[derive(Clone, Copy)]
53struct PullOptions<'a> {
54 local_thread: Option<&'a str>,
55 depth: Option<u32>,
56 target_state: Option<StateId>,
57 materialization: PullMaterialization,
58}
59
60struct PullWantPlan {
61 wants: Vec<ObjectDescriptor>,
62 transfer_plan: RepositoryTransferPlan<ObjectInfo>,
63 wanted_types: WantedTypes,
64 want_full_closure: bool,
65}
66
67type WantedTypes = HashMap<PackObjectId, Vec<ObjectType>>;
68
69struct GitLanePushPlan {
70 local_revision_address: String,
71 pack: Option<GitPackPushPlan>,
78 ref_updates: Vec<GitRefUpdateTransfer>,
84}
85
86#[derive(Clone)]
87struct GitPackPushPlan {
88 transfer_id: String,
89 pack_id: Vec<u8>,
90 pack_size: u64,
91 #[cfg_attr(not(test), allow(dead_code))]
96 roots: Vec<GitObjectId>,
97 pack_file: Arc<Mutex<NamedTempFile>>,
100}
101
102const PUSH_FULL_DESCRIPTOR_OBJECT_THRESHOLD: usize = 512;
103const PULL_PACK_SPOOL_OBJECT_THRESHOLD: usize = 512;
104const NATIVE_PACK_DRAIN_OBJECT_INTERVAL: usize = 32;
105const NATIVE_PACK_OBJECT_PREFETCH_LIMIT: usize = 32;
106const NATIVE_PACK_OBJECT_LOAD_WORKER_LIMIT: usize = 8;
107
108#[derive(Debug, Clone, Default)]
109pub struct PullObjectMix {
110 pub blobs: usize,
111 pub trees: usize,
112 pub states: usize,
113 pub actions: usize,
114 pub redactions: usize,
115 pub state_visibilities: usize,
116 pub state_attachments: usize,
117}
118
119#[derive(Debug, Clone)]
120pub struct HostedRefEntry {
121 pub name: String,
122 pub state_id: StateId,
123 pub is_thread: bool,
124 pub revision_address: String,
125}
126
127impl PullObjectMix {
128 fn record(&mut self, obj_type: ObjectType) {
129 match obj_type.bucket() {
130 ObjectTypeBucket::Blob => self.blobs += 1,
131 ObjectTypeBucket::Tree => self.trees += 1,
132 ObjectTypeBucket::State => self.states += 1,
133 ObjectTypeBucket::Action => self.actions += 1,
134 ObjectTypeBucket::Redaction => self.redactions += 1,
135 ObjectTypeBucket::StateVisibility => self.state_visibilities += 1,
136 ObjectTypeBucket::StateAttachment => self.state_attachments += 1,
137 }
138 }
139
140 pub fn total(&self) -> usize {
141 self.blobs
142 + self.trees
143 + self.states
144 + self.actions
145 + self.redactions
146 + self.state_visibilities
147 + self.state_attachments
148 }
149}
150
151#[derive(Debug, Clone, Default)]
152pub struct PullProfile {
153 pub ready_wait: Duration,
154 pub receive_and_apply: Duration,
155 pub decode: Duration,
156 pub store_receive_object: Duration,
157 pub metadata_sync: Duration,
158 pub pack_decode_apply: Duration,
159 pub raw_decode_apply: Duration,
160 pub pack_decode: Duration,
161 pub raw_decode: Duration,
162 pub bytes_received: usize,
163 pub pack_bytes_received: usize,
164 pub raw_bytes_received: usize,
165 pub objects_received: usize,
166 pub object_mix: PullObjectMix,
167}
168
169impl HostedGrpcClient {
170 fn stream_opening_proof(
171 &self,
172 stream_id: &str,
173 route: &str,
174 repository: &str,
175 resume_cursor: &str,
176 ) -> Result<StreamOpeningProof, ProtocolError> {
177 use crypto::Signer as _;
178
179 let signer = self.device_signer()?.ok_or_else(|| {
180 ProtocolError::AuthenticationFailed(
181 "hosted sync requires a stable device signing identity".to_string(),
182 )
183 })?;
184 let identity = self.stable_signing_identity()?;
185 let capability_context = Vec::new();
186 let canonical = grpc::signing::stream_open_bytes(
187 identity,
188 stream_id,
189 route,
190 repository,
191 resume_cursor,
192 &capability_context,
193 );
194 let signature = signer
195 .sign(&canonical)
196 .map_err(|err| ProtocolError::AuthenticationFailed(err.to_string()))?;
197 Ok(StreamOpeningProof {
198 stream_id: stream_id.to_string(),
199 route: route.to_string(),
200 repository: super::helpers::repository_ref(repository),
201 resume_cursor: resume_cursor.to_string(),
202 capability_context,
203 nonce: Vec::new(),
204 signature,
205 })
206 }
207
208 pub async fn list_refs(&mut self, repo_path: &str) -> Result<Vec<RefEntry>, ProtocolError> {
209 Ok(self
210 .list_refs_with_revision_addresses(repo_path)
211 .await?
212 .into_iter()
213 .map(|entry| RefEntry {
214 name: entry.name,
215 state_id: entry.state_id,
216 is_thread: entry.is_thread,
217 })
218 .collect())
219 }
220
221 pub async fn list_refs_with_revision_addresses(
222 &mut self,
223 repo_path: &str,
224 ) -> Result<Vec<HostedRefEntry>, ProtocolError> {
225 let mut request = Request::new(ListRefsRequest {
226 repo_path: super::helpers::repository_ref(repo_path),
227 });
228 self.apply_signed_auth(
229 &mut request,
230 "/heddle.api.v1alpha1.RepoSyncService/ListRefs",
231 )?;
232 let response = self
233 .inner
234 .list_refs(request)
235 .await
236 .map_err(status_to_protocol_error)?
237 .into_inner();
238 response
239 .refs
240 .into_iter()
241 .map(|entry| {
242 Ok(HostedRefEntry {
243 name: entry.name,
244 state_id: super::helpers::parse_proto_state_id(entry.state_id)?.ok_or_else(
245 || ProtocolError::InvalidState("ref is missing its state ID".to_string()),
246 )?,
247 is_thread: entry.is_thread,
248 revision_address: entry.revision_address,
249 })
250 })
251 .collect()
252 }
253
254 #[allow(clippy::too_many_arguments)]
255 pub async fn update_ref(
256 &mut self,
257 repo_path: &str,
258 name: &str,
259 is_thread: bool,
260 old_value: Option<StateId>,
261 new_value: StateId,
262 force: bool,
263 thread_metadata: Option<&SyncedThreadMetadata>,
264 client_operation_id: String,
265 ) -> Result<RefUpdated, ProtocolError> {
266 let operation_id = ClientOperationId::for_required_method(
267 "heddle.api.v1alpha1.RepoSyncService/UpdateRef",
268 client_operation_id,
269 )?;
270 let mut request = Request::new(UpdateRefRequest {
271 repo_path: super::helpers::repository_ref(repo_path),
272 name: name.to_string(),
273 is_thread,
274 force,
275 old_value: old_value
276 .map(|value| value.to_string_full())
277 .unwrap_or_default(),
278 new_value: new_value.to_string_full(),
279 thread_metadata: thread_metadata.map(to_proto_thread_metadata),
280 old_revision_address: old_value
281 .map(|value| RevisionAddress::heddle(value).to_string())
282 .unwrap_or_default(),
283 new_revision_address: RevisionAddress::heddle(new_value).to_string(),
284 client_operation_id: operation_id.to_wire(),
285 });
286 self.apply_signed_auth(
287 &mut request,
288 "/heddle.api.v1alpha1.RepoSyncService/UpdateRef",
289 )?;
290 let response = self
291 .inner
292 .update_ref(request)
293 .await
294 .map_err(status_to_protocol_error)?
295 .into_inner();
296 Ok(RefUpdated {
297 success: response.success,
298 old_value: if response.old_value.is_empty() {
299 None
300 } else {
301 Some(
302 StateId::parse(&response.old_value)
303 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?,
304 )
305 },
306 error: (!response.error.is_empty()).then_some(response.error),
307 })
308 }
309
310 pub async fn push(
311 &mut self,
312 repo: &Repository,
313 repo_path: &str,
314 local_state: StateId,
315 target_thread: &str,
316 force: bool,
317 client_operation_id: String,
318 ) -> Result<PushComplete, ProtocolError> {
319 let operation_id = ClientOperationId::caller_or_fresh(
320 "heddle.api.v1alpha1.RepoSyncService/Push",
321 client_operation_id,
322 );
323 self.push_with_revision(
324 repo,
325 repo_path,
326 local_state,
327 target_thread,
328 force,
329 operation_id,
330 RevisionAddress::heddle(local_state).to_string(),
331 None,
332 &Progress::null(),
333 )
334 .await
335 }
336
337 #[allow(clippy::too_many_arguments)]
352 pub async fn push_git_overlay_mirror(
353 &mut self,
354 repo: &Repository,
355 repo_path: &str,
356 local_state: StateId,
357 target_thread: &str,
358 force: bool,
359 progress: &Progress,
360 client_operation_id: String,
361 ) -> Result<PushComplete, ProtocolError> {
362 let operation_id = ClientOperationId::caller_or_fresh(
363 "heddle.api.v1alpha1.RepoSyncService/Push",
364 client_operation_id,
365 );
366 progress.set_phase("packing refs");
367 let remote_ref_expectations = self.git_mirror_ref_expectations(repo_path).await?;
368 let git_lane = build_git_mirror_push_plan(
369 repo,
370 self.transport.chunk_size.max(1),
371 &remote_ref_expectations,
372 )?;
373 let local_revision_address = git_lane.local_revision_address.clone();
374 self.push_with_revision(
375 repo,
376 repo_path,
377 local_state,
378 target_thread,
379 force,
380 operation_id,
381 local_revision_address,
382 Some(git_lane),
383 progress,
384 )
385 .await
386 }
387
388 async fn git_mirror_ref_expectations(
392 &mut self,
393 repo_path: &str,
394 ) -> Result<HashMap<String, GitRefRemoteExpectation>, ProtocolError> {
395 let remote_refs = self.list_refs_with_revision_addresses(repo_path).await?;
396 let mut expectations = HashMap::with_capacity(remote_refs.len());
397 for entry in remote_refs {
398 let expectation = parse_git_ref_expectation(&entry.revision_address)?;
399 expectations.insert(entry.name, expectation);
400 }
401 Ok(expectations)
402 }
403
404 #[allow(clippy::too_many_arguments)]
405 async fn push_with_revision(
406 &mut self,
407 repo: &Repository,
408 repo_path: &str,
409 local_state: StateId,
410 target_thread: &str,
411 force: bool,
412 operation_id: ClientOperationId,
413 local_revision_address: String,
414 git_lane: Option<GitLanePushPlan>,
415 progress: &Progress,
416 ) -> Result<PushComplete, ProtocolError> {
417 let _ = self.transport.chunk_size;
418 let _ = self.transport.resume_attempts;
419 let git_lane_intent = if git_lane.is_some() {
424 GitLaneTransferIntent::ExistingImplementation
425 } else {
426 GitLaneTransferIntent::HeddleObjectsOnly
427 };
428 let closure = wire::enumerate_state_closure_transfer_with_options(
429 repo.store(),
430 local_state,
431 wire::StateClosureOptions::default(),
432 PUSH_FULL_DESCRIPTOR_OBJECT_THRESHOLD,
433 )?;
434 let object_plan =
435 RepositoryTransferPlan::from_planned_objects(closure.planned_objects, git_lane_intent);
436 let full_objects = closure.full_objects;
437 let object_count = full_objects
438 .as_ref()
439 .map_or(object_plan.stats.total_objects, std::vec::Vec::len);
440 let transfer_id = push_transfer_id(repo_path, local_state, target_thread);
441 let transport_mode = preferred_transport_mode(&self.transport, object_count);
442 let thread_metadata = load_thread_metadata(repo, target_thread, local_state)?;
443 let request_message = PushClientFrame {
444 frame: Some(push_client_frame::Frame::Request(Box::new(PushRequest {
445 repo_path: super::helpers::repository_ref(repo_path),
446 local_state: super::helpers::proto_state_id(local_state),
447 target_thread: target_thread.to_string(),
448 create_thread: true,
449 force,
450 objects: full_objects.as_ref().map_or_else(
451 || {
452 object_plan
453 .partitions
454 .iter()
455 .map(to_proto_planned_object)
456 .collect()
457 },
458 |objects| objects.iter().map(to_proto_object_info).collect(),
459 ),
460 transfer: Some(self.transport.transfer_checkpoint_with_mode(
461 transfer_id.clone(),
462 transport_mode,
463 0,
464 0,
465 false,
466 )),
467 partial_fetch_status: partial_fetch_status_for_repo(repo),
468 allow_partial_fetch: true,
469 thread_metadata: thread_metadata
470 .map(|metadata| to_proto_thread_metadata(&metadata)),
471 local_revision_address,
472 }))),
473 client_operation_id: operation_id.to_wire(),
474 };
475
476 let (tx, rx) = mpsc::channel(self.transport.max_inflight_objects.max(4));
477 tx.send(PushClientFrame {
478 frame: Some(push_client_frame::Frame::Open(self.stream_opening_proof(
479 &transfer_id,
480 "/heddle.api.v1alpha1.RepoSyncService/Push",
481 repo_path,
482 "",
483 )?)),
484 client_operation_id: operation_id.to_wire(),
485 })
486 .await
487 .map_err(|_| ProtocolError::InvalidState("failed to open push stream".to_string()))?;
488 tx.send(request_message).await.map_err(|_| {
489 ProtocolError::InvalidState("failed to initialize push stream".to_string())
490 })?;
491 let mut request = Request::new(ReceiverStream::new(rx));
492 self.apply_auth(&mut request, "/heddle.api.v1alpha1.RepoSyncService/Push")?;
493 let mut response = self
494 .inner
495 .push(request)
496 .await
497 .map_err(status_to_protocol_error)?
498 .into_inner();
499
500 let ready = match response.message().await.map_err(status_to_protocol_error)? {
501 Some(PushServerFrame {
502 frame: Some(push_server_frame::Frame::Ready(ready)),
503 }) => ready,
504 _ => {
505 return Err(ProtocolError::InvalidState(
506 "expected PushReady from gRPC server".to_string(),
507 ));
508 }
509 };
510 let object_index = match full_objects {
511 Some(objects) => objects
512 .into_iter()
513 .map(|info| (descriptor_id_from_info(&info), info))
514 .collect::<HashMap<_, _>>(),
515 None => object_plan
516 .partitions
517 .iter()
518 .map(|object| {
519 (
520 descriptor_id_from_plan(object),
521 object_info_from_plan(object),
522 )
523 })
524 .collect::<HashMap<_, _>>(),
525 };
526
527 let ready_transport_mode = ready
528 .transfer
529 .as_ref()
530 .and_then(|transfer| TransportMode::try_from(transfer.transport_mode).ok())
531 .unwrap_or(transport_mode);
532 let wanted_infos = ready
533 .want_objects
534 .into_iter()
535 .map(|want| {
536 object_index
537 .get(&descriptor_id(&want))
538 .cloned()
539 .ok_or_else(|| {
540 ProtocolError::InvalidState("server requested unknown object".to_string())
541 })
542 })
543 .collect::<Result<Vec<_>, _>>()?;
544
545 let wanted_plan = RepositoryTransferPlan::from_object_infos(wanted_infos, git_lane_intent);
546
547 if !wanted_plan.partitions.packable_objects.is_empty() {
548 send_native_pack_streaming_messages(
549 &tx,
550 repo,
551 &wanted_plan.partitions.packable_objects,
552 PushWireIdentities {
553 transfer_id: &transfer_id,
554 client_operation_id: operation_id.as_str(),
555 },
556 self.transport.chunk_size.max(1),
557 &self.transport,
558 ready_transport_mode,
559 )
560 .await?;
561 }
562
563 for info in wanted_plan.partitions.sidecar_objects {
564 let message = sidecar_push_message(repo, info, operation_id.as_str())?;
565 tx.send(message).await.map_err(|_| {
566 ProtocolError::InvalidState("push stream closed unexpectedly".to_string())
567 })?;
568 }
569
570 if let Some(git_lane) = git_lane {
571 if let Some(pack) = git_lane.pack.as_ref() {
577 send_git_pack_streaming_messages(
578 &tx,
579 pack,
580 self.transport.chunk_size.max(1),
581 progress,
582 operation_id.as_str(),
583 )
584 .await?;
585 } else {
586 progress.set_phase("no new objects to pack");
587 }
588 progress.set_phase(format!("writing {} refs", git_lane.ref_updates.len()));
589 for ref_update in git_lane.ref_updates {
590 tx.send(git_lane_push_message(
591 git_lane_transfer::Body::RefUpdate(ref_update),
592 operation_id.as_str(),
593 ))
594 .await
595 .map_err(|_| {
596 ProtocolError::InvalidState("push stream closed unexpectedly".to_string())
597 })?;
598 }
599 }
600 drop(tx);
601
602 let result = match response.message().await.map_err(status_to_protocol_error)? {
603 Some(PushServerFrame {
604 frame: Some(push_server_frame::Frame::Complete(complete)),
605 }) => PushComplete {
606 success: complete.success,
607 new_state: super::helpers::parse_proto_state_id(complete.new_state)?,
608 error: (!complete.error.is_empty()).then_some(complete.error),
609 transfer_id: complete
610 .transfer
611 .as_ref()
612 .map(|transfer| transfer.transfer_id.clone())
613 .unwrap_or_default(),
614 transport_mode: complete
615 .transfer
616 .as_ref()
617 .map(|transfer| transport_mode_name(transfer.transport_mode))
618 .unwrap_or("raw-objects")
619 .to_string(),
620 resume_offset: complete
621 .transfer
622 .as_ref()
623 .map(|transfer| transfer.resume_offset)
624 .unwrap_or_default(),
625 chunk_index: complete
626 .transfer
627 .as_ref()
628 .map(|transfer| transfer.chunk_index)
629 .unwrap_or_default(),
630 checkpoint: complete
631 .transfer
632 .as_ref()
633 .map(|transfer| transfer.checkpoint.clone())
634 .unwrap_or_default(),
635 is_complete: complete
636 .transfer
637 .as_ref()
638 .map(|transfer| transfer.is_complete)
639 .unwrap_or(false),
640 },
641 _ => {
642 return Err(ProtocolError::InvalidState(
643 "expected PushComplete from gRPC server".to_string(),
644 ));
645 }
646 };
647
648 if result.success {
649 self.sync_remote_markers(repo, repo_path, local_state)
650 .await?;
651 }
652 Ok(result)
653 }
654
655 pub async fn pull(
656 &mut self,
657 repo: &Repository,
658 repo_path: &str,
659 remote_thread: &str,
660 local_thread: Option<&str>,
661 ) -> Result<PullComplete, ProtocolError> {
662 self.pull_with_options(
663 repo,
664 repo_path,
665 remote_thread,
666 PullOptions {
667 local_thread,
668 depth: None,
669 target_state: None,
670 materialization: PullMaterialization::Full,
671 },
672 )
673 .await
674 }
675
676 pub async fn pull_profiled(
677 &mut self,
678 repo: &Repository,
679 repo_path: &str,
680 remote_thread: &str,
681 local_thread: Option<&str>,
682 ) -> Result<(PullComplete, PullProfile), ProtocolError> {
683 self.pull_exchange(
684 repo,
685 repo_path,
686 remote_thread,
687 PullOptions {
688 local_thread,
689 depth: None,
690 target_state: None,
691 materialization: PullMaterialization::Full,
692 },
693 )
694 .await
695 .map(|exchange| (exchange.result, exchange.profile))
696 }
697
698 pub async fn pull_partial(
699 &mut self,
700 repo: &Repository,
701 repo_path: &str,
702 remote_thread: &str,
703 local_thread: Option<&str>,
704 ) -> Result<PullComplete, ProtocolError> {
705 self.pull_with_options(
706 repo,
707 repo_path,
708 remote_thread,
709 PullOptions {
710 local_thread,
711 depth: None,
712 target_state: None,
713 materialization: PullMaterialization::Lazy,
714 },
715 )
716 .await
717 }
718
719 pub async fn pull_with_depth_and_materialization(
720 &mut self,
721 repo: &Repository,
722 repo_path: &str,
723 remote_thread: &str,
724 local_thread: Option<&str>,
725 depth: Option<u32>,
726 materialization: PullMaterialization,
727 ) -> Result<PullComplete, ProtocolError> {
728 self.pull_with_options(
729 repo,
730 repo_path,
731 remote_thread,
732 PullOptions {
733 local_thread,
734 depth,
735 target_state: None,
736 materialization,
737 },
738 )
739 .await
740 }
741
742 pub async fn pull_with_depth(
743 &mut self,
744 repo: &Repository,
745 repo_path: &str,
746 remote_thread: &str,
747 local_thread: Option<&str>,
748 depth: Option<u32>,
749 ) -> Result<PullComplete, ProtocolError> {
750 self.pull_with_depth_and_materialization(
751 repo,
752 repo_path,
753 remote_thread,
754 local_thread,
755 depth,
756 PullMaterialization::Full,
757 )
758 .await
759 }
760
761 pub async fn fetch_state(
762 &mut self,
763 repo: &Repository,
764 repo_path: &str,
765 remote_thread: &str,
766 target_state: StateId,
767 ) -> Result<usize, ProtocolError> {
768 self.pull_exchange(
769 repo,
770 repo_path,
771 remote_thread,
772 PullOptions {
773 local_thread: None,
774 depth: None,
775 target_state: Some(target_state),
776 materialization: PullMaterialization::Full,
777 },
778 )
779 .await
780 .map(|exchange| exchange.object_count)
781 }
782
783 pub async fn fetch_state_partial(
784 &mut self,
785 repo: &Repository,
786 repo_path: &str,
787 remote_thread: &str,
788 target_state: StateId,
789 ) -> Result<usize, ProtocolError> {
790 self.pull_exchange(
791 repo,
792 repo_path,
793 remote_thread,
794 PullOptions {
795 local_thread: None,
796 depth: None,
797 target_state: Some(target_state),
798 materialization: PullMaterialization::Lazy,
799 },
800 )
801 .await
802 .map(|exchange| exchange.object_count)
803 }
804
805 pub async fn hydrate_blob_at_path(
806 &mut self,
807 repo: &Repository,
808 repo_path: &str,
809 reference: &str,
810 path: &str,
811 ) -> Result<objects::object::Blob, ProtocolError> {
812 let mut request = Request::new(GetBlobRequest {
813 repo_path: super::helpers::repository_ref(repo_path),
814 r#ref: reference.to_string(),
815 path: path.to_string(),
816 });
817 self.apply_signed_auth(
818 &mut request,
819 "/heddle.api.v1alpha1.RepositoryService/GetBlob",
820 )?;
821 let response = self
822 .content
823 .get_blob(request)
824 .await
825 .map_err(status_to_protocol_error)?
826 .into_inner();
827
828 let content = super::helpers::decode_blob_content(response.content, response.is_binary)?;
829 let blob = objects::object::Blob::new(content);
830 repo.store().put_blob(&blob)?;
831 repo.clear_missing_blob(&blob.hash())?;
832 Ok(blob)
833 }
834
835 pub async fn hydrate_missing_blobs_for_state(
836 &mut self,
837 repo: &Repository,
838 repo_path: &str,
839 remote_thread: &str,
840 target_state: StateId,
841 ) -> Result<usize, ProtocolError> {
842 let exchange = self
843 .pull_exchange(
844 repo,
845 repo_path,
846 remote_thread,
847 PullOptions {
848 local_thread: None,
849 depth: None,
850 target_state: Some(target_state),
851 materialization: PullMaterialization::Full,
852 },
853 )
854 .await?;
855 clear_missing_blobs_for_state(repo, target_state)?;
856 Ok(exchange.object_count)
857 }
858
859 async fn pull_with_options(
860 &mut self,
861 repo: &Repository,
862 repo_path: &str,
863 remote_thread: &str,
864 options: PullOptions<'_>,
865 ) -> Result<PullComplete, ProtocolError> {
866 self.pull_exchange(repo, repo_path, remote_thread, options)
867 .await
868 .map(|exchange| exchange.result)
869 }
870
871 async fn pull_exchange(
872 &mut self,
873 repo: &Repository,
874 repo_path: &str,
875 remote_thread: &str,
876 options: PullOptions<'_>,
877 ) -> Result<PullExchange, ProtocolError> {
878 let exchange_start = Instant::now();
879 let mut exclude_states = Vec::new();
880 let advertised_head = if let Some(local_thread) = options.local_thread {
891 locally_complete_local_thread_head(repo, local_thread, options.target_state)?
892 } else {
893 locally_complete_pull_head(repo, remote_thread, options.target_state)?
894 };
895 if let Some(head) = advertised_head {
896 exclude_states.push(head);
897 }
898 let allow_partial_fetch = options.materialization.allows_partial_fetch();
899 let fresh_full_pull =
900 supports_compact_full_pull(repo, allow_partial_fetch, &exclude_states)?;
901
902 let transfer_id = pull_transfer_id(
903 repo_path,
904 remote_thread,
905 options.local_thread,
906 options.depth,
907 options.target_state,
908 );
909 let request_message = PullClientFrame {
910 frame: Some(pull_client_frame::Frame::Request(PullRequest {
911 repo_path: super::helpers::repository_ref(repo_path),
912 remote_thread: remote_thread.to_string(),
913 local_thread: options.local_thread.unwrap_or_default().to_string(),
914 target_state: options
915 .target_state
916 .and_then(super::helpers::proto_state_id),
917 depth: options.depth.unwrap_or_default(),
918 exclude_states: exclude_states
919 .iter()
920 .copied()
921 .filter_map(super::helpers::proto_state_id)
922 .collect(),
923 transfer: Some(self.transport.transfer_checkpoint_with_mode(
924 transfer_id.clone(),
925 TransportMode::NativePack,
926 0,
927 0,
928 false,
929 )),
930 partial_fetch_status: partial_fetch_status_for_repo(repo),
931 allow_partial_fetch,
932 fresh_full_pull,
933 target_revision_address: options
934 .target_state
935 .map(|state| RevisionAddress::heddle(state).to_string())
936 .unwrap_or_default(),
937 })),
938 };
939
940 let (tx, rx) = mpsc::channel(self.transport.max_inflight_objects.max(4));
941 tx.send(PullClientFrame {
942 frame: Some(pull_client_frame::Frame::Open(self.stream_opening_proof(
943 &transfer_id,
944 "/heddle.api.v1alpha1.RepoSyncService/Pull",
945 repo_path,
946 "",
947 )?)),
948 })
949 .await
950 .map_err(|_| ProtocolError::InvalidState("failed to open pull stream".to_string()))?;
951 tx.send(request_message).await.map_err(|_| {
952 ProtocolError::InvalidState("failed to initialize pull stream".to_string())
953 })?;
954 let mut request = Request::new(ReceiverStream::new(rx));
955 self.apply_auth(&mut request, "/heddle.api.v1alpha1.RepoSyncService/Pull")?;
956 let mut response = self
957 .inner
958 .pull(request)
959 .await
960 .map_err(status_to_protocol_error)?
961 .into_inner();
962
963 let ready = match response.message().await.map_err(status_to_protocol_error)? {
964 Some(PullServerFrame {
965 frame: Some(pull_server_frame::Frame::Ready(ready)),
966 }) => ready,
967 _ => {
968 return Err(ProtocolError::InvalidState(
969 "expected PullReady from gRPC server".to_string(),
970 ));
971 }
972 };
973 let mut profile = PullProfile {
974 ready_wait: exchange_start.elapsed(),
975 ..PullProfile::default()
976 };
977 let remote_state = super::helpers::parse_proto_state_id(ready.remote_state)?
978 .ok_or_else(|| ProtocolError::InvalidState("missing remote state".to_string()))?;
979 let advertised_object_count = ready.objects_to_fetch.len();
980 let PullWantPlan {
981 wants,
982 transfer_plan,
983 wanted_types,
984 want_full_closure,
985 } = plan_pull_wants(
986 repo,
987 &remote_state,
988 ready.full_closure_available,
989 ready.objects_to_fetch,
990 allow_partial_fetch,
991 )?;
992 let native_pack_required = native_pack_required_for_pull(want_full_closure, &transfer_plan);
993
994 tx.send(PullClientFrame {
995 frame: Some(pull_client_frame::Frame::Want(WantObjects {
996 objects: wants.clone(),
997 want_full_closure,
998 transfer: Some(self.transport.transfer_checkpoint_with_mode(
999 transfer_id.clone(),
1000 TransportMode::NativePack,
1001 0,
1002 0,
1003 false,
1004 )),
1005 })),
1006 })
1007 .await
1008 .map_err(|_| ProtocolError::InvalidState("pull stream closed unexpectedly".to_string()))?;
1009 drop(tx);
1010
1011 let receive_start = Instant::now();
1012 let use_pack_spool = advertised_object_count > PULL_PACK_SPOOL_OBJECT_THRESHOLD;
1013 let mut pack_state = wire::PackChunkState::default();
1014 let mut pack_spool = if use_pack_spool {
1015 Some(wire::PackChunkSpool::new_in(repo.heddle_dir())?)
1016 } else {
1017 None
1018 };
1019 let mut git_lane_repo = None;
1020 let mut git_pack_state = GitPackPullInstallState::default();
1021 let mut received = 0usize;
1022 while let Some(message) = response.message().await.map_err(status_to_protocol_error)? {
1023 match message.frame {
1024 Some(pull_server_frame::Frame::Pack(chunk)) => {
1025 profile.bytes_received =
1026 profile.bytes_received.saturating_add(chunk.data.len());
1027 profile.pack_bytes_received =
1028 profile.pack_bytes_received.saturating_add(chunk.data.len());
1029 let transfer = chunk.transfer.as_ref().ok_or_else(|| {
1030 ProtocolError::InvalidState(
1031 "native pack chunk missing transfer checkpoint".to_string(),
1032 )
1033 })?;
1034 let stream_kind = PackStreamKind::try_from(chunk.stream_kind)
1035 .unwrap_or(PackStreamKind::Unspecified);
1036 if stream_kind == PackStreamKind::Unspecified {
1037 return Err(ProtocolError::InvalidState(
1038 "native pack chunk missing stream kind".to_string(),
1039 ));
1040 }
1041 let decode_start = Instant::now();
1042 if let Some(pack_spool) = pack_spool.as_mut() {
1043 pack_spool.receive_chunk(
1044 stream_kind == PackStreamKind::Index,
1045 transfer.resume_offset,
1046 transfer.chunk_index,
1047 transfer.is_complete,
1048 &chunk.data,
1049 chunk.is_final_chunk,
1050 )?;
1051 } else {
1052 wire::receive_pack_chunk(
1053 &mut pack_state,
1054 stream_kind == PackStreamKind::Index,
1055 transfer.resume_offset,
1056 transfer.chunk_index,
1057 transfer.is_complete,
1058 &chunk.data,
1059 chunk.is_final_chunk,
1060 )?;
1061 }
1062 let decode_elapsed = decode_start.elapsed();
1063 profile.pack_decode += decode_elapsed;
1064 profile.pack_decode_apply += decode_elapsed;
1065 profile.decode += decode_elapsed;
1066 }
1067 Some(pull_server_frame::Frame::Redaction(transfer)) => {
1068 wire::check_received_transfer_blob_size(
1074 transfer.redactions_blob.len(),
1075 wire::MAX_RECEIVED_REDACTIONS_BLOB_SIZE,
1076 "redactions",
1077 )?;
1078 profile.bytes_received = profile
1079 .bytes_received
1080 .saturating_add(transfer.redactions_blob.len());
1081 profile.object_mix.record(ObjectType::Redaction);
1082 let blob = ContentHash::from_hex(&transfer.blob_hash).map_err(|err| {
1083 ProtocolError::InvalidState(format!(
1084 "RedactionTransfer.blob_hash is not a valid content hash: {err}"
1085 ))
1086 })?;
1087 let decode_start = Instant::now();
1088 repo.accept_wire_redactions(blob, &transfer.redactions_blob)
1089 .map_err(|err| {
1090 ProtocolError::InvalidState(format!(
1091 "accept_wire_redactions for blob {}: {err}",
1092 transfer.blob_hash
1093 ))
1094 })?;
1095 let decode_elapsed = decode_start.elapsed();
1096 profile.store_receive_object += decode_elapsed;
1097 }
1098 Some(pull_server_frame::Frame::StateVisibility(transfer)) => {
1099 wire::check_received_transfer_blob_size(
1100 transfer.state_visibility_blob.len(),
1101 wire::MAX_RECEIVED_STATE_VISIBILITY_BLOB_SIZE,
1102 "state-visibility",
1103 )?;
1104 profile.bytes_received = profile
1105 .bytes_received
1106 .saturating_add(transfer.state_visibility_blob.len());
1107 profile.object_mix.record(ObjectType::StateVisibility);
1108 let state = transfer
1109 .state_id
1110 .as_ref()
1111 .ok_or_else(|| {
1112 ProtocolError::InvalidState(
1113 "StateVisibilityTransfer.state_id is required".to_string(),
1114 )
1115 })
1116 .and_then(|state| {
1117 StateId::try_from_slice(&state.value).map_err(|err| {
1118 ProtocolError::InvalidState(format!(
1119 "StateVisibilityTransfer.state_id is not a valid StateId: {err}"
1120 ))
1121 })
1122 })?;
1123 let decode_start = Instant::now();
1124 repo.accept_wire_state_visibility(state, &transfer.state_visibility_blob)
1125 .map_err(|err| {
1126 ProtocolError::InvalidState(format!(
1127 "accept_wire_state_visibility for state {}: {err}",
1128 state
1129 ))
1130 })?;
1131 let decode_elapsed = decode_start.elapsed();
1132 profile.store_receive_object += decode_elapsed;
1133 }
1134 Some(pull_server_frame::Frame::GitLane(transfer)) => {
1135 let decode_start = Instant::now();
1136 profile.bytes_received = profile
1137 .bytes_received
1138 .saturating_add(git_lane_transfer_size(&transfer));
1139 accept_git_lane_pull_transfer(
1140 repo,
1141 &mut git_lane_repo,
1142 &mut git_pack_state,
1143 transfer,
1144 )?;
1145 let decode_elapsed = decode_start.elapsed();
1146 profile.store_receive_object += decode_elapsed;
1147 }
1148 Some(pull_server_frame::Frame::Complete(complete)) => {
1149 profile.receive_and_apply = receive_start.elapsed();
1150 git_pack_state.ensure_idle()?;
1151 let final_state = super::helpers::parse_proto_state_id(complete.new_state)?;
1152
1153 if complete.success {
1154 if native_pack_required {
1155 let store_start = Instant::now();
1156 let installed_ids = if let Some(pack_spool) = pack_spool.as_mut() {
1157 if !pack_spool.is_complete() {
1158 return Err(ProtocolError::InvalidState(
1159 "pull completed before native pack stream finished"
1160 .to_string(),
1161 ));
1162 }
1163 pack_spool.install_into(repo.store())?
1164 } else {
1165 if !pack_state.is_complete() {
1166 return Err(ProtocolError::InvalidState(
1167 "pull completed before native pack stream finished"
1168 .to_string(),
1169 ));
1170 }
1171 wire::install_received_pack(
1172 repo.store(),
1173 &pack_state.pack_data,
1174 &pack_state.index_data,
1175 )?
1176 };
1177 profile.store_receive_object += store_start.elapsed();
1178 received = installed_ids.len();
1179 for id in installed_ids {
1180 match (id, wanted_packable_type(&wanted_types, &id)) {
1181 (PackObjectId::Hash(hash), Some(ObjectType::Blob)) => {
1182 profile.object_mix.record(ObjectType::Blob);
1183 repo.clear_missing_blob(&hash)?;
1184 }
1185 (_, Some(obj_type)) => {
1186 profile.object_mix.record(obj_type);
1187 }
1188 (PackObjectId::StateId(_), None) => {
1189 profile.object_mix.record(ObjectType::State);
1190 }
1191 (PackObjectId::Hash(hash), None) => {
1192 let inferred =
1193 infer_installed_hash_object_type(repo, &hash)?;
1194 profile.object_mix.record(inferred);
1195 }
1196 }
1197 }
1198 }
1199
1200 let metadata_start = Instant::now();
1201 if let Some(local_thread) = options.local_thread
1202 && let Some(state) = final_state
1203 {
1204 repo.refs()
1205 .set_thread(&ThreadName::from(local_thread), &state)?;
1206 }
1207 if let Some(state) = final_state
1208 && allow_partial_fetch
1209 {
1210 mark_missing_blobs_for_state(repo, state)?;
1211 } else if final_state.is_some() {
1212 let _ = repo.clear_all_missing_blobs()?;
1213 }
1214 let synced_markers = complete
1215 .transfer
1216 .as_ref()
1217 .map(|transfer| apply_marker_snapshot(repo, &transfer.checkpoint))
1218 .transpose()?
1219 .unwrap_or(false);
1220 if !synced_markers {
1221 self.sync_local_markers(repo, repo_path).await?;
1222 }
1223 profile.metadata_sync = metadata_start.elapsed();
1224 profile.objects_received = received;
1225 return Ok(PullExchange {
1226 result: PullComplete {
1227 success: true,
1228 final_state,
1229 error: None,
1230 transfer_id: complete
1231 .transfer
1232 .as_ref()
1233 .map(|transfer| transfer.transfer_id.clone())
1234 .unwrap_or_default(),
1235 transport_mode: complete
1236 .transfer
1237 .as_ref()
1238 .map(|transfer| transport_mode_name(transfer.transport_mode))
1239 .unwrap_or("native-pack")
1240 .to_string(),
1241 resume_offset: complete
1242 .transfer
1243 .as_ref()
1244 .map(|transfer| transfer.resume_offset)
1245 .unwrap_or_default(),
1246 chunk_index: complete
1247 .transfer
1248 .as_ref()
1249 .map(|transfer| transfer.chunk_index)
1250 .unwrap_or_default(),
1251 checkpoint: complete
1252 .transfer
1253 .as_ref()
1254 .map(|transfer| transfer.checkpoint.clone())
1255 .unwrap_or_default(),
1256 is_complete: complete
1257 .transfer
1258 .as_ref()
1259 .map(|transfer| transfer.is_complete)
1260 .unwrap_or(false),
1261 },
1262 object_count: received,
1263 profile,
1264 });
1265 }
1266
1267 profile.objects_received = received;
1268 return Ok(PullExchange {
1269 result: PullComplete {
1270 success: false,
1271 final_state,
1272 error: (!complete.error.is_empty()).then_some(complete.error),
1273 transfer_id: complete
1274 .transfer
1275 .as_ref()
1276 .map(|transfer| transfer.transfer_id.clone())
1277 .unwrap_or_default(),
1278 transport_mode: complete
1279 .transfer
1280 .as_ref()
1281 .map(|transfer| transport_mode_name(transfer.transport_mode))
1282 .unwrap_or("native-pack")
1283 .to_string(),
1284 resume_offset: complete
1285 .transfer
1286 .as_ref()
1287 .map(|transfer| transfer.resume_offset)
1288 .unwrap_or_default(),
1289 chunk_index: complete
1290 .transfer
1291 .as_ref()
1292 .map(|transfer| transfer.chunk_index)
1293 .unwrap_or_default(),
1294 checkpoint: complete
1295 .transfer
1296 .as_ref()
1297 .map(|transfer| transfer.checkpoint.clone())
1298 .unwrap_or_default(),
1299 is_complete: complete
1300 .transfer
1301 .as_ref()
1302 .map(|transfer| transfer.is_complete)
1303 .unwrap_or(false),
1304 },
1305 object_count: received,
1306 profile,
1307 });
1308 }
1309 _ => {}
1310 }
1311 }
1312
1313 Err(ProtocolError::InvalidState(format!(
1314 "pull stream ended unexpectedly after receiving {received} packed objects"
1315 )))
1316 }
1317}
1318
1319fn redaction_push_message(
1320 repo: &Repository,
1321 info: wire::ObjectInfo,
1322 client_operation_id: &str,
1323) -> Result<PushClientFrame, ProtocolError> {
1324 let wire::ObjectId::Hash(blob) = info.id else {
1325 return Err(ProtocolError::InvalidState(
1326 "wanted Redaction must be keyed by ObjectId::Hash(content_hash)".to_string(),
1327 ));
1328 };
1329 let hex = blob.to_hex();
1330 let bytes = repo
1335 .store()
1336 .get_redactions_bytes_for_blob(&blob)
1337 .map_err(|err| {
1338 ProtocolError::InvalidState(format!("load redactions sidecar for {}: {err}", hex))
1339 })?
1340 .ok_or_else(|| {
1341 ProtocolError::InvalidState(format!(
1342 "server wants redaction for blob {} but sender has no sidecar",
1343 hex
1344 ))
1345 })?;
1346 Ok(PushClientFrame {
1347 frame: Some(push_client_frame::Frame::Redaction(RedactionTransfer {
1348 blob_hash: hex,
1349 redactions_blob: bytes,
1350 })),
1351 client_operation_id: client_operation_id.to_string(),
1352 })
1353}
1354
1355fn native_pack_required_for_pull(
1356 want_full_closure: bool,
1357 transfer_plan: &RepositoryTransferPlan<ObjectInfo>,
1358) -> bool {
1359 transfer_plan.requires_native_pack(want_full_closure)
1360}
1361
1362fn object_info_from_plan(object: &PlannedObject) -> ObjectInfo {
1363 ObjectInfo {
1364 id: object.id.clone(),
1365 obj_type: object.obj_type,
1366 size: 0,
1367 delta_base: None,
1368 }
1369}
1370
1371fn to_proto_planned_object(object: &PlannedObject) -> ObjectDescriptor {
1372 object_descriptor_with_status(
1373 &object_info_from_plan(object),
1374 ObjectAvailabilityStatus::Present,
1375 "",
1376 )
1377}
1378
1379fn descriptor_id_from_plan(object: &PlannedObject) -> (String, String) {
1380 let id = match &object.id {
1381 wire::ObjectId::Hash(hash) => hash.to_hex(),
1382 wire::ObjectId::StateId(state_id) => state_id.to_string_full(),
1383 wire::ObjectId::StateAttachment { state, id } => {
1384 format!("{}:{}", state.to_string_full(), id.as_hash().to_hex())
1385 }
1386 };
1387 (id, object_type_name(object.obj_type).to_string())
1388}
1389
1390fn record_wanted_type(wanted_types: &mut WantedTypes, pack_id: PackObjectId, obj_type: ObjectType) {
1391 let types = wanted_types.entry(pack_id).or_default();
1392 if !types.contains(&obj_type) {
1393 types.push(obj_type);
1394 }
1395}
1396
1397fn wanted_packable_type(wanted_types: &WantedTypes, pack_id: &PackObjectId) -> Option<ObjectType> {
1398 wanted_types
1399 .get(pack_id)
1400 .and_then(|types| types.iter().copied().find(|obj_type| obj_type.packable()))
1401}
1402
1403fn sidecar_push_message(
1404 repo: &Repository,
1405 info: wire::ObjectInfo,
1406 client_operation_id: &str,
1407) -> Result<PushClientFrame, ProtocolError> {
1408 match info.obj_type {
1409 ObjectType::Redaction => redaction_push_message(repo, info, client_operation_id),
1410 ObjectType::StateVisibility => {
1411 state_visibility_push_message(repo, info, client_operation_id)
1412 }
1413 obj_type => Err(ProtocolError::InvalidState(format!(
1414 "{obj_type:?} is not an out-of-pack sidecar object"
1415 ))),
1416 }
1417}
1418
1419fn state_visibility_push_message(
1420 repo: &Repository,
1421 info: wire::ObjectInfo,
1422 client_operation_id: &str,
1423) -> Result<PushClientFrame, ProtocolError> {
1424 let wire::ObjectId::StateId(state) = info.id else {
1425 return Err(ProtocolError::InvalidState(
1426 "wanted StateVisibility must be keyed by ObjectId::StateId(state)".to_string(),
1427 ));
1428 };
1429 let state_id = state.to_string_full();
1430 let bytes = repo
1431 .get_state_visibility_bytes_for_state(&state)
1432 .map_err(|err| {
1433 ProtocolError::InvalidState(format!(
1434 "load state-visibility sidecar for {}: {err}",
1435 state_id
1436 ))
1437 })?
1438 .ok_or_else(|| {
1439 ProtocolError::InvalidState(format!(
1440 "server wants state visibility for state {} but sender has no sidecar",
1441 state_id
1442 ))
1443 })?;
1444 Ok(PushClientFrame {
1445 frame: Some(push_client_frame::Frame::StateVisibility(
1446 StateVisibilityTransfer {
1447 state_id: super::helpers::proto_state_id(state),
1448 state_visibility_blob: bytes,
1449 },
1450 )),
1451 client_operation_id: client_operation_id.to_string(),
1452 })
1453}
1454
1455fn load_thread_metadata(
1456 repo: &Repository,
1457 target_thread: &str,
1458 local_state: StateId,
1459) -> Result<Option<SyncedThreadMetadata>, ProtocolError> {
1460 let thread_manager = ThreadManager::new(repo.heddle_dir());
1461 Ok(thread_manager.find_synced_record_by_thread(repo, target_thread, Some(local_state))?)
1462}
1463
1464fn plan_pull_wants(
1465 repo: &Repository,
1466 remote_state: &StateId,
1467 full_closure_available: bool,
1468 objects_to_fetch: Vec<ObjectDescriptor>,
1469 allow_partial_fetch: bool,
1470) -> Result<PullWantPlan, ProtocolError> {
1471 if full_closure_available {
1472 return Ok(PullWantPlan {
1473 wants: Vec::new(),
1474 transfer_plan: RepositoryTransferPlan::from_object_infos(
1475 Vec::<ObjectInfo>::new(),
1476 GitLaneTransferIntent::HeddleObjectsOnly,
1477 ),
1478 wanted_types: HashMap::new(),
1479 want_full_closure: true,
1480 });
1481 }
1482 let request_full_closure =
1483 should_request_full_closure(repo, remote_state, allow_partial_fetch)?;
1484 let mut wants = Vec::with_capacity(objects_to_fetch.len());
1485 let mut wanted_infos = Vec::with_capacity(objects_to_fetch.len());
1486 let mut wanted_types = HashMap::with_capacity(objects_to_fetch.len());
1487
1488 for descriptor in objects_to_fetch {
1489 let info = parse_descriptor_to_info(descriptor)?;
1490 let pack_id = match &info.id {
1491 wire::ObjectId::Hash(hash) => PackObjectId::Hash(*hash),
1492 wire::ObjectId::StateId(state_id) => PackObjectId::StateId(*state_id),
1493 wire::ObjectId::StateAttachment { id, .. } => PackObjectId::Hash(*id.as_hash()),
1494 };
1495 let include = if request_full_closure {
1496 true
1497 } else {
1498 let has = wire::has_object(repo.store(), &info)?;
1499 !(has || (allow_partial_fetch && matches!(info.obj_type, ObjectType::Blob)))
1500 };
1501
1502 if include {
1503 record_wanted_type(&mut wanted_types, pack_id, info.obj_type);
1504 wants.push(object_descriptor_with_status(
1505 &info,
1506 ObjectAvailabilityStatus::Missing,
1507 "requested by client",
1508 ));
1509 wanted_infos.push(info);
1510 }
1511 }
1512
1513 Ok(PullWantPlan {
1514 wants,
1515 transfer_plan: RepositoryTransferPlan::from_object_infos(
1516 wanted_infos,
1517 GitLaneTransferIntent::HeddleObjectsOnly,
1518 ),
1519 wanted_types,
1520 want_full_closure: false,
1521 })
1522}
1523
1524fn supports_compact_full_pull(
1525 repo: &Repository,
1526 allow_partial_fetch: bool,
1527 exclude_states: &[StateId],
1528) -> Result<bool, ProtocolError> {
1529 if allow_partial_fetch || !exclude_states.is_empty() {
1530 return Ok(false);
1531 }
1532 repo_looks_fresh(repo)
1533}
1534
1535fn locally_complete_pull_head(
1558 repo: &Repository,
1559 remote_thread: &str,
1560 target_state: Option<StateId>,
1561) -> Result<Option<StateId>, ProtocolError> {
1562 locally_complete_thread_head(repo, remote_thread, target_state)
1563}
1564
1565fn locally_complete_local_thread_head(
1574 repo: &Repository,
1575 local_thread: &str,
1576 target_state: Option<StateId>,
1577) -> Result<Option<StateId>, ProtocolError> {
1578 locally_complete_thread_head(repo, local_thread, target_state)
1579}
1580
1581fn locally_complete_thread_head(
1601 repo: &Repository,
1602 thread: &str,
1603 target_state: Option<StateId>,
1604) -> Result<Option<StateId>, ProtocolError> {
1605 if target_state.is_some() {
1608 return Ok(None);
1609 }
1610 if !repo.missing_blobs()?.is_empty() {
1613 return Ok(None);
1614 }
1615 let Some(head) = repo.refs().get_thread(&ThreadName::from(thread))? else {
1616 return Ok(None);
1619 };
1620 match wire::enumerate_state_closure(repo.store(), head) {
1625 Ok(_) => Ok(Some(head)),
1626 Err(ProtocolError::ObjectNotFound(_)) => Ok(None),
1627 Err(err) => Err(err),
1628 }
1629}
1630
1631fn should_request_full_closure(
1632 repo: &Repository,
1633 remote_state: &StateId,
1634 allow_partial_fetch: bool,
1635) -> Result<bool, ProtocolError> {
1636 if allow_partial_fetch || repo.store().has_state(remote_state)? {
1637 return Ok(false);
1638 }
1639 repo_looks_fresh(repo)
1640}
1641
1642fn repo_looks_fresh(repo: &Repository) -> Result<bool, ProtocolError> {
1643 if repo.head()?.is_some() {
1644 return Ok(false);
1645 }
1646 if !repo.refs().list_threads()?.is_empty() || !repo.refs().list_markers()?.is_empty() {
1647 return Ok(false);
1648 }
1649 Ok(repo.missing_blobs()?.is_empty())
1650}
1651
1652fn infer_installed_hash_object_type(
1653 repo: &Repository,
1654 hash: &ContentHash,
1655) -> Result<ObjectType, ProtocolError> {
1656 let store = repo.store();
1657 if store.get_tree(hash)?.is_some() {
1658 return Ok(ObjectType::Tree);
1659 }
1660 if store
1661 .get_action(&objects::object::ActionId::from_hash(*hash))?
1662 .is_some()
1663 {
1664 return Ok(ObjectType::Action);
1665 }
1666 Ok(ObjectType::Blob)
1667}
1668
1669fn apply_marker_snapshot(repo: &Repository, checkpoint: &[u8]) -> Result<bool, ProtocolError> {
1670 const HEADER: &str = "heddle-markers-v1\n";
1671 if checkpoint.is_empty() {
1672 return Ok(false);
1673 }
1674 let payload = std::str::from_utf8(checkpoint)
1675 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
1676 let Some(lines) = payload.strip_prefix(HEADER) else {
1677 return Ok(false);
1678 };
1679
1680 for line in lines.lines() {
1681 if line.is_empty() {
1682 continue;
1683 }
1684 let Some((name, state_id)) = line.split_once('\t') else {
1685 return Err(ProtocolError::InvalidState(
1686 "invalid marker snapshot line".to_string(),
1687 ));
1688 };
1689 let state_id =
1690 StateId::parse(state_id).map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
1691 if !repo.store().has_state(&state_id)? {
1692 continue;
1693 }
1694 let name = MarkerName::from(name);
1695 match repo.refs().get_marker(&name)? {
1696 Some(existing) if existing == state_id => {}
1697 Some(existing) => repo.refs().set_marker_cas(
1698 &name,
1699 refs::RefExpectation::Value(existing),
1700 &state_id,
1701 )?,
1702 None => repo.refs().create_marker(&name, &state_id)?,
1703 }
1704 }
1705
1706 Ok(true)
1707}
1708
1709fn state_id_string_to_bytes(s: &str) -> Vec<u8> {
1710 if s.is_empty() {
1711 return Vec::new();
1712 }
1713 objects::object::StateId::parse(s)
1714 .map(|id| id.as_bytes().to_vec())
1715 .unwrap_or_default()
1716}
1717
1718fn to_proto_thread_metadata(metadata: &SyncedThreadMetadata) -> ThreadMetadata {
1719 ThreadMetadata {
1720 name: metadata.thread.clone(),
1721 target_thread: metadata.target_thread.clone(),
1722 parent_thread: metadata.parent_thread.clone(),
1723 task: metadata.task.clone(),
1724 thread_mode: metadata.mode.to_string(),
1725 thread_state: metadata.state.to_string(),
1726 freshness: metadata.freshness.to_string(),
1727 base_state: StateId::parse(&metadata.base_state)
1728 .ok()
1729 .and_then(super::helpers::proto_state_id),
1730 base_root: state_id_string_to_bytes(&metadata.base_root),
1731 current_state: metadata.current_state.as_deref().and_then(|state| {
1732 StateId::parse(state)
1733 .ok()
1734 .and_then(super::helpers::proto_state_id)
1735 }),
1736 merged_state: metadata.merged_state.as_deref().and_then(|state| {
1737 StateId::parse(state)
1738 .ok()
1739 .and_then(super::helpers::proto_state_id)
1740 }),
1741 changed_paths: metadata.changed_paths.clone(),
1742 impact_categories: metadata
1743 .impact_categories
1744 .iter()
1745 .map(ToString::to_string)
1746 .collect(),
1747 heavy_impact_paths: metadata.heavy_impact_paths.clone(),
1748 promotion_suggested: metadata.promotion_suggested,
1749 verification_summary: Some(ThreadVerificationSummary {
1750 tests_passed: metadata.verification_summary.tests_passed,
1751 tests_failed: metadata
1752 .verification_summary
1753 .tests_failed
1754 .unwrap_or_default(),
1755 coverage_pct: metadata.verification_summary.coverage_pct,
1756 lint_warnings: metadata.verification_summary.lint_warnings,
1757 }),
1758 confidence_summary: Some(ThreadConfidenceSummary {
1759 value: metadata.confidence_summary.value,
1760 band: metadata
1761 .confidence_summary
1762 .band
1763 .as_ref()
1764 .map(ToString::to_string),
1765 }),
1766 integration_policy_result: Some(ThreadIntegrationPolicy {
1767 status: metadata
1768 .integration_policy_result
1769 .status
1770 .clone()
1771 .unwrap_or_default(),
1772 reason: metadata
1773 .integration_policy_result
1774 .reason
1775 .clone()
1776 .unwrap_or_default(),
1777 }),
1778 created_at: Some(prost_types::Timestamp {
1779 seconds: metadata.created_at.timestamp(),
1780 nanos: metadata.created_at.timestamp_subsec_nanos() as i32,
1781 }),
1782 updated_at: Some(prost_types::Timestamp {
1783 seconds: metadata.updated_at.timestamp(),
1784 nanos: metadata.updated_at.timestamp_subsec_nanos() as i32,
1785 }),
1786 }
1787}
1788
1789struct PullExchange {
1790 result: PullComplete,
1791 object_count: usize,
1792 profile: PullProfile,
1793}
1794
1795fn mark_missing_blobs_for_state(repo: &Repository, state_id: StateId) -> Result<(), ProtocolError> {
1796 let state = repo
1797 .store()
1798 .get_state(&state_id)?
1799 .ok_or_else(|| ProtocolError::ObjectNotFound(state_id.to_string_full()))?;
1800 let mut missing = wire::missing_blobs_in_tree(repo.store(), state.tree)?;
1801 for attachment in repo.list_state_attachments(&state_id)? {
1802 match attachment.body {
1803 objects::object::StateAttachmentBody::Context(root) => {
1804 missing.extend(wire::missing_blobs_in_tree(repo.store(), root)?);
1805 }
1806 objects::object::StateAttachmentBody::RiskSignals(hash)
1807 | objects::object::StateAttachmentBody::ReviewSignatures(hash)
1808 | objects::object::StateAttachmentBody::Discussions(hash)
1809 | objects::object::StateAttachmentBody::StructuredConflicts(hash)
1810 if !repo.store().has_blob(&hash)? =>
1811 {
1812 missing.push(hash)
1813 }
1814 _ => {}
1815 }
1816 }
1817 missing
1818 .into_iter()
1819 .try_for_each(|hash| repo.record_missing_blob(hash).map_err(ProtocolError::from))
1820}
1821
1822fn clear_missing_blobs_for_state(
1823 repo: &Repository,
1824 state_id: StateId,
1825) -> Result<(), ProtocolError> {
1826 let state = repo
1827 .store()
1828 .get_state(&state_id)?
1829 .ok_or_else(|| ProtocolError::ObjectNotFound(state_id.to_string_full()))?;
1830 let mut missing = wire::missing_blobs_in_tree(repo.store(), state.tree)?;
1831 for attachment in repo.list_state_attachments(&state_id)? {
1832 match attachment.body {
1833 objects::object::StateAttachmentBody::Context(root) => {
1834 missing.extend(wire::missing_blobs_in_tree(repo.store(), root)?);
1835 }
1836 objects::object::StateAttachmentBody::RiskSignals(hash)
1837 | objects::object::StateAttachmentBody::ReviewSignatures(hash)
1838 | objects::object::StateAttachmentBody::Discussions(hash)
1839 | objects::object::StateAttachmentBody::StructuredConflicts(hash) => missing.push(hash),
1840 _ => {}
1841 }
1842 }
1843 missing
1844 .into_iter()
1845 .try_for_each(|hash| repo.clear_missing_blob(&hash).map_err(ProtocolError::from))
1846}
1847
1848fn partial_fetch_status_for_repo(repo: &Repository) -> i32 {
1849 match repo.missing_blobs() {
1850 Ok(missing) if !missing.is_empty() => PartialFetchStatus::Enabled as i32,
1851 Ok(_) => PartialFetchStatus::Disabled as i32,
1852 Err(_) => PartialFetchStatus::Unspecified as i32,
1853 }
1854}
1855
1856fn pull_transfer_id(
1857 repo_path: &str,
1858 remote_thread: &str,
1859 local_thread: Option<&str>,
1860 depth: Option<u32>,
1861 target_state: Option<StateId>,
1862) -> String {
1863 format!(
1864 "pull:{repo_path}:{remote_thread}:{}:{depth:?}:{}",
1865 local_thread.unwrap_or_default(),
1866 target_state
1867 .map(|value| value.to_string_full())
1868 .unwrap_or_default()
1869 )
1870}
1871
1872fn push_transfer_id(repo_path: &str, local_state: StateId, target_thread: &str) -> String {
1873 format!(
1874 "push:{repo_path}:{}:{target_thread}",
1875 local_state.to_string_full()
1876 )
1877}
1878
1879fn build_git_mirror_push_plan(
1893 repo: &Repository,
1894 chunk_size: usize,
1895 remote_ref_expectations: &HashMap<String, GitRefRemoteExpectation>,
1896) -> Result<GitLanePushPlan, ProtocolError> {
1897 if repo.capability() != RepositoryCapability::GitOverlay {
1898 return Err(ProtocolError::InvalidState(
1899 "Git remote mirror pushes require a git-overlay repository".to_string(),
1900 ));
1901 }
1902 let git_repo = repo
1903 .git_overlay_sley_repository()
1904 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
1905 .ok_or_else(|| {
1906 ProtocolError::InvalidState("git-overlay repository has no Git store".to_string())
1907 })?;
1908 build_git_mirror_plan_from_sley(&git_repo, chunk_size, remote_ref_expectations)
1909}
1910
1911fn build_git_mirror_plan_from_sley(
1914 git_repo: &SleyRepository,
1915 chunk_size: usize,
1916 remote_ref_expectations: &HashMap<String, GitRefRemoteExpectation>,
1917) -> Result<GitLanePushPlan, ProtocolError> {
1918 let refs = git_repo
1919 .references()
1920 .list_refs()
1921 .map_err(|err| ProtocolError::InvalidState(format!("list git-overlay refs: {err}")))?;
1922
1923 let mut roots: Vec<GitObjectId> = Vec::new();
1924 let mut ref_updates: Vec<GitRefUpdateTransfer> = Vec::new();
1925 let mut newest_root: Option<GitObjectId> = None;
1926
1927 for reference in refs {
1928 if GitRefName::new(&reference.name).is_local_only() {
1942 continue;
1943 }
1944
1945 let target_oid = match reference.target {
1950 ReferenceTarget::Direct(oid) => oid,
1951 ReferenceTarget::Symbolic(_) => continue,
1952 };
1953
1954 let object = git_repo.read_object(&target_oid).map_err(|err| {
1958 ProtocolError::InvalidState(format!(
1959 "git-overlay ref {} target {} is not readable: {err}",
1960 reference.name,
1961 target_oid.to_hex()
1962 ))
1963 })?;
1964
1965 let peeled_oid = if object.object_type == sley::GitObjectType::Tag {
1966 let peeled = git_repo.peel_to_object_oid(target_oid).map_err(|err| {
1967 ProtocolError::InvalidState(format!(
1968 "peel git-overlay tag ref {}: {err}",
1969 reference.name
1970 ))
1971 })?;
1972 Some(peeled)
1973 } else {
1974 None
1975 };
1976
1977 roots.push(target_oid);
1981 newest_root.get_or_insert(target_oid);
1982
1983 let expectation = remote_ref_expectations
1984 .get(&reference.name)
1985 .cloned()
1986 .unwrap_or(GitRefRemoteExpectation::Missing);
1987
1988 let kind = grpc_git_ref_kind(GitRefName::new(&reference.name).wire_kind());
1989 let mut message =
1990 git_ref_update_message(&reference.name, kind, target_oid, peeled_oid, None);
1991 apply_git_ref_expectation_value(&mut message, &expectation);
1992 ref_updates.push(message);
1993 }
1994
1995 if roots.is_empty() {
1996 return Err(ProtocolError::InvalidState(
1997 "git-overlay repository has no direct refs to mirror".to_string(),
1998 ));
1999 }
2000
2001 let format = git_repo.object_format();
2012 let mut have_boundary: Vec<GitObjectId> = Vec::new();
2013 for expectation in remote_ref_expectations.values() {
2014 if let GitRefRemoteExpectation::Value(oid_bytes) = expectation
2015 && let Ok(oid) = GitObjectId::from_raw(format, oid_bytes)
2016 {
2017 have_boundary.push(oid);
2021 }
2022 }
2023
2024 let pack = build_git_lane_multi_root_pack_plan(git_repo, roots, have_boundary, chunk_size)?;
2025
2026 let local_revision_address = newest_root
2029 .map(|oid| RevisionAddress::git_commit(oid.to_hex()).to_string())
2030 .unwrap_or_default();
2031
2032 Ok(GitLanePushPlan {
2033 local_revision_address,
2034 pack,
2035 ref_updates,
2036 })
2037}
2038
2039fn grpc_git_ref_kind(kind: ClassifiedGitRefKind) -> GrpcGitRefKind {
2040 match kind {
2041 ClassifiedGitRefKind::Branch => GrpcGitRefKind::Branch,
2042 ClassifiedGitRefKind::Tag => GrpcGitRefKind::Tag,
2043 ClassifiedGitRefKind::Note => GrpcGitRefKind::Note,
2044 ClassifiedGitRefKind::Other => GrpcGitRefKind::Other,
2045 }
2046}
2047
2048fn build_git_lane_multi_root_pack_plan(
2054 git_repo: &SleyRepository,
2055 roots: Vec<GitObjectId>,
2056 excluded: Vec<GitObjectId>,
2057 chunk_size: usize,
2058) -> Result<Option<GitPackPushPlan>, ProtocolError> {
2059 if roots.is_empty() {
2060 return Err(ProtocolError::InvalidState(
2061 "cannot plan a Git pack with no roots".to_string(),
2062 ));
2063 }
2064 let Some(plan) = git_repo
2065 .reachable_pack_plan()
2066 .roots(roots.iter().copied())
2067 .exclusions(excluded)
2068 .build()
2069 .map_err(|err| {
2070 ProtocolError::InvalidState(format!("plan reachable Git pack stream: {err}"))
2071 })?
2072 else {
2073 return Ok(None);
2076 };
2077 let pack_file = NamedTempFile::new()
2078 .map_err(|err| ProtocolError::InvalidState(format!("create Git pack tempfile: {err}")))?;
2079 let prepared = plan.prepare_to_file(pack_file.path()).map_err(|err| {
2080 ProtocolError::InvalidState(format!("write reachable Git pack tempfile: {err}"))
2081 })?;
2082 let pack_size = prepared.summary.pack_size;
2083 let checksum = prepared.summary.checksum;
2084 if pack_size > wire::MAX_RECEIVED_GIT_PACK_SIZE {
2085 return Err(ProtocolError::InvalidState(format!(
2086 "Git pack exceeds maximum transfer size of {} bytes; multi-pack split for repos over this size is a follow-up (plan §B.2)",
2087 wire::MAX_RECEIVED_GIT_PACK_SIZE
2088 )));
2089 }
2090 let chunk_size = chunk_size.max(1) as u64;
2091 let chunk_count = pack_size.div_ceil(chunk_size);
2092 if chunk_count > u32::MAX as u64 {
2093 return Err(ProtocolError::InvalidState(
2094 "Git pack chunk count exceeds u32".to_string(),
2095 ));
2096 }
2097 let transfer_id = format!("git-pack:{}", checksum.to_hex());
2098 Ok(Some(GitPackPushPlan {
2099 transfer_id,
2100 pack_id: checksum.as_bytes().to_vec(),
2101 pack_size,
2102 roots,
2103 pack_file: Arc::new(Mutex::new(pack_file)),
2104 }))
2105}
2106
2107async fn send_git_pack_streaming_messages(
2108 tx: &mpsc::Sender<PushClientFrame>,
2109 pack: &GitPackPushPlan,
2110 chunk_size: usize,
2111 progress: &Progress,
2112 client_operation_id: &str,
2113) -> Result<(), ProtocolError> {
2114 let tx = tx.clone();
2115 let pack = pack.clone();
2116 let progress = progress.clone();
2117 let client_operation_id = client_operation_id.to_string();
2118 tokio::task::spawn_blocking(move || {
2119 stream_git_pack_messages_blocking(tx, pack, chunk_size, progress, client_operation_id)
2120 })
2121 .await
2122 .map_err(|err| ProtocolError::InvalidState(format!("Git pack streaming task failed: {err}")))?
2123}
2124
2125fn stream_git_pack_messages_blocking(
2126 tx: mpsc::Sender<PushClientFrame>,
2127 pack: GitPackPushPlan,
2128 chunk_size: usize,
2129 progress: Progress,
2130 client_operation_id: String,
2131) -> Result<(), ProtocolError> {
2132 let mut writer = GitPackPushMessageWriter::new(
2133 tx,
2134 pack.transfer_id.clone(),
2135 pack.pack_id.clone(),
2136 pack.pack_size,
2137 chunk_size,
2138 progress,
2139 client_operation_id,
2140 );
2141 let mut pack_file = pack
2142 .pack_file
2143 .lock()
2144 .map_err(|err| ProtocolError::InvalidState(format!("lock Git pack tempfile: {err}")))?;
2145 pack_file
2146 .seek(SeekFrom::Start(0))
2147 .map_err(|err| ProtocolError::InvalidState(format!("rewind Git pack tempfile: {err}")))?;
2148 let streamed = io::copy(&mut pack_file.as_file_mut(), &mut writer)
2149 .map_err(|err| ProtocolError::InvalidState(format!("stream Git pack tempfile: {err}")))?;
2150 if streamed != pack.pack_size {
2151 return Err(ProtocolError::InvalidState(format!(
2152 "Git pack stream changed while sending; expected {} bytes/{}, streamed {} bytes",
2153 pack.pack_size,
2154 hex::encode(&pack.pack_id),
2155 streamed
2156 )));
2157 }
2158 writer.finish()?;
2159 Ok(())
2160}
2161
2162struct GitPackPushMessageWriter {
2163 tx: mpsc::Sender<PushClientFrame>,
2164 transfer_id: String,
2165 pack_id: Vec<u8>,
2166 pack_size: u64,
2167 chunk_size: usize,
2168 buffer: Vec<u8>,
2169 offset: u64,
2170 chunk_index: u32,
2171 client_operation_id: String,
2172 progress: Progress,
2175 last_progress_pct: u64,
2179}
2180
2181impl GitPackPushMessageWriter {
2182 fn new(
2183 tx: mpsc::Sender<PushClientFrame>,
2184 transfer_id: String,
2185 pack_id: Vec<u8>,
2186 pack_size: u64,
2187 chunk_size: usize,
2188 progress: Progress,
2189 client_operation_id: String,
2190 ) -> Self {
2191 let chunk_size = chunk_size.max(1);
2192 Self {
2193 tx,
2194 transfer_id,
2195 pack_id,
2196 pack_size,
2197 chunk_size,
2198 buffer: Vec::with_capacity(chunk_size),
2199 offset: 0,
2200 chunk_index: 0,
2201 client_operation_id,
2202 progress,
2203 last_progress_pct: u64::MAX,
2204 }
2205 }
2206
2207 fn send_buffer(&mut self) -> io::Result<()> {
2208 if self.buffer.is_empty() {
2209 return Ok(());
2210 }
2211 let chunk = std::mem::take(&mut self.buffer);
2212 let next_offset = self.offset.checked_add(chunk.len() as u64).ok_or_else(|| {
2213 io::Error::new(io::ErrorKind::InvalidData, "Git pack offset overflow")
2214 })?;
2215 if next_offset > self.pack_size {
2216 return Err(io::Error::new(
2217 io::ErrorKind::InvalidData,
2218 format!(
2219 "Git pack stream exceeded planned size {}; got at least {}",
2220 self.pack_size, next_offset
2221 ),
2222 ));
2223 }
2224 let is_final_chunk = next_offset == self.pack_size;
2225 self.tx
2226 .blocking_send(git_lane_push_message(
2227 git_lane_transfer::Body::Pack(GitPackTransfer {
2228 transfer_id: self.transfer_id.clone(),
2229 offset: self.offset,
2230 chunk_index: self.chunk_index,
2231 is_final_chunk,
2232 pack_size: self.pack_size,
2233 pack_chunk: chunk,
2234 pack_id: self.pack_id.clone(),
2235 }),
2236 &self.client_operation_id,
2237 ))
2238 .map_err(|_| {
2239 io::Error::new(io::ErrorKind::BrokenPipe, "push stream closed unexpectedly")
2240 })?;
2241 self.offset = next_offset;
2242 self.chunk_index = self.chunk_index.checked_add(1).ok_or_else(|| {
2243 io::Error::new(io::ErrorKind::InvalidData, "Git pack chunk index overflow")
2244 })?;
2245 self.report_upload_progress();
2246 Ok(())
2247 }
2248
2249 fn report_upload_progress(&mut self) {
2254 if !self.progress.is_active() {
2255 return;
2256 }
2257 let pct = self.offset.saturating_mul(100) / self.pack_size.max(1);
2258 if pct == self.last_progress_pct {
2259 return;
2260 }
2261 self.last_progress_pct = pct;
2262 self.progress.set_phase(format!(
2263 "uploading {}/{} bytes ({pct}%)",
2264 self.offset, self.pack_size
2265 ));
2266 }
2267
2268 fn finish(mut self) -> Result<(), ProtocolError> {
2269 self.send_buffer().map_err(|err| {
2270 ProtocolError::InvalidState(format!("send final Git pack chunk: {err}"))
2271 })?;
2272 if self.offset != self.pack_size {
2273 return Err(ProtocolError::InvalidState(format!(
2274 "Git pack stream length mismatch: expected {}, got {}",
2275 self.pack_size, self.offset
2276 )));
2277 }
2278 Ok(())
2279 }
2280}
2281
2282impl Write for GitPackPushMessageWriter {
2283 fn write(&mut self, mut buf: &[u8]) -> io::Result<usize> {
2284 let written = buf.len();
2285 while !buf.is_empty() {
2286 if self.offset == self.pack_size {
2287 return Err(io::Error::new(
2288 io::ErrorKind::InvalidData,
2289 format!("Git pack stream wrote past planned size {}", self.pack_size),
2290 ));
2291 }
2292 let capacity = self
2293 .chunk_size
2294 .checked_sub(self.buffer.len())
2295 .ok_or_else(|| {
2296 io::Error::new(io::ErrorKind::InvalidData, "Git pack chunk buffer overflow")
2297 })?;
2298 let take = capacity.min(buf.len());
2299 self.buffer.extend_from_slice(&buf[..take]);
2300 buf = &buf[take..];
2301 if self.buffer.len() == self.chunk_size {
2302 self.send_buffer()?;
2303 }
2304 }
2305 Ok(written)
2306 }
2307
2308 fn flush(&mut self) -> io::Result<()> {
2309 Ok(())
2310 }
2311}
2312
2313fn git_ref_update_message(
2320 name: &str,
2321 kind: GrpcGitRefKind,
2322 target_oid: GitObjectId,
2323 peeled_oid: Option<GitObjectId>,
2324 checkpoint: Option<GitCheckpointTransfer>,
2325) -> GitRefUpdateTransfer {
2326 GitRefUpdateTransfer {
2327 name: name.to_string(),
2328 kind: kind as i32,
2329 target_oid: proto_git_oid(&target_oid),
2330 peeled_oid: peeled_oid.as_ref().and_then(proto_git_oid),
2331 expected_missing: false,
2332 expected_target_oid: None,
2333 checkpoint,
2334 }
2335}
2336
2337fn apply_git_ref_expectation_value(
2338 update: &mut GitRefUpdateTransfer,
2339 expectation: &GitRefRemoteExpectation,
2340) {
2341 match expectation {
2342 GitRefRemoteExpectation::Missing => {
2343 update.expected_missing = true;
2344 update.expected_target_oid = None;
2345 }
2346 GitRefRemoteExpectation::Value(oid) => {
2347 update.expected_missing = false;
2348 update.expected_target_oid = proto_git_oid_bytes(oid);
2349 }
2350 }
2351}
2352
2353#[derive(Clone, Debug, PartialEq, Eq)]
2354enum GitRefRemoteExpectation {
2355 Missing,
2356 Value(Vec<u8>),
2357}
2358
2359fn parse_git_ref_expectation(
2360 remote_revision_address: &str,
2361) -> Result<GitRefRemoteExpectation, ProtocolError> {
2362 if remote_revision_address.is_empty() {
2363 return Ok(GitRefRemoteExpectation::Missing);
2364 }
2365
2366 match remote_revision_address.parse::<RevisionAddress>() {
2367 Ok(RevisionAddress::Heddle(_)) => Ok(GitRefRemoteExpectation::Missing),
2368 Ok(RevisionAddress::GitCommit(oid)) => hex::decode(&oid)
2369 .map(GitRefRemoteExpectation::Value)
2370 .map_err(|err| {
2371 ProtocolError::InvalidState(format!(
2372 "server returned invalid Git remote_revision_address: {err}"
2373 ))
2374 }),
2375 Err(err) => Err(ProtocolError::InvalidState(format!(
2376 "server returned invalid remote_revision_address {remote_revision_address:?}: {err}"
2377 ))),
2378 }
2379}
2380
2381fn git_lane_push_message(
2382 body: git_lane_transfer::Body,
2383 client_operation_id: &str,
2384) -> PushClientFrame {
2385 PushClientFrame {
2386 frame: Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
2387 body: Some(body),
2388 })),
2389 client_operation_id: client_operation_id.to_string(),
2390 }
2391}
2392
2393fn git_lane_transfer_size(transfer: &GitLaneTransfer) -> usize {
2394 match transfer.body.as_ref() {
2395 Some(git_lane_transfer::Body::Pack(pack)) => pack.pack_chunk.len(),
2396 Some(git_lane_transfer::Body::RefUpdate(update)) => update
2397 .target_oid
2398 .as_ref()
2399 .map_or(0, |oid| oid.digest.len())
2400 .saturating_add(update.peeled_oid.as_ref().map_or(0, |oid| oid.digest.len()))
2401 .saturating_add(
2402 update
2403 .expected_target_oid
2404 .as_ref()
2405 .map_or(0, |oid| oid.digest.len()),
2406 )
2407 .saturating_add(
2408 update
2409 .checkpoint
2410 .as_ref()
2411 .map(git_checkpoint_transfer_size)
2412 .unwrap_or_default(),
2413 ),
2414 Some(git_lane_transfer::Body::Checkpoint(checkpoint)) => {
2415 git_checkpoint_transfer_size(checkpoint)
2416 }
2417 None => 0,
2418 }
2419}
2420
2421fn git_checkpoint_transfer_size(checkpoint: &GitCheckpointTransfer) -> usize {
2422 checkpoint
2423 .heddle_state_id
2424 .as_ref()
2425 .map_or(0, |state| state.value.len())
2426 .saturating_add(
2427 checkpoint
2428 .git_commit_oid
2429 .as_ref()
2430 .map_or(0, |oid| oid.digest.len()),
2431 )
2432 .saturating_add(checkpoint.thread.len())
2433 .saturating_add(checkpoint.metadata_json.len())
2434}
2435
2436#[derive(Default)]
2437struct GitPackPullInstallState {
2438 active: Option<GitPackPullInstall>,
2439}
2440
2441impl GitPackPullInstallState {
2442 fn ensure_idle(&self) -> Result<(), ProtocolError> {
2443 if self.active.is_none() {
2444 Ok(())
2445 } else {
2446 Err(ProtocolError::InvalidState(
2447 "Git pack transfer ended before final chunk".to_string(),
2448 ))
2449 }
2450 }
2451
2452 fn receive_chunk(
2453 &mut self,
2454 git_repo: &SleyRepository,
2455 pack: GitPackTransfer,
2456 ) -> Result<(), ProtocolError> {
2457 if pack.transfer_id.is_empty() {
2458 return Err(ProtocolError::InvalidState(
2459 "Git pack transfer_id is required".to_string(),
2460 ));
2461 }
2462 if pack.pack_size > wire::MAX_RECEIVED_GIT_PACK_SIZE {
2463 return Err(ProtocolError::InvalidState(format!(
2464 "Git pack exceeds maximum transfer size of {} bytes",
2465 wire::MAX_RECEIVED_GIT_PACK_SIZE
2466 )));
2467 }
2468 if pack.pack_chunk.is_empty() {
2469 return Err(ProtocolError::InvalidState(
2470 "Git pack chunk must not be empty".to_string(),
2471 ));
2472 }
2473 let pack_id =
2474 GitObjectId::from_raw(git_repo.object_format(), &pack.pack_id).map_err(|err| {
2475 ProtocolError::InvalidState(format!("GitPackTransfer.pack_id: {err}"))
2476 })?;
2477 if self.active.is_none() {
2478 if pack.offset != 0 {
2479 return Err(ProtocolError::InvalidState(format!(
2480 "Git pack offset mismatch: expected 0, got {}",
2481 pack.offset
2482 )));
2483 }
2484 if pack.chunk_index != 0 {
2485 return Err(ProtocolError::InvalidState(format!(
2486 "Git pack chunk index mismatch: expected 0, got {}",
2487 pack.chunk_index
2488 )));
2489 }
2490 let writer = git_repo
2491 .objects()
2492 .begin_raw_pack_install(pack_id, pack.pack_size)
2493 .map_err(|err| {
2494 ProtocolError::InvalidState(format!("begin Git pack install: {err}"))
2495 })?;
2496 self.active = Some(GitPackPullInstall {
2497 transfer_id: pack.transfer_id.clone(),
2498 pack_id,
2499 pack_size: pack.pack_size,
2500 next_offset: 0,
2501 next_chunk_index: 0,
2502 writer,
2503 });
2504 }
2505
2506 let active = self.active.as_mut().ok_or_else(|| {
2507 ProtocolError::InvalidState("Git pack install not active".to_string())
2508 })?;
2509 active.receive_chunk(&pack, pack_id)?;
2510 if pack.is_final_chunk {
2511 let active = self.active.take().ok_or_else(|| {
2512 ProtocolError::InvalidState("Git pack install not active".to_string())
2513 })?;
2514 active.finish()?;
2515 git_repo.refresh_objects();
2516 }
2517 Ok(())
2518 }
2519}
2520
2521struct GitPackPullInstall {
2522 transfer_id: String,
2523 pack_id: GitObjectId,
2524 pack_size: u64,
2525 next_offset: u64,
2526 next_chunk_index: u32,
2527 writer: sley::plumbing::sley_odb::RawPackStreamingInstall,
2528}
2529
2530impl GitPackPullInstall {
2531 fn receive_chunk(
2532 &mut self,
2533 pack: &GitPackTransfer,
2534 pack_id: GitObjectId,
2535 ) -> Result<(), ProtocolError> {
2536 if self.transfer_id != pack.transfer_id {
2537 return Err(ProtocolError::InvalidState(format!(
2538 "Git pack transfer id changed from {:?} to {:?}",
2539 self.transfer_id, pack.transfer_id
2540 )));
2541 }
2542 if self.pack_id != pack_id {
2543 return Err(ProtocolError::InvalidState(
2544 "Git pack id changed during transfer".to_string(),
2545 ));
2546 }
2547 if self.pack_size != pack.pack_size {
2548 return Err(ProtocolError::InvalidState(
2549 "Git pack size changed during transfer".to_string(),
2550 ));
2551 }
2552 if pack.offset != self.next_offset {
2553 return Err(ProtocolError::InvalidState(format!(
2554 "Git pack offset mismatch: expected {}, got {}",
2555 self.next_offset, pack.offset
2556 )));
2557 }
2558 if pack.chunk_index != self.next_chunk_index {
2559 return Err(ProtocolError::InvalidState(format!(
2560 "Git pack chunk index mismatch: expected {}, got {}",
2561 self.next_chunk_index, pack.chunk_index
2562 )));
2563 }
2564 let chunk_len = u64::try_from(pack.pack_chunk.len()).map_err(|_| {
2565 ProtocolError::InvalidState("Git pack chunk length exceeds u64".to_string())
2566 })?;
2567 let next_offset = self
2568 .next_offset
2569 .checked_add(chunk_len)
2570 .ok_or_else(|| ProtocolError::InvalidState("Git pack offset overflow".to_string()))?;
2571 if next_offset > self.pack_size {
2572 return Err(ProtocolError::InvalidState(
2573 "Git pack chunk exceeds declared pack size".to_string(),
2574 ));
2575 }
2576 self.writer.write_all(&pack.pack_chunk).map_err(|err| {
2577 ProtocolError::InvalidState(format!("write streamed Git pack chunk: {err}"))
2578 })?;
2579 self.next_offset = next_offset;
2580 self.next_chunk_index = self.next_chunk_index.checked_add(1).ok_or_else(|| {
2581 ProtocolError::InvalidState("Git pack chunk index overflow".to_string())
2582 })?;
2583 if pack.is_final_chunk {
2584 if self.next_offset != self.pack_size {
2585 return Err(ProtocolError::InvalidState(format!(
2586 "Git pack final size mismatch: declared {}, received {}",
2587 self.pack_size, self.next_offset
2588 )));
2589 }
2590 } else if self.next_offset == self.pack_size {
2591 return Err(ProtocolError::InvalidState(
2592 "Git pack reached declared size without final chunk marker".to_string(),
2593 ));
2594 }
2595 Ok(())
2596 }
2597
2598 fn finish(self) -> Result<(), ProtocolError> {
2599 self.writer
2600 .finish()
2601 .map_err(|err| ProtocolError::InvalidState(format!("install Git pack: {err}")))?;
2602 Ok(())
2603 }
2604}
2605
2606fn accept_git_lane_pull_transfer(
2607 repo: &Repository,
2608 git_repo: &mut Option<SleyRepository>,
2609 git_pack_state: &mut GitPackPullInstallState,
2610 transfer: GitLaneTransfer,
2611) -> Result<(), ProtocolError> {
2612 if repo.capability() != RepositoryCapability::GitOverlay {
2613 return Err(ProtocolError::InvalidState(format!(
2614 "received git-lane pull transfer for non-GitOverlay repository (capability {:?})",
2615 repo.capability()
2616 )));
2617 }
2618 match transfer.body {
2619 Some(git_lane_transfer::Body::Pack(pack)) => {
2620 accept_git_lane_pack(repo, git_repo, git_pack_state, pack)
2621 }
2622 Some(git_lane_transfer::Body::RefUpdate(update)) => {
2623 accept_git_lane_ref_update(repo, git_repo, update)
2624 }
2625 Some(git_lane_transfer::Body::Checkpoint(checkpoint)) => {
2626 record_git_lane_checkpoint(repo, git_lane_sley_repository(repo, git_repo)?, checkpoint)
2627 }
2628 None => Err(ProtocolError::InvalidState(
2629 "GitLaneTransfer body is required".to_string(),
2630 )),
2631 }
2632}
2633
2634fn accept_git_lane_pack(
2635 repo: &Repository,
2636 git_repo: &mut Option<SleyRepository>,
2637 git_pack_state: &mut GitPackPullInstallState,
2638 pack: GitPackTransfer,
2639) -> Result<(), ProtocolError> {
2640 let git_repo = git_lane_sley_repository(repo, git_repo)?;
2641 git_pack_state.receive_chunk(git_repo, pack)?;
2642 Ok(())
2643}
2644
2645fn accept_git_lane_ref_update(
2655 repo: &Repository,
2656 git_repo: &mut Option<SleyRepository>,
2657 update: GitRefUpdateTransfer,
2658) -> Result<(), ProtocolError> {
2659 let git_repo = git_lane_sley_repository(repo, git_repo)?;
2660 let target = git_oid_from_proto(
2661 git_repo,
2662 "GitRefUpdateTransfer.target_oid",
2663 update.target_oid.as_ref(),
2664 )?;
2665 git_repo.read_commit(&target).map_err(|err| {
2666 ProtocolError::InvalidState(format!(
2667 "Git ref {} target commit {} is not present after pack receive: {err}",
2668 update.name,
2669 target.to_hex()
2670 ))
2671 })?;
2672 let refs = git_repo.references();
2673 let mut tx = refs.transaction();
2674 tx.update_to(
2675 update.name.clone(),
2676 ReferenceTarget::Direct(target),
2677 RefPrecondition::Any,
2678 None,
2679 );
2680 tx.commit().map_err(|err| {
2681 ProtocolError::InvalidState(format!("update Git ref {}: {err}", update.name))
2682 })?;
2683 if let Some(checkpoint) = update.checkpoint {
2684 record_git_lane_checkpoint(repo, git_repo, checkpoint)?;
2685 }
2686 Ok(())
2687}
2688
2689fn record_git_lane_checkpoint(
2690 repo: &Repository,
2691 git_repo: &SleyRepository,
2692 checkpoint: GitCheckpointTransfer,
2693) -> Result<(), ProtocolError> {
2694 let state = checkpoint
2695 .heddle_state_id
2696 .as_ref()
2697 .ok_or_else(|| {
2698 ProtocolError::InvalidState(
2699 "GitCheckpointTransfer.heddle_state_id is required".to_string(),
2700 )
2701 })
2702 .and_then(|state| {
2703 StateId::try_from_slice(&state.value)
2704 .map_err(|err| ProtocolError::InvalidState(err.to_string()))
2705 })?;
2706 let commit_oid = git_oid_from_proto(
2707 git_repo,
2708 "GitCheckpointTransfer.git_commit_oid",
2709 checkpoint.git_commit_oid.as_ref(),
2710 )?;
2711 let commit_hex = commit_oid.to_hex();
2712 if repo
2713 .latest_git_checkpoint_for_state(&state)
2714 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
2715 .is_some_and(|record| record.git_commit == commit_hex)
2716 {
2717 return Ok(());
2718 }
2719 let summary = serde_json::from_str::<serde_json::Value>(&checkpoint.metadata_json)
2720 .ok()
2721 .and_then(|metadata| {
2722 metadata
2723 .get("summary")
2724 .and_then(serde_json::Value::as_str)
2725 .map(str::to_string)
2726 })
2727 .unwrap_or_else(|| "pulled Git checkpoint".to_string());
2728 repo.record_git_checkpoint(&state, commit_hex, summary)
2729 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
2730 Ok(())
2731}
2732
2733fn git_lane_sley_repository<'a>(
2734 repo: &Repository,
2735 git_repo: &'a mut Option<SleyRepository>,
2736) -> Result<&'a SleyRepository, ProtocolError> {
2737 if git_repo.is_none() {
2738 *git_repo = Some(
2739 repo.git_overlay_sley_repository()
2740 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
2741 .ok_or_else(|| {
2742 ProtocolError::InvalidState(
2743 "git-overlay repository has no Git store".to_string(),
2744 )
2745 })?,
2746 );
2747 }
2748 match git_repo.as_ref() {
2749 Some(git_repo) => Ok(git_repo),
2750 None => Err(ProtocolError::InvalidState(
2751 "git-overlay repository has no Git store".to_string(),
2752 )),
2753 }
2754}
2755
2756fn git_oid_from_bytes(
2757 git_repo: &SleyRepository,
2758 field: &str,
2759 bytes: &[u8],
2760) -> Result<GitObjectId, ProtocolError> {
2761 GitObjectId::from_raw(git_repo.object_format(), bytes)
2762 .map_err(|err| ProtocolError::InvalidState(format!("{field}: {err}")))
2763}
2764
2765fn git_oid_from_proto(
2766 git_repo: &SleyRepository,
2767 field: &str,
2768 oid: Option<&ProtoGitObjectId>,
2769) -> Result<GitObjectId, ProtocolError> {
2770 let oid = oid.ok_or_else(|| ProtocolError::InvalidState(format!("{field} is required")))?;
2771 git_oid_from_bytes(git_repo, field, &oid.digest)
2772}
2773
2774fn proto_git_oid(oid: &GitObjectId) -> Option<ProtoGitObjectId> {
2775 proto_git_oid_bytes(oid.as_bytes())
2776}
2777
2778fn proto_git_oid_bytes(bytes: &[u8]) -> Option<ProtoGitObjectId> {
2779 let algorithm = match bytes.len() {
2780 20 => GitObjectAlgorithm::Sha1,
2781 32 => GitObjectAlgorithm::Sha256,
2782 _ => return None,
2783 };
2784 Some(ProtoGitObjectId {
2785 algorithm: algorithm as i32,
2786 digest: bytes.to_vec(),
2787 })
2788}
2789
2790#[derive(Clone, Copy)]
2791struct PushWireIdentities<'a> {
2792 transfer_id: &'a str,
2793 client_operation_id: &'a str,
2794}
2795
2796async fn send_native_pack_streaming_messages(
2797 tx: &mpsc::Sender<PushClientFrame>,
2798 repo: &Repository,
2799 objects: &[ObjectInfo],
2800 identities: PushWireIdentities<'_>,
2801 chunk_size: usize,
2802 transport: &super::helpers::HostedTransportPolicy,
2803 transport_mode: TransportMode,
2804) -> Result<(), ProtocolError> {
2805 let object_count = u64::try_from(objects.len()).map_err(|_| {
2806 ProtocolError::InvalidState("native pack object count exceeds u64".to_string())
2807 })?;
2808 let mut writer = wire::NativePackStreamingWriter::new_in(repo.heddle_dir(), object_count)?;
2809 let mut pack_reader = wire::GrowingPackChunkReader::open(writer.pack_path(), chunk_size)?;
2810 let (loaded_tx, mut loaded_rx) = mpsc::channel::<(
2811 usize,
2812 Result<wire::ObjectData, ProtocolError>,
2813 )>(NATIVE_PACK_OBJECT_PREFETCH_LIMIT);
2814 let store = repo.store().clone();
2815 let object_plan = objects.to_vec();
2816 let loader = tokio::task::spawn_blocking(move || {
2817 load_native_pack_objects_parallel(store, object_plan, loaded_tx);
2818 });
2819
2820 let mut next_index = 0usize;
2821 let mut pending = BTreeMap::new();
2822 while next_index < objects.len() {
2823 let (index, object) = loaded_rx.recv().await.ok_or_else(|| {
2824 ProtocolError::InvalidState(
2825 "native pack object loader stopped before sending all objects".to_string(),
2826 )
2827 })?;
2828 pending.insert(index, object);
2829
2830 while let Some(object) = pending.remove(&next_index) {
2831 let object = object?;
2832 let should_drain = object.data.len() >= chunk_size
2833 || (next_index + 1).is_multiple_of(NATIVE_PACK_DRAIN_OBJECT_INTERVAL);
2834 writer.add_object_data(object)?;
2835 if should_drain {
2836 writer.flush_pack()?;
2837 drain_growing_native_pack_stream(
2838 tx,
2839 &mut pack_reader,
2840 false,
2841 PackStreamKind::Pack,
2842 identities,
2843 transport,
2844 transport_mode,
2845 )
2846 .await?;
2847 }
2848 next_index += 1;
2849 }
2850 }
2851 loader.await.map_err(|err| {
2852 ProtocolError::InvalidState(format!("native pack object loader task failed: {err}"))
2853 })?;
2854
2855 let bundle = writer.finish()?;
2856 drain_growing_native_pack_stream(
2857 tx,
2858 &mut pack_reader,
2859 true,
2860 PackStreamKind::Pack,
2861 identities,
2862 transport,
2863 transport_mode,
2864 )
2865 .await?;
2866 send_native_pack_file_stream(
2867 tx,
2868 &bundle.index_path,
2869 PackStreamKind::Index,
2870 identities,
2871 chunk_size,
2872 transport,
2873 transport_mode,
2874 )
2875 .await
2876}
2877
2878fn load_native_pack_objects_parallel(
2879 store: AnyStore,
2880 objects: Vec<ObjectInfo>,
2881 tx: mpsc::Sender<(usize, Result<wire::ObjectData, ProtocolError>)>,
2882) {
2883 if objects.is_empty() {
2884 return;
2885 }
2886 let worker_count = native_pack_object_load_worker_count(objects.len());
2887 let objects = Arc::new(objects);
2888 let next_index = Arc::new(AtomicUsize::new(0));
2889
2890 std::thread::scope(|scope| {
2891 for _ in 0..worker_count {
2892 let store = store.clone();
2893 let objects = Arc::clone(&objects);
2894 let next_index = Arc::clone(&next_index);
2895 let tx = tx.clone();
2896 scope.spawn(move || {
2897 loop {
2898 let index = next_index.fetch_add(1, Ordering::Relaxed);
2899 let Some(info) = objects.get(index) else {
2900 break;
2901 };
2902 let object = wire::load_object_data(&store, &info.id, info.obj_type);
2903 if tx.blocking_send((index, object)).is_err() {
2904 break;
2905 }
2906 }
2907 });
2908 }
2909 });
2910}
2911
2912fn native_pack_object_load_worker_count(object_count: usize) -> usize {
2913 let available = std::thread::available_parallelism()
2914 .map(usize::from)
2915 .unwrap_or(1);
2916 object_count
2917 .min(available)
2918 .clamp(1, NATIVE_PACK_OBJECT_LOAD_WORKER_LIMIT)
2919}
2920
2921async fn drain_growing_native_pack_stream(
2922 tx: &mpsc::Sender<PushClientFrame>,
2923 reader: &mut wire::GrowingPackChunkReader,
2924 final_stream: bool,
2925 stream_kind: PackStreamKind,
2926 identities: PushWireIdentities<'_>,
2927 transport: &super::helpers::HostedTransportPolicy,
2928 transport_mode: TransportMode,
2929) -> Result<(), ProtocolError> {
2930 while let Some((offset, chunk_index, data, is_final_chunk)) =
2931 reader.next_available_chunk(final_stream)?
2932 {
2933 send_pack_chunk(
2934 tx,
2935 stream_kind,
2936 data,
2937 identities,
2938 transport,
2939 transport_mode,
2940 chunk_index,
2941 offset,
2942 is_final_chunk,
2943 )
2944 .await?;
2945 }
2946 Ok(())
2947}
2948
2949async fn send_native_pack_file_stream(
2950 tx: &mpsc::Sender<PushClientFrame>,
2951 path: &std::path::Path,
2952 stream_kind: PackStreamKind,
2953 identities: PushWireIdentities<'_>,
2954 chunk_size: usize,
2955 transport: &super::helpers::HostedTransportPolicy,
2956 transport_mode: TransportMode,
2957) -> Result<(), ProtocolError> {
2958 let mut reader = wire::PackFileChunkReader::open(path, chunk_size)?;
2959 while let Some((offset, chunk_index, data, is_final_chunk)) = reader.next_chunk()? {
2960 send_pack_chunk(
2961 tx,
2962 stream_kind,
2963 data,
2964 identities,
2965 transport,
2966 transport_mode,
2967 chunk_index,
2968 offset,
2969 is_final_chunk,
2970 )
2971 .await?;
2972 }
2973 Ok(())
2974}
2975
2976#[allow(clippy::too_many_arguments)]
2977async fn send_pack_chunk(
2978 tx: &mpsc::Sender<PushClientFrame>,
2979 stream_kind: PackStreamKind,
2980 data: Vec<u8>,
2981 identities: PushWireIdentities<'_>,
2982 transport: &super::helpers::HostedTransportPolicy,
2983 transport_mode: TransportMode,
2984 chunk_index: u32,
2985 offset: u64,
2986 is_final_chunk: bool,
2987) -> Result<(), ProtocolError> {
2988 let chunk_length = data.len().min(u32::MAX as usize) as u32;
2989 tx.send(PushClientFrame {
2990 frame: Some(push_client_frame::Frame::Pack(PackChunk {
2991 stream_kind: stream_kind as i32,
2992 data,
2993 transfer: Some(transport.transfer_checkpoint_with_mode(
2994 identities.transfer_id,
2995 transport_mode,
2996 chunk_index,
2997 offset,
2998 is_final_chunk,
2999 )),
3000 chunk_length,
3001 is_final_chunk,
3002 })),
3003 client_operation_id: identities.client_operation_id.to_string(),
3004 })
3005 .await
3006 .map_err(|_| ProtocolError::InvalidState("push stream closed unexpectedly".to_string()))
3007}
3008
3009fn preferred_transport_mode(
3010 transport: &super::helpers::HostedTransportPolicy,
3011 object_count: usize,
3012) -> TransportMode {
3013 let _ = transport;
3014 let _ = object_count;
3015 TransportMode::NativePack
3016}
3017
3018#[cfg(test)]
3019mod tests {
3020 use std::collections::HashSet;
3021
3022 use chrono::{TimeZone, Utc};
3023 use cli_shared::ClientConfig;
3024 use grpc::heddle::api::v1alpha1::{
3025 ListRefsRequest, ListRefsResponse, PullComplete as GrpcPullComplete, PullReady,
3026 PullServerFrame, PushServerFrame, StateId as ProtoStateId, TransferCheckpoint,
3027 UpdateRefRequest, UpdateRefResponse, pull_server_frame, push_client_frame,
3028 repo_sync_service_server::{RepoSyncService, RepoSyncServiceServer},
3029 };
3030 use objects::{
3031 object::{
3032 Attribution, Blob, ContentHash, Principal, Redaction, State, StateId, StateVisibility,
3033 StateVisibilityBlob, Tree, TreeEntry, VisibilityTier,
3034 },
3035 store::ObjectStore,
3036 };
3037 use tempfile::TempDir;
3038 use tonic::{Response, Status, transport::Server};
3039 use wire::{ObjectId, ObjectInfo};
3040
3041 use super::*;
3042 use crate::grpc_hosted::helpers::{
3043 descriptor_id_from_info, proto_state_id, to_proto_object_info,
3044 };
3045
3046 fn temp_repo() -> (TempDir, Repository) {
3047 let dir = TempDir::new().expect("tempdir");
3048 let repo = Repository::init_default(dir.path()).expect("init repo");
3049 (dir, repo)
3050 }
3051
3052 fn proto_oid_bytes(oid: &Option<ProtoGitObjectId>) -> Option<&[u8]> {
3053 oid.as_ref().map(|oid| oid.digest.as_slice())
3054 }
3055
3056 fn signed_test_config() -> ClientConfig {
3057 let signer = crypto::Ed25519Signer::generate().expect("generate test signing identity");
3058 ClientConfig::default()
3059 .with_auth_proof_key_pem(signer.to_pem().expect("export test key"))
3060 .with_authenticated_principal("principal:alice")
3061 }
3062
3063 fn temp_repo_unseeded() -> (TempDir, Repository) {
3067 let dir = TempDir::new().expect("tempdir");
3068 let repo = Repository::init(dir.path()).expect("init repo");
3069 (dir, repo)
3070 }
3071
3072 fn redaction_info(blob: ContentHash) -> ObjectInfo {
3073 ObjectInfo {
3074 id: ObjectId::Hash(blob),
3075 obj_type: ObjectType::Redaction,
3076 size: 0,
3077 delta_base: None,
3078 }
3079 }
3080
3081 fn state_info(state: StateId) -> ObjectInfo {
3082 ObjectInfo {
3083 id: ObjectId::StateId(state),
3084 obj_type: ObjectType::State,
3085 size: 0,
3086 delta_base: None,
3087 }
3088 }
3089
3090 fn state_visibility_info(state: StateId) -> ObjectInfo {
3091 ObjectInfo {
3092 id: ObjectId::StateId(state),
3093 obj_type: ObjectType::StateVisibility,
3094 size: 0,
3095 delta_base: None,
3096 }
3097 }
3098
3099 fn sample_blob() -> ContentHash {
3100 ContentHash::from_bytes([7u8; 32])
3101 }
3102
3103 #[test]
3104 fn descriptor_id_from_info_matches_proto_encode_path() {
3105 let infos = [
3106 redaction_info(sample_blob()),
3107 state_info(StateId::from_bytes([3u8; 32])),
3108 state_visibility_info(StateId::from_bytes([9u8; 32])),
3109 ];
3110 for info in infos {
3111 assert_eq!(
3112 descriptor_id_from_info(&info),
3113 descriptor_id(&to_proto_object_info(&info)),
3114 "keying path must stay byte-identical to the throwaway-encode path",
3115 );
3116 }
3117 }
3118
3119 #[test]
3120 fn push_operation_identity_is_not_the_resumable_transfer_identity() {
3121 let transfer_id = push_transfer_id(
3122 "acme/widgets",
3123 StateId::from_bytes([5; 32]),
3124 "feature/retry",
3125 );
3126 let operation_id =
3127 ClientOperationId::caller_or_fresh("heddle.api.v1alpha1.RepoSyncService/Push", "");
3128 let next_operation_id =
3129 ClientOperationId::caller_or_fresh("heddle.api.v1alpha1.RepoSyncService/Push", "");
3130
3131 assert_ne!(operation_id.as_str(), transfer_id);
3132 assert_ne!(operation_id, next_operation_id);
3133 }
3134
3135 #[tokio::test]
3136 async fn push_and_pull_opening_proofs_are_bound_to_the_authenticated_principal() {
3137 use crypto::Signer as _;
3138
3139 let config = signed_test_config();
3140 let signer = crypto::Ed25519Signer::from_pem(
3141 config
3142 .auth_proof_key_pem
3143 .as_deref()
3144 .expect("test proof key"),
3145 )
3146 .expect("test signer");
3147 let channel = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
3148 let client = HostedGrpcClient::from_channel(channel, &config).expect("test client");
3149 for (stream_id, route) in [
3150 (
3151 "push-transfer-1",
3152 "/heddle.api.v1alpha1.RepoSyncService/Push",
3153 ),
3154 (
3155 "pull-transfer-1",
3156 "/heddle.api.v1alpha1.RepoSyncService/Pull",
3157 ),
3158 ] {
3159 let proof = client
3160 .stream_opening_proof(stream_id, route, "acme/widgets", "")
3161 .expect("opening proof");
3162 let canonical = grpc::signing::stream_open_bytes(
3163 "principal:alice",
3164 &proof.stream_id,
3165 &proof.route,
3166 "acme/widgets",
3167 &proof.resume_cursor,
3168 &proof.capability_context,
3169 );
3170
3171 crypto::verify_payload_signature(
3172 &canonical,
3173 "ed25519",
3174 signer.public_key(),
3175 &proof.signature,
3176 )
3177 .expect("stream proof must verify under the contract principal identity");
3178 }
3179
3180 let implementation = include_str!("sync.rs")
3181 .split("#[cfg(test)]")
3182 .next()
3183 .expect("sync implementation section");
3184 assert_eq!(
3185 implementation.matches("self.stream_opening_proof(").count(),
3186 2,
3187 "Push and Pull must both route their opening frame through the shared proof chokepoint"
3188 );
3189 }
3190
3191 #[test]
3192 fn git_lane_messages_pack_reachable_commit_graph_and_attach_checkpoint() {
3193 let dir = TempDir::new().expect("tempdir");
3194 let git = sley::Repository::init(dir.path()).expect("init git");
3195 let blob_oid = git.write_blob(b"hello\n").expect("write blob");
3196 let tree = sley::TreeObject {
3197 entries: vec![sley::plumbing::sley_object::TreeEntry {
3198 mode: 0o100644,
3199 name: sley::BString::from(b"hello.txt"),
3200 oid: blob_oid,
3201 }],
3202 };
3203 let tree_oid = git
3204 .write_raw_object(sley::GitObjectType::Tree, tree.write())
3205 .expect("write tree");
3206 let commit = sley::CommitObject {
3207 tree: tree_oid,
3208 parents: Vec::new(),
3209 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3210 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3211 encoding: None,
3212 message: b"checkpoint\n".to_vec(),
3213 };
3214 let commit_oid = git
3215 .write_raw_object(sley::GitObjectType::Commit, commit.write())
3216 .expect("write commit");
3217
3218 let pack =
3219 build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64 * 1024)
3220 .expect("build git lane pack plan")
3221 .expect("non-empty pack plan");
3222 assert_eq!(pack.pack_id.len(), git.object_format().raw_len());
3223 let (tx, mut rx) = mpsc::channel(8);
3224 stream_git_pack_messages_blocking(
3225 tx,
3226 pack.clone(),
3227 64 * 1024,
3228 Progress::null(),
3229 "test-push-op".to_string(),
3230 )
3231 .expect("stream git lane pack");
3232 let mut pack_bytes = Vec::new();
3233 let mut chunks = Vec::new();
3234 while let Some(chunk) = rx.blocking_recv() {
3235 assert_eq!(chunk.client_operation_id, "test-push-op");
3236 let Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
3237 body: Some(git_lane_transfer::Body::Pack(pack)),
3238 })) = chunk.frame
3239 else {
3240 panic!("expected git pack chunk");
3241 };
3242 pack_bytes.extend_from_slice(&pack.pack_chunk);
3243 chunks.push(pack);
3244 }
3245 assert!(!chunks.is_empty());
3246 assert!(chunks.last().expect("pack chunk").is_final_chunk);
3247 assert_eq!(pack_bytes.len() as u64, pack.pack_size);
3248 let indexed = sley::plumbing::sley_odb::index_raw_pack(&pack_bytes, git.object_format())
3249 .expect("generated pack should index");
3250 assert_eq!(indexed.pack_id.as_bytes(), pack.pack_id.as_slice());
3251 assert_eq!(indexed.objects.len(), 3);
3252
3253 let state = StateId::from_bytes([9u8; 32]);
3254 let ref_message = git_ref_update_message(
3255 "refs/heads/main",
3256 GrpcGitRefKind::Branch,
3257 commit_oid,
3258 None,
3259 Some(GitCheckpointTransfer {
3260 heddle_state_id: proto_state_id(state),
3261 git_commit_oid: proto_git_oid(&commit_oid),
3262 thread: "main".to_string(),
3263 metadata_json: String::new(),
3264 }),
3265 );
3266 let update = ref_message;
3267 assert_eq!(update.name, "refs/heads/main");
3268 assert_eq!(update.kind, GrpcGitRefKind::Branch as i32);
3269 assert_eq!(
3270 update.target_oid.as_ref().map(|oid| oid.digest.as_slice()),
3271 Some(commit_oid.as_bytes())
3272 );
3273 let checkpoint = update.checkpoint.expect("checkpoint");
3274 assert_eq!(
3275 checkpoint
3276 .heddle_state_id
3277 .as_ref()
3278 .map(|state| state.value.as_slice()),
3279 Some(state.as_bytes().as_slice())
3280 );
3281 assert_eq!(
3282 checkpoint
3283 .git_commit_oid
3284 .as_ref()
3285 .map(|oid| oid.digest.as_slice()),
3286 Some(commit_oid.as_bytes())
3287 );
3288 assert_eq!(checkpoint.thread, "main");
3289 }
3290
3291 fn sample_ref_update_message(commit_oid: GitObjectId) -> PushClientFrame {
3292 git_lane_push_message(
3293 git_lane_transfer::Body::RefUpdate(git_ref_update_message(
3294 "refs/heads/main",
3295 GrpcGitRefKind::Branch,
3296 commit_oid,
3297 None,
3298 Some(GitCheckpointTransfer {
3299 heddle_state_id: proto_state_id(StateId::from_bytes([9u8; 32])),
3300 git_commit_oid: proto_git_oid(&commit_oid),
3301 thread: "main".to_string(),
3302 metadata_json: String::new(),
3303 }),
3304 )),
3305 "test-push-op",
3306 )
3307 }
3308
3309 fn write_commit(git: &sley::Repository, seed: &str) -> GitObjectId {
3312 let blob_oid = git
3313 .write_blob(format!("content-{seed}\n").as_bytes())
3314 .expect("write blob");
3315 let tree = sley::TreeObject {
3316 entries: vec![sley::plumbing::sley_object::TreeEntry {
3317 mode: 0o100644,
3318 name: sley::BString::from(format!("{seed}.txt").into_bytes()),
3319 oid: blob_oid,
3320 }],
3321 };
3322 let tree_oid = git
3323 .write_raw_object(sley::GitObjectType::Tree, tree.write())
3324 .expect("write tree");
3325 let commit = sley::CommitObject {
3326 tree: tree_oid,
3327 parents: Vec::new(),
3328 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3329 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3330 encoding: None,
3331 message: format!("commit {seed}\n").into_bytes(),
3332 };
3333 git.write_raw_object(sley::GitObjectType::Commit, commit.write())
3334 .expect("write commit")
3335 }
3336
3337 fn set_ref(git: &mut sley::Repository, name: &str, target: GitObjectId) {
3338 let refs = git.references();
3339 let mut tx = refs.transaction();
3340 tx.update_to(
3341 name.to_string(),
3342 ReferenceTarget::Direct(target),
3343 RefPrecondition::Any,
3344 None,
3345 );
3346 tx.commit().expect("commit ref");
3347 }
3348
3349 fn mirror_ref_updates(plan: &GitLanePushPlan) -> Vec<GitRefUpdateTransfer> {
3350 plan.ref_updates.clone()
3351 }
3352
3353 fn write_child_commit(git: &sley::Repository, parent: GitObjectId, seed: &str) -> GitObjectId {
3357 let blob_oid = git
3358 .write_blob(format!("content-{seed}\n").as_bytes())
3359 .expect("write blob");
3360 let tree = sley::TreeObject {
3361 entries: vec![sley::plumbing::sley_object::TreeEntry {
3362 mode: 0o100644,
3363 name: sley::BString::from(format!("{seed}.txt").into_bytes()),
3364 oid: blob_oid,
3365 }],
3366 };
3367 let tree_oid = git
3368 .write_raw_object(sley::GitObjectType::Tree, tree.write())
3369 .expect("write tree");
3370 let commit = sley::CommitObject {
3371 tree: tree_oid,
3372 parents: vec![parent],
3373 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3374 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3375 encoding: None,
3376 message: format!("commit {seed}\n").into_bytes(),
3377 };
3378 git.write_raw_object(sley::GitObjectType::Commit, commit.write())
3379 .expect("write commit")
3380 }
3381
3382 fn value_expectation(oid: GitObjectId) -> GitRefRemoteExpectation {
3383 GitRefRemoteExpectation::Value(oid.as_bytes().to_vec())
3384 }
3385
3386 #[test]
3395 fn git_mirror_plan_packs_only_wanted_objects_against_have_boundary() {
3396 let dir = TempDir::new().expect("tempdir");
3397 let mut git = sley::Repository::init(dir.path()).expect("init git");
3398
3399 let base = write_commit(&git, "base-with-a-reasonably-large-payload");
3402 let mut tip = base;
3403 for i in 0..8 {
3404 tip = write_child_commit(&git, tip, &format!("layer-{i}"));
3405 }
3406 let descendant = write_child_commit(&git, tip, "descendant-adds-one-object");
3407 set_ref(&mut git, "refs/heads/main", descendant);
3408
3409 let full =
3412 build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new()).expect("full plan");
3413 let full_pack = full.pack.as_ref().expect("fresh server produces a pack");
3414 let full_size = full_pack.pack_size;
3415 assert!(full_size > 0, "baseline full pack must be non-empty");
3416
3417 let mut have_tip = HashMap::new();
3421 have_tip.insert("refs/heads/main".to_string(), value_expectation(tip));
3422 let delta =
3423 build_git_mirror_plan_from_sley(&git, 64 * 1024, &have_tip).expect("delta plan");
3424 let delta_pack = delta
3425 .pack
3426 .as_ref()
3427 .expect("descendant still has one new object to pack");
3428 let delta_size = delta_pack.pack_size;
3429 assert!(
3430 delta_size * 4 < full_size,
3431 "want-only packing must be dramatically smaller than a full re-pack \
3432 (delta {delta_size} bytes vs full {full_size} bytes)",
3433 );
3434
3435 let mut have_target = HashMap::new();
3439 have_target.insert("refs/heads/main".to_string(), value_expectation(descendant));
3440 let ref_only =
3441 build_git_mirror_plan_from_sley(&git, 64 * 1024, &have_target).expect("ref-only plan");
3442 assert!(
3443 ref_only.pack.is_none(),
3444 "pushing a ref the server already has must send NO pack",
3445 );
3446 assert_eq!(
3448 ref_only.ref_updates.len(),
3449 1,
3450 "the ref update still ships even with an empty want set",
3451 );
3452 }
3453
3454 #[test]
3457 fn git_lane_reachable_pack_plan_matches_legacy_writer_checksum() {
3458 let dir = TempDir::new().expect("tempdir");
3459 let git = sley::Repository::init(dir.path()).expect("init git");
3460 let commit_oid = write_commit(&git, "byte-identity");
3461
3462 let pack_plan =
3463 build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64 * 1024)
3464 .expect("build pack plan")
3465 .expect("non-empty pack plan");
3466
3467 let mut legacy_pack = Vec::new();
3468 let legacy = sley::plumbing::sley_odb::write_reachable_pack_to_writer(
3469 git.objects().as_ref(),
3470 git.object_format(),
3471 std::iter::once(commit_oid),
3472 &HashSet::new(),
3473 &mut legacy_pack,
3474 )
3475 .expect("legacy reachable pack")
3476 .expect("legacy pack summary");
3477
3478 assert_eq!(pack_plan.pack_id, legacy.checksum.as_bytes().to_vec());
3479 assert_eq!(pack_plan.pack_size, legacy.pack_size);
3480 assert_eq!(pack_plan.pack_size, legacy_pack.len() as u64);
3481
3482 let mut planned_pack = Vec::new();
3483 pack_plan
3484 .pack_file
3485 .lock()
3486 .expect("lock pack tempfile")
3487 .seek(SeekFrom::Start(0))
3488 .expect("rewind pack tempfile");
3489 io::copy(
3490 &mut pack_plan
3491 .pack_file
3492 .lock()
3493 .expect("lock pack tempfile")
3494 .as_file_mut(),
3495 &mut planned_pack,
3496 )
3497 .expect("read planned pack");
3498 assert_eq!(planned_pack, legacy_pack);
3499 }
3500
3501 #[test]
3505 fn git_mirror_plan_builds_one_pack_and_checkpointless_updates_for_all_refs() {
3506 let dir = TempDir::new().expect("tempdir");
3507 let mut git = sley::Repository::init(dir.path()).expect("init git");
3508
3509 let main_commit = write_commit(&git, "main");
3510 let feature_commit = write_commit(&git, "feature");
3511 let tagged_commit = write_commit(&git, "tagged");
3512
3513 set_ref(&mut git, "refs/heads/main", main_commit);
3514 set_ref(&mut git, "refs/heads/feature", feature_commit);
3515
3516 let tag = sley::TagObject {
3518 object: tagged_commit,
3519 object_type: sley::GitObjectType::Commit,
3520 name: b"v1".to_vec(),
3521 tagger: Some(b"Tester <test@example.com> 1700000000 +0000".to_vec()),
3522 message: b"release v1\n".to_vec(),
3523 raw_body: None,
3524 };
3525 let tag_oid = git
3526 .write_raw_object(sley::GitObjectType::Tag, tag.write())
3527 .expect("write tag");
3528 set_ref(&mut git, "refs/tags/v1", tag_oid);
3529
3530 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3531 .expect("build mirror plan");
3532
3533 let updates = mirror_ref_updates(&plan);
3534 assert_eq!(updates.len(), 3, "one ref update per direct ref");
3535
3536 assert!(
3539 updates.iter().all(|u| u.checkpoint.is_none()),
3540 "mirror mode signals via checkpoint: None on every ref update",
3541 );
3542
3543 let by_name: HashMap<&str, &GitRefUpdateTransfer> =
3544 updates.iter().map(|u| (u.name.as_str(), u)).collect();
3545
3546 let main = by_name["refs/heads/main"];
3547 assert_eq!(main.kind, GrpcGitRefKind::Branch as i32);
3548 assert_eq!(
3549 proto_oid_bytes(&main.target_oid),
3550 Some(main_commit.as_bytes())
3551 );
3552 assert!(main.peeled_oid.is_none(), "commit refs are not peeled");
3553
3554 let feature = by_name["refs/heads/feature"];
3555 assert_eq!(feature.kind, GrpcGitRefKind::Branch as i32);
3556 assert_eq!(
3557 proto_oid_bytes(&feature.target_oid),
3558 Some(feature_commit.as_bytes())
3559 );
3560
3561 let tag_update = by_name["refs/tags/v1"];
3562 assert_eq!(tag_update.kind, GrpcGitRefKind::Tag as i32);
3563 assert_eq!(
3564 proto_oid_bytes(&tag_update.target_oid),
3565 Some(tag_oid.as_bytes())
3566 );
3567 assert_eq!(
3568 proto_oid_bytes(&tag_update.peeled_oid),
3569 Some(tagged_commit.as_bytes()),
3570 "annotated tag ref is peeled to its underlying commit",
3571 );
3572
3573 let plan_pack = plan.pack.as_ref().expect("full pack for fresh server");
3577 let mut pack_roots: Vec<GitObjectId> = plan_pack.roots.clone();
3578 pack_roots.sort_by_key(|oid| oid.to_hex());
3579 let mut expected_roots = vec![main_commit, feature_commit, tag_oid];
3580 expected_roots.sort_by_key(|oid| oid.to_hex());
3581 assert_eq!(
3582 pack_roots, expected_roots,
3583 "pack roots == resolved ref targets",
3584 );
3585
3586 let (tx, mut rx) = mpsc::channel(64);
3589 stream_git_pack_messages_blocking(
3590 tx,
3591 plan_pack.clone(),
3592 64 * 1024,
3593 Progress::null(),
3594 "test-push-op".to_string(),
3595 )
3596 .expect("stream mirror pack");
3597 let mut pack_bytes = Vec::new();
3598 while let Some(message) = rx.blocking_recv() {
3599 if let Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
3600 body: Some(git_lane_transfer::Body::Pack(pack)),
3601 })) = message.frame
3602 {
3603 pack_bytes.extend_from_slice(&pack.pack_chunk);
3604 }
3605 }
3606 assert_eq!(
3607 pack_bytes.len() as u64,
3608 plan_pack.pack_size,
3609 "streamed pack size must match plan",
3610 );
3611 let indexed = sley::plumbing::sley_odb::index_raw_pack(&pack_bytes, git.object_format())
3612 .expect("mirror pack indexes");
3613 let packed: HashSet<Vec<u8>> = indexed
3614 .objects
3615 .iter()
3616 .map(|obj| obj.oid.as_bytes().to_vec())
3617 .collect();
3618 for oid in [main_commit, feature_commit, tagged_commit, tag_oid] {
3619 assert!(
3620 packed.contains(oid.as_bytes()),
3621 "pack must contain {}",
3622 oid.to_hex()
3623 );
3624 }
3625 }
3626
3627 #[test]
3630 fn git_mirror_plan_skips_symbolic_refs() {
3631 let dir = TempDir::new().expect("tempdir");
3632 let mut git = sley::Repository::init(dir.path()).expect("init git");
3633 let main_commit = write_commit(&git, "main");
3634 set_ref(&mut git, "refs/heads/main", main_commit);
3635 {
3638 let refs = git.references();
3639 let mut tx = refs.transaction();
3640 tx.update_to(
3641 "HEAD".to_string(),
3642 ReferenceTarget::Symbolic("refs/heads/main".to_string()),
3643 RefPrecondition::Any,
3644 None,
3645 );
3646 tx.commit().expect("set HEAD");
3647 }
3648
3649 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3650 .expect("build mirror plan");
3651 let updates = mirror_ref_updates(&plan);
3652 assert_eq!(updates.len(), 1, "only the direct ref is pushed");
3653 assert_eq!(updates[0].name, "refs/heads/main");
3654 }
3655
3656 #[test]
3659 fn git_mirror_plan_applies_per_ref_remote_expectations() {
3660 let dir = TempDir::new().expect("tempdir");
3661 let mut git = sley::Repository::init(dir.path()).expect("init git");
3662 let main_commit = write_commit(&git, "main");
3663 let feature_commit = write_commit(&git, "feature");
3664 set_ref(&mut git, "refs/heads/main", main_commit);
3665 set_ref(&mut git, "refs/heads/feature", feature_commit);
3666
3667 let remote_oid = "89abcdef012345670123456789abcdef01234567";
3668 let mut expectations = HashMap::new();
3669 expectations.insert(
3670 "refs/heads/main".to_string(),
3671 GitRefRemoteExpectation::Value(hex::decode(remote_oid).expect("hex")),
3672 );
3673 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &expectations)
3676 .expect("build mirror plan");
3677 let updates = mirror_ref_updates(&plan);
3678 let by_name: HashMap<&str, &GitRefUpdateTransfer> =
3679 updates.iter().map(|u| (u.name.as_str(), u)).collect();
3680
3681 let main = by_name["refs/heads/main"];
3682 assert!(!main.expected_missing);
3683 assert_eq!(
3684 hex::encode(proto_oid_bytes(&main.expected_target_oid).expect("expected oid")),
3685 remote_oid
3686 );
3687
3688 let feature = by_name["refs/heads/feature"];
3689 assert!(
3690 feature.expected_missing,
3691 "ref absent from ListRefs is expected-missing (create)",
3692 );
3693 }
3694
3695 #[test]
3699 fn git_mirror_plan_includes_pull_request_refs() {
3700 let dir = TempDir::new().expect("tempdir");
3701 let mut git = sley::Repository::init(dir.path()).expect("init git");
3702 let pr_commit = write_commit(&git, "pr");
3703 set_ref(&mut git, "refs/pull/42/head", pr_commit);
3704
3705 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3706 .expect("build mirror plan");
3707 let updates = mirror_ref_updates(&plan);
3708 let pull = updates
3709 .iter()
3710 .find(|update| update.name == "refs/pull/42/head")
3711 .expect("refs/pull/* ref must be mirrored");
3712 assert_eq!(
3713 pull.kind,
3714 GrpcGitRefKind::Other as i32,
3715 "refs/pull/* classifies as Other",
3716 );
3717 assert_eq!(
3718 proto_oid_bytes(&pull.target_oid),
3719 Some(pr_commit.as_bytes())
3720 );
3721 assert!(
3722 pull.checkpoint.is_none(),
3723 "mirror ref updates are checkpoint-less",
3724 );
3725 assert!(pull.peeled_oid.is_none(), "a commit ref is not peeled");
3726 }
3727
3728 #[test]
3733 fn git_mirror_plan_excludes_local_only_refs_and_keeps_content() {
3734 let dir = TempDir::new().expect("tempdir");
3735 let mut git = sley::Repository::init(dir.path()).expect("init git");
3736
3737 let main_commit = write_commit(&git, "main");
3738 let notes_commit = write_commit(&git, "notes");
3739 let stash_commit = write_commit(&git, "stash");
3740 let remote_commit = write_commit(&git, "remote");
3741 let original_commit = write_commit(&git, "original");
3742 let replace_commit = write_commit(&git, "replace");
3743
3744 set_ref(&mut git, "refs/heads/main", main_commit);
3746 set_ref(&mut git, "refs/notes/heddle", notes_commit);
3747
3748 set_ref(&mut git, "refs/stash", stash_commit);
3750 set_ref(&mut git, "refs/remotes/origin/main", remote_commit);
3751 set_ref(&mut git, "refs/original/refs/heads/main", original_commit);
3752 set_ref(&mut git, "refs/replace/deadbeef", replace_commit);
3753
3754 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3755 .expect("build mirror plan");
3756 let names: Vec<&str> = plan
3757 .ref_updates
3758 .iter()
3759 .map(|update| update.name.as_str())
3760 .collect();
3761
3762 assert!(
3763 names.contains(&"refs/heads/main"),
3764 "content branch is mirrored: {names:?}",
3765 );
3766 assert!(
3767 names.contains(&"refs/notes/heddle"),
3768 "heddle state-note ref is mirrored: {names:?}",
3769 );
3770 for excluded in [
3771 "refs/stash",
3772 "refs/remotes/origin/main",
3773 "refs/original/refs/heads/main",
3774 "refs/replace/deadbeef",
3775 ] {
3776 assert!(
3777 !names.contains(&excluded),
3778 "local-only ref {excluded} must NOT be mirrored: {names:?}",
3779 );
3780 }
3781 assert_eq!(
3782 names.len(),
3783 2,
3784 "exactly the two content refs ship: {names:?}"
3785 );
3786 }
3787
3788 #[test]
3793 fn git_mirror_plan_ignores_dangling_local_only_ref() {
3794 let dir = TempDir::new().expect("tempdir");
3795 let mut git = sley::Repository::init(dir.path()).expect("init git");
3796
3797 let main_commit = write_commit(&git, "main");
3798 set_ref(&mut git, "refs/heads/main", main_commit);
3799
3800 let missing = GitObjectId::from_hex(
3802 sley::ObjectFormat::Sha1,
3803 "0123456789abcdef0123456789abcdef01234567",
3804 )
3805 .expect("oid");
3806 set_ref(&mut git, "refs/original/refs/heads/main", missing);
3807
3808 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3809 .expect("dangling local-only ref must not fail the mirror plan");
3810 let names: Vec<&str> = plan
3811 .ref_updates
3812 .iter()
3813 .map(|update| update.name.as_str())
3814 .collect();
3815 assert_eq!(
3816 names,
3817 vec!["refs/heads/main"],
3818 "only content ships: {names:?}"
3819 );
3820 }
3821
3822 #[test]
3823 fn git_ref_expectation_marks_missing_when_remote_has_no_git_revision() {
3824 let commit_oid = GitObjectId::from_hex(
3825 sley::ObjectFormat::Sha1,
3826 "0123456789abcdef0123456789abcdef01234567",
3827 )
3828 .expect("oid");
3829 let message = sample_ref_update_message(commit_oid);
3830
3831 let expectation = parse_git_ref_expectation("").expect("missing expectation");
3832 let update = git_ref_update_from_message(&message);
3833 let mut update = update.clone();
3834 apply_git_ref_expectation_value(&mut update, &expectation);
3835 assert!(update.expected_missing);
3836 assert!(update.expected_target_oid.is_none());
3837 }
3838
3839 #[test]
3840 fn git_ref_expectation_uses_remote_git_revision_oid() {
3841 let commit_oid = GitObjectId::from_hex(
3842 sley::ObjectFormat::Sha1,
3843 "0123456789abcdef0123456789abcdef01234567",
3844 )
3845 .expect("oid");
3846 let remote_oid = "89abcdef012345670123456789abcdef01234567";
3847 let message = sample_ref_update_message(commit_oid);
3848
3849 let expectation =
3850 parse_git_ref_expectation(&format!("git:{remote_oid}")).expect("git expectation");
3851 let update = git_ref_update_from_message(&message);
3852 let mut update = update.clone();
3853 apply_git_ref_expectation_value(&mut update, &expectation);
3854 assert!(!update.expected_missing);
3855 assert_eq!(
3856 hex::encode(proto_oid_bytes(&update.expected_target_oid).expect("expected oid")),
3857 remote_oid
3858 );
3859 }
3860
3861 #[test]
3862 fn git_pack_stream_writer_emits_ordered_chunks() {
3863 let (tx, mut rx) = mpsc::channel(4);
3864 let mut writer = GitPackPushMessageWriter::new(
3865 tx,
3866 "git-pack:test".to_string(),
3867 vec![0x42; 20],
3868 10,
3869 4,
3870 Progress::null(),
3871 "test-push-op".to_string(),
3872 );
3873 writer.write_all(b"abcdefghij").expect("write pack bytes");
3874 writer.finish().expect("finish pack stream");
3875
3876 let mut chunks = Vec::new();
3877 while let Some(message) = rx.blocking_recv() {
3878 assert_eq!(message.client_operation_id, "test-push-op");
3879 let Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
3880 body: Some(git_lane_transfer::Body::Pack(pack)),
3881 })) = message.frame
3882 else {
3883 panic!("expected git pack chunk");
3884 };
3885 chunks.push(pack);
3886 }
3887
3888 assert_eq!(chunks.len(), 3);
3889 assert_eq!(chunks[0].offset, 0);
3890 assert_eq!(chunks[0].chunk_index, 0);
3891 assert!(!chunks[0].is_final_chunk);
3892 assert_eq!(chunks[0].pack_chunk.as_slice(), b"abcd");
3893 assert_eq!(chunks[1].offset, 4);
3894 assert_eq!(chunks[1].chunk_index, 1);
3895 assert!(!chunks[1].is_final_chunk);
3896 assert_eq!(chunks[1].pack_chunk.as_slice(), b"efgh");
3897 assert_eq!(chunks[2].offset, 8);
3898 assert_eq!(chunks[2].chunk_index, 2);
3899 assert!(chunks[2].is_final_chunk);
3900 assert_eq!(chunks[2].pack_chunk.as_slice(), b"ij");
3901 assert!(
3902 chunks
3903 .iter()
3904 .all(|chunk| chunk.pack_id.as_slice() == &[0x42; 20][..])
3905 );
3906 }
3907
3908 #[derive(Default)]
3911 struct PhaseCapturingSink {
3912 phases: std::sync::Mutex<Vec<String>>,
3913 }
3914
3915 impl PhaseCapturingSink {
3916 fn phases(&self) -> Vec<String> {
3917 self.phases.lock().unwrap().clone()
3918 }
3919 }
3920
3921 impl objects::Sink for PhaseCapturingSink {
3922 fn render(&self, snap: objects::ProgressSnapshot) {
3923 self.phases.lock().unwrap().push(snap.phase);
3924 }
3925 }
3926
3927 #[test]
3931 fn git_pack_stream_reports_upload_progress() {
3932 let dir = TempDir::new().expect("tempdir");
3933 let git = sley::Repository::init(dir.path()).expect("init git");
3934 let commit_oid = write_commit(&git, "progress");
3935 let pack = build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64)
3937 .expect("build pack plan")
3938 .expect("non-empty pack plan");
3939
3940 let sink = std::sync::Arc::new(PhaseCapturingSink::default());
3941 struct Forward(std::sync::Arc<PhaseCapturingSink>);
3942 impl objects::Sink for Forward {
3943 fn render(&self, snap: objects::ProgressSnapshot) {
3944 self.0.render(snap);
3945 }
3946 }
3947 let progress = Progress::with_sink(Box::new(Forward(std::sync::Arc::clone(&sink))));
3948
3949 let (tx, mut rx) = mpsc::channel(1024);
3950 stream_git_pack_messages_blocking(
3951 tx,
3952 pack.clone(),
3953 64,
3954 progress.clone(),
3955 "test-push-op".to_string(),
3956 )
3957 .expect("stream pack");
3958 while rx.blocking_recv().is_some() {}
3959
3960 let phases = sink.phases();
3961 assert!(
3962 phases.iter().any(|phase| phase.contains("uploading")),
3963 "pack streamer must drive an 'uploading' progress phase; saw {phases:?}",
3964 );
3965 let last_uploading = phases
3966 .iter()
3967 .rev()
3968 .find(|phase| phase.contains("uploading"))
3969 .cloned();
3970 assert!(
3971 last_uploading
3972 .as_deref()
3973 .is_some_and(|phase| phase.contains(&pack.pack_size.to_string())),
3974 "final uploading phase must show the full pack size ({} bytes); saw {last_uploading:?}",
3975 pack.pack_size,
3976 );
3977 }
3978
3979 #[test]
3980 fn git_pack_pull_install_state_streams_pack_into_sley_store() {
3981 let source_dir = TempDir::new().expect("source tempdir");
3982 let source = sley::Repository::init(source_dir.path()).expect("init source git");
3983 let blob_oid = source.write_blob(b"streamed pull pack\n").expect("blob");
3984 let pack = sley::plumbing::sley_odb::build_reachable_pack(
3985 source.objects().as_ref(),
3986 source.object_format(),
3987 [blob_oid],
3988 &HashSet::new(),
3989 )
3990 .expect("build pack")
3991 .expect("reachable pack");
3992
3993 let dest_dir = TempDir::new().expect("dest tempdir");
3994 let dest = sley::Repository::init(dest_dir.path()).expect("init dest git");
3995 let mut state = GitPackPullInstallState::default();
3996 let chunk_size = 7usize;
3997 let mut offset = 0u64;
3998 for (chunk_index, chunk) in pack.pack.chunks(chunk_size).enumerate() {
3999 let next_offset = offset + chunk.len() as u64;
4000 state
4001 .receive_chunk(
4002 &dest,
4003 GitPackTransfer {
4004 transfer_id: "git-pack:test".to_string(),
4005 offset,
4006 chunk_index: chunk_index as u32,
4007 is_final_chunk: next_offset == pack.pack.len() as u64,
4008 pack_size: pack.pack.len() as u64,
4009 pack_chunk: chunk.to_vec(),
4010 pack_id: pack.checksum.as_bytes().to_vec(),
4011 },
4012 )
4013 .expect("receive chunk");
4014 offset = next_offset;
4015 }
4016
4017 state.ensure_idle().expect("stream should finish");
4018 let object = dest.read_object(&blob_oid).expect("read installed object");
4019 assert_eq!(object.body.as_slice(), b"streamed pull pack\n");
4020 }
4021
4022 #[test]
4023 fn accept_git_lane_pull_transfer_errors_for_non_overlay_repo() {
4024 let (_dir, repo) = temp_repo();
4025 assert_ne!(
4026 repo.capability(),
4027 RepositoryCapability::GitOverlay,
4028 "init_default repo must be non-GitOverlay for this guard test",
4029 );
4030 let mut git_repo = None;
4031 let mut state = GitPackPullInstallState::default();
4032 let transfer = GitLaneTransfer {
4033 body: Some(git_lane_transfer::Body::Pack(GitPackTransfer {
4034 transfer_id: "git-pack:test".to_string(),
4035 offset: 0,
4036 chunk_index: 0,
4037 is_final_chunk: true,
4038 pack_size: 0,
4039 pack_chunk: Vec::new(),
4040 pack_id: Vec::new(),
4041 })),
4042 };
4043 let err = accept_git_lane_pull_transfer(&repo, &mut git_repo, &mut state, transfer)
4044 .expect_err("git-lane pull to a non-GitOverlay repo must fail loud, not silently drop");
4045 assert!(
4046 matches!(err, ProtocolError::InvalidState(_)),
4047 "expected InvalidState protocol error, got {err:?}",
4048 );
4049 }
4050
4051 fn git_ref_update_from_message(message: &PushClientFrame) -> &GitRefUpdateTransfer {
4052 let Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
4053 body: Some(git_lane_transfer::Body::RefUpdate(update)),
4054 })) = message.frame.as_ref()
4055 else {
4056 panic!("expected git ref update message");
4057 };
4058 update
4059 }
4060
4061 fn loose_tree_path(repo: &Repository, hash: &ContentHash) -> std::path::PathBuf {
4062 let hex = hash.to_hex();
4063 let (prefix, rest) = hex.split_at(2);
4064 repo.heddle_dir()
4065 .join("objects")
4066 .join("trees")
4067 .join(prefix)
4068 .join(rest)
4069 }
4070
4071 fn sample_redaction(blob: ContentHash) -> Redaction {
4072 Redaction {
4073 redacted_blob: blob,
4074 state: StateId::from_bytes([1u8; 32]),
4075 path: "config/secrets.toml".into(),
4076 reason: "leaked credential".into(),
4077 redactor: Principal {
4078 name: "Grace Hopper".into(),
4079 email: "grace@example.com".into(),
4080 },
4081 redacted_at: Utc.with_ymd_and_hms(2026, 5, 10, 14, 33, 0).unwrap(),
4082 signature: None,
4083 purged_at: None,
4084 supersedes: None,
4085 }
4086 }
4087
4088 fn sample_state_visibility(state: StateId) -> StateVisibility {
4089 StateVisibility {
4090 state,
4091 tier: VisibilityTier::Restricted {
4092 scope_label: "security-embargo".into(),
4093 },
4094 embargo_until: None,
4095 declarer: Principal {
4096 name: "Grace Hopper".into(),
4097 email: "grace@example.com".into(),
4098 },
4099 declared_at: Utc.with_ymd_and_hms(2026, 6, 1, 12, 0, 0).unwrap(),
4100 signature: None,
4101 supersedes: None,
4102 }
4103 }
4104
4105 #[test]
4106 fn non_packable_object_types_are_in_out_of_pack_transfer_partition() {
4107 for obj_type in wire::native_pack_excluded_object_types() {
4108 assert!(
4109 wire::TransferPartitions::<ObjectInfo>::is_sidecar_object_type(*obj_type),
4110 "{obj_type:?} is excluded from native packs but missing from the out-of-pack transfer partition"
4111 );
4112 }
4113 }
4114
4115 #[test]
4116 fn native_pack_required_tracks_packable_pull_wants() {
4117 let blob = sample_blob();
4118 let state = StateId::from_bytes([9u8; 32]);
4119
4120 let sidecar_only = RepositoryTransferPlan::from_object_infos(
4121 vec![state_visibility_info(state)],
4122 GitLaneTransferIntent::HeddleObjectsOnly,
4123 );
4124 assert!(!native_pack_required_for_pull(false, &sidecar_only));
4125
4126 let redaction_only = RepositoryTransferPlan::from_object_infos(
4127 vec![redaction_info(blob)],
4128 GitLaneTransferIntent::HeddleObjectsOnly,
4129 );
4130 assert!(!native_pack_required_for_pull(false, &redaction_only));
4131
4132 let packable = RepositoryTransferPlan::from_object_infos(
4133 vec![ObjectInfo {
4134 id: ObjectId::Hash(blob),
4135 obj_type: ObjectType::Blob,
4136 size: 0,
4137 delta_base: None,
4138 }],
4139 GitLaneTransferIntent::HeddleObjectsOnly,
4140 );
4141 assert!(native_pack_required_for_pull(false, &packable));
4142
4143 let state_with_sidecar = RepositoryTransferPlan::from_object_infos(
4144 vec![state_info(state), state_visibility_info(state)],
4145 GitLaneTransferIntent::HeddleObjectsOnly,
4146 );
4147 assert!(native_pack_required_for_pull(false, &state_with_sidecar));
4148 let empty = RepositoryTransferPlan::from_object_infos(
4149 Vec::<ObjectInfo>::new(),
4150 GitLaneTransferIntent::HeddleObjectsOnly,
4151 );
4152 assert!(native_pack_required_for_pull(true, &empty));
4153 }
4154
4155 #[test]
4156 fn plan_pull_wants_accumulates_state_and_visibility_for_same_state_id() {
4157 let (_dir, repo) = temp_repo();
4158 let state = StateId::from_bytes([9u8; 32]);
4159 let plan = plan_pull_wants(
4160 &repo,
4161 &state,
4162 false,
4163 vec![
4164 object_descriptor_with_status(
4165 &state_info(state),
4166 ObjectAvailabilityStatus::Missing,
4167 "missing state",
4168 ),
4169 object_descriptor_with_status(
4170 &state_visibility_info(state),
4171 ObjectAvailabilityStatus::Missing,
4172 "missing state visibility",
4173 ),
4174 ],
4175 false,
4176 )
4177 .expect("plan pull wants");
4178
4179 let wanted = plan
4180 .wanted_types
4181 .get(&PackObjectId::StateId(state))
4182 .expect("same StateId want entry");
4183 assert_eq!(
4184 wanted.as_slice(),
4185 &[ObjectType::State, ObjectType::StateVisibility]
4186 );
4187 assert!(native_pack_required_for_pull(
4188 plan.want_full_closure,
4189 &plan.transfer_plan
4190 ));
4191 }
4192
4193 #[derive(Clone)]
4194 struct SidecarOnlyPullService {
4195 state: StateId,
4196 state_visibility_blob: Vec<u8>,
4197 }
4198
4199 #[tonic::async_trait]
4200 impl RepoSyncService for SidecarOnlyPullService {
4201 async fn list_refs(
4202 &self,
4203 _request: tonic::Request<ListRefsRequest>,
4204 ) -> Result<Response<ListRefsResponse>, Status> {
4205 Ok(Response::new(ListRefsResponse::default()))
4206 }
4207
4208 async fn update_ref(
4209 &self,
4210 _request: tonic::Request<UpdateRefRequest>,
4211 ) -> Result<Response<UpdateRefResponse>, Status> {
4212 Ok(Response::new(UpdateRefResponse::default()))
4213 }
4214
4215 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushServerFrame, Status>>;
4216
4217 async fn push(
4218 &self,
4219 _request: tonic::Request<tonic::Streaming<PushClientFrame>>,
4220 ) -> Result<Response<Self::PushStream>, Status> {
4221 let (_tx, rx) = mpsc::channel(1);
4222 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4223 rx,
4224 )))
4225 }
4226
4227 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullServerFrame, Status>>;
4228
4229 async fn pull(
4230 &self,
4231 request: tonic::Request<tonic::Streaming<PullClientFrame>>,
4232 ) -> Result<Response<Self::PullStream>, Status> {
4233 let state = self.state;
4234 let state_visibility_blob = self.state_visibility_blob.clone();
4235 let (tx, rx) = mpsc::channel(4);
4236
4237 tokio::spawn(async move {
4238 let mut inbound = request.into_inner();
4239 if !matches!(
4240 inbound.message().await,
4241 Ok(Some(PullClientFrame {
4242 frame: Some(pull_client_frame::Frame::Open(_)),
4243 }))
4244 ) {
4245 let _ = tx
4246 .send(Err(Status::invalid_argument("expected signed pull open")))
4247 .await;
4248 return;
4249 }
4250 match inbound.message().await {
4251 Ok(Some(PullClientFrame {
4252 frame: Some(pull_client_frame::Frame::Request(_)),
4253 })) => {}
4254 other => {
4255 let _ = tx
4256 .send(Err(Status::invalid_argument(format!(
4257 "expected pull request, got {other:?}"
4258 ))))
4259 .await;
4260 return;
4261 }
4262 }
4263
4264 let descriptor = object_descriptor_with_status(
4265 &state_visibility_info(state),
4266 ObjectAvailabilityStatus::Missing,
4267 "missing state visibility",
4268 );
4269 let ready = PullServerFrame {
4270 frame: Some(pull_server_frame::Frame::Ready(PullReady {
4271 remote_state: proto_state_id(state),
4272 objects_to_fetch: vec![descriptor],
4273 transfer: None,
4274 partial_fetch_status: PartialFetchStatus::Disabled as i32,
4275 missing_objects: Vec::new(),
4276 full_closure_available: false,
4277 object_count: 1,
4278 remote_revision_address: RevisionAddress::heddle(state).to_string(),
4279 })),
4280 };
4281 if tx.send(Ok(ready)).await.is_err() {
4282 return;
4283 }
4284
4285 match inbound.message().await {
4286 Ok(Some(PullClientFrame {
4287 frame: Some(pull_client_frame::Frame::Want(want)),
4288 })) if !want.want_full_closure
4289 && want.objects.len() == 1
4290 && want.objects[0].object_type == "state_visibility" => {}
4291 other => {
4292 let _ = tx
4293 .send(Err(Status::invalid_argument(format!(
4294 "expected sidecar-only want, got {other:?}"
4295 ))))
4296 .await;
4297 return;
4298 }
4299 }
4300
4301 let transfer = PullServerFrame {
4302 frame: Some(pull_server_frame::Frame::StateVisibility(
4303 StateVisibilityTransfer {
4304 state_id: proto_state_id(state),
4305 state_visibility_blob,
4306 },
4307 )),
4308 };
4309 if tx.send(Ok(transfer)).await.is_err() {
4310 return;
4311 }
4312
4313 let complete = PullServerFrame {
4314 frame: Some(pull_server_frame::Frame::Complete(GrpcPullComplete {
4315 success: true,
4316 new_state: proto_state_id(state),
4317 error: String::new(),
4318 transfer: Some(TransferCheckpoint {
4319 transfer_id: "sidecar-only-test".to_string(),
4320 transport_mode: TransportMode::NativePack as i32,
4321 resume_offset: 0,
4322 chunk_index: 0,
4323 checkpoint: b"heddle-markers-v1\n".to_vec(),
4324 is_complete: true,
4325 }),
4326 new_revision_address: RevisionAddress::heddle(state).to_string(),
4327 })),
4328 };
4329 let _ = tx.send(Ok(complete)).await;
4330 });
4331
4332 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4333 rx,
4334 )))
4335 }
4336 }
4337
4338 async fn connect_sidecar_only_service(
4339 service: SidecarOnlyPullService,
4340 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
4341 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
4342 Ok(listener) => listener,
4343 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
4344 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
4345 return None;
4346 }
4347 Err(err) => panic!("bind test server: {err}"),
4348 };
4349 let addr = listener.local_addr().expect("local addr");
4350 let incoming = futures::stream::unfold(listener, |listener| async {
4351 match listener.accept().await {
4352 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
4353 Err(err) => Some((Err(err), listener)),
4354 }
4355 });
4356
4357 let handle = tokio::spawn(async move {
4358 Server::builder()
4359 .add_service(RepoSyncServiceServer::new(service))
4360 .serve_with_incoming(incoming)
4361 .await
4362 .expect("serve sidecar-only test service");
4363 });
4364
4365 let client = HostedGrpcClient::connect(addr, &signed_test_config())
4366 .await
4367 .expect("connect client");
4368 Some((client, handle))
4369 }
4370
4371 #[tokio::test]
4372 async fn state_visibility_sidecar_only_pull_completes_without_native_pack() {
4373 let (_dir, repo) = temp_repo();
4374 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4375 let state = State::new_snapshot(
4376 tree_hash,
4377 vec![],
4378 Attribution::human(Principal {
4379 name: "Grace Hopper".into(),
4380 email: "grace@example.com".into(),
4381 }),
4382 );
4383 let state_id = state.state_id;
4384 repo.store().put_state(&state).expect("put state");
4385 assert!(
4386 repo.get_state_visibility_bytes_for_state(&state_id)
4387 .expect("load local sidecar")
4388 .is_none(),
4389 "test starts with state present and StateVisibility sidecar absent"
4390 );
4391
4392 let state_visibility_blob =
4393 StateVisibilityBlob::new(vec![sample_state_visibility(state_id)])
4394 .encode()
4395 .expect("encode state visibility blob");
4396 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4397 state: state_id,
4398 state_visibility_blob,
4399 })
4400 .await
4401 else {
4402 return;
4403 };
4404
4405 let exchange = tokio::time::timeout(
4406 Duration::from_secs(5),
4407 client.pull_exchange(
4408 &repo,
4409 "owner/repo",
4410 "main",
4411 PullOptions {
4412 local_thread: None,
4413 depth: None,
4414 target_state: Some(state_id),
4415 materialization: PullMaterialization::Full,
4416 },
4417 ),
4418 )
4419 .await
4420 .expect("sidecar-only pull must not hang waiting for native pack")
4421 .expect("sidecar-only pull succeeds");
4422 server.abort();
4423
4424 assert!(exchange.result.success);
4425 assert_eq!(exchange.object_count, 0);
4426 assert_eq!(exchange.profile.pack_bytes_received, 0);
4427 assert_eq!(exchange.profile.object_mix.state_visibilities, 1);
4428 assert!(
4429 repo.get_state_visibility_for_state(&state_id)
4430 .expect("load accepted sidecar")
4431 .has_record(),
4432 "pull must accept the out-of-pack StateVisibility sidecar"
4433 );
4434 }
4435
4436 #[tokio::test]
4440 async fn legitimate_large_sidecar_blob_decodes_and_installs() {
4441 let (_dir, repo) = temp_repo();
4442 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4443 let state = State::new_snapshot(
4444 tree_hash,
4445 vec![],
4446 Attribution::human(Principal {
4447 name: "Grace Hopper".into(),
4448 email: "grace@example.com".into(),
4449 }),
4450 );
4451 let state_id = state.state_id;
4452 repo.store().put_state(&state).expect("put state");
4453
4454 let mut record = sample_state_visibility(state_id);
4457 record.tier = VisibilityTier::Restricted {
4458 scope_label: "x".repeat(8 * 1024 * 1024),
4459 };
4460 let state_visibility_blob = StateVisibilityBlob::new(vec![record])
4461 .encode()
4462 .expect("encode large state visibility blob");
4463 assert!(
4464 state_visibility_blob.len() > 4 * 1024 * 1024,
4465 "blob must exceed tonic's 4 MiB default to exercise the raised decode limit"
4466 );
4467 assert!(
4468 (state_visibility_blob.len() as u64) <= wire::MAX_RECEIVED_STATE_VISIBILITY_BLOB_SIZE,
4469 "blob must stay within the legitimate sidecar receive cap"
4470 );
4471
4472 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4473 state: state_id,
4474 state_visibility_blob,
4475 })
4476 .await
4477 else {
4478 return;
4479 };
4480
4481 let exchange = tokio::time::timeout(
4482 Duration::from_secs(30),
4483 client.pull_exchange(
4484 &repo,
4485 "owner/repo",
4486 "main",
4487 PullOptions {
4488 local_thread: None,
4489 depth: None,
4490 target_state: Some(state_id),
4491 materialization: PullMaterialization::Full,
4492 },
4493 ),
4494 )
4495 .await
4496 .expect("large-sidecar pull must not hang")
4497 .expect("large but legitimate sidecar pull succeeds");
4498 server.abort();
4499
4500 assert!(exchange.result.success);
4501 assert!(
4502 repo.get_state_visibility_for_state(&state_id)
4503 .expect("load accepted sidecar")
4504 .has_record(),
4505 "pull must accept a legitimately-large StateVisibility sidecar"
4506 );
4507 }
4508
4509 #[tokio::test]
4513 async fn oversized_sidecar_blob_rejected_at_grpc_decode_boundary() {
4514 let (_dir, repo) = temp_repo();
4515 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4516 let state = State::new_snapshot(
4517 tree_hash,
4518 vec![],
4519 Attribution::human(Principal {
4520 name: "Grace Hopper".into(),
4521 email: "grace@example.com".into(),
4522 }),
4523 );
4524 let state_id = state.state_id;
4525 repo.store().put_state(&state).expect("put state");
4526
4527 let oversized = vec![0u8; wire::MAX_PULL_DECODE_MESSAGE_SIZE + 1];
4530 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4531 state: state_id,
4532 state_visibility_blob: oversized,
4533 })
4534 .await
4535 else {
4536 return;
4537 };
4538
4539 let result = tokio::time::timeout(
4540 Duration::from_secs(30),
4541 client.pull_exchange(
4542 &repo,
4543 "owner/repo",
4544 "main",
4545 PullOptions {
4546 local_thread: None,
4547 depth: None,
4548 target_state: Some(state_id),
4549 materialization: PullMaterialization::Full,
4550 },
4551 ),
4552 )
4553 .await
4554 .expect("oversized-sidecar pull must not hang");
4555 server.abort();
4556
4557 let err = match result {
4558 Err(err) => err,
4559 Ok(_) => panic!("oversized sidecar PullClientFrame must be rejected at decode"),
4560 };
4561 let message = err.to_string();
4562 assert!(
4563 !message.contains("exceeds receive size limit"),
4564 "rejection must come from the decode-size limit, before the post-decode check: {message}"
4565 );
4566 assert!(
4567 repo.get_state_visibility_for_state(&state_id)
4568 .expect("load sidecar")
4569 .latest()
4570 .expect("resolve visibility")
4571 .is_none(),
4572 "an oversized sidecar must never be installed"
4573 );
4574 }
4575
4576 #[derive(Clone)]
4577 struct StateAndVisibilityPullService {
4578 state: StateId,
4579 pack_bundle: wire::NativePackBundle,
4580 state_visibility_blob: Vec<u8>,
4581 }
4582
4583 #[tonic::async_trait]
4584 impl RepoSyncService for StateAndVisibilityPullService {
4585 async fn list_refs(
4586 &self,
4587 _request: tonic::Request<ListRefsRequest>,
4588 ) -> Result<Response<ListRefsResponse>, Status> {
4589 Ok(Response::new(ListRefsResponse::default()))
4590 }
4591
4592 async fn update_ref(
4593 &self,
4594 _request: tonic::Request<UpdateRefRequest>,
4595 ) -> Result<Response<UpdateRefResponse>, Status> {
4596 Ok(Response::new(UpdateRefResponse::default()))
4597 }
4598
4599 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushServerFrame, Status>>;
4600
4601 async fn push(
4602 &self,
4603 _request: tonic::Request<tonic::Streaming<PushClientFrame>>,
4604 ) -> Result<Response<Self::PushStream>, Status> {
4605 let (_tx, rx) = mpsc::channel(1);
4606 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4607 rx,
4608 )))
4609 }
4610
4611 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullServerFrame, Status>>;
4612
4613 async fn pull(
4614 &self,
4615 request: tonic::Request<tonic::Streaming<PullClientFrame>>,
4616 ) -> Result<Response<Self::PullStream>, Status> {
4617 let state = self.state;
4618 let pack_bundle = self.pack_bundle.clone();
4619 let state_visibility_blob = self.state_visibility_blob.clone();
4620 let (tx, rx) = mpsc::channel(8);
4621
4622 tokio::spawn(async move {
4623 let mut inbound = request.into_inner();
4624 if !matches!(
4625 inbound.message().await,
4626 Ok(Some(PullClientFrame {
4627 frame: Some(pull_client_frame::Frame::Open(_)),
4628 }))
4629 ) {
4630 let _ = tx
4631 .send(Err(Status::invalid_argument("expected signed pull open")))
4632 .await;
4633 return;
4634 }
4635 match inbound.message().await {
4636 Ok(Some(PullClientFrame {
4637 frame: Some(pull_client_frame::Frame::Request(_)),
4638 })) => {}
4639 other => {
4640 let _ = tx
4641 .send(Err(Status::invalid_argument(format!(
4642 "expected pull request, got {other:?}"
4643 ))))
4644 .await;
4645 return;
4646 }
4647 }
4648
4649 let ready = PullServerFrame {
4650 frame: Some(pull_server_frame::Frame::Ready(PullReady {
4651 remote_state: proto_state_id(state),
4652 objects_to_fetch: vec![
4653 object_descriptor_with_status(
4654 &state_info(state),
4655 ObjectAvailabilityStatus::Missing,
4656 "missing state",
4657 ),
4658 object_descriptor_with_status(
4659 &state_visibility_info(state),
4660 ObjectAvailabilityStatus::Missing,
4661 "missing state visibility",
4662 ),
4663 ],
4664 transfer: None,
4665 partial_fetch_status: PartialFetchStatus::Disabled as i32,
4666 missing_objects: Vec::new(),
4667 full_closure_available: false,
4668 object_count: 2,
4669 remote_revision_address: RevisionAddress::heddle(state).to_string(),
4670 })),
4671 };
4672 if tx.send(Ok(ready)).await.is_err() {
4673 return;
4674 }
4675
4676 match inbound.message().await {
4677 Ok(Some(PullClientFrame {
4678 frame: Some(pull_client_frame::Frame::Want(want)),
4679 })) if !want.want_full_closure
4680 && want.objects.len() == 2
4681 && want
4682 .objects
4683 .iter()
4684 .any(|object| object.object_type == "state")
4685 && want
4686 .objects
4687 .iter()
4688 .any(|object| object.object_type == "state_visibility") => {}
4689 other => {
4690 let _ = tx
4691 .send(Err(Status::invalid_argument(format!(
4692 "expected state + sidecar wants, got {other:?}"
4693 ))))
4694 .await;
4695 return;
4696 }
4697 }
4698
4699 for message in
4700 encode_pull_native_pack_messages(&pack_bundle, "state-and-visibility-test", 16)
4701 {
4702 if tx.send(Ok(message)).await.is_err() {
4703 return;
4704 }
4705 }
4706
4707 let transfer = PullServerFrame {
4708 frame: Some(pull_server_frame::Frame::StateVisibility(
4709 StateVisibilityTransfer {
4710 state_id: proto_state_id(state),
4711 state_visibility_blob,
4712 },
4713 )),
4714 };
4715 if tx.send(Ok(transfer)).await.is_err() {
4716 return;
4717 }
4718
4719 let complete = PullServerFrame {
4720 frame: Some(pull_server_frame::Frame::Complete(GrpcPullComplete {
4721 success: true,
4722 new_state: proto_state_id(state),
4723 error: String::new(),
4724 transfer: Some(TransferCheckpoint {
4725 transfer_id: "state-and-visibility-test".to_string(),
4726 transport_mode: TransportMode::NativePack as i32,
4727 resume_offset: 0,
4728 chunk_index: 0,
4729 checkpoint: b"heddle-markers-v1\n".to_vec(),
4730 is_complete: true,
4731 }),
4732 new_revision_address: RevisionAddress::heddle(state).to_string(),
4733 })),
4734 };
4735 let _ = tx.send(Ok(complete)).await;
4736 });
4737
4738 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4739 rx,
4740 )))
4741 }
4742 }
4743
4744 fn encode_pull_native_pack_messages(
4745 bundle: &wire::NativePackBundle,
4746 transfer_id: &str,
4747 chunk_size: usize,
4748 ) -> Vec<PullServerFrame> {
4749 let mut messages = Vec::new();
4750 let chunk_size = chunk_size.max(1);
4751
4752 let pack_total_chunks = wire::chunk_count(bundle.pack_data.len(), chunk_size);
4753 for chunk_index in 0..pack_total_chunks.max(1) {
4754 let Some((start, len)) =
4755 wire::chunk_bounds(bundle.pack_data.len(), chunk_size, chunk_index)
4756 else {
4757 break;
4758 };
4759 messages.push(PullServerFrame {
4760 frame: Some(pull_server_frame::Frame::Pack(PackChunk {
4761 stream_kind: PackStreamKind::Pack as i32,
4762 data: bundle.pack_data[start..start + len].to_vec(),
4763 transfer: Some(TransferCheckpoint {
4764 transfer_id: transfer_id.to_string(),
4765 transport_mode: TransportMode::NativePack as i32,
4766 resume_offset: start as u64,
4767 chunk_index: chunk_index as u32,
4768 checkpoint: Vec::new(),
4769 is_complete: chunk_index + 1 == pack_total_chunks,
4770 }),
4771 chunk_length: len as u32,
4772 is_final_chunk: chunk_index + 1 == pack_total_chunks,
4773 })),
4774 });
4775 }
4776
4777 let index_total_chunks = wire::chunk_count(bundle.index_data.len(), chunk_size);
4778 for chunk_index in 0..index_total_chunks.max(1) {
4779 let Some((start, len)) =
4780 wire::chunk_bounds(bundle.index_data.len(), chunk_size, chunk_index)
4781 else {
4782 break;
4783 };
4784 messages.push(PullServerFrame {
4785 frame: Some(pull_server_frame::Frame::Pack(PackChunk {
4786 stream_kind: PackStreamKind::Index as i32,
4787 data: bundle.index_data[start..start + len].to_vec(),
4788 transfer: Some(TransferCheckpoint {
4789 transfer_id: transfer_id.to_string(),
4790 transport_mode: TransportMode::NativePack as i32,
4791 resume_offset: start as u64,
4792 chunk_index: chunk_index as u32,
4793 checkpoint: Vec::new(),
4794 is_complete: chunk_index + 1 == index_total_chunks,
4795 }),
4796 chunk_length: len as u32,
4797 is_final_chunk: chunk_index + 1 == index_total_chunks,
4798 })),
4799 });
4800 }
4801
4802 messages
4803 }
4804
4805 async fn connect_state_and_visibility_service(
4806 service: StateAndVisibilityPullService,
4807 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
4808 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
4809 Ok(listener) => listener,
4810 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
4811 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
4812 return None;
4813 }
4814 Err(err) => panic!("bind test server: {err}"),
4815 };
4816 let addr = listener.local_addr().expect("local addr");
4817 let incoming = futures::stream::unfold(listener, |listener| async {
4818 match listener.accept().await {
4819 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
4820 Err(err) => Some((Err(err), listener)),
4821 }
4822 });
4823
4824 let handle = tokio::spawn(async move {
4825 Server::builder()
4826 .add_service(RepoSyncServiceServer::new(service))
4827 .serve_with_incoming(incoming)
4828 .await
4829 .expect("serve state-and-visibility test service");
4830 });
4831
4832 let client = HostedGrpcClient::connect(addr, &signed_test_config())
4833 .await
4834 .expect("connect client");
4835 Some((client, handle))
4836 }
4837
4838 #[tokio::test]
4839 async fn state_and_visibility_same_state_id_pull_requests_pack_and_sidecar() {
4840 let (_source_dir, source_repo) = temp_repo();
4841 let (_target_dir, target_repo) = temp_repo();
4842 let tree_hash = source_repo
4843 .store()
4844 .put_tree(&Tree::new())
4845 .expect("put tree");
4846 let state = State::new_snapshot(
4847 tree_hash,
4848 vec![],
4849 Attribution::human(Principal {
4850 name: "Grace Hopper".into(),
4851 email: "grace@example.com".into(),
4852 }),
4853 );
4854 let state_id = state.state_id;
4855 source_repo
4856 .store()
4857 .put_state(&state)
4858 .expect("put source state");
4859 let state_visibility_blob =
4860 StateVisibilityBlob::new(vec![sample_state_visibility(state_id)])
4861 .encode()
4862 .expect("encode state visibility blob");
4863 source_repo
4864 .accept_wire_state_visibility(state_id, &state_visibility_blob)
4865 .expect("put source state visibility");
4866 let pack_bundle = wire::build_native_pack(source_repo.store(), &[state_info(state_id)])
4867 .expect("build state pack");
4868
4869 assert!(
4870 target_repo
4871 .store()
4872 .get_state(&state_id)
4873 .expect("load target state")
4874 .is_none(),
4875 "test starts with state absent"
4876 );
4877 assert!(
4878 target_repo
4879 .get_state_visibility_bytes_for_state(&state_id)
4880 .expect("load target sidecar")
4881 .is_none(),
4882 "test starts with StateVisibility sidecar absent"
4883 );
4884
4885 let Some((mut client, server)) =
4886 connect_state_and_visibility_service(StateAndVisibilityPullService {
4887 state: state_id,
4888 pack_bundle,
4889 state_visibility_blob,
4890 })
4891 .await
4892 else {
4893 return;
4894 };
4895
4896 let exchange = tokio::time::timeout(
4897 Duration::from_secs(5),
4898 client.pull_exchange(
4899 &target_repo,
4900 "owner/repo",
4901 "main",
4902 PullOptions {
4903 local_thread: None,
4904 depth: None,
4905 target_state: Some(state_id),
4906 materialization: PullMaterialization::Full,
4907 },
4908 ),
4909 )
4910 .await
4911 .expect("state + sidecar pull must not hang waiting for native pack")
4912 .expect("state + sidecar pull succeeds");
4913 server.abort();
4914
4915 assert!(exchange.result.success);
4916 assert_eq!(exchange.object_count, 1);
4917 assert!(exchange.profile.pack_bytes_received > 0);
4918 assert_eq!(exchange.profile.object_mix.states, 1);
4919 assert_eq!(exchange.profile.object_mix.state_visibilities, 1);
4920 assert!(
4921 target_repo
4922 .store()
4923 .get_state(&state_id)
4924 .expect("load installed state")
4925 .is_some(),
4926 "native pack must install the State"
4927 );
4928 assert!(
4929 target_repo
4930 .get_state_visibility_for_state(&state_id)
4931 .expect("load accepted sidecar")
4932 .has_record(),
4933 "pull must accept the out-of-pack StateVisibility sidecar"
4934 );
4935 }
4936
4937 #[test]
4938 fn missing_blobs_in_tree_treats_absent_tree_as_empty() {
4939 let (_dir, repo) = temp_repo();
4940 let absent_tree = ContentHash::from_bytes([99u8; 32]);
4941
4942 let missing = wire::missing_blobs_in_tree(repo.store(), absent_tree)
4943 .expect("absent tree is not an error");
4944
4945 assert!(missing.is_empty());
4946 }
4947
4948 #[test]
4949 fn missing_blobs_in_tree_reports_only_genuinely_missing_blobs() {
4950 let (_dir, repo) = temp_repo();
4951 let present_blob = Blob::from("already local");
4952 let present_hash = repo.store().put_blob(&present_blob).expect("put blob");
4953 let missing_hash = ContentHash::from_bytes([42u8; 32]);
4954 let tree = Tree::from_entries(vec![
4955 TreeEntry::file("local.txt", present_hash, false).expect("present entry"),
4956 TreeEntry::file("remote.txt", missing_hash, false).expect("missing entry"),
4957 ]);
4958 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
4959
4960 let missing =
4961 wire::missing_blobs_in_tree(repo.store(), tree_hash).expect("collect missing blobs");
4962
4963 assert_eq!(missing, vec![missing_hash]);
4964 }
4965
4966 #[test]
4967 fn missing_blobs_in_tree_reports_corrupt_tree_read() {
4968 let (_dir, repo) = temp_repo();
4969 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4970 std::fs::write(loose_tree_path(&repo, &tree_hash), [0xc1]).expect("corrupt tree");
4971 repo.store().clear_recent_caches();
4972
4973 let err = wire::missing_blobs_in_tree(repo.store(), tree_hash)
4974 .expect_err("corrupt tree must fail");
4975
4976 assert!(matches!(err, ProtocolError::InvalidState(_)));
4977 assert!(
4978 err.to_string().contains(&format!(
4979 "load tree {} while collecting lazy hydration missing blobs",
4980 tree_hash.to_hex()
4981 )),
4982 "unexpected error: {err}"
4983 );
4984 }
4985
4986 #[test]
4987 fn redaction_push_message_uses_hex_keyed_sidecar_payload() {
4988 let (_dir, repo) = temp_repo();
4989 let blob = sample_blob();
4990 repo.put_redaction(sample_redaction(blob))
4991 .expect("put redaction");
4992 let expected_bytes = repo
4993 .store()
4994 .get_redactions_bytes_for_blob(&blob)
4995 .expect("load sidecar")
4996 .expect("sidecar exists");
4997
4998 let message =
4999 redaction_push_message(&repo, redaction_info(blob), "test-push-op").expect("message");
5000 assert_eq!(message.client_operation_id, "test-push-op");
5001
5002 let Some(push_client_frame::Frame::Redaction(transfer)) = message.frame else {
5003 panic!("expected redaction transfer");
5004 };
5005 assert_eq!(transfer.blob_hash, blob.to_hex());
5006 assert_eq!(transfer.redactions_blob, expected_bytes);
5007 }
5008
5009 #[test]
5010 fn redaction_push_message_reports_missing_sidecar_with_blob_hex() {
5011 let (_dir, repo) = temp_repo();
5012 let blob = sample_blob();
5013
5014 let err = redaction_push_message(&repo, redaction_info(blob), "test-push-op")
5015 .expect_err("missing sidecar");
5016
5017 assert!(matches!(err, ProtocolError::InvalidState(_)));
5018 assert!(
5019 err.to_string().contains(&format!(
5020 "server wants redaction for blob {} but sender has no sidecar",
5021 blob.to_hex()
5022 )),
5023 "unexpected error: {err}"
5024 );
5025 }
5026
5027 #[test]
5028 fn redaction_push_message_reports_sidecar_load_error_with_blob_hex() {
5029 let (_dir, repo) = temp_repo();
5030 let blob = sample_blob();
5031 let redaction_path = repo
5032 .heddle_dir()
5033 .join("redactions")
5034 .join(format!("{}.bin", blob.to_hex()));
5035 std::fs::create_dir_all(&redaction_path).expect("directory at redaction path");
5036
5037 let err = redaction_push_message(&repo, redaction_info(blob), "test-push-op")
5038 .expect_err("load error");
5039
5040 assert!(matches!(err, ProtocolError::InvalidState(_)));
5041 assert!(
5042 err.to_string()
5043 .contains(&format!("load redactions sidecar for {}:", blob.to_hex())),
5044 "unexpected error: {err}"
5045 );
5046 }
5047
5048 #[test]
5049 fn state_visibility_push_message_uses_state_keyed_sidecar_payload() {
5050 let (_dir, repo) = temp_repo();
5051 let state = StateId::from_bytes([17u8; 32]);
5052 repo.put_state_visibility(sample_state_visibility(state))
5053 .expect("put state visibility");
5054 let expected_bytes = repo
5055 .get_state_visibility_bytes_for_state(&state)
5056 .expect("load sidecar")
5057 .expect("sidecar exists");
5058
5059 let message =
5060 state_visibility_push_message(&repo, state_visibility_info(state), "test-push-op")
5061 .expect("message");
5062 assert_eq!(message.client_operation_id, "test-push-op");
5063
5064 let Some(push_client_frame::Frame::StateVisibility(transfer)) = message.frame else {
5065 panic!("expected state visibility transfer");
5066 };
5067 assert_eq!(
5068 transfer
5069 .state_id
5070 .as_ref()
5071 .map(|state| state.value.as_slice()),
5072 Some(state.as_bytes().as_slice())
5073 );
5074 assert_eq!(transfer.state_visibility_blob, expected_bytes);
5075 }
5076
5077 fn sample_attribution() -> Attribution {
5080 Attribution::human(Principal {
5081 name: "Grace Hopper".into(),
5082 email: "grace@example.com".into(),
5083 })
5084 }
5085
5086 fn put_state_with_blob(repo: &Repository, contents: &str, parents: Vec<StateId>) -> StateId {
5089 let blob = Blob::from(contents);
5090 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
5091 let tree = Tree::from_entries(vec![
5092 TreeEntry::file(format!("{contents}.txt"), blob_hash, false).expect("tree entry"),
5093 ]);
5094 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
5095 let state = State::new_snapshot(tree_hash, parents, sample_attribution());
5096 let state_id = state.state_id;
5097 repo.store().put_state(&state).expect("put state");
5098 state_id
5099 }
5100
5101 #[test]
5102 fn locally_complete_pull_head_advertises_fully_present_thread_head() {
5103 let (_dir, repo) = temp_repo();
5104 let head = put_state_with_blob(&repo, "alpha", vec![]);
5105 repo.refs()
5106 .set_thread(&ThreadName::from("main"), &head)
5107 .expect("set thread");
5108
5109 let advertised =
5110 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
5111 assert_eq!(
5112 advertised,
5113 Some(head),
5114 "a thread head whose full closure is present locally must be advertised"
5115 );
5116 }
5117
5118 #[test]
5119 fn locally_complete_pull_head_skips_unknown_thread() {
5120 let (_dir, repo) = temp_repo_unseeded();
5124 let advertised =
5125 locally_complete_pull_head(&repo, "nonexistent", None).expect("completeness check");
5126 assert_eq!(advertised, None);
5127 }
5128
5129 #[test]
5130 fn locally_complete_pull_head_skips_when_target_state_override_in_play() {
5131 let (_dir, repo) = temp_repo();
5132 let head = put_state_with_blob(&repo, "alpha", vec![]);
5133 repo.refs()
5134 .set_thread(&ThreadName::from("main"), &head)
5135 .expect("set thread");
5136
5137 let advertised =
5140 locally_complete_pull_head(&repo, "main", Some(head)).expect("completeness check");
5141 assert_eq!(advertised, None);
5142 }
5143
5144 #[test]
5145 fn locally_complete_pull_head_skips_repo_with_missing_blobs() {
5146 let (_dir, repo) = temp_repo();
5151 let head = put_state_with_blob(&repo, "alpha", vec![]);
5152 repo.refs()
5153 .set_thread(&ThreadName::from("main"), &head)
5154 .expect("set thread");
5155 repo.record_missing_blob(ContentHash::from_bytes([88u8; 32]))
5156 .expect("record missing blob");
5157
5158 let advertised =
5159 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
5160 assert_eq!(
5161 advertised, None,
5162 "a repo carrying missing blobs must never advertise a head"
5163 );
5164 }
5165
5166 #[test]
5167 fn locally_complete_pull_head_skips_head_with_incomplete_closure() {
5168 let (_dir, repo) = temp_repo();
5173 let absent_parent = StateId::from_bytes([21; 32]);
5174 let blob = Blob::from("orphan");
5175 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
5176 let tree = Tree::from_entries(vec![
5177 TreeEntry::file("orphan.txt", blob_hash, false).expect("tree entry"),
5178 ]);
5179 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
5180 let child = State::new_snapshot(tree_hash, vec![absent_parent], sample_attribution());
5182 let child_id = child.state_id;
5183 repo.store().put_state(&child).expect("put child state");
5184 repo.refs()
5185 .set_thread(&ThreadName::from("main"), &child_id)
5186 .expect("set thread");
5187
5188 let advertised =
5189 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
5190 assert_eq!(
5191 advertised, None,
5192 "a head whose closure has an absent parent state must not be advertised"
5193 );
5194 }
5195
5196 #[test]
5197 fn locally_complete_local_thread_head_advertises_fully_present_thread_head() {
5198 let (_dir, repo) = temp_repo();
5201 let head = put_state_with_blob(&repo, "alpha", vec![]);
5202 repo.refs()
5203 .set_thread(&ThreadName::from("feature"), &head)
5204 .expect("set thread");
5205
5206 let advertised =
5207 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
5208 assert_eq!(
5209 advertised,
5210 Some(head),
5211 "an explicit local-thread head whose full closure is present must be advertised"
5212 );
5213 }
5214
5215 #[test]
5216 fn locally_complete_local_thread_head_skips_unknown_thread() {
5217 let (_dir, repo) = temp_repo_unseeded();
5220 let advertised = locally_complete_local_thread_head(&repo, "nonexistent", None)
5221 .expect("completeness check");
5222 assert_eq!(advertised, None);
5223 }
5224
5225 #[test]
5226 fn locally_complete_local_thread_head_skips_when_target_state_override_in_play() {
5227 let (_dir, repo) = temp_repo();
5228 let head = put_state_with_blob(&repo, "alpha", vec![]);
5229 repo.refs()
5230 .set_thread(&ThreadName::from("feature"), &head)
5231 .expect("set thread");
5232
5233 let advertised = locally_complete_local_thread_head(&repo, "feature", Some(head))
5236 .expect("completeness check");
5237 assert_eq!(advertised, None);
5238 }
5239
5240 #[test]
5241 fn locally_complete_local_thread_head_skips_repo_with_missing_blobs() {
5242 let (_dir, repo) = temp_repo();
5245 let head = put_state_with_blob(&repo, "alpha", vec![]);
5246 repo.refs()
5247 .set_thread(&ThreadName::from("feature"), &head)
5248 .expect("set thread");
5249 repo.record_missing_blob(ContentHash::from_bytes([88u8; 32]))
5250 .expect("record missing blob");
5251
5252 let advertised =
5253 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
5254 assert_eq!(
5255 advertised, None,
5256 "a repo carrying missing blobs must never advertise an explicit local-thread head"
5257 );
5258 }
5259
5260 #[test]
5261 fn locally_complete_local_thread_head_skips_head_with_incomplete_closure() {
5262 let (_dir, repo) = temp_repo();
5267 let absent_parent = StateId::from_bytes([22; 32]);
5268 let blob = Blob::from("orphan");
5269 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
5270 let tree = Tree::from_entries(vec![
5271 TreeEntry::file("orphan.txt", blob_hash, false).expect("tree entry"),
5272 ]);
5273 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
5274 let child = State::new_snapshot(tree_hash, vec![absent_parent], sample_attribution());
5275 let child_id = child.state_id;
5276 repo.store().put_state(&child).expect("put child state");
5277 repo.refs()
5278 .set_thread(&ThreadName::from("feature"), &child_id)
5279 .expect("set thread");
5280
5281 let advertised =
5282 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
5283 assert_eq!(
5284 advertised, None,
5285 "an explicit local-thread head whose closure has an absent parent must not be advertised"
5286 );
5287 }
5288
5289 #[derive(Clone)]
5295 struct DeltaAwarePullService {
5296 remote_state: StateId,
5297 full_closure: Vec<ObjectInfo>,
5303 delta_objects: Vec<ObjectInfo>,
5304 known_parent: StateId,
5305 full_pack: wire::NativePackBundle,
5306 delta_pack: wire::NativePackBundle,
5307 captured_exclude: std::sync::Arc<std::sync::Mutex<Option<Vec<ProtoStateId>>>>,
5308 }
5309
5310 #[tonic::async_trait]
5311 impl RepoSyncService for DeltaAwarePullService {
5312 async fn list_refs(
5313 &self,
5314 _request: tonic::Request<ListRefsRequest>,
5315 ) -> Result<Response<ListRefsResponse>, Status> {
5316 Ok(Response::new(ListRefsResponse::default()))
5317 }
5318
5319 async fn update_ref(
5320 &self,
5321 _request: tonic::Request<UpdateRefRequest>,
5322 ) -> Result<Response<UpdateRefResponse>, Status> {
5323 Ok(Response::new(UpdateRefResponse::default()))
5324 }
5325
5326 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushServerFrame, Status>>;
5327
5328 async fn push(
5329 &self,
5330 _request: tonic::Request<tonic::Streaming<PushClientFrame>>,
5331 ) -> Result<Response<Self::PushStream>, Status> {
5332 let (_tx, rx) = mpsc::channel(1);
5333 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
5334 rx,
5335 )))
5336 }
5337
5338 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullServerFrame, Status>>;
5339
5340 async fn pull(
5341 &self,
5342 request: tonic::Request<tonic::Streaming<PullClientFrame>>,
5343 ) -> Result<Response<Self::PullStream>, Status> {
5344 let svc = self.clone();
5345 let (tx, rx) = mpsc::channel(16);
5346
5347 tokio::spawn(async move {
5348 let mut inbound = request.into_inner();
5349 if !matches!(
5350 inbound.message().await,
5351 Ok(Some(PullClientFrame {
5352 frame: Some(pull_client_frame::Frame::Open(_)),
5353 }))
5354 ) {
5355 let _ = tx
5356 .send(Err(Status::invalid_argument("expected signed pull open")))
5357 .await;
5358 return;
5359 }
5360 let exclude = match inbound.message().await {
5361 Ok(Some(PullClientFrame {
5362 frame: Some(pull_client_frame::Frame::Request(req)),
5363 })) => req.exclude_states,
5364 other => {
5365 let _ = tx
5366 .send(Err(Status::invalid_argument(format!(
5367 "expected pull request, got {other:?}"
5368 ))))
5369 .await;
5370 return;
5371 }
5372 };
5373 *svc.captured_exclude.lock().unwrap() = Some(exclude.clone());
5374
5375 let remote_proto = proto_state_id(svc.remote_state).expect("valid remote state");
5376 let parent_proto = proto_state_id(svc.known_parent).expect("valid parent state");
5377
5378 let (objects, pack, short_circuit) = if exclude.contains(&remote_proto) {
5380 (Vec::new(), None, true)
5382 } else if exclude.contains(&parent_proto) {
5383 (
5385 svc.delta_objects.clone(),
5386 Some(svc.delta_pack.clone()),
5387 false,
5388 )
5389 } else {
5390 (svc.full_closure.clone(), Some(svc.full_pack.clone()), false)
5392 };
5393
5394 let descriptors: Vec<_> = objects
5395 .iter()
5396 .map(|info| {
5397 object_descriptor_with_status(
5398 info,
5399 ObjectAvailabilityStatus::Missing,
5400 "requested by client",
5401 )
5402 })
5403 .collect();
5404
5405 let ready = PullServerFrame {
5406 frame: Some(pull_server_frame::Frame::Ready(PullReady {
5407 remote_state: proto_state_id(svc.remote_state),
5408 objects_to_fetch: descriptors,
5409 transfer: None,
5410 partial_fetch_status: PartialFetchStatus::Disabled as i32,
5411 missing_objects: Vec::new(),
5412 full_closure_available: false,
5413 object_count: objects.len() as u32,
5414 remote_revision_address: RevisionAddress::heddle(svc.remote_state)
5415 .to_string(),
5416 })),
5417 };
5418 if tx.send(Ok(ready)).await.is_err() {
5419 return;
5420 }
5421
5422 match inbound.message().await {
5424 Ok(Some(PullClientFrame {
5425 frame: Some(pull_client_frame::Frame::Want(_)),
5426 })) => {}
5427 other => {
5428 let _ = tx
5429 .send(Err(Status::invalid_argument(format!(
5430 "expected want, got {other:?}"
5431 ))))
5432 .await;
5433 return;
5434 }
5435 }
5436
5437 if let Some(pack) = pack
5438 && !short_circuit
5439 {
5440 for message in encode_pull_native_pack_messages(&pack, "delta-aware-test", 64) {
5441 if tx.send(Ok(message)).await.is_err() {
5442 return;
5443 }
5444 }
5445 }
5446
5447 let complete = PullServerFrame {
5448 frame: Some(pull_server_frame::Frame::Complete(GrpcPullComplete {
5449 success: true,
5450 new_state: proto_state_id(svc.remote_state),
5451 error: String::new(),
5452 transfer: Some(TransferCheckpoint {
5453 transfer_id: "delta-aware-test".to_string(),
5454 transport_mode: TransportMode::NativePack as i32,
5455 resume_offset: 0,
5456 chunk_index: 0,
5457 checkpoint: b"heddle-markers-v1\n".to_vec(),
5458 is_complete: true,
5459 }),
5460 new_revision_address: RevisionAddress::heddle(svc.remote_state).to_string(),
5461 })),
5462 };
5463 let _ = tx.send(Ok(complete)).await;
5464 });
5465
5466 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
5467 rx,
5468 )))
5469 }
5470 }
5471
5472 async fn connect_delta_aware_service(
5473 service: DeltaAwarePullService,
5474 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
5475 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
5476 Ok(listener) => listener,
5477 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
5478 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
5479 return None;
5480 }
5481 Err(err) => panic!("bind test server: {err}"),
5482 };
5483 let addr = listener.local_addr().expect("local addr");
5484 let incoming = futures::stream::unfold(listener, |listener| async {
5485 match listener.accept().await {
5486 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
5487 Err(err) => Some((Err(err), listener)),
5488 }
5489 });
5490 let handle = tokio::spawn(async move {
5491 Server::builder()
5492 .add_service(RepoSyncServiceServer::new(service))
5493 .serve_with_incoming(incoming)
5494 .await
5495 .expect("serve delta-aware test service");
5496 });
5497 let client = HostedGrpcClient::connect(addr, &signed_test_config())
5498 .await
5499 .expect("connect client");
5500 Some((client, handle))
5501 }
5502
5503 #[tokio::test]
5504 async fn warm_bare_pull_advertises_head_and_fires_zero_delta_short_circuit() {
5505 let (_dir, repo) = temp_repo();
5509 let head = put_state_with_blob(&repo, "tip", vec![]);
5510 repo.refs()
5511 .set_thread(&ThreadName::from("main"), &head)
5512 .expect("set thread");
5513
5514 let full_closure =
5515 wire::enumerate_state_closure(repo.store(), head).expect("enumerate closure");
5516 let full_pack =
5517 wire::build_native_pack(repo.store(), &full_closure).expect("build full pack");
5518 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5519
5520 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5521 remote_state: head,
5522 full_closure,
5523 delta_objects: Vec::new(),
5524 known_parent: StateId::from_bytes([23; 32]),
5525 full_pack: full_pack.clone(),
5526 delta_pack: full_pack,
5527 captured_exclude: captured.clone(),
5528 })
5529 .await
5530 else {
5531 return;
5532 };
5533
5534 let exchange = tokio::time::timeout(
5535 Duration::from_secs(5),
5536 client.pull_exchange(
5537 &repo,
5538 "owner/repo",
5539 "main",
5540 PullOptions {
5541 local_thread: None,
5542 depth: None,
5543 target_state: None,
5544 materialization: PullMaterialization::Full,
5545 },
5546 ),
5547 )
5548 .await
5549 .expect("warm bare pull must not hang")
5550 .expect("warm bare pull succeeds");
5551 server.abort();
5552
5553 let advertised = captured.lock().unwrap().clone().expect("request captured");
5554 assert_eq!(
5555 advertised,
5556 vec![proto_state_id(head).expect("valid state")],
5557 "a bare pull at the tip must advertise that tip in exclude_states"
5558 );
5559 assert!(exchange.result.success);
5560 assert_eq!(
5561 exchange.object_count, 0,
5562 "the zero-delta short-circuit must transfer no objects, not the full closure"
5563 );
5564 }
5565
5566 #[tokio::test]
5567 async fn behind_client_bare_pull_receives_exactly_the_missing_delta() {
5568 let (_src_dir, src_repo) = temp_repo_unseeded();
5575 let parent = put_state_with_blob(&src_repo, "base", vec![]);
5576 let child = put_state_with_blob(&src_repo, "feature", vec![parent]);
5577 let child_closure =
5578 wire::enumerate_state_closure(src_repo.store(), child).expect("child closure");
5579
5580 let (_dir, repo) = temp_repo_unseeded();
5584 let parent_closure =
5585 wire::enumerate_state_closure(src_repo.store(), parent).expect("parent closure");
5586 let parent_pack =
5587 wire::build_native_pack(src_repo.store(), &parent_closure).expect("parent pack");
5588 wire::install_received_pack(
5589 repo.store(),
5590 &parent_pack.pack_data,
5591 &parent_pack.index_data,
5592 )
5593 .expect("install parent closure into client");
5594 repo.refs()
5595 .set_thread(&ThreadName::from("main"), &parent)
5596 .expect("set thread to parent");
5597 wire::enumerate_state_closure(repo.store(), parent).expect("client holds parent closure");
5599 assert!(
5601 repo.store()
5602 .get_state(&child)
5603 .expect("probe child")
5604 .is_none(),
5605 "client must start without the child state"
5606 );
5607 let parent_clone = parent;
5608 let full_pack =
5609 wire::build_native_pack(src_repo.store(), &child_closure).expect("full pack");
5610 let delta_objects = wire::enumerate_state_closure_with_options(
5613 src_repo.store(),
5614 child,
5615 wire::StateClosureOptions {
5616 depth: None,
5617 exclude_states: vec![parent_clone],
5618 },
5619 )
5620 .expect("delta closure");
5621 assert!(
5622 !delta_objects.is_empty() && delta_objects.len() < child_closure.len(),
5623 "delta must be a strict, non-empty subset of the full closure"
5624 );
5625 let delta_pack =
5626 wire::build_native_pack(src_repo.store(), &delta_objects).expect("delta pack");
5627
5628 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5629 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5630 remote_state: child,
5631 full_closure: child_closure.clone(),
5632 delta_objects: delta_objects.clone(),
5633 known_parent: parent_clone,
5634 full_pack,
5635 delta_pack,
5636 captured_exclude: captured.clone(),
5637 })
5638 .await
5639 else {
5640 return;
5641 };
5642
5643 let exchange = tokio::time::timeout(
5644 Duration::from_secs(5),
5645 client.pull_exchange(
5646 &repo,
5647 "owner/repo",
5648 "main",
5649 PullOptions {
5650 local_thread: None,
5651 depth: None,
5652 target_state: None,
5653 materialization: PullMaterialization::Full,
5654 },
5655 ),
5656 )
5657 .await
5658 .expect("behind-client pull must not hang")
5659 .expect("behind-client pull succeeds");
5660 server.abort();
5661
5662 let advertised = captured.lock().unwrap().clone().expect("request captured");
5663 assert_eq!(
5664 advertised,
5665 vec![proto_state_id(parent).expect("valid state")],
5666 "a behind client must advertise the parent head it holds"
5667 );
5668 assert!(exchange.result.success);
5669 let reassembled = wire::enumerate_state_closure(repo.store(), child)
5673 .expect("the client must hold the complete child closure after a delta pull");
5674 assert_eq!(
5675 reassembled.len(),
5676 child_closure.len(),
5677 "delta pull must leave the client with the full child closure, no gaps"
5678 );
5679 }
5680
5681 #[tokio::test]
5682 async fn fresh_bare_pull_advertises_nothing_and_gets_full_closure() {
5683 let (_dir, repo) = temp_repo_unseeded();
5687 let (_src_dir, src_repo) = temp_repo_unseeded();
5688 let remote = put_state_with_blob(&src_repo, "seed", vec![]);
5689 let full_closure =
5690 wire::enumerate_state_closure(src_repo.store(), remote).expect("closure");
5691 let full_pack =
5692 wire::build_native_pack(src_repo.store(), &full_closure).expect("full pack");
5693
5694 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5695 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5696 remote_state: remote,
5697 full_closure: full_closure.clone(),
5698 delta_objects: Vec::new(),
5699 known_parent: StateId::from_bytes([24; 32]),
5700 full_pack: full_pack.clone(),
5701 delta_pack: full_pack,
5702 captured_exclude: captured.clone(),
5703 })
5704 .await
5705 else {
5706 return;
5707 };
5708
5709 let exchange = tokio::time::timeout(
5710 Duration::from_secs(5),
5711 client.pull_exchange(
5712 &repo,
5713 "owner/repo",
5714 "main",
5715 PullOptions {
5716 local_thread: None,
5717 depth: None,
5718 target_state: None,
5719 materialization: PullMaterialization::Full,
5720 },
5721 ),
5722 )
5723 .await
5724 .expect("fresh bare pull must not hang")
5725 .expect("fresh bare pull succeeds");
5726 server.abort();
5727
5728 let advertised = captured.lock().unwrap().clone().expect("request captured");
5729 assert!(
5730 advertised.is_empty(),
5731 "a fresh repo must advertise no exclude_states"
5732 );
5733 assert!(exchange.result.success);
5734 assert_eq!(
5735 exchange.object_count,
5736 full_closure.len(),
5737 "a fresh pull must receive the full closure, nothing wrongly excluded"
5738 );
5739 assert!(
5740 wire::enumerate_state_closure(repo.store(), remote).is_ok(),
5741 "fresh pull must install the complete closure"
5742 );
5743 }
5744
5745 #[tokio::test]
5746 async fn explicit_local_thread_incomplete_closure_does_not_advertise_and_repairs() {
5747 let (_src_dir, src_repo) = temp_repo_unseeded();
5757 let parent = put_state_with_blob(&src_repo, "base", vec![]);
5758 let child = put_state_with_blob(&src_repo, "feature", vec![parent]);
5759 let child_closure =
5760 wire::enumerate_state_closure(src_repo.store(), child).expect("child closure");
5761 let full_pack =
5762 wire::build_native_pack(src_repo.store(), &child_closure).expect("full pack");
5763
5764 let (_dir, repo) = temp_repo_unseeded();
5769 let child_only = wire::enumerate_state_closure_with_options(
5770 src_repo.store(),
5771 child,
5772 wire::StateClosureOptions {
5773 depth: None,
5774 exclude_states: vec![parent],
5775 },
5776 )
5777 .expect("child-only objects");
5778 let child_only_pack =
5779 wire::build_native_pack(src_repo.store(), &child_only).expect("child-only pack");
5780 wire::install_received_pack(
5781 repo.store(),
5782 &child_only_pack.pack_data,
5783 &child_only_pack.index_data,
5784 )
5785 .expect("install child-only objects");
5786 repo.refs()
5787 .set_thread(&ThreadName::from("feature"), &child)
5788 .expect("set local thread to child");
5789 assert!(
5792 matches!(
5793 wire::enumerate_state_closure(repo.store(), child),
5794 Err(ProtocolError::ObjectNotFound(_))
5795 ),
5796 "the explicit thread head's closure must start incomplete"
5797 );
5798
5799 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5800 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5801 remote_state: child,
5802 full_closure: child_closure.clone(),
5803 delta_objects: Vec::new(),
5804 known_parent: parent,
5805 full_pack: full_pack.clone(),
5806 delta_pack: full_pack,
5807 captured_exclude: captured.clone(),
5808 })
5809 .await
5810 else {
5811 return;
5812 };
5813
5814 let exchange = tokio::time::timeout(
5815 Duration::from_secs(5),
5816 client.pull_exchange(
5817 &repo,
5818 "owner/repo",
5819 "main",
5820 PullOptions {
5821 local_thread: Some("feature"),
5822 depth: None,
5823 target_state: None,
5824 materialization: PullMaterialization::Full,
5825 },
5826 ),
5827 )
5828 .await
5829 .expect("explicit-thread pull must not hang")
5830 .expect("explicit-thread pull succeeds");
5831 server.abort();
5832
5833 let advertised = captured.lock().unwrap().clone().expect("request captured");
5834 assert!(
5835 advertised.is_empty(),
5836 "an explicit --local-thread with an incomplete closure must advertise nothing, \
5837 falling back to a full pull (got {advertised:?})"
5838 );
5839 assert!(exchange.result.success);
5840 let reassembled = wire::enumerate_state_closure(repo.store(), child)
5842 .expect("the client must hold the complete child closure after the fallback full pull");
5843 assert_eq!(
5844 reassembled.len(),
5845 child_closure.len(),
5846 "the full-pull fallback must leave the client with the complete closure, no gaps"
5847 );
5848 }
5849
5850 #[tokio::test]
5851 async fn explicit_local_thread_complete_closure_advertises_and_fires_short_circuit() {
5852 let (_dir, repo) = temp_repo();
5857 let head = put_state_with_blob(&repo, "tip", vec![]);
5858 repo.refs()
5859 .set_thread(&ThreadName::from("feature"), &head)
5860 .expect("set local thread");
5861 wire::enumerate_state_closure(repo.store(), head)
5863 .expect("client holds the complete thread closure");
5864
5865 let full_closure =
5866 wire::enumerate_state_closure(repo.store(), head).expect("enumerate closure");
5867 let full_pack =
5868 wire::build_native_pack(repo.store(), &full_closure).expect("build full pack");
5869 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5870
5871 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5872 remote_state: head,
5873 full_closure,
5874 delta_objects: Vec::new(),
5875 known_parent: StateId::from_bytes([25; 32]),
5876 full_pack: full_pack.clone(),
5877 delta_pack: full_pack,
5878 captured_exclude: captured.clone(),
5879 })
5880 .await
5881 else {
5882 return;
5883 };
5884
5885 let exchange = tokio::time::timeout(
5886 Duration::from_secs(5),
5887 client.pull_exchange(
5888 &repo,
5889 "owner/repo",
5890 "main",
5891 PullOptions {
5892 local_thread: Some("feature"),
5893 depth: None,
5894 target_state: None,
5895 materialization: PullMaterialization::Full,
5896 },
5897 ),
5898 )
5899 .await
5900 .expect("explicit-thread pull must not hang")
5901 .expect("explicit-thread pull succeeds");
5902 server.abort();
5903
5904 let advertised = captured.lock().unwrap().clone().expect("request captured");
5905 assert_eq!(
5906 advertised,
5907 vec![proto_state_id(head).expect("valid state")],
5908 "an explicit --local-thread with a complete closure must advertise its head"
5909 );
5910 assert!(exchange.result.success);
5911 assert_eq!(
5912 exchange.object_count, 0,
5913 "advertising the complete head must fire the zero-delta short-circuit, not a full pull"
5914 );
5915 }
5916}