1use std::{
2 collections::{BTreeMap, HashMap},
3 io::{self, Seek, SeekFrom, Write},
4 sync::{
5 Arc, Mutex,
6 atomic::{AtomicUsize, Ordering},
7 },
8 time::{Duration, Instant},
9};
10
11use grpc::heddle::api::v1alpha1::{
12 GetBlobRequest, GitCheckpointTransfer, GitLaneTransfer, GitObjectAlgorithm,
13 GitObjectId as ProtoGitObjectId, GitPackTransfer, GitRefKind as GrpcGitRefKind,
14 GitRefUpdateTransfer, ListRefsRequest, ObjectAvailabilityStatus, ObjectDescriptor, PackChunk,
15 PackStreamKind, PartialFetchStatus, PullClientFrame, PullRequest, PullServerFrame,
16 PushClientFrame, PushRequest, PushServerFrame, RedactionTransfer, StateAttachmentTransfer,
17 StateVisibilityTransfer,
18 StreamOpeningProof, ThreadConfidenceSummary, ThreadIntegrationPolicy, ThreadMetadata,
19 ThreadVerificationSummary, TransportMode, UpdateRefRequest, WantObjects, git_lane_transfer,
20 pull_client_frame, pull_server_frame, push_client_frame, push_server_frame,
21};
22use objects::{
23 Progress,
24 object::{ContentHash, MarkerName, StateId, ThreadName},
25 store::{AnyStore, ObjectStore, PackObjectId},
26};
27use repo::{
28 GitRefKind as ClassifiedGitRefKind, GitRefName, Repository, RepositoryCapability,
29 RevisionAddress, SyncedThreadMetadata, ThreadManager,
30};
31use sley::{
32 ObjectId as GitObjectId, RefPrecondition, ReferenceTarget, Repository as SleyRepository,
33};
34use tempfile::NamedTempFile;
35use tokio::sync::mpsc;
36use tokio_stream::wrappers::ReceiverStream;
37use tonic::Request;
38use wire::{
39 GitLaneTransferIntent, ObjectInfo, ObjectType, ObjectTypeBucket, PlannedObject, ProtocolError,
40 PullComplete, PushComplete, RefEntry, RefUpdated, RepositoryTransferPlan,
41};
42
43use super::{
44 HostedGrpcClient, PullMaterialization,
45 helpers::{
46 descriptor_id, descriptor_id_from_info, object_descriptor_with_status, object_type_name,
47 parse_descriptor_to_info, status_to_protocol_error, to_proto_object_info,
48 transport_mode_name,
49 },
50 operation_id::ClientOperationId,
51};
52
53#[derive(Clone, Copy)]
54struct PullOptions<'a> {
55 local_thread: Option<&'a str>,
56 depth: Option<u32>,
57 target_state: Option<StateId>,
58 materialization: PullMaterialization,
59}
60
61struct PullWantPlan {
62 wants: Vec<ObjectDescriptor>,
63 transfer_plan: RepositoryTransferPlan<ObjectInfo>,
64 wanted_types: WantedTypes,
65 want_full_closure: bool,
66}
67
68type WantedTypes = HashMap<PackObjectId, Vec<ObjectType>>;
69
70struct GitLanePushPlan {
71 local_revision_address: String,
72 pack: Option<GitPackPushPlan>,
79 ref_updates: Vec<GitRefUpdateTransfer>,
85}
86
87#[derive(Clone)]
88struct GitPackPushPlan {
89 transfer_id: String,
90 pack_id: Vec<u8>,
91 pack_size: u64,
92 #[cfg_attr(not(test), allow(dead_code))]
97 roots: Vec<GitObjectId>,
98 pack_file: Arc<Mutex<NamedTempFile>>,
101}
102
103const PUSH_FULL_DESCRIPTOR_OBJECT_THRESHOLD: usize = 512;
104const PULL_PACK_SPOOL_OBJECT_THRESHOLD: usize = 512;
105const NATIVE_PACK_DRAIN_OBJECT_INTERVAL: usize = 32;
106const NATIVE_PACK_OBJECT_PREFETCH_LIMIT: usize = 32;
107const NATIVE_PACK_OBJECT_LOAD_WORKER_LIMIT: usize = 8;
108
109#[derive(Debug, Clone, Default)]
110pub struct PullObjectMix {
111 pub blobs: usize,
112 pub trees: usize,
113 pub states: usize,
114 pub actions: usize,
115 pub redactions: usize,
116 pub state_visibilities: usize,
117 pub state_attachments: usize,
118}
119
120#[derive(Debug, Clone)]
121pub struct HostedRefEntry {
122 pub name: String,
123 pub state_id: StateId,
124 pub is_thread: bool,
125 pub revision_address: String,
126}
127
128impl PullObjectMix {
129 fn record(&mut self, obj_type: ObjectType) {
130 match obj_type.bucket() {
131 ObjectTypeBucket::Blob => self.blobs += 1,
132 ObjectTypeBucket::Tree => self.trees += 1,
133 ObjectTypeBucket::State => self.states += 1,
134 ObjectTypeBucket::Action => self.actions += 1,
135 ObjectTypeBucket::Redaction => self.redactions += 1,
136 ObjectTypeBucket::StateVisibility => self.state_visibilities += 1,
137 ObjectTypeBucket::StateAttachment => self.state_attachments += 1,
138 }
139 }
140
141 pub fn total(&self) -> usize {
142 self.blobs
143 + self.trees
144 + self.states
145 + self.actions
146 + self.redactions
147 + self.state_visibilities
148 + self.state_attachments
149 }
150}
151
152#[derive(Debug, Clone, Default)]
153pub struct PullProfile {
154 pub ready_wait: Duration,
155 pub receive_and_apply: Duration,
156 pub decode: Duration,
157 pub store_receive_object: Duration,
158 pub metadata_sync: Duration,
159 pub pack_decode_apply: Duration,
160 pub raw_decode_apply: Duration,
161 pub pack_decode: Duration,
162 pub raw_decode: Duration,
163 pub bytes_received: usize,
164 pub pack_bytes_received: usize,
165 pub raw_bytes_received: usize,
166 pub objects_received: usize,
167 pub object_mix: PullObjectMix,
168}
169
170impl HostedGrpcClient {
171 fn stream_opening_proof(
172 &self,
173 stream_id: &str,
174 route: &str,
175 repository: &str,
176 resume_cursor: &str,
177 ) -> Result<StreamOpeningProof, ProtocolError> {
178 use crypto::Signer as _;
179
180 let signer = self.device_signer()?.ok_or_else(|| {
181 ProtocolError::AuthenticationFailed(
182 "hosted sync requires a stable device signing identity".to_string(),
183 )
184 })?;
185 let identity = self.stable_signing_identity()?;
186 let capability_context = Vec::new();
187 let canonical = grpc::signing::stream_open_bytes(
188 identity,
189 stream_id,
190 route,
191 repository,
192 resume_cursor,
193 &capability_context,
194 );
195 let signature = signer
196 .sign(&canonical)
197 .map_err(|err| ProtocolError::AuthenticationFailed(err.to_string()))?;
198 Ok(StreamOpeningProof {
199 stream_id: stream_id.to_string(),
200 route: route.to_string(),
201 repository: super::helpers::repository_ref(repository),
202 resume_cursor: resume_cursor.to_string(),
203 capability_context,
204 nonce: Vec::new(),
205 signature,
206 })
207 }
208
209 pub async fn list_refs(&mut self, repo_path: &str) -> Result<Vec<RefEntry>, ProtocolError> {
210 Ok(self
211 .list_refs_with_revision_addresses(repo_path)
212 .await?
213 .into_iter()
214 .map(|entry| RefEntry {
215 name: entry.name,
216 state_id: entry.state_id,
217 is_thread: entry.is_thread,
218 })
219 .collect())
220 }
221
222 pub async fn list_refs_with_revision_addresses(
223 &mut self,
224 repo_path: &str,
225 ) -> Result<Vec<HostedRefEntry>, ProtocolError> {
226 let mut request = Request::new(ListRefsRequest {
227 repo_path: super::helpers::repository_ref(repo_path),
228 });
229 self.apply_signed_auth(
230 &mut request,
231 "/heddle.api.v1alpha1.RepoSyncService/ListRefs",
232 )?;
233 let response = self
234 .inner
235 .list_refs(request)
236 .await
237 .map_err(status_to_protocol_error)?
238 .into_inner();
239 response
240 .refs
241 .into_iter()
242 .map(|entry| {
243 Ok(HostedRefEntry {
244 name: entry.name,
245 state_id: super::helpers::parse_proto_state_id(entry.state_id)?.ok_or_else(
246 || ProtocolError::InvalidState("ref is missing its state ID".to_string()),
247 )?,
248 is_thread: entry.is_thread,
249 revision_address: entry.revision_address,
250 })
251 })
252 .collect()
253 }
254
255 #[allow(clippy::too_many_arguments)]
256 pub async fn update_ref(
257 &mut self,
258 repo_path: &str,
259 name: &str,
260 is_thread: bool,
261 old_value: Option<StateId>,
262 new_value: StateId,
263 force: bool,
264 thread_metadata: Option<&SyncedThreadMetadata>,
265 client_operation_id: String,
266 ) -> Result<RefUpdated, ProtocolError> {
267 let operation_id = ClientOperationId::for_required_method(
268 "heddle.api.v1alpha1.RepoSyncService/UpdateRef",
269 client_operation_id,
270 )?;
271 let mut request = Request::new(UpdateRefRequest {
272 repo_path: super::helpers::repository_ref(repo_path),
273 name: name.to_string(),
274 is_thread,
275 force,
276 old_value: old_value
277 .map(|value| value.to_string_full())
278 .unwrap_or_default(),
279 new_value: new_value.to_string_full(),
280 thread_metadata: thread_metadata.map(to_proto_thread_metadata),
281 old_revision_address: old_value
282 .map(|value| RevisionAddress::heddle(value).to_string())
283 .unwrap_or_default(),
284 new_revision_address: RevisionAddress::heddle(new_value).to_string(),
285 client_operation_id: operation_id.to_wire(),
286 });
287 self.apply_signed_auth(
288 &mut request,
289 "/heddle.api.v1alpha1.RepoSyncService/UpdateRef",
290 )?;
291 let response = self
292 .inner
293 .update_ref(request)
294 .await
295 .map_err(status_to_protocol_error)?
296 .into_inner();
297 Ok(RefUpdated {
298 success: response.success,
299 old_value: if response.old_value.is_empty() {
300 None
301 } else {
302 Some(
303 StateId::parse(&response.old_value)
304 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?,
305 )
306 },
307 error: (!response.error.is_empty()).then_some(response.error),
308 })
309 }
310
311 pub async fn push(
312 &mut self,
313 repo: &Repository,
314 repo_path: &str,
315 local_state: StateId,
316 target_thread: &str,
317 force: bool,
318 client_operation_id: String,
319 ) -> Result<PushComplete, ProtocolError> {
320 let operation_id = ClientOperationId::caller_or_fresh(
321 "heddle.api.v1alpha1.RepoSyncService/Push",
322 client_operation_id,
323 );
324 self.push_with_revision(
325 repo,
326 repo_path,
327 local_state,
328 target_thread,
329 force,
330 operation_id,
331 RevisionAddress::heddle(local_state).to_string(),
332 None,
333 &Progress::null(),
334 )
335 .await
336 }
337
338 #[allow(clippy::too_many_arguments)]
353 pub async fn push_git_overlay_mirror(
354 &mut self,
355 repo: &Repository,
356 repo_path: &str,
357 local_state: StateId,
358 target_thread: &str,
359 force: bool,
360 progress: &Progress,
361 client_operation_id: String,
362 ) -> Result<PushComplete, ProtocolError> {
363 let operation_id = ClientOperationId::caller_or_fresh(
364 "heddle.api.v1alpha1.RepoSyncService/Push",
365 client_operation_id,
366 );
367 progress.set_phase("packing refs");
368 let remote_ref_expectations = self.git_mirror_ref_expectations(repo_path).await?;
369 let git_lane = build_git_mirror_push_plan(
370 repo,
371 self.transport.chunk_size.max(1),
372 &remote_ref_expectations,
373 )?;
374 let local_revision_address = git_lane.local_revision_address.clone();
375 self.push_with_revision(
376 repo,
377 repo_path,
378 local_state,
379 target_thread,
380 force,
381 operation_id,
382 local_revision_address,
383 Some(git_lane),
384 progress,
385 )
386 .await
387 }
388
389 async fn git_mirror_ref_expectations(
393 &mut self,
394 repo_path: &str,
395 ) -> Result<HashMap<String, GitRefRemoteExpectation>, ProtocolError> {
396 let remote_refs = self.list_refs_with_revision_addresses(repo_path).await?;
397 let mut expectations = HashMap::with_capacity(remote_refs.len());
398 for entry in remote_refs {
399 let expectation = parse_git_ref_expectation(&entry.revision_address)?;
400 expectations.insert(entry.name, expectation);
401 }
402 Ok(expectations)
403 }
404
405 #[allow(clippy::too_many_arguments)]
406 async fn push_with_revision(
407 &mut self,
408 repo: &Repository,
409 repo_path: &str,
410 local_state: StateId,
411 target_thread: &str,
412 force: bool,
413 operation_id: ClientOperationId,
414 local_revision_address: String,
415 git_lane: Option<GitLanePushPlan>,
416 progress: &Progress,
417 ) -> Result<PushComplete, ProtocolError> {
418 let _ = self.transport.chunk_size;
419 let _ = self.transport.resume_attempts;
420 let git_lane_intent = if git_lane.is_some() {
425 GitLaneTransferIntent::ExistingImplementation
426 } else {
427 GitLaneTransferIntent::HeddleObjectsOnly
428 };
429 let closure = wire::enumerate_state_closure_transfer_with_options(
430 repo.store(),
431 local_state,
432 wire::StateClosureOptions::default(),
433 PUSH_FULL_DESCRIPTOR_OBJECT_THRESHOLD,
434 )?;
435 let object_plan =
436 RepositoryTransferPlan::from_planned_objects(closure.planned_objects, git_lane_intent);
437 let full_objects = closure.full_objects;
438 let object_count = full_objects
439 .as_ref()
440 .map_or(object_plan.stats.total_objects, std::vec::Vec::len);
441 let transfer_id = push_transfer_id(repo_path, local_state, target_thread);
442 let transport_mode = preferred_transport_mode(&self.transport, object_count);
443 let thread_metadata = load_thread_metadata(repo, target_thread, local_state)?;
444 let request_message = PushClientFrame {
445 frame: Some(push_client_frame::Frame::Request(Box::new(PushRequest {
446 repo_path: super::helpers::repository_ref(repo_path),
447 local_state: super::helpers::proto_state_id(local_state),
448 target_thread: target_thread.to_string(),
449 create_thread: true,
450 force,
451 objects: full_objects.as_ref().map_or_else(
452 || {
453 object_plan
454 .partitions
455 .iter()
456 .map(to_proto_planned_object)
457 .collect()
458 },
459 |objects| objects.iter().map(to_proto_object_info).collect(),
460 ),
461 transfer: Some(self.transport.transfer_checkpoint_with_mode(
462 transfer_id.clone(),
463 transport_mode,
464 0,
465 0,
466 false,
467 )),
468 partial_fetch_status: partial_fetch_status_for_repo(repo),
469 allow_partial_fetch: true,
470 thread_metadata: thread_metadata
471 .map(|metadata| to_proto_thread_metadata(&metadata)),
472 local_revision_address,
473 }))),
474 client_operation_id: operation_id.to_wire(),
475 };
476
477 let (tx, rx) = mpsc::channel(self.transport.max_inflight_objects.max(4));
478 tx.send(PushClientFrame {
479 frame: Some(push_client_frame::Frame::Open(self.stream_opening_proof(
480 &transfer_id,
481 "/heddle.api.v1alpha1.RepoSyncService/Push",
482 repo_path,
483 "",
484 )?)),
485 client_operation_id: operation_id.to_wire(),
486 })
487 .await
488 .map_err(|_| ProtocolError::InvalidState("failed to open push stream".to_string()))?;
489 tx.send(request_message).await.map_err(|_| {
490 ProtocolError::InvalidState("failed to initialize push stream".to_string())
491 })?;
492 let mut request = Request::new(ReceiverStream::new(rx));
493 self.apply_auth(&mut request, "/heddle.api.v1alpha1.RepoSyncService/Push")?;
494 let mut response = self
495 .inner
496 .push(request)
497 .await
498 .map_err(status_to_protocol_error)?
499 .into_inner();
500
501 let ready = match response.message().await.map_err(status_to_protocol_error)? {
502 Some(PushServerFrame {
503 frame: Some(push_server_frame::Frame::Ready(ready)),
504 }) => ready,
505 _ => {
506 return Err(ProtocolError::InvalidState(
507 "expected PushReady from gRPC server".to_string(),
508 ));
509 }
510 };
511 let object_index = match full_objects {
512 Some(objects) => objects
513 .into_iter()
514 .map(|info| (descriptor_id_from_info(&info), info))
515 .collect::<HashMap<_, _>>(),
516 None => object_plan
517 .partitions
518 .iter()
519 .map(|object| {
520 (
521 descriptor_id_from_plan(object),
522 object_info_from_plan(object),
523 )
524 })
525 .collect::<HashMap<_, _>>(),
526 };
527
528 let ready_transport_mode = ready
529 .transfer
530 .as_ref()
531 .and_then(|transfer| TransportMode::try_from(transfer.transport_mode).ok())
532 .unwrap_or(transport_mode);
533 let wanted_infos = ready
534 .want_objects
535 .into_iter()
536 .map(|want| {
537 object_index
538 .get(&descriptor_id(&want))
539 .cloned()
540 .ok_or_else(|| {
541 ProtocolError::InvalidState("server requested unknown object".to_string())
542 })
543 })
544 .collect::<Result<Vec<_>, _>>()?;
545
546 let wanted_plan = RepositoryTransferPlan::from_object_infos(wanted_infos, git_lane_intent);
547
548 let (pack_objects, push_sidecar_objects): (Vec<_>, Vec<_>) = wanted_plan
556 .partitions
557 .packable_objects
558 .into_iter()
559 .partition(|info| info.obj_type.packable_for_push());
560
561 if !pack_objects.is_empty() {
562 send_native_pack_streaming_messages(
563 &tx,
564 repo,
565 &pack_objects,
566 PushWireIdentities {
567 transfer_id: &transfer_id,
568 client_operation_id: operation_id.as_str(),
569 },
570 self.transport.chunk_size.max(1),
571 &self.transport,
572 ready_transport_mode,
573 )
574 .await?;
575 }
576
577 for info in wanted_plan
578 .partitions
579 .sidecar_objects
580 .into_iter()
581 .chain(push_sidecar_objects)
582 {
583 let message = sidecar_push_message(repo, info, operation_id.as_str())?;
584 tx.send(message).await.map_err(|_| {
585 ProtocolError::InvalidState("push stream closed unexpectedly".to_string())
586 })?;
587 }
588
589 if let Some(git_lane) = git_lane {
590 if let Some(pack) = git_lane.pack.as_ref() {
596 send_git_pack_streaming_messages(
597 &tx,
598 pack,
599 self.transport.chunk_size.max(1),
600 progress,
601 operation_id.as_str(),
602 )
603 .await?;
604 } else {
605 progress.set_phase("no new objects to pack");
606 }
607 progress.set_phase(format!("writing {} refs", git_lane.ref_updates.len()));
608 for ref_update in git_lane.ref_updates {
609 tx.send(git_lane_push_message(
610 git_lane_transfer::Body::RefUpdate(ref_update),
611 operation_id.as_str(),
612 ))
613 .await
614 .map_err(|_| {
615 ProtocolError::InvalidState("push stream closed unexpectedly".to_string())
616 })?;
617 }
618 }
619 drop(tx);
620
621 let result = match response.message().await.map_err(status_to_protocol_error)? {
622 Some(PushServerFrame {
623 frame: Some(push_server_frame::Frame::Complete(complete)),
624 }) => PushComplete {
625 success: complete.success,
626 new_state: super::helpers::parse_proto_state_id(complete.new_state)?,
627 error: (!complete.error.is_empty()).then_some(complete.error),
628 transfer_id: complete
629 .transfer
630 .as_ref()
631 .map(|transfer| transfer.transfer_id.clone())
632 .unwrap_or_default(),
633 transport_mode: complete
634 .transfer
635 .as_ref()
636 .map(|transfer| transport_mode_name(transfer.transport_mode))
637 .unwrap_or("raw-objects")
638 .to_string(),
639 resume_offset: complete
640 .transfer
641 .as_ref()
642 .map(|transfer| transfer.resume_offset)
643 .unwrap_or_default(),
644 chunk_index: complete
645 .transfer
646 .as_ref()
647 .map(|transfer| transfer.chunk_index)
648 .unwrap_or_default(),
649 checkpoint: complete
650 .transfer
651 .as_ref()
652 .map(|transfer| transfer.checkpoint.clone())
653 .unwrap_or_default(),
654 is_complete: complete
655 .transfer
656 .as_ref()
657 .map(|transfer| transfer.is_complete)
658 .unwrap_or(false),
659 },
660 _ => {
661 return Err(ProtocolError::InvalidState(
662 "expected PushComplete from gRPC server".to_string(),
663 ));
664 }
665 };
666
667 if result.success {
668 self.sync_remote_markers(repo, repo_path, local_state)
669 .await?;
670 }
671 Ok(result)
672 }
673
674 pub async fn pull(
675 &mut self,
676 repo: &Repository,
677 repo_path: &str,
678 remote_thread: &str,
679 local_thread: Option<&str>,
680 ) -> Result<PullComplete, ProtocolError> {
681 self.pull_with_options(
682 repo,
683 repo_path,
684 remote_thread,
685 PullOptions {
686 local_thread,
687 depth: None,
688 target_state: None,
689 materialization: PullMaterialization::Full,
690 },
691 )
692 .await
693 }
694
695 pub async fn pull_profiled(
696 &mut self,
697 repo: &Repository,
698 repo_path: &str,
699 remote_thread: &str,
700 local_thread: Option<&str>,
701 ) -> Result<(PullComplete, PullProfile), ProtocolError> {
702 self.pull_exchange(
703 repo,
704 repo_path,
705 remote_thread,
706 PullOptions {
707 local_thread,
708 depth: None,
709 target_state: None,
710 materialization: PullMaterialization::Full,
711 },
712 )
713 .await
714 .map(|exchange| (exchange.result, exchange.profile))
715 }
716
717 pub async fn pull_partial(
718 &mut self,
719 repo: &Repository,
720 repo_path: &str,
721 remote_thread: &str,
722 local_thread: Option<&str>,
723 ) -> Result<PullComplete, ProtocolError> {
724 self.pull_with_options(
725 repo,
726 repo_path,
727 remote_thread,
728 PullOptions {
729 local_thread,
730 depth: None,
731 target_state: None,
732 materialization: PullMaterialization::Lazy,
733 },
734 )
735 .await
736 }
737
738 pub async fn pull_with_depth_and_materialization(
739 &mut self,
740 repo: &Repository,
741 repo_path: &str,
742 remote_thread: &str,
743 local_thread: Option<&str>,
744 depth: Option<u32>,
745 materialization: PullMaterialization,
746 ) -> Result<PullComplete, ProtocolError> {
747 self.pull_with_options(
748 repo,
749 repo_path,
750 remote_thread,
751 PullOptions {
752 local_thread,
753 depth,
754 target_state: None,
755 materialization,
756 },
757 )
758 .await
759 }
760
761 pub async fn pull_with_depth(
762 &mut self,
763 repo: &Repository,
764 repo_path: &str,
765 remote_thread: &str,
766 local_thread: Option<&str>,
767 depth: Option<u32>,
768 ) -> Result<PullComplete, ProtocolError> {
769 self.pull_with_depth_and_materialization(
770 repo,
771 repo_path,
772 remote_thread,
773 local_thread,
774 depth,
775 PullMaterialization::Full,
776 )
777 .await
778 }
779
780 pub async fn fetch_state(
781 &mut self,
782 repo: &Repository,
783 repo_path: &str,
784 remote_thread: &str,
785 target_state: StateId,
786 ) -> Result<usize, ProtocolError> {
787 self.pull_exchange(
788 repo,
789 repo_path,
790 remote_thread,
791 PullOptions {
792 local_thread: None,
793 depth: None,
794 target_state: Some(target_state),
795 materialization: PullMaterialization::Full,
796 },
797 )
798 .await
799 .map(|exchange| exchange.object_count)
800 }
801
802 pub async fn fetch_state_partial(
803 &mut self,
804 repo: &Repository,
805 repo_path: &str,
806 remote_thread: &str,
807 target_state: StateId,
808 ) -> Result<usize, ProtocolError> {
809 self.pull_exchange(
810 repo,
811 repo_path,
812 remote_thread,
813 PullOptions {
814 local_thread: None,
815 depth: None,
816 target_state: Some(target_state),
817 materialization: PullMaterialization::Lazy,
818 },
819 )
820 .await
821 .map(|exchange| exchange.object_count)
822 }
823
824 pub async fn hydrate_blob_at_path(
825 &mut self,
826 repo: &Repository,
827 repo_path: &str,
828 reference: &str,
829 path: &str,
830 ) -> Result<objects::object::Blob, ProtocolError> {
831 let mut request = Request::new(GetBlobRequest {
832 repo_path: super::helpers::repository_ref(repo_path),
833 r#ref: reference.to_string(),
834 path: path.to_string(),
835 });
836 self.apply_signed_auth(
837 &mut request,
838 "/heddle.api.v1alpha1.RepositoryService/GetBlob",
839 )?;
840 let response = self
841 .content
842 .get_blob(request)
843 .await
844 .map_err(status_to_protocol_error)?
845 .into_inner();
846
847 let content = super::helpers::decode_blob_content(response.content, response.is_binary)?;
848 let blob = objects::object::Blob::new(content);
849 repo.store().put_blob(&blob)?;
850 repo.clear_missing_blob(&blob.hash())?;
851 Ok(blob)
852 }
853
854 pub async fn hydrate_missing_blobs_for_state(
855 &mut self,
856 repo: &Repository,
857 repo_path: &str,
858 remote_thread: &str,
859 target_state: StateId,
860 ) -> Result<usize, ProtocolError> {
861 let exchange = self
862 .pull_exchange(
863 repo,
864 repo_path,
865 remote_thread,
866 PullOptions {
867 local_thread: None,
868 depth: None,
869 target_state: Some(target_state),
870 materialization: PullMaterialization::Full,
871 },
872 )
873 .await?;
874 clear_missing_blobs_for_state(repo, target_state)?;
875 Ok(exchange.object_count)
876 }
877
878 async fn pull_with_options(
879 &mut self,
880 repo: &Repository,
881 repo_path: &str,
882 remote_thread: &str,
883 options: PullOptions<'_>,
884 ) -> Result<PullComplete, ProtocolError> {
885 self.pull_exchange(repo, repo_path, remote_thread, options)
886 .await
887 .map(|exchange| exchange.result)
888 }
889
890 async fn pull_exchange(
891 &mut self,
892 repo: &Repository,
893 repo_path: &str,
894 remote_thread: &str,
895 options: PullOptions<'_>,
896 ) -> Result<PullExchange, ProtocolError> {
897 let exchange_start = Instant::now();
898 let mut exclude_states = Vec::new();
899 let advertised_head = if let Some(local_thread) = options.local_thread {
910 locally_complete_local_thread_head(repo, local_thread, options.target_state)?
911 } else {
912 locally_complete_pull_head(repo, remote_thread, options.target_state)?
913 };
914 if let Some(head) = advertised_head {
915 exclude_states.push(head);
916 }
917 let allow_partial_fetch = options.materialization.allows_partial_fetch();
918 let fresh_full_pull =
919 supports_compact_full_pull(repo, allow_partial_fetch, &exclude_states)?;
920
921 let transfer_id = pull_transfer_id(
922 repo_path,
923 remote_thread,
924 options.local_thread,
925 options.depth,
926 options.target_state,
927 );
928 let request_message = PullClientFrame {
929 frame: Some(pull_client_frame::Frame::Request(PullRequest {
930 repo_path: super::helpers::repository_ref(repo_path),
931 remote_thread: remote_thread.to_string(),
932 local_thread: options.local_thread.unwrap_or_default().to_string(),
933 target_state: options
934 .target_state
935 .and_then(super::helpers::proto_state_id),
936 depth: options.depth.unwrap_or_default(),
937 exclude_states: exclude_states
938 .iter()
939 .copied()
940 .filter_map(super::helpers::proto_state_id)
941 .collect(),
942 transfer: Some(self.transport.transfer_checkpoint_with_mode(
943 transfer_id.clone(),
944 TransportMode::NativePack,
945 0,
946 0,
947 false,
948 )),
949 partial_fetch_status: partial_fetch_status_for_repo(repo),
950 allow_partial_fetch,
951 fresh_full_pull,
952 target_revision_address: options
953 .target_state
954 .map(|state| RevisionAddress::heddle(state).to_string())
955 .unwrap_or_default(),
956 })),
957 };
958
959 let (tx, rx) = mpsc::channel(self.transport.max_inflight_objects.max(4));
960 tx.send(PullClientFrame {
961 frame: Some(pull_client_frame::Frame::Open(self.stream_opening_proof(
962 &transfer_id,
963 "/heddle.api.v1alpha1.RepoSyncService/Pull",
964 repo_path,
965 "",
966 )?)),
967 })
968 .await
969 .map_err(|_| ProtocolError::InvalidState("failed to open pull stream".to_string()))?;
970 tx.send(request_message).await.map_err(|_| {
971 ProtocolError::InvalidState("failed to initialize pull stream".to_string())
972 })?;
973 let mut request = Request::new(ReceiverStream::new(rx));
974 self.apply_auth(&mut request, "/heddle.api.v1alpha1.RepoSyncService/Pull")?;
975 let mut response = self
976 .inner
977 .pull(request)
978 .await
979 .map_err(status_to_protocol_error)?
980 .into_inner();
981
982 let ready = match response.message().await.map_err(status_to_protocol_error)? {
983 Some(PullServerFrame {
984 frame: Some(pull_server_frame::Frame::Ready(ready)),
985 }) => ready,
986 _ => {
987 return Err(ProtocolError::InvalidState(
988 "expected PullReady from gRPC server".to_string(),
989 ));
990 }
991 };
992 let mut profile = PullProfile {
993 ready_wait: exchange_start.elapsed(),
994 ..PullProfile::default()
995 };
996 let remote_state = super::helpers::parse_proto_state_id(ready.remote_state)?
997 .ok_or_else(|| ProtocolError::InvalidState("missing remote state".to_string()))?;
998 let advertised_object_count = ready.objects_to_fetch.len();
999 let PullWantPlan {
1000 wants,
1001 transfer_plan,
1002 wanted_types,
1003 want_full_closure,
1004 } = plan_pull_wants(
1005 repo,
1006 &remote_state,
1007 ready.full_closure_available,
1008 ready.objects_to_fetch,
1009 allow_partial_fetch,
1010 )?;
1011 let native_pack_required = native_pack_required_for_pull(want_full_closure, &transfer_plan);
1012
1013 tx.send(PullClientFrame {
1014 frame: Some(pull_client_frame::Frame::Want(WantObjects {
1015 objects: wants.clone(),
1016 want_full_closure,
1017 transfer: Some(self.transport.transfer_checkpoint_with_mode(
1018 transfer_id.clone(),
1019 TransportMode::NativePack,
1020 0,
1021 0,
1022 false,
1023 )),
1024 })),
1025 })
1026 .await
1027 .map_err(|_| ProtocolError::InvalidState("pull stream closed unexpectedly".to_string()))?;
1028 drop(tx);
1029
1030 let receive_start = Instant::now();
1031 let use_pack_spool = advertised_object_count > PULL_PACK_SPOOL_OBJECT_THRESHOLD;
1032 let mut pack_state = wire::PackChunkState::default();
1033 let mut pack_spool = if use_pack_spool {
1034 Some(wire::PackChunkSpool::new_in(repo.heddle_dir())?)
1035 } else {
1036 None
1037 };
1038 let mut git_lane_repo = None;
1039 let mut git_pack_state = GitPackPullInstallState::default();
1040 let mut received = 0usize;
1041 while let Some(message) = response.message().await.map_err(status_to_protocol_error)? {
1042 match message.frame {
1043 Some(pull_server_frame::Frame::Pack(chunk)) => {
1044 profile.bytes_received =
1045 profile.bytes_received.saturating_add(chunk.data.len());
1046 profile.pack_bytes_received =
1047 profile.pack_bytes_received.saturating_add(chunk.data.len());
1048 let transfer = chunk.transfer.as_ref().ok_or_else(|| {
1049 ProtocolError::InvalidState(
1050 "native pack chunk missing transfer checkpoint".to_string(),
1051 )
1052 })?;
1053 let stream_kind = PackStreamKind::try_from(chunk.stream_kind)
1054 .unwrap_or(PackStreamKind::Unspecified);
1055 if stream_kind == PackStreamKind::Unspecified {
1056 return Err(ProtocolError::InvalidState(
1057 "native pack chunk missing stream kind".to_string(),
1058 ));
1059 }
1060 let decode_start = Instant::now();
1061 if let Some(pack_spool) = pack_spool.as_mut() {
1062 pack_spool.receive_chunk(
1063 stream_kind == PackStreamKind::Index,
1064 transfer.resume_offset,
1065 transfer.chunk_index,
1066 transfer.is_complete,
1067 &chunk.data,
1068 chunk.is_final_chunk,
1069 )?;
1070 } else {
1071 wire::receive_pack_chunk(
1072 &mut pack_state,
1073 stream_kind == PackStreamKind::Index,
1074 transfer.resume_offset,
1075 transfer.chunk_index,
1076 transfer.is_complete,
1077 &chunk.data,
1078 chunk.is_final_chunk,
1079 )?;
1080 }
1081 let decode_elapsed = decode_start.elapsed();
1082 profile.pack_decode += decode_elapsed;
1083 profile.pack_decode_apply += decode_elapsed;
1084 profile.decode += decode_elapsed;
1085 }
1086 Some(pull_server_frame::Frame::Redaction(transfer)) => {
1087 wire::check_received_transfer_blob_size(
1093 transfer.redactions_blob.len(),
1094 wire::MAX_RECEIVED_REDACTIONS_BLOB_SIZE,
1095 "redactions",
1096 )?;
1097 profile.bytes_received = profile
1098 .bytes_received
1099 .saturating_add(transfer.redactions_blob.len());
1100 profile.object_mix.record(ObjectType::Redaction);
1101 let blob = ContentHash::from_hex(&transfer.blob_hash).map_err(|err| {
1102 ProtocolError::InvalidState(format!(
1103 "RedactionTransfer.blob_hash is not a valid content hash: {err}"
1104 ))
1105 })?;
1106 let decode_start = Instant::now();
1107 repo.accept_wire_redactions(blob, &transfer.redactions_blob)
1108 .map_err(|err| {
1109 ProtocolError::InvalidState(format!(
1110 "accept_wire_redactions for blob {}: {err}",
1111 transfer.blob_hash
1112 ))
1113 })?;
1114 let decode_elapsed = decode_start.elapsed();
1115 profile.store_receive_object += decode_elapsed;
1116 }
1117 Some(pull_server_frame::Frame::StateVisibility(transfer)) => {
1118 wire::check_received_transfer_blob_size(
1119 transfer.state_visibility_blob.len(),
1120 wire::MAX_RECEIVED_STATE_VISIBILITY_BLOB_SIZE,
1121 "state-visibility",
1122 )?;
1123 profile.bytes_received = profile
1124 .bytes_received
1125 .saturating_add(transfer.state_visibility_blob.len());
1126 profile.object_mix.record(ObjectType::StateVisibility);
1127 let state = transfer
1128 .state_id
1129 .as_ref()
1130 .ok_or_else(|| {
1131 ProtocolError::InvalidState(
1132 "StateVisibilityTransfer.state_id is required".to_string(),
1133 )
1134 })
1135 .and_then(|state| {
1136 StateId::try_from_slice(&state.value).map_err(|err| {
1137 ProtocolError::InvalidState(format!(
1138 "StateVisibilityTransfer.state_id is not a valid StateId: {err}"
1139 ))
1140 })
1141 })?;
1142 let decode_start = Instant::now();
1143 repo.accept_wire_state_visibility(state, &transfer.state_visibility_blob)
1144 .map_err(|err| {
1145 ProtocolError::InvalidState(format!(
1146 "accept_wire_state_visibility for state {}: {err}",
1147 state
1148 ))
1149 })?;
1150 let decode_elapsed = decode_start.elapsed();
1151 profile.store_receive_object += decode_elapsed;
1152 }
1153 Some(pull_server_frame::Frame::GitLane(transfer)) => {
1154 let decode_start = Instant::now();
1155 profile.bytes_received = profile
1156 .bytes_received
1157 .saturating_add(git_lane_transfer_size(&transfer));
1158 accept_git_lane_pull_transfer(
1159 repo,
1160 &mut git_lane_repo,
1161 &mut git_pack_state,
1162 transfer,
1163 )?;
1164 let decode_elapsed = decode_start.elapsed();
1165 profile.store_receive_object += decode_elapsed;
1166 }
1167 Some(pull_server_frame::Frame::Complete(complete)) => {
1168 profile.receive_and_apply = receive_start.elapsed();
1169 git_pack_state.ensure_idle()?;
1170 let final_state = super::helpers::parse_proto_state_id(complete.new_state)?;
1171
1172 if complete.success {
1173 if native_pack_required {
1174 let store_start = Instant::now();
1175 let installed_ids = if let Some(pack_spool) = pack_spool.as_mut() {
1176 if !pack_spool.is_complete() {
1177 return Err(ProtocolError::InvalidState(
1178 "pull completed before native pack stream finished"
1179 .to_string(),
1180 ));
1181 }
1182 pack_spool.install_into(repo.store())?
1183 } else {
1184 if !pack_state.is_complete() {
1185 return Err(ProtocolError::InvalidState(
1186 "pull completed before native pack stream finished"
1187 .to_string(),
1188 ));
1189 }
1190 wire::install_received_pack(
1191 repo.store(),
1192 &pack_state.pack_data,
1193 &pack_state.index_data,
1194 )?
1195 };
1196 profile.store_receive_object += store_start.elapsed();
1197 received = installed_ids.len();
1198 for id in installed_ids {
1199 match (id, wanted_packable_type(&wanted_types, &id)) {
1200 (PackObjectId::Hash(hash), Some(ObjectType::Blob)) => {
1201 profile.object_mix.record(ObjectType::Blob);
1202 repo.clear_missing_blob(&hash)?;
1203 }
1204 (_, Some(obj_type)) => {
1205 profile.object_mix.record(obj_type);
1206 }
1207 (PackObjectId::StateId(_), None) => {
1208 profile.object_mix.record(ObjectType::State);
1209 }
1210 (PackObjectId::Hash(hash), None) => {
1211 let inferred =
1212 infer_installed_hash_object_type(repo, &hash)?;
1213 profile.object_mix.record(inferred);
1214 }
1215 }
1216 }
1217 }
1218
1219 let metadata_start = Instant::now();
1220 if let Some(local_thread) = options.local_thread
1221 && let Some(state) = final_state
1222 {
1223 repo.refs()
1224 .set_thread(&ThreadName::from(local_thread), &state)?;
1225 }
1226 if let Some(state) = final_state
1227 && allow_partial_fetch
1228 {
1229 mark_missing_blobs_for_state(repo, state)?;
1230 } else if final_state.is_some() {
1231 let _ = repo.clear_all_missing_blobs()?;
1232 }
1233 let synced_markers = complete
1234 .transfer
1235 .as_ref()
1236 .map(|transfer| apply_marker_snapshot(repo, &transfer.checkpoint))
1237 .transpose()?
1238 .unwrap_or(false);
1239 if !synced_markers {
1240 self.sync_local_markers(repo, repo_path).await?;
1241 }
1242 profile.metadata_sync = metadata_start.elapsed();
1243 profile.objects_received = received;
1244 return Ok(PullExchange {
1245 result: PullComplete {
1246 success: true,
1247 final_state,
1248 error: None,
1249 transfer_id: complete
1250 .transfer
1251 .as_ref()
1252 .map(|transfer| transfer.transfer_id.clone())
1253 .unwrap_or_default(),
1254 transport_mode: complete
1255 .transfer
1256 .as_ref()
1257 .map(|transfer| transport_mode_name(transfer.transport_mode))
1258 .unwrap_or("native-pack")
1259 .to_string(),
1260 resume_offset: complete
1261 .transfer
1262 .as_ref()
1263 .map(|transfer| transfer.resume_offset)
1264 .unwrap_or_default(),
1265 chunk_index: complete
1266 .transfer
1267 .as_ref()
1268 .map(|transfer| transfer.chunk_index)
1269 .unwrap_or_default(),
1270 checkpoint: complete
1271 .transfer
1272 .as_ref()
1273 .map(|transfer| transfer.checkpoint.clone())
1274 .unwrap_or_default(),
1275 is_complete: complete
1276 .transfer
1277 .as_ref()
1278 .map(|transfer| transfer.is_complete)
1279 .unwrap_or(false),
1280 },
1281 object_count: received,
1282 profile,
1283 });
1284 }
1285
1286 profile.objects_received = received;
1287 return Ok(PullExchange {
1288 result: PullComplete {
1289 success: false,
1290 final_state,
1291 error: (!complete.error.is_empty()).then_some(complete.error),
1292 transfer_id: complete
1293 .transfer
1294 .as_ref()
1295 .map(|transfer| transfer.transfer_id.clone())
1296 .unwrap_or_default(),
1297 transport_mode: complete
1298 .transfer
1299 .as_ref()
1300 .map(|transfer| transport_mode_name(transfer.transport_mode))
1301 .unwrap_or("native-pack")
1302 .to_string(),
1303 resume_offset: complete
1304 .transfer
1305 .as_ref()
1306 .map(|transfer| transfer.resume_offset)
1307 .unwrap_or_default(),
1308 chunk_index: complete
1309 .transfer
1310 .as_ref()
1311 .map(|transfer| transfer.chunk_index)
1312 .unwrap_or_default(),
1313 checkpoint: complete
1314 .transfer
1315 .as_ref()
1316 .map(|transfer| transfer.checkpoint.clone())
1317 .unwrap_or_default(),
1318 is_complete: complete
1319 .transfer
1320 .as_ref()
1321 .map(|transfer| transfer.is_complete)
1322 .unwrap_or(false),
1323 },
1324 object_count: received,
1325 profile,
1326 });
1327 }
1328 _ => {}
1329 }
1330 }
1331
1332 Err(ProtocolError::InvalidState(format!(
1333 "pull stream ended unexpectedly after receiving {received} packed objects"
1334 )))
1335 }
1336}
1337
1338fn redaction_push_message(
1339 repo: &Repository,
1340 info: wire::ObjectInfo,
1341 client_operation_id: &str,
1342) -> Result<PushClientFrame, ProtocolError> {
1343 let wire::ObjectId::Hash(blob) = info.id else {
1344 return Err(ProtocolError::InvalidState(
1345 "wanted Redaction must be keyed by ObjectId::Hash(content_hash)".to_string(),
1346 ));
1347 };
1348 let hex = blob.to_hex();
1349 let bytes = repo
1354 .store()
1355 .get_redactions_bytes_for_blob(&blob)
1356 .map_err(|err| {
1357 ProtocolError::InvalidState(format!("load redactions sidecar for {}: {err}", hex))
1358 })?
1359 .ok_or_else(|| {
1360 ProtocolError::InvalidState(format!(
1361 "server wants redaction for blob {} but sender has no sidecar",
1362 hex
1363 ))
1364 })?;
1365 Ok(PushClientFrame {
1366 frame: Some(push_client_frame::Frame::Redaction(RedactionTransfer {
1367 blob_hash: hex,
1368 redactions_blob: bytes,
1369 })),
1370 client_operation_id: client_operation_id.to_string(),
1371 })
1372}
1373
1374fn native_pack_required_for_pull(
1375 want_full_closure: bool,
1376 transfer_plan: &RepositoryTransferPlan<ObjectInfo>,
1377) -> bool {
1378 transfer_plan.requires_native_pack(want_full_closure)
1379}
1380
1381fn object_info_from_plan(object: &PlannedObject) -> ObjectInfo {
1382 ObjectInfo {
1383 id: object.id.clone(),
1384 obj_type: object.obj_type,
1385 size: 0,
1386 delta_base: None,
1387 }
1388}
1389
1390fn to_proto_planned_object(object: &PlannedObject) -> ObjectDescriptor {
1391 object_descriptor_with_status(
1392 &object_info_from_plan(object),
1393 ObjectAvailabilityStatus::Present,
1394 "",
1395 )
1396}
1397
1398fn descriptor_id_from_plan(object: &PlannedObject) -> (String, String) {
1399 let id = match &object.id {
1400 wire::ObjectId::Hash(hash) => hash.to_hex(),
1401 wire::ObjectId::StateId(state_id) => state_id.to_string_full(),
1402 wire::ObjectId::StateAttachment { state, id, kind: _ } => {
1403 format!("{}:{}", state.to_string_full(), id.as_hash().to_hex())
1404 }
1405 };
1406 (id, object_type_name(object.obj_type).to_string())
1407}
1408
1409fn record_wanted_type(wanted_types: &mut WantedTypes, pack_id: PackObjectId, obj_type: ObjectType) {
1410 let types = wanted_types.entry(pack_id).or_default();
1411 if !types.contains(&obj_type) {
1412 types.push(obj_type);
1413 }
1414}
1415
1416fn wanted_packable_type(wanted_types: &WantedTypes, pack_id: &PackObjectId) -> Option<ObjectType> {
1417 wanted_types
1418 .get(pack_id)
1419 .and_then(|types| types.iter().copied().find(|obj_type| obj_type.packable()))
1420}
1421
1422fn sidecar_push_message(
1423 repo: &Repository,
1424 info: wire::ObjectInfo,
1425 client_operation_id: &str,
1426) -> Result<PushClientFrame, ProtocolError> {
1427 match info.obj_type {
1428 ObjectType::Redaction => redaction_push_message(repo, info, client_operation_id),
1429 ObjectType::StateVisibility => {
1430 state_visibility_push_message(repo, info, client_operation_id)
1431 }
1432 ObjectType::StateAttachment => {
1433 state_attachment_push_message(repo, info, client_operation_id)
1434 }
1435 obj_type => Err(ProtocolError::InvalidState(format!(
1436 "{obj_type:?} is not an out-of-pack sidecar object"
1437 ))),
1438 }
1439}
1440
1441fn state_attachment_push_message(
1458 repo: &Repository,
1459 info: wire::ObjectInfo,
1460 client_operation_id: &str,
1461) -> Result<PushClientFrame, ProtocolError> {
1462 let record = wire::load_object_data(repo.store(), &info.id, ObjectType::StateAttachment)?;
1465 let wire::ObjectId::StateAttachment { state, id, kind } = info.id else {
1466 return Err(ProtocolError::InvalidState(
1467 "wanted StateAttachment must be keyed by ObjectId::StateAttachment".to_string(),
1468 ));
1469 };
1470 Ok(PushClientFrame {
1471 frame: Some(push_client_frame::Frame::StateAttachment(
1472 StateAttachmentTransfer {
1473 state_id: super::helpers::proto_state_id(state),
1474 attachment_id: id.as_hash().to_hex(),
1475 attachment_kind: super::helpers::attachment_kind_to_proto(kind) as i32,
1476 attachment_object: record.data,
1477 },
1478 )),
1479 client_operation_id: client_operation_id.to_string(),
1480 })
1481}
1482
1483fn state_visibility_push_message(
1484 repo: &Repository,
1485 info: wire::ObjectInfo,
1486 client_operation_id: &str,
1487) -> Result<PushClientFrame, ProtocolError> {
1488 let wire::ObjectId::StateId(state) = info.id else {
1489 return Err(ProtocolError::InvalidState(
1490 "wanted StateVisibility must be keyed by ObjectId::StateId(state)".to_string(),
1491 ));
1492 };
1493 let state_id = state.to_string_full();
1494 let bytes = repo
1495 .get_state_visibility_bytes_for_state(&state)
1496 .map_err(|err| {
1497 ProtocolError::InvalidState(format!(
1498 "load state-visibility sidecar for {}: {err}",
1499 state_id
1500 ))
1501 })?
1502 .ok_or_else(|| {
1503 ProtocolError::InvalidState(format!(
1504 "server wants state visibility for state {} but sender has no sidecar",
1505 state_id
1506 ))
1507 })?;
1508 Ok(PushClientFrame {
1509 frame: Some(push_client_frame::Frame::StateVisibility(
1510 StateVisibilityTransfer {
1511 state_id: super::helpers::proto_state_id(state),
1512 state_visibility_blob: bytes,
1513 },
1514 )),
1515 client_operation_id: client_operation_id.to_string(),
1516 })
1517}
1518
1519fn load_thread_metadata(
1520 repo: &Repository,
1521 target_thread: &str,
1522 local_state: StateId,
1523) -> Result<Option<SyncedThreadMetadata>, ProtocolError> {
1524 let thread_manager = ThreadManager::new(repo.heddle_dir());
1525 Ok(thread_manager.find_synced_record_by_thread(repo, target_thread, Some(local_state))?)
1526}
1527
1528fn plan_pull_wants(
1529 repo: &Repository,
1530 remote_state: &StateId,
1531 full_closure_available: bool,
1532 objects_to_fetch: Vec<ObjectDescriptor>,
1533 allow_partial_fetch: bool,
1534) -> Result<PullWantPlan, ProtocolError> {
1535 if full_closure_available {
1536 return Ok(PullWantPlan {
1537 wants: Vec::new(),
1538 transfer_plan: RepositoryTransferPlan::from_object_infos(
1539 Vec::<ObjectInfo>::new(),
1540 GitLaneTransferIntent::HeddleObjectsOnly,
1541 ),
1542 wanted_types: HashMap::new(),
1543 want_full_closure: true,
1544 });
1545 }
1546 let request_full_closure =
1547 should_request_full_closure(repo, remote_state, allow_partial_fetch)?;
1548 let mut wants = Vec::with_capacity(objects_to_fetch.len());
1549 let mut wanted_infos = Vec::with_capacity(objects_to_fetch.len());
1550 let mut wanted_types = HashMap::with_capacity(objects_to_fetch.len());
1551
1552 for descriptor in objects_to_fetch {
1553 let info = parse_descriptor_to_info(descriptor)?;
1554 let pack_id = match &info.id {
1555 wire::ObjectId::Hash(hash) => PackObjectId::Hash(*hash),
1556 wire::ObjectId::StateId(state_id) => PackObjectId::StateId(*state_id),
1557 wire::ObjectId::StateAttachment { id, .. } => PackObjectId::Hash(*id.as_hash()),
1558 };
1559 let include = if request_full_closure {
1560 true
1561 } else {
1562 let has = wire::has_object(repo.store(), &info)?;
1563 !(has || (allow_partial_fetch && matches!(info.obj_type, ObjectType::Blob)))
1564 };
1565
1566 if include {
1567 record_wanted_type(&mut wanted_types, pack_id, info.obj_type);
1568 wants.push(object_descriptor_with_status(
1569 &info,
1570 ObjectAvailabilityStatus::Missing,
1571 "requested by client",
1572 ));
1573 wanted_infos.push(info);
1574 }
1575 }
1576
1577 Ok(PullWantPlan {
1578 wants,
1579 transfer_plan: RepositoryTransferPlan::from_object_infos(
1580 wanted_infos,
1581 GitLaneTransferIntent::HeddleObjectsOnly,
1582 ),
1583 wanted_types,
1584 want_full_closure: false,
1585 })
1586}
1587
1588fn supports_compact_full_pull(
1589 repo: &Repository,
1590 allow_partial_fetch: bool,
1591 exclude_states: &[StateId],
1592) -> Result<bool, ProtocolError> {
1593 if allow_partial_fetch || !exclude_states.is_empty() {
1594 return Ok(false);
1595 }
1596 repo_looks_fresh(repo)
1597}
1598
1599fn locally_complete_pull_head(
1622 repo: &Repository,
1623 remote_thread: &str,
1624 target_state: Option<StateId>,
1625) -> Result<Option<StateId>, ProtocolError> {
1626 locally_complete_thread_head(repo, remote_thread, target_state)
1627}
1628
1629fn locally_complete_local_thread_head(
1638 repo: &Repository,
1639 local_thread: &str,
1640 target_state: Option<StateId>,
1641) -> Result<Option<StateId>, ProtocolError> {
1642 locally_complete_thread_head(repo, local_thread, target_state)
1643}
1644
1645fn locally_complete_thread_head(
1665 repo: &Repository,
1666 thread: &str,
1667 target_state: Option<StateId>,
1668) -> Result<Option<StateId>, ProtocolError> {
1669 if target_state.is_some() {
1672 return Ok(None);
1673 }
1674 if !repo.missing_blobs()?.is_empty() {
1677 return Ok(None);
1678 }
1679 let Some(head) = repo.refs().get_thread(&ThreadName::from(thread))? else {
1680 return Ok(None);
1683 };
1684 match wire::enumerate_state_closure(repo.store(), head) {
1689 Ok(_) => Ok(Some(head)),
1690 Err(ProtocolError::ObjectNotFound(_)) => Ok(None),
1691 Err(err) => Err(err),
1692 }
1693}
1694
1695fn should_request_full_closure(
1696 repo: &Repository,
1697 remote_state: &StateId,
1698 allow_partial_fetch: bool,
1699) -> Result<bool, ProtocolError> {
1700 if allow_partial_fetch || repo.store().has_state(remote_state)? {
1701 return Ok(false);
1702 }
1703 repo_looks_fresh(repo)
1704}
1705
1706fn repo_looks_fresh(repo: &Repository) -> Result<bool, ProtocolError> {
1707 if repo.head()?.is_some() {
1708 return Ok(false);
1709 }
1710 if !repo.refs().list_threads()?.is_empty() || !repo.refs().list_markers()?.is_empty() {
1711 return Ok(false);
1712 }
1713 Ok(repo.missing_blobs()?.is_empty())
1714}
1715
1716fn infer_installed_hash_object_type(
1717 repo: &Repository,
1718 hash: &ContentHash,
1719) -> Result<ObjectType, ProtocolError> {
1720 let store = repo.store();
1721 if store.get_tree(hash)?.is_some() {
1722 return Ok(ObjectType::Tree);
1723 }
1724 if store
1725 .get_action(&objects::object::ActionId::from_hash(*hash))?
1726 .is_some()
1727 {
1728 return Ok(ObjectType::Action);
1729 }
1730 Ok(ObjectType::Blob)
1731}
1732
1733fn apply_marker_snapshot(repo: &Repository, checkpoint: &[u8]) -> Result<bool, ProtocolError> {
1734 const HEADER: &str = "heddle-markers-v1\n";
1735 if checkpoint.is_empty() {
1736 return Ok(false);
1737 }
1738 let payload = std::str::from_utf8(checkpoint)
1739 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
1740 let Some(lines) = payload.strip_prefix(HEADER) else {
1741 return Ok(false);
1742 };
1743
1744 for line in lines.lines() {
1745 if line.is_empty() {
1746 continue;
1747 }
1748 let Some((name, state_id)) = line.split_once('\t') else {
1749 return Err(ProtocolError::InvalidState(
1750 "invalid marker snapshot line".to_string(),
1751 ));
1752 };
1753 let state_id =
1754 StateId::parse(state_id).map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
1755 if !repo.store().has_state(&state_id)? {
1756 continue;
1757 }
1758 let name = MarkerName::from(name);
1759 match repo.refs().get_marker(&name)? {
1760 Some(existing) if existing == state_id => {}
1761 Some(existing) => repo.refs().set_marker_cas(
1762 &name,
1763 refs::RefExpectation::Value(existing),
1764 &state_id,
1765 )?,
1766 None => repo.refs().create_marker(&name, &state_id)?,
1767 }
1768 }
1769
1770 Ok(true)
1771}
1772
1773fn state_id_string_to_bytes(s: &str) -> Vec<u8> {
1774 if s.is_empty() {
1775 return Vec::new();
1776 }
1777 objects::object::StateId::parse(s)
1778 .map(|id| id.as_bytes().to_vec())
1779 .unwrap_or_default()
1780}
1781
1782fn to_proto_thread_metadata(metadata: &SyncedThreadMetadata) -> ThreadMetadata {
1783 ThreadMetadata {
1784 name: metadata.thread.clone(),
1785 target_thread: metadata.target_thread.clone(),
1786 parent_thread: metadata.parent_thread.clone(),
1787 task: metadata.task.clone(),
1788 thread_mode: metadata.mode.to_string(),
1789 thread_state: metadata.state.to_string(),
1790 freshness: metadata.freshness.to_string(),
1791 base_state: StateId::parse(&metadata.base_state)
1792 .ok()
1793 .and_then(super::helpers::proto_state_id),
1794 base_root: state_id_string_to_bytes(&metadata.base_root),
1795 current_state: metadata.current_state.as_deref().and_then(|state| {
1796 StateId::parse(state)
1797 .ok()
1798 .and_then(super::helpers::proto_state_id)
1799 }),
1800 merged_state: metadata.merged_state.as_deref().and_then(|state| {
1801 StateId::parse(state)
1802 .ok()
1803 .and_then(super::helpers::proto_state_id)
1804 }),
1805 changed_paths: metadata.changed_paths.clone(),
1806 impact_categories: metadata
1807 .impact_categories
1808 .iter()
1809 .map(ToString::to_string)
1810 .collect(),
1811 heavy_impact_paths: metadata.heavy_impact_paths.clone(),
1812 promotion_suggested: metadata.promotion_suggested,
1813 verification_summary: Some(ThreadVerificationSummary {
1814 tests_passed: metadata.verification_summary.tests_passed,
1815 tests_failed: metadata
1816 .verification_summary
1817 .tests_failed
1818 .unwrap_or_default(),
1819 coverage_pct: metadata.verification_summary.coverage_pct,
1820 lint_warnings: metadata.verification_summary.lint_warnings,
1821 }),
1822 confidence_summary: Some(ThreadConfidenceSummary {
1823 value: metadata.confidence_summary.value,
1824 band: metadata
1825 .confidence_summary
1826 .band
1827 .as_ref()
1828 .map(ToString::to_string),
1829 }),
1830 integration_policy_result: Some(ThreadIntegrationPolicy {
1831 status: metadata
1832 .integration_policy_result
1833 .status
1834 .clone()
1835 .unwrap_or_default(),
1836 reason: metadata
1837 .integration_policy_result
1838 .reason
1839 .clone()
1840 .unwrap_or_default(),
1841 }),
1842 created_at: Some(prost_types::Timestamp {
1843 seconds: metadata.created_at.timestamp(),
1844 nanos: metadata.created_at.timestamp_subsec_nanos() as i32,
1845 }),
1846 updated_at: Some(prost_types::Timestamp {
1847 seconds: metadata.updated_at.timestamp(),
1848 nanos: metadata.updated_at.timestamp_subsec_nanos() as i32,
1849 }),
1850 }
1851}
1852
1853struct PullExchange {
1854 result: PullComplete,
1855 object_count: usize,
1856 profile: PullProfile,
1857}
1858
1859fn mark_missing_blobs_for_state(repo: &Repository, state_id: StateId) -> Result<(), ProtocolError> {
1860 let state = repo
1861 .store()
1862 .get_state(&state_id)?
1863 .ok_or_else(|| ProtocolError::ObjectNotFound(state_id.to_string_full()))?;
1864 let mut missing = wire::missing_blobs_in_tree(repo.store(), state.tree)?;
1865 for attachment in repo.list_state_attachments(&state_id)? {
1866 match attachment.body {
1867 objects::object::StateAttachmentBody::Context(root) => {
1868 missing.extend(wire::missing_blobs_in_tree(repo.store(), root)?);
1869 }
1870 objects::object::StateAttachmentBody::RiskSignals(hash)
1871 | objects::object::StateAttachmentBody::ReviewSignatures(hash)
1872 | objects::object::StateAttachmentBody::Discussions(hash)
1873 | objects::object::StateAttachmentBody::StructuredConflicts(hash)
1874 if !repo.store().has_blob(&hash)? =>
1875 {
1876 missing.push(hash)
1877 }
1878 _ => {}
1879 }
1880 }
1881 missing
1882 .into_iter()
1883 .try_for_each(|hash| repo.record_missing_blob(hash).map_err(ProtocolError::from))
1884}
1885
1886fn clear_missing_blobs_for_state(
1887 repo: &Repository,
1888 state_id: StateId,
1889) -> Result<(), ProtocolError> {
1890 let state = repo
1891 .store()
1892 .get_state(&state_id)?
1893 .ok_or_else(|| ProtocolError::ObjectNotFound(state_id.to_string_full()))?;
1894 let mut missing = wire::missing_blobs_in_tree(repo.store(), state.tree)?;
1895 for attachment in repo.list_state_attachments(&state_id)? {
1896 match attachment.body {
1897 objects::object::StateAttachmentBody::Context(root) => {
1898 missing.extend(wire::missing_blobs_in_tree(repo.store(), root)?);
1899 }
1900 objects::object::StateAttachmentBody::RiskSignals(hash)
1901 | objects::object::StateAttachmentBody::ReviewSignatures(hash)
1902 | objects::object::StateAttachmentBody::Discussions(hash)
1903 | objects::object::StateAttachmentBody::StructuredConflicts(hash) => missing.push(hash),
1904 _ => {}
1905 }
1906 }
1907 missing
1908 .into_iter()
1909 .try_for_each(|hash| repo.clear_missing_blob(&hash).map_err(ProtocolError::from))
1910}
1911
1912fn partial_fetch_status_for_repo(repo: &Repository) -> i32 {
1913 match repo.missing_blobs() {
1914 Ok(missing) if !missing.is_empty() => PartialFetchStatus::Enabled as i32,
1915 Ok(_) => PartialFetchStatus::Disabled as i32,
1916 Err(_) => PartialFetchStatus::Unspecified as i32,
1917 }
1918}
1919
1920fn pull_transfer_id(
1921 repo_path: &str,
1922 remote_thread: &str,
1923 local_thread: Option<&str>,
1924 depth: Option<u32>,
1925 target_state: Option<StateId>,
1926) -> String {
1927 format!(
1928 "pull:{repo_path}:{remote_thread}:{}:{depth:?}:{}",
1929 local_thread.unwrap_or_default(),
1930 target_state
1931 .map(|value| value.to_string_full())
1932 .unwrap_or_default()
1933 )
1934}
1935
1936fn push_transfer_id(repo_path: &str, local_state: StateId, target_thread: &str) -> String {
1937 format!(
1938 "push:{repo_path}:{}:{target_thread}",
1939 local_state.to_string_full()
1940 )
1941}
1942
1943fn build_git_mirror_push_plan(
1957 repo: &Repository,
1958 chunk_size: usize,
1959 remote_ref_expectations: &HashMap<String, GitRefRemoteExpectation>,
1960) -> Result<GitLanePushPlan, ProtocolError> {
1961 if repo.capability() != RepositoryCapability::GitOverlay {
1962 return Err(ProtocolError::InvalidState(
1963 "Git remote mirror pushes require a git-overlay repository".to_string(),
1964 ));
1965 }
1966 let git_repo = repo
1967 .git_overlay_sley_repository()
1968 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
1969 .ok_or_else(|| {
1970 ProtocolError::InvalidState("git-overlay repository has no Git store".to_string())
1971 })?;
1972 build_git_mirror_plan_from_sley(&git_repo, chunk_size, remote_ref_expectations)
1973}
1974
1975fn build_git_mirror_plan_from_sley(
1978 git_repo: &SleyRepository,
1979 chunk_size: usize,
1980 remote_ref_expectations: &HashMap<String, GitRefRemoteExpectation>,
1981) -> Result<GitLanePushPlan, ProtocolError> {
1982 let refs = git_repo
1983 .references()
1984 .list_refs()
1985 .map_err(|err| ProtocolError::InvalidState(format!("list git-overlay refs: {err}")))?;
1986
1987 let mut roots: Vec<GitObjectId> = Vec::new();
1988 let mut ref_updates: Vec<GitRefUpdateTransfer> = Vec::new();
1989 let mut newest_root: Option<GitObjectId> = None;
1990
1991 for reference in refs {
1992 if GitRefName::new(&reference.name).is_local_only() {
2006 continue;
2007 }
2008
2009 let target_oid = match reference.target {
2014 ReferenceTarget::Direct(oid) => oid,
2015 ReferenceTarget::Symbolic(_) => continue,
2016 };
2017
2018 let object = git_repo.read_object(&target_oid).map_err(|err| {
2022 ProtocolError::InvalidState(format!(
2023 "git-overlay ref {} target {} is not readable: {err}",
2024 reference.name,
2025 target_oid.to_hex()
2026 ))
2027 })?;
2028
2029 let peeled_oid = if object.object_type == sley::GitObjectType::Tag {
2030 let peeled = git_repo.peel_to_object_oid(target_oid).map_err(|err| {
2031 ProtocolError::InvalidState(format!(
2032 "peel git-overlay tag ref {}: {err}",
2033 reference.name
2034 ))
2035 })?;
2036 Some(peeled)
2037 } else {
2038 None
2039 };
2040
2041 roots.push(target_oid);
2045 newest_root.get_or_insert(target_oid);
2046
2047 let expectation = remote_ref_expectations
2048 .get(&reference.name)
2049 .cloned()
2050 .unwrap_or(GitRefRemoteExpectation::Missing);
2051
2052 let kind = grpc_git_ref_kind(GitRefName::new(&reference.name).wire_kind());
2053 let mut message =
2054 git_ref_update_message(&reference.name, kind, target_oid, peeled_oid, None);
2055 apply_git_ref_expectation_value(&mut message, &expectation);
2056 ref_updates.push(message);
2057 }
2058
2059 if roots.is_empty() {
2060 return Err(ProtocolError::InvalidState(
2061 "git-overlay repository has no direct refs to mirror".to_string(),
2062 ));
2063 }
2064
2065 let format = git_repo.object_format();
2076 let mut have_boundary: Vec<GitObjectId> = Vec::new();
2077 for expectation in remote_ref_expectations.values() {
2078 if let GitRefRemoteExpectation::Value(oid_bytes) = expectation
2079 && let Ok(oid) = GitObjectId::from_raw(format, oid_bytes)
2080 {
2081 have_boundary.push(oid);
2085 }
2086 }
2087
2088 let pack = build_git_lane_multi_root_pack_plan(git_repo, roots, have_boundary, chunk_size)?;
2089
2090 let local_revision_address = newest_root
2093 .map(|oid| RevisionAddress::git_commit(oid.to_hex()).to_string())
2094 .unwrap_or_default();
2095
2096 Ok(GitLanePushPlan {
2097 local_revision_address,
2098 pack,
2099 ref_updates,
2100 })
2101}
2102
2103fn grpc_git_ref_kind(kind: ClassifiedGitRefKind) -> GrpcGitRefKind {
2104 match kind {
2105 ClassifiedGitRefKind::Branch => GrpcGitRefKind::Branch,
2106 ClassifiedGitRefKind::Tag => GrpcGitRefKind::Tag,
2107 ClassifiedGitRefKind::Note => GrpcGitRefKind::Note,
2108 ClassifiedGitRefKind::Other => GrpcGitRefKind::Other,
2109 }
2110}
2111
2112fn build_git_lane_multi_root_pack_plan(
2118 git_repo: &SleyRepository,
2119 roots: Vec<GitObjectId>,
2120 excluded: Vec<GitObjectId>,
2121 chunk_size: usize,
2122) -> Result<Option<GitPackPushPlan>, ProtocolError> {
2123 if roots.is_empty() {
2124 return Err(ProtocolError::InvalidState(
2125 "cannot plan a Git pack with no roots".to_string(),
2126 ));
2127 }
2128 let Some(plan) = git_repo
2129 .reachable_pack_plan()
2130 .roots(roots.iter().copied())
2131 .exclusions(excluded)
2132 .build()
2133 .map_err(|err| {
2134 ProtocolError::InvalidState(format!("plan reachable Git pack stream: {err}"))
2135 })?
2136 else {
2137 return Ok(None);
2140 };
2141 let pack_file = NamedTempFile::new()
2142 .map_err(|err| ProtocolError::InvalidState(format!("create Git pack tempfile: {err}")))?;
2143 let prepared = plan.prepare_to_file(pack_file.path()).map_err(|err| {
2144 ProtocolError::InvalidState(format!("write reachable Git pack tempfile: {err}"))
2145 })?;
2146 let pack_size = prepared.summary.pack_size;
2147 let checksum = prepared.summary.checksum;
2148 if pack_size > wire::MAX_RECEIVED_GIT_PACK_SIZE {
2149 return Err(ProtocolError::InvalidState(format!(
2150 "Git pack exceeds maximum transfer size of {} bytes; multi-pack split for repos over this size is a follow-up (plan §B.2)",
2151 wire::MAX_RECEIVED_GIT_PACK_SIZE
2152 )));
2153 }
2154 let chunk_size = chunk_size.max(1) as u64;
2155 let chunk_count = pack_size.div_ceil(chunk_size);
2156 if chunk_count > u32::MAX as u64 {
2157 return Err(ProtocolError::InvalidState(
2158 "Git pack chunk count exceeds u32".to_string(),
2159 ));
2160 }
2161 let transfer_id = format!("git-pack:{}", checksum.to_hex());
2162 Ok(Some(GitPackPushPlan {
2163 transfer_id,
2164 pack_id: checksum.as_bytes().to_vec(),
2165 pack_size,
2166 roots,
2167 pack_file: Arc::new(Mutex::new(pack_file)),
2168 }))
2169}
2170
2171async fn send_git_pack_streaming_messages(
2172 tx: &mpsc::Sender<PushClientFrame>,
2173 pack: &GitPackPushPlan,
2174 chunk_size: usize,
2175 progress: &Progress,
2176 client_operation_id: &str,
2177) -> Result<(), ProtocolError> {
2178 let tx = tx.clone();
2179 let pack = pack.clone();
2180 let progress = progress.clone();
2181 let client_operation_id = client_operation_id.to_string();
2182 tokio::task::spawn_blocking(move || {
2183 stream_git_pack_messages_blocking(tx, pack, chunk_size, progress, client_operation_id)
2184 })
2185 .await
2186 .map_err(|err| ProtocolError::InvalidState(format!("Git pack streaming task failed: {err}")))?
2187}
2188
2189fn stream_git_pack_messages_blocking(
2190 tx: mpsc::Sender<PushClientFrame>,
2191 pack: GitPackPushPlan,
2192 chunk_size: usize,
2193 progress: Progress,
2194 client_operation_id: String,
2195) -> Result<(), ProtocolError> {
2196 let mut writer = GitPackPushMessageWriter::new(
2197 tx,
2198 pack.transfer_id.clone(),
2199 pack.pack_id.clone(),
2200 pack.pack_size,
2201 chunk_size,
2202 progress,
2203 client_operation_id,
2204 );
2205 let mut pack_file = pack
2206 .pack_file
2207 .lock()
2208 .map_err(|err| ProtocolError::InvalidState(format!("lock Git pack tempfile: {err}")))?;
2209 pack_file
2210 .seek(SeekFrom::Start(0))
2211 .map_err(|err| ProtocolError::InvalidState(format!("rewind Git pack tempfile: {err}")))?;
2212 let streamed = io::copy(&mut pack_file.as_file_mut(), &mut writer)
2213 .map_err(|err| ProtocolError::InvalidState(format!("stream Git pack tempfile: {err}")))?;
2214 if streamed != pack.pack_size {
2215 return Err(ProtocolError::InvalidState(format!(
2216 "Git pack stream changed while sending; expected {} bytes/{}, streamed {} bytes",
2217 pack.pack_size,
2218 hex::encode(&pack.pack_id),
2219 streamed
2220 )));
2221 }
2222 writer.finish()?;
2223 Ok(())
2224}
2225
2226struct GitPackPushMessageWriter {
2227 tx: mpsc::Sender<PushClientFrame>,
2228 transfer_id: String,
2229 pack_id: Vec<u8>,
2230 pack_size: u64,
2231 chunk_size: usize,
2232 buffer: Vec<u8>,
2233 offset: u64,
2234 chunk_index: u32,
2235 client_operation_id: String,
2236 progress: Progress,
2239 last_progress_pct: u64,
2243}
2244
2245impl GitPackPushMessageWriter {
2246 fn new(
2247 tx: mpsc::Sender<PushClientFrame>,
2248 transfer_id: String,
2249 pack_id: Vec<u8>,
2250 pack_size: u64,
2251 chunk_size: usize,
2252 progress: Progress,
2253 client_operation_id: String,
2254 ) -> Self {
2255 let chunk_size = chunk_size.max(1);
2256 Self {
2257 tx,
2258 transfer_id,
2259 pack_id,
2260 pack_size,
2261 chunk_size,
2262 buffer: Vec::with_capacity(chunk_size),
2263 offset: 0,
2264 chunk_index: 0,
2265 client_operation_id,
2266 progress,
2267 last_progress_pct: u64::MAX,
2268 }
2269 }
2270
2271 fn send_buffer(&mut self) -> io::Result<()> {
2272 if self.buffer.is_empty() {
2273 return Ok(());
2274 }
2275 let chunk = std::mem::take(&mut self.buffer);
2276 let next_offset = self.offset.checked_add(chunk.len() as u64).ok_or_else(|| {
2277 io::Error::new(io::ErrorKind::InvalidData, "Git pack offset overflow")
2278 })?;
2279 if next_offset > self.pack_size {
2280 return Err(io::Error::new(
2281 io::ErrorKind::InvalidData,
2282 format!(
2283 "Git pack stream exceeded planned size {}; got at least {}",
2284 self.pack_size, next_offset
2285 ),
2286 ));
2287 }
2288 let is_final_chunk = next_offset == self.pack_size;
2289 self.tx
2290 .blocking_send(git_lane_push_message(
2291 git_lane_transfer::Body::Pack(GitPackTransfer {
2292 transfer_id: self.transfer_id.clone(),
2293 offset: self.offset,
2294 chunk_index: self.chunk_index,
2295 is_final_chunk,
2296 pack_size: self.pack_size,
2297 pack_chunk: chunk,
2298 pack_id: self.pack_id.clone(),
2299 }),
2300 &self.client_operation_id,
2301 ))
2302 .map_err(|_| {
2303 io::Error::new(io::ErrorKind::BrokenPipe, "push stream closed unexpectedly")
2304 })?;
2305 self.offset = next_offset;
2306 self.chunk_index = self.chunk_index.checked_add(1).ok_or_else(|| {
2307 io::Error::new(io::ErrorKind::InvalidData, "Git pack chunk index overflow")
2308 })?;
2309 self.report_upload_progress();
2310 Ok(())
2311 }
2312
2313 fn report_upload_progress(&mut self) {
2318 if !self.progress.is_active() {
2319 return;
2320 }
2321 let pct = self.offset.saturating_mul(100) / self.pack_size.max(1);
2322 if pct == self.last_progress_pct {
2323 return;
2324 }
2325 self.last_progress_pct = pct;
2326 self.progress.set_phase(format!(
2327 "uploading {}/{} bytes ({pct}%)",
2328 self.offset, self.pack_size
2329 ));
2330 }
2331
2332 fn finish(mut self) -> Result<(), ProtocolError> {
2333 self.send_buffer().map_err(|err| {
2334 ProtocolError::InvalidState(format!("send final Git pack chunk: {err}"))
2335 })?;
2336 if self.offset != self.pack_size {
2337 return Err(ProtocolError::InvalidState(format!(
2338 "Git pack stream length mismatch: expected {}, got {}",
2339 self.pack_size, self.offset
2340 )));
2341 }
2342 Ok(())
2343 }
2344}
2345
2346impl Write for GitPackPushMessageWriter {
2347 fn write(&mut self, mut buf: &[u8]) -> io::Result<usize> {
2348 let written = buf.len();
2349 while !buf.is_empty() {
2350 if self.offset == self.pack_size {
2351 return Err(io::Error::new(
2352 io::ErrorKind::InvalidData,
2353 format!("Git pack stream wrote past planned size {}", self.pack_size),
2354 ));
2355 }
2356 let capacity = self
2357 .chunk_size
2358 .checked_sub(self.buffer.len())
2359 .ok_or_else(|| {
2360 io::Error::new(io::ErrorKind::InvalidData, "Git pack chunk buffer overflow")
2361 })?;
2362 let take = capacity.min(buf.len());
2363 self.buffer.extend_from_slice(&buf[..take]);
2364 buf = &buf[take..];
2365 if self.buffer.len() == self.chunk_size {
2366 self.send_buffer()?;
2367 }
2368 }
2369 Ok(written)
2370 }
2371
2372 fn flush(&mut self) -> io::Result<()> {
2373 Ok(())
2374 }
2375}
2376
2377fn git_ref_update_message(
2384 name: &str,
2385 kind: GrpcGitRefKind,
2386 target_oid: GitObjectId,
2387 peeled_oid: Option<GitObjectId>,
2388 checkpoint: Option<GitCheckpointTransfer>,
2389) -> GitRefUpdateTransfer {
2390 GitRefUpdateTransfer {
2391 name: name.to_string(),
2392 kind: kind as i32,
2393 target_oid: proto_git_oid(&target_oid),
2394 peeled_oid: peeled_oid.as_ref().and_then(proto_git_oid),
2395 expected_missing: false,
2396 expected_target_oid: None,
2397 checkpoint,
2398 }
2399}
2400
2401fn apply_git_ref_expectation_value(
2402 update: &mut GitRefUpdateTransfer,
2403 expectation: &GitRefRemoteExpectation,
2404) {
2405 match expectation {
2406 GitRefRemoteExpectation::Missing => {
2407 update.expected_missing = true;
2408 update.expected_target_oid = None;
2409 }
2410 GitRefRemoteExpectation::Value(oid) => {
2411 update.expected_missing = false;
2412 update.expected_target_oid = proto_git_oid_bytes(oid);
2413 }
2414 }
2415}
2416
2417#[derive(Clone, Debug, PartialEq, Eq)]
2418enum GitRefRemoteExpectation {
2419 Missing,
2420 Value(Vec<u8>),
2421}
2422
2423fn parse_git_ref_expectation(
2424 remote_revision_address: &str,
2425) -> Result<GitRefRemoteExpectation, ProtocolError> {
2426 if remote_revision_address.is_empty() {
2427 return Ok(GitRefRemoteExpectation::Missing);
2428 }
2429
2430 match remote_revision_address.parse::<RevisionAddress>() {
2431 Ok(RevisionAddress::Heddle(_)) => Ok(GitRefRemoteExpectation::Missing),
2432 Ok(RevisionAddress::GitCommit(oid)) => hex::decode(&oid)
2433 .map(GitRefRemoteExpectation::Value)
2434 .map_err(|err| {
2435 ProtocolError::InvalidState(format!(
2436 "server returned invalid Git remote_revision_address: {err}"
2437 ))
2438 }),
2439 Err(err) => Err(ProtocolError::InvalidState(format!(
2440 "server returned invalid remote_revision_address {remote_revision_address:?}: {err}"
2441 ))),
2442 }
2443}
2444
2445fn git_lane_push_message(
2446 body: git_lane_transfer::Body,
2447 client_operation_id: &str,
2448) -> PushClientFrame {
2449 PushClientFrame {
2450 frame: Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
2451 body: Some(body),
2452 })),
2453 client_operation_id: client_operation_id.to_string(),
2454 }
2455}
2456
2457fn git_lane_transfer_size(transfer: &GitLaneTransfer) -> usize {
2458 match transfer.body.as_ref() {
2459 Some(git_lane_transfer::Body::Pack(pack)) => pack.pack_chunk.len(),
2460 Some(git_lane_transfer::Body::RefUpdate(update)) => update
2461 .target_oid
2462 .as_ref()
2463 .map_or(0, |oid| oid.digest.len())
2464 .saturating_add(update.peeled_oid.as_ref().map_or(0, |oid| oid.digest.len()))
2465 .saturating_add(
2466 update
2467 .expected_target_oid
2468 .as_ref()
2469 .map_or(0, |oid| oid.digest.len()),
2470 )
2471 .saturating_add(
2472 update
2473 .checkpoint
2474 .as_ref()
2475 .map(git_checkpoint_transfer_size)
2476 .unwrap_or_default(),
2477 ),
2478 Some(git_lane_transfer::Body::Checkpoint(checkpoint)) => {
2479 git_checkpoint_transfer_size(checkpoint)
2480 }
2481 None => 0,
2482 }
2483}
2484
2485fn git_checkpoint_transfer_size(checkpoint: &GitCheckpointTransfer) -> usize {
2486 checkpoint
2487 .heddle_state_id
2488 .as_ref()
2489 .map_or(0, |state| state.value.len())
2490 .saturating_add(
2491 checkpoint
2492 .git_commit_oid
2493 .as_ref()
2494 .map_or(0, |oid| oid.digest.len()),
2495 )
2496 .saturating_add(checkpoint.thread.len())
2497 .saturating_add(checkpoint.metadata_json.len())
2498}
2499
2500#[derive(Default)]
2501struct GitPackPullInstallState {
2502 active: Option<GitPackPullInstall>,
2503}
2504
2505impl GitPackPullInstallState {
2506 fn ensure_idle(&self) -> Result<(), ProtocolError> {
2507 if self.active.is_none() {
2508 Ok(())
2509 } else {
2510 Err(ProtocolError::InvalidState(
2511 "Git pack transfer ended before final chunk".to_string(),
2512 ))
2513 }
2514 }
2515
2516 fn receive_chunk(
2517 &mut self,
2518 git_repo: &SleyRepository,
2519 pack: GitPackTransfer,
2520 ) -> Result<(), ProtocolError> {
2521 if pack.transfer_id.is_empty() {
2522 return Err(ProtocolError::InvalidState(
2523 "Git pack transfer_id is required".to_string(),
2524 ));
2525 }
2526 if pack.pack_size > wire::MAX_RECEIVED_GIT_PACK_SIZE {
2527 return Err(ProtocolError::InvalidState(format!(
2528 "Git pack exceeds maximum transfer size of {} bytes",
2529 wire::MAX_RECEIVED_GIT_PACK_SIZE
2530 )));
2531 }
2532 if pack.pack_chunk.is_empty() {
2533 return Err(ProtocolError::InvalidState(
2534 "Git pack chunk must not be empty".to_string(),
2535 ));
2536 }
2537 let pack_id =
2538 GitObjectId::from_raw(git_repo.object_format(), &pack.pack_id).map_err(|err| {
2539 ProtocolError::InvalidState(format!("GitPackTransfer.pack_id: {err}"))
2540 })?;
2541 if self.active.is_none() {
2542 if pack.offset != 0 {
2543 return Err(ProtocolError::InvalidState(format!(
2544 "Git pack offset mismatch: expected 0, got {}",
2545 pack.offset
2546 )));
2547 }
2548 if pack.chunk_index != 0 {
2549 return Err(ProtocolError::InvalidState(format!(
2550 "Git pack chunk index mismatch: expected 0, got {}",
2551 pack.chunk_index
2552 )));
2553 }
2554 let writer = git_repo
2555 .objects()
2556 .begin_raw_pack_install(pack_id, pack.pack_size)
2557 .map_err(|err| {
2558 ProtocolError::InvalidState(format!("begin Git pack install: {err}"))
2559 })?;
2560 self.active = Some(GitPackPullInstall {
2561 transfer_id: pack.transfer_id.clone(),
2562 pack_id,
2563 pack_size: pack.pack_size,
2564 next_offset: 0,
2565 next_chunk_index: 0,
2566 writer,
2567 });
2568 }
2569
2570 let active = self.active.as_mut().ok_or_else(|| {
2571 ProtocolError::InvalidState("Git pack install not active".to_string())
2572 })?;
2573 active.receive_chunk(&pack, pack_id)?;
2574 if pack.is_final_chunk {
2575 let active = self.active.take().ok_or_else(|| {
2576 ProtocolError::InvalidState("Git pack install not active".to_string())
2577 })?;
2578 active.finish()?;
2579 git_repo.refresh_objects();
2580 }
2581 Ok(())
2582 }
2583}
2584
2585struct GitPackPullInstall {
2586 transfer_id: String,
2587 pack_id: GitObjectId,
2588 pack_size: u64,
2589 next_offset: u64,
2590 next_chunk_index: u32,
2591 writer: sley::plumbing::sley_odb::RawPackStreamingInstall,
2592}
2593
2594impl GitPackPullInstall {
2595 fn receive_chunk(
2596 &mut self,
2597 pack: &GitPackTransfer,
2598 pack_id: GitObjectId,
2599 ) -> Result<(), ProtocolError> {
2600 if self.transfer_id != pack.transfer_id {
2601 return Err(ProtocolError::InvalidState(format!(
2602 "Git pack transfer id changed from {:?} to {:?}",
2603 self.transfer_id, pack.transfer_id
2604 )));
2605 }
2606 if self.pack_id != pack_id {
2607 return Err(ProtocolError::InvalidState(
2608 "Git pack id changed during transfer".to_string(),
2609 ));
2610 }
2611 if self.pack_size != pack.pack_size {
2612 return Err(ProtocolError::InvalidState(
2613 "Git pack size changed during transfer".to_string(),
2614 ));
2615 }
2616 if pack.offset != self.next_offset {
2617 return Err(ProtocolError::InvalidState(format!(
2618 "Git pack offset mismatch: expected {}, got {}",
2619 self.next_offset, pack.offset
2620 )));
2621 }
2622 if pack.chunk_index != self.next_chunk_index {
2623 return Err(ProtocolError::InvalidState(format!(
2624 "Git pack chunk index mismatch: expected {}, got {}",
2625 self.next_chunk_index, pack.chunk_index
2626 )));
2627 }
2628 let chunk_len = u64::try_from(pack.pack_chunk.len()).map_err(|_| {
2629 ProtocolError::InvalidState("Git pack chunk length exceeds u64".to_string())
2630 })?;
2631 let next_offset = self
2632 .next_offset
2633 .checked_add(chunk_len)
2634 .ok_or_else(|| ProtocolError::InvalidState("Git pack offset overflow".to_string()))?;
2635 if next_offset > self.pack_size {
2636 return Err(ProtocolError::InvalidState(
2637 "Git pack chunk exceeds declared pack size".to_string(),
2638 ));
2639 }
2640 self.writer.write_all(&pack.pack_chunk).map_err(|err| {
2641 ProtocolError::InvalidState(format!("write streamed Git pack chunk: {err}"))
2642 })?;
2643 self.next_offset = next_offset;
2644 self.next_chunk_index = self.next_chunk_index.checked_add(1).ok_or_else(|| {
2645 ProtocolError::InvalidState("Git pack chunk index overflow".to_string())
2646 })?;
2647 if pack.is_final_chunk {
2648 if self.next_offset != self.pack_size {
2649 return Err(ProtocolError::InvalidState(format!(
2650 "Git pack final size mismatch: declared {}, received {}",
2651 self.pack_size, self.next_offset
2652 )));
2653 }
2654 } else if self.next_offset == self.pack_size {
2655 return Err(ProtocolError::InvalidState(
2656 "Git pack reached declared size without final chunk marker".to_string(),
2657 ));
2658 }
2659 Ok(())
2660 }
2661
2662 fn finish(self) -> Result<(), ProtocolError> {
2663 self.writer
2664 .finish()
2665 .map_err(|err| ProtocolError::InvalidState(format!("install Git pack: {err}")))?;
2666 Ok(())
2667 }
2668}
2669
2670fn accept_git_lane_pull_transfer(
2671 repo: &Repository,
2672 git_repo: &mut Option<SleyRepository>,
2673 git_pack_state: &mut GitPackPullInstallState,
2674 transfer: GitLaneTransfer,
2675) -> Result<(), ProtocolError> {
2676 if repo.capability() != RepositoryCapability::GitOverlay {
2677 return Err(ProtocolError::InvalidState(format!(
2678 "received git-lane pull transfer for non-GitOverlay repository (capability {:?})",
2679 repo.capability()
2680 )));
2681 }
2682 match transfer.body {
2683 Some(git_lane_transfer::Body::Pack(pack)) => {
2684 accept_git_lane_pack(repo, git_repo, git_pack_state, pack)
2685 }
2686 Some(git_lane_transfer::Body::RefUpdate(update)) => {
2687 accept_git_lane_ref_update(repo, git_repo, update)
2688 }
2689 Some(git_lane_transfer::Body::Checkpoint(checkpoint)) => {
2690 record_git_lane_checkpoint(repo, git_lane_sley_repository(repo, git_repo)?, checkpoint)
2691 }
2692 None => Err(ProtocolError::InvalidState(
2693 "GitLaneTransfer body is required".to_string(),
2694 )),
2695 }
2696}
2697
2698fn accept_git_lane_pack(
2699 repo: &Repository,
2700 git_repo: &mut Option<SleyRepository>,
2701 git_pack_state: &mut GitPackPullInstallState,
2702 pack: GitPackTransfer,
2703) -> Result<(), ProtocolError> {
2704 let git_repo = git_lane_sley_repository(repo, git_repo)?;
2705 git_pack_state.receive_chunk(git_repo, pack)?;
2706 Ok(())
2707}
2708
2709fn accept_git_lane_ref_update(
2719 repo: &Repository,
2720 git_repo: &mut Option<SleyRepository>,
2721 update: GitRefUpdateTransfer,
2722) -> Result<(), ProtocolError> {
2723 let git_repo = git_lane_sley_repository(repo, git_repo)?;
2724 let target = git_oid_from_proto(
2725 git_repo,
2726 "GitRefUpdateTransfer.target_oid",
2727 update.target_oid.as_ref(),
2728 )?;
2729 git_repo.read_commit(&target).map_err(|err| {
2730 ProtocolError::InvalidState(format!(
2731 "Git ref {} target commit {} is not present after pack receive: {err}",
2732 update.name,
2733 target.to_hex()
2734 ))
2735 })?;
2736 let refs = git_repo.references();
2737 let mut tx = refs.transaction();
2738 tx.update_to(
2739 update.name.clone(),
2740 ReferenceTarget::Direct(target),
2741 RefPrecondition::Any,
2742 None,
2743 );
2744 tx.commit().map_err(|err| {
2745 ProtocolError::InvalidState(format!("update Git ref {}: {err}", update.name))
2746 })?;
2747 if let Some(checkpoint) = update.checkpoint {
2748 record_git_lane_checkpoint(repo, git_repo, checkpoint)?;
2749 }
2750 Ok(())
2751}
2752
2753fn record_git_lane_checkpoint(
2754 repo: &Repository,
2755 git_repo: &SleyRepository,
2756 checkpoint: GitCheckpointTransfer,
2757) -> Result<(), ProtocolError> {
2758 let state = checkpoint
2759 .heddle_state_id
2760 .as_ref()
2761 .ok_or_else(|| {
2762 ProtocolError::InvalidState(
2763 "GitCheckpointTransfer.heddle_state_id is required".to_string(),
2764 )
2765 })
2766 .and_then(|state| {
2767 StateId::try_from_slice(&state.value)
2768 .map_err(|err| ProtocolError::InvalidState(err.to_string()))
2769 })?;
2770 let commit_oid = git_oid_from_proto(
2771 git_repo,
2772 "GitCheckpointTransfer.git_commit_oid",
2773 checkpoint.git_commit_oid.as_ref(),
2774 )?;
2775 let commit_hex = commit_oid.to_hex();
2776 if repo
2777 .latest_git_checkpoint_for_state(&state)
2778 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
2779 .is_some_and(|record| record.git_commit == commit_hex)
2780 {
2781 return Ok(());
2782 }
2783 let summary = serde_json::from_str::<serde_json::Value>(&checkpoint.metadata_json)
2784 .ok()
2785 .and_then(|metadata| {
2786 metadata
2787 .get("summary")
2788 .and_then(serde_json::Value::as_str)
2789 .map(str::to_string)
2790 })
2791 .unwrap_or_else(|| "pulled Git checkpoint".to_string());
2792 repo.record_git_checkpoint(&state, commit_hex, summary)
2793 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?;
2794 Ok(())
2795}
2796
2797fn git_lane_sley_repository<'a>(
2798 repo: &Repository,
2799 git_repo: &'a mut Option<SleyRepository>,
2800) -> Result<&'a SleyRepository, ProtocolError> {
2801 if git_repo.is_none() {
2802 *git_repo = Some(
2803 repo.git_overlay_sley_repository()
2804 .map_err(|err| ProtocolError::InvalidState(err.to_string()))?
2805 .ok_or_else(|| {
2806 ProtocolError::InvalidState(
2807 "git-overlay repository has no Git store".to_string(),
2808 )
2809 })?,
2810 );
2811 }
2812 match git_repo.as_ref() {
2813 Some(git_repo) => Ok(git_repo),
2814 None => Err(ProtocolError::InvalidState(
2815 "git-overlay repository has no Git store".to_string(),
2816 )),
2817 }
2818}
2819
2820fn git_oid_from_bytes(
2821 git_repo: &SleyRepository,
2822 field: &str,
2823 bytes: &[u8],
2824) -> Result<GitObjectId, ProtocolError> {
2825 GitObjectId::from_raw(git_repo.object_format(), bytes)
2826 .map_err(|err| ProtocolError::InvalidState(format!("{field}: {err}")))
2827}
2828
2829fn git_oid_from_proto(
2830 git_repo: &SleyRepository,
2831 field: &str,
2832 oid: Option<&ProtoGitObjectId>,
2833) -> Result<GitObjectId, ProtocolError> {
2834 let oid = oid.ok_or_else(|| ProtocolError::InvalidState(format!("{field} is required")))?;
2835 git_oid_from_bytes(git_repo, field, &oid.digest)
2836}
2837
2838fn proto_git_oid(oid: &GitObjectId) -> Option<ProtoGitObjectId> {
2839 proto_git_oid_bytes(oid.as_bytes())
2840}
2841
2842fn proto_git_oid_bytes(bytes: &[u8]) -> Option<ProtoGitObjectId> {
2843 let algorithm = match bytes.len() {
2844 20 => GitObjectAlgorithm::Sha1,
2845 32 => GitObjectAlgorithm::Sha256,
2846 _ => return None,
2847 };
2848 Some(ProtoGitObjectId {
2849 algorithm: algorithm as i32,
2850 digest: bytes.to_vec(),
2851 })
2852}
2853
2854#[derive(Clone, Copy)]
2855struct PushWireIdentities<'a> {
2856 transfer_id: &'a str,
2857 client_operation_id: &'a str,
2858}
2859
2860async fn send_native_pack_streaming_messages(
2861 tx: &mpsc::Sender<PushClientFrame>,
2862 repo: &Repository,
2863 objects: &[ObjectInfo],
2864 identities: PushWireIdentities<'_>,
2865 chunk_size: usize,
2866 transport: &super::helpers::HostedTransportPolicy,
2867 transport_mode: TransportMode,
2868) -> Result<(), ProtocolError> {
2869 let object_count = u64::try_from(objects.len()).map_err(|_| {
2870 ProtocolError::InvalidState("native pack object count exceeds u64".to_string())
2871 })?;
2872 let mut writer = wire::NativePackStreamingWriter::new_in(repo.heddle_dir(), object_count)?;
2873 let mut pack_reader = wire::GrowingPackChunkReader::open(writer.pack_path(), chunk_size)?;
2874 let (loaded_tx, mut loaded_rx) = mpsc::channel::<(
2875 usize,
2876 Result<wire::ObjectData, ProtocolError>,
2877 )>(NATIVE_PACK_OBJECT_PREFETCH_LIMIT);
2878 let store = repo.store().clone();
2879 let object_plan = objects.to_vec();
2880 let loader = tokio::task::spawn_blocking(move || {
2881 load_native_pack_objects_parallel(store, object_plan, loaded_tx);
2882 });
2883
2884 let mut next_index = 0usize;
2885 let mut pending = BTreeMap::new();
2886 while next_index < objects.len() {
2887 let (index, object) = loaded_rx.recv().await.ok_or_else(|| {
2888 ProtocolError::InvalidState(
2889 "native pack object loader stopped before sending all objects".to_string(),
2890 )
2891 })?;
2892 pending.insert(index, object);
2893
2894 while let Some(object) = pending.remove(&next_index) {
2895 let object = object?;
2896 let should_drain = object.data.len() >= chunk_size
2897 || (next_index + 1).is_multiple_of(NATIVE_PACK_DRAIN_OBJECT_INTERVAL);
2898 writer.add_object_data(object)?;
2899 if should_drain {
2900 writer.flush_pack()?;
2901 drain_growing_native_pack_stream(
2902 tx,
2903 &mut pack_reader,
2904 false,
2905 PackStreamKind::Pack,
2906 identities,
2907 transport,
2908 transport_mode,
2909 )
2910 .await?;
2911 }
2912 next_index += 1;
2913 }
2914 }
2915 loader.await.map_err(|err| {
2916 ProtocolError::InvalidState(format!("native pack object loader task failed: {err}"))
2917 })?;
2918
2919 let bundle = writer.finish()?;
2920 drain_growing_native_pack_stream(
2921 tx,
2922 &mut pack_reader,
2923 true,
2924 PackStreamKind::Pack,
2925 identities,
2926 transport,
2927 transport_mode,
2928 )
2929 .await?;
2930 send_native_pack_file_stream(
2931 tx,
2932 &bundle.index_path,
2933 PackStreamKind::Index,
2934 identities,
2935 chunk_size,
2936 transport,
2937 transport_mode,
2938 )
2939 .await
2940}
2941
2942fn load_native_pack_objects_parallel(
2943 store: AnyStore,
2944 objects: Vec<ObjectInfo>,
2945 tx: mpsc::Sender<(usize, Result<wire::ObjectData, ProtocolError>)>,
2946) {
2947 if objects.is_empty() {
2948 return;
2949 }
2950 let worker_count = native_pack_object_load_worker_count(objects.len());
2951 let objects = Arc::new(objects);
2952 let next_index = Arc::new(AtomicUsize::new(0));
2953
2954 std::thread::scope(|scope| {
2955 for _ in 0..worker_count {
2956 let store = store.clone();
2957 let objects = Arc::clone(&objects);
2958 let next_index = Arc::clone(&next_index);
2959 let tx = tx.clone();
2960 scope.spawn(move || {
2961 loop {
2962 let index = next_index.fetch_add(1, Ordering::Relaxed);
2963 let Some(info) = objects.get(index) else {
2964 break;
2965 };
2966 let object = wire::load_object_data(&store, &info.id, info.obj_type);
2967 if tx.blocking_send((index, object)).is_err() {
2968 break;
2969 }
2970 }
2971 });
2972 }
2973 });
2974}
2975
2976fn native_pack_object_load_worker_count(object_count: usize) -> usize {
2977 let available = std::thread::available_parallelism()
2978 .map(usize::from)
2979 .unwrap_or(1);
2980 object_count
2981 .min(available)
2982 .clamp(1, NATIVE_PACK_OBJECT_LOAD_WORKER_LIMIT)
2983}
2984
2985async fn drain_growing_native_pack_stream(
2986 tx: &mpsc::Sender<PushClientFrame>,
2987 reader: &mut wire::GrowingPackChunkReader,
2988 final_stream: bool,
2989 stream_kind: PackStreamKind,
2990 identities: PushWireIdentities<'_>,
2991 transport: &super::helpers::HostedTransportPolicy,
2992 transport_mode: TransportMode,
2993) -> Result<(), ProtocolError> {
2994 while let Some((offset, chunk_index, data, is_final_chunk)) =
2995 reader.next_available_chunk(final_stream)?
2996 {
2997 send_pack_chunk(
2998 tx,
2999 stream_kind,
3000 data,
3001 identities,
3002 transport,
3003 transport_mode,
3004 chunk_index,
3005 offset,
3006 is_final_chunk,
3007 )
3008 .await?;
3009 }
3010 Ok(())
3011}
3012
3013async fn send_native_pack_file_stream(
3014 tx: &mpsc::Sender<PushClientFrame>,
3015 path: &std::path::Path,
3016 stream_kind: PackStreamKind,
3017 identities: PushWireIdentities<'_>,
3018 chunk_size: usize,
3019 transport: &super::helpers::HostedTransportPolicy,
3020 transport_mode: TransportMode,
3021) -> Result<(), ProtocolError> {
3022 let mut reader = wire::PackFileChunkReader::open(path, chunk_size)?;
3023 while let Some((offset, chunk_index, data, is_final_chunk)) = reader.next_chunk()? {
3024 send_pack_chunk(
3025 tx,
3026 stream_kind,
3027 data,
3028 identities,
3029 transport,
3030 transport_mode,
3031 chunk_index,
3032 offset,
3033 is_final_chunk,
3034 )
3035 .await?;
3036 }
3037 Ok(())
3038}
3039
3040#[allow(clippy::too_many_arguments)]
3041async fn send_pack_chunk(
3042 tx: &mpsc::Sender<PushClientFrame>,
3043 stream_kind: PackStreamKind,
3044 data: Vec<u8>,
3045 identities: PushWireIdentities<'_>,
3046 transport: &super::helpers::HostedTransportPolicy,
3047 transport_mode: TransportMode,
3048 chunk_index: u32,
3049 offset: u64,
3050 is_final_chunk: bool,
3051) -> Result<(), ProtocolError> {
3052 let chunk_length = data.len().min(u32::MAX as usize) as u32;
3053 tx.send(PushClientFrame {
3054 frame: Some(push_client_frame::Frame::Pack(PackChunk {
3055 stream_kind: stream_kind as i32,
3056 data,
3057 transfer: Some(transport.transfer_checkpoint_with_mode(
3058 identities.transfer_id,
3059 transport_mode,
3060 chunk_index,
3061 offset,
3062 is_final_chunk,
3063 )),
3064 chunk_length,
3065 is_final_chunk,
3066 })),
3067 client_operation_id: identities.client_operation_id.to_string(),
3068 })
3069 .await
3070 .map_err(|_| ProtocolError::InvalidState("push stream closed unexpectedly".to_string()))
3071}
3072
3073fn preferred_transport_mode(
3074 transport: &super::helpers::HostedTransportPolicy,
3075 object_count: usize,
3076) -> TransportMode {
3077 let _ = transport;
3078 let _ = object_count;
3079 TransportMode::NativePack
3080}
3081
3082#[cfg(test)]
3083mod tests {
3084 use std::collections::HashSet;
3085
3086 use chrono::{TimeZone, Utc};
3087 use cli_shared::ClientConfig;
3088 use grpc::heddle::api::v1alpha1::{
3089 ListRefsRequest, ListRefsResponse, PullComplete as GrpcPullComplete, PullReady,
3090 PullServerFrame, PushServerFrame, StateId as ProtoStateId, TransferCheckpoint,
3091 UpdateRefRequest, UpdateRefResponse, pull_server_frame, push_client_frame,
3092 repo_sync_service_server::{RepoSyncService, RepoSyncServiceServer},
3093 };
3094 use objects::{
3095 object::{
3096 Attribution, Blob, ContentHash, Principal, Redaction, State, StateId, StateVisibility,
3097 StateVisibilityBlob, Tree, TreeEntry, VisibilityTier,
3098 },
3099 store::ObjectStore,
3100 };
3101 use tempfile::TempDir;
3102 use tonic::{Response, Status, transport::Server};
3103 use wire::{ObjectId, ObjectInfo};
3104
3105 use super::*;
3106 use crate::grpc_hosted::helpers::{
3107 descriptor_id_from_info, proto_state_id, to_proto_object_info,
3108 };
3109
3110 fn temp_repo() -> (TempDir, Repository) {
3111 let dir = TempDir::new().expect("tempdir");
3112 let repo = Repository::init_default(dir.path()).expect("init repo");
3113 (dir, repo)
3114 }
3115
3116 fn proto_oid_bytes(oid: &Option<ProtoGitObjectId>) -> Option<&[u8]> {
3117 oid.as_ref().map(|oid| oid.digest.as_slice())
3118 }
3119
3120 fn signed_test_config() -> ClientConfig {
3121 let signer = crypto::Ed25519Signer::generate().expect("generate test signing identity");
3122 ClientConfig::default()
3123 .with_auth_proof_key_pem(signer.to_pem().expect("export test key"))
3124 .with_authenticated_principal("principal:alice")
3125 }
3126
3127 fn temp_repo_unseeded() -> (TempDir, Repository) {
3131 let dir = TempDir::new().expect("tempdir");
3132 let repo = Repository::init(dir.path()).expect("init repo");
3133 (dir, repo)
3134 }
3135
3136 fn redaction_info(blob: ContentHash) -> ObjectInfo {
3137 ObjectInfo {
3138 id: ObjectId::Hash(blob),
3139 obj_type: ObjectType::Redaction,
3140 size: 0,
3141 delta_base: None,
3142 }
3143 }
3144
3145 fn state_info(state: StateId) -> ObjectInfo {
3146 ObjectInfo {
3147 id: ObjectId::StateId(state),
3148 obj_type: ObjectType::State,
3149 size: 0,
3150 delta_base: None,
3151 }
3152 }
3153
3154 fn state_visibility_info(state: StateId) -> ObjectInfo {
3155 ObjectInfo {
3156 id: ObjectId::StateId(state),
3157 obj_type: ObjectType::StateVisibility,
3158 size: 0,
3159 delta_base: None,
3160 }
3161 }
3162
3163 fn sample_blob() -> ContentHash {
3164 ContentHash::from_bytes([7u8; 32])
3165 }
3166
3167 #[test]
3168 fn descriptor_id_from_info_matches_proto_encode_path() {
3169 let infos = [
3170 redaction_info(sample_blob()),
3171 state_info(StateId::from_bytes([3u8; 32])),
3172 state_visibility_info(StateId::from_bytes([9u8; 32])),
3173 ];
3174 for info in infos {
3175 assert_eq!(
3176 descriptor_id_from_info(&info),
3177 descriptor_id(&to_proto_object_info(&info)),
3178 "keying path must stay byte-identical to the throwaway-encode path",
3179 );
3180 }
3181 }
3182
3183 #[test]
3184 fn push_operation_identity_is_not_the_resumable_transfer_identity() {
3185 let transfer_id = push_transfer_id(
3186 "acme/widgets",
3187 StateId::from_bytes([5; 32]),
3188 "feature/retry",
3189 );
3190 let operation_id =
3191 ClientOperationId::caller_or_fresh("heddle.api.v1alpha1.RepoSyncService/Push", "");
3192 let next_operation_id =
3193 ClientOperationId::caller_or_fresh("heddle.api.v1alpha1.RepoSyncService/Push", "");
3194
3195 assert_ne!(operation_id.as_str(), transfer_id);
3196 assert_ne!(operation_id, next_operation_id);
3197 }
3198
3199 #[tokio::test]
3200 async fn push_and_pull_opening_proofs_are_bound_to_the_authenticated_principal() {
3201 use crypto::Signer as _;
3202
3203 let config = signed_test_config();
3204 let signer = crypto::Ed25519Signer::from_pem(
3205 config
3206 .auth_proof_key_pem
3207 .as_deref()
3208 .expect("test proof key"),
3209 )
3210 .expect("test signer");
3211 let channel = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
3212 let client = HostedGrpcClient::from_channel(channel, &config).expect("test client");
3213 for (stream_id, route) in [
3214 (
3215 "push-transfer-1",
3216 "/heddle.api.v1alpha1.RepoSyncService/Push",
3217 ),
3218 (
3219 "pull-transfer-1",
3220 "/heddle.api.v1alpha1.RepoSyncService/Pull",
3221 ),
3222 ] {
3223 let proof = client
3224 .stream_opening_proof(stream_id, route, "acme/widgets", "")
3225 .expect("opening proof");
3226 let canonical = grpc::signing::stream_open_bytes(
3227 "principal:alice",
3228 &proof.stream_id,
3229 &proof.route,
3230 "acme/widgets",
3231 &proof.resume_cursor,
3232 &proof.capability_context,
3233 );
3234
3235 crypto::verify_payload_signature(
3236 &canonical,
3237 "ed25519",
3238 signer.public_key(),
3239 &proof.signature,
3240 )
3241 .expect("stream proof must verify under the contract principal identity");
3242 }
3243
3244 let implementation = include_str!("sync.rs")
3245 .split("#[cfg(test)]")
3246 .next()
3247 .expect("sync implementation section");
3248 assert_eq!(
3249 implementation.matches("self.stream_opening_proof(").count(),
3250 2,
3251 "Push and Pull must both route their opening frame through the shared proof chokepoint"
3252 );
3253 }
3254
3255 #[test]
3256 fn git_lane_messages_pack_reachable_commit_graph_and_attach_checkpoint() {
3257 let dir = TempDir::new().expect("tempdir");
3258 let git = sley::Repository::init(dir.path()).expect("init git");
3259 let blob_oid = git.write_blob(b"hello\n").expect("write blob");
3260 let tree = sley::TreeObject {
3261 entries: vec![sley::plumbing::sley_object::TreeEntry {
3262 mode: 0o100644,
3263 name: sley::BString::from(b"hello.txt"),
3264 oid: blob_oid,
3265 }],
3266 };
3267 let tree_oid = git
3268 .write_raw_object(sley::GitObjectType::Tree, tree.write())
3269 .expect("write tree");
3270 let commit = sley::CommitObject {
3271 tree: tree_oid,
3272 parents: Vec::new(),
3273 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3274 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3275 encoding: None,
3276 message: b"checkpoint\n".to_vec(),
3277 };
3278 let commit_oid = git
3279 .write_raw_object(sley::GitObjectType::Commit, commit.write())
3280 .expect("write commit");
3281
3282 let pack =
3283 build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64 * 1024)
3284 .expect("build git lane pack plan")
3285 .expect("non-empty pack plan");
3286 assert_eq!(pack.pack_id.len(), git.object_format().raw_len());
3287 let (tx, mut rx) = mpsc::channel(8);
3288 stream_git_pack_messages_blocking(
3289 tx,
3290 pack.clone(),
3291 64 * 1024,
3292 Progress::null(),
3293 "test-push-op".to_string(),
3294 )
3295 .expect("stream git lane pack");
3296 let mut pack_bytes = Vec::new();
3297 let mut chunks = Vec::new();
3298 while let Some(chunk) = rx.blocking_recv() {
3299 assert_eq!(chunk.client_operation_id, "test-push-op");
3300 let Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
3301 body: Some(git_lane_transfer::Body::Pack(pack)),
3302 })) = chunk.frame
3303 else {
3304 panic!("expected git pack chunk");
3305 };
3306 pack_bytes.extend_from_slice(&pack.pack_chunk);
3307 chunks.push(pack);
3308 }
3309 assert!(!chunks.is_empty());
3310 assert!(chunks.last().expect("pack chunk").is_final_chunk);
3311 assert_eq!(pack_bytes.len() as u64, pack.pack_size);
3312 let indexed = sley::plumbing::sley_odb::index_raw_pack(&pack_bytes, git.object_format())
3313 .expect("generated pack should index");
3314 assert_eq!(indexed.pack_id.as_bytes(), pack.pack_id.as_slice());
3315 assert_eq!(indexed.objects.len(), 3);
3316
3317 let state = StateId::from_bytes([9u8; 32]);
3318 let ref_message = git_ref_update_message(
3319 "refs/heads/main",
3320 GrpcGitRefKind::Branch,
3321 commit_oid,
3322 None,
3323 Some(GitCheckpointTransfer {
3324 heddle_state_id: proto_state_id(state),
3325 git_commit_oid: proto_git_oid(&commit_oid),
3326 thread: "main".to_string(),
3327 metadata_json: String::new(),
3328 }),
3329 );
3330 let update = ref_message;
3331 assert_eq!(update.name, "refs/heads/main");
3332 assert_eq!(update.kind, GrpcGitRefKind::Branch as i32);
3333 assert_eq!(
3334 update.target_oid.as_ref().map(|oid| oid.digest.as_slice()),
3335 Some(commit_oid.as_bytes())
3336 );
3337 let checkpoint = update.checkpoint.expect("checkpoint");
3338 assert_eq!(
3339 checkpoint
3340 .heddle_state_id
3341 .as_ref()
3342 .map(|state| state.value.as_slice()),
3343 Some(state.as_bytes().as_slice())
3344 );
3345 assert_eq!(
3346 checkpoint
3347 .git_commit_oid
3348 .as_ref()
3349 .map(|oid| oid.digest.as_slice()),
3350 Some(commit_oid.as_bytes())
3351 );
3352 assert_eq!(checkpoint.thread, "main");
3353 }
3354
3355 fn sample_ref_update_message(commit_oid: GitObjectId) -> PushClientFrame {
3356 git_lane_push_message(
3357 git_lane_transfer::Body::RefUpdate(git_ref_update_message(
3358 "refs/heads/main",
3359 GrpcGitRefKind::Branch,
3360 commit_oid,
3361 None,
3362 Some(GitCheckpointTransfer {
3363 heddle_state_id: proto_state_id(StateId::from_bytes([9u8; 32])),
3364 git_commit_oid: proto_git_oid(&commit_oid),
3365 thread: "main".to_string(),
3366 metadata_json: String::new(),
3367 }),
3368 )),
3369 "test-push-op",
3370 )
3371 }
3372
3373 fn write_commit(git: &sley::Repository, seed: &str) -> GitObjectId {
3376 let blob_oid = git
3377 .write_blob(format!("content-{seed}\n").as_bytes())
3378 .expect("write blob");
3379 let tree = sley::TreeObject {
3380 entries: vec![sley::plumbing::sley_object::TreeEntry {
3381 mode: 0o100644,
3382 name: sley::BString::from(format!("{seed}.txt").into_bytes()),
3383 oid: blob_oid,
3384 }],
3385 };
3386 let tree_oid = git
3387 .write_raw_object(sley::GitObjectType::Tree, tree.write())
3388 .expect("write tree");
3389 let commit = sley::CommitObject {
3390 tree: tree_oid,
3391 parents: Vec::new(),
3392 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3393 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3394 encoding: None,
3395 message: format!("commit {seed}\n").into_bytes(),
3396 };
3397 git.write_raw_object(sley::GitObjectType::Commit, commit.write())
3398 .expect("write commit")
3399 }
3400
3401 fn set_ref(git: &mut sley::Repository, name: &str, target: GitObjectId) {
3402 let refs = git.references();
3403 let mut tx = refs.transaction();
3404 tx.update_to(
3405 name.to_string(),
3406 ReferenceTarget::Direct(target),
3407 RefPrecondition::Any,
3408 None,
3409 );
3410 tx.commit().expect("commit ref");
3411 }
3412
3413 fn mirror_ref_updates(plan: &GitLanePushPlan) -> Vec<GitRefUpdateTransfer> {
3414 plan.ref_updates.clone()
3415 }
3416
3417 fn write_child_commit(git: &sley::Repository, parent: GitObjectId, seed: &str) -> GitObjectId {
3421 let blob_oid = git
3422 .write_blob(format!("content-{seed}\n").as_bytes())
3423 .expect("write blob");
3424 let tree = sley::TreeObject {
3425 entries: vec![sley::plumbing::sley_object::TreeEntry {
3426 mode: 0o100644,
3427 name: sley::BString::from(format!("{seed}.txt").into_bytes()),
3428 oid: blob_oid,
3429 }],
3430 };
3431 let tree_oid = git
3432 .write_raw_object(sley::GitObjectType::Tree, tree.write())
3433 .expect("write tree");
3434 let commit = sley::CommitObject {
3435 tree: tree_oid,
3436 parents: vec![parent],
3437 author: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3438 committer: b"Tester <test@example.com> 1700000000 +0000".to_vec(),
3439 encoding: None,
3440 message: format!("commit {seed}\n").into_bytes(),
3441 };
3442 git.write_raw_object(sley::GitObjectType::Commit, commit.write())
3443 .expect("write commit")
3444 }
3445
3446 fn value_expectation(oid: GitObjectId) -> GitRefRemoteExpectation {
3447 GitRefRemoteExpectation::Value(oid.as_bytes().to_vec())
3448 }
3449
3450 #[test]
3459 fn git_mirror_plan_packs_only_wanted_objects_against_have_boundary() {
3460 let dir = TempDir::new().expect("tempdir");
3461 let mut git = sley::Repository::init(dir.path()).expect("init git");
3462
3463 let base = write_commit(&git, "base-with-a-reasonably-large-payload");
3466 let mut tip = base;
3467 for i in 0..8 {
3468 tip = write_child_commit(&git, tip, &format!("layer-{i}"));
3469 }
3470 let descendant = write_child_commit(&git, tip, "descendant-adds-one-object");
3471 set_ref(&mut git, "refs/heads/main", descendant);
3472
3473 let full =
3476 build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new()).expect("full plan");
3477 let full_pack = full.pack.as_ref().expect("fresh server produces a pack");
3478 let full_size = full_pack.pack_size;
3479 assert!(full_size > 0, "baseline full pack must be non-empty");
3480
3481 let mut have_tip = HashMap::new();
3485 have_tip.insert("refs/heads/main".to_string(), value_expectation(tip));
3486 let delta =
3487 build_git_mirror_plan_from_sley(&git, 64 * 1024, &have_tip).expect("delta plan");
3488 let delta_pack = delta
3489 .pack
3490 .as_ref()
3491 .expect("descendant still has one new object to pack");
3492 let delta_size = delta_pack.pack_size;
3493 assert!(
3494 delta_size * 4 < full_size,
3495 "want-only packing must be dramatically smaller than a full re-pack \
3496 (delta {delta_size} bytes vs full {full_size} bytes)",
3497 );
3498
3499 let mut have_target = HashMap::new();
3503 have_target.insert("refs/heads/main".to_string(), value_expectation(descendant));
3504 let ref_only =
3505 build_git_mirror_plan_from_sley(&git, 64 * 1024, &have_target).expect("ref-only plan");
3506 assert!(
3507 ref_only.pack.is_none(),
3508 "pushing a ref the server already has must send NO pack",
3509 );
3510 assert_eq!(
3512 ref_only.ref_updates.len(),
3513 1,
3514 "the ref update still ships even with an empty want set",
3515 );
3516 }
3517
3518 #[test]
3521 fn git_lane_reachable_pack_plan_matches_legacy_writer_checksum() {
3522 let dir = TempDir::new().expect("tempdir");
3523 let git = sley::Repository::init(dir.path()).expect("init git");
3524 let commit_oid = write_commit(&git, "byte-identity");
3525
3526 let pack_plan =
3527 build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64 * 1024)
3528 .expect("build pack plan")
3529 .expect("non-empty pack plan");
3530
3531 let mut legacy_pack = Vec::new();
3532 let legacy = sley::plumbing::sley_odb::write_reachable_pack_to_writer(
3533 git.objects().as_ref(),
3534 git.object_format(),
3535 std::iter::once(commit_oid),
3536 &HashSet::new(),
3537 &mut legacy_pack,
3538 )
3539 .expect("legacy reachable pack")
3540 .expect("legacy pack summary");
3541
3542 assert_eq!(pack_plan.pack_id, legacy.checksum.as_bytes().to_vec());
3543 assert_eq!(pack_plan.pack_size, legacy.pack_size);
3544 assert_eq!(pack_plan.pack_size, legacy_pack.len() as u64);
3545
3546 let mut planned_pack = Vec::new();
3547 pack_plan
3548 .pack_file
3549 .lock()
3550 .expect("lock pack tempfile")
3551 .seek(SeekFrom::Start(0))
3552 .expect("rewind pack tempfile");
3553 io::copy(
3554 &mut pack_plan
3555 .pack_file
3556 .lock()
3557 .expect("lock pack tempfile")
3558 .as_file_mut(),
3559 &mut planned_pack,
3560 )
3561 .expect("read planned pack");
3562 assert_eq!(planned_pack, legacy_pack);
3563 }
3564
3565 #[test]
3569 fn git_mirror_plan_builds_one_pack_and_checkpointless_updates_for_all_refs() {
3570 let dir = TempDir::new().expect("tempdir");
3571 let mut git = sley::Repository::init(dir.path()).expect("init git");
3572
3573 let main_commit = write_commit(&git, "main");
3574 let feature_commit = write_commit(&git, "feature");
3575 let tagged_commit = write_commit(&git, "tagged");
3576
3577 set_ref(&mut git, "refs/heads/main", main_commit);
3578 set_ref(&mut git, "refs/heads/feature", feature_commit);
3579
3580 let tag = sley::TagObject {
3582 object: tagged_commit,
3583 object_type: sley::GitObjectType::Commit,
3584 name: b"v1".to_vec(),
3585 tagger: Some(b"Tester <test@example.com> 1700000000 +0000".to_vec()),
3586 message: b"release v1\n".to_vec(),
3587 raw_body: None,
3588 };
3589 let tag_oid = git
3590 .write_raw_object(sley::GitObjectType::Tag, tag.write())
3591 .expect("write tag");
3592 set_ref(&mut git, "refs/tags/v1", tag_oid);
3593
3594 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3595 .expect("build mirror plan");
3596
3597 let updates = mirror_ref_updates(&plan);
3598 assert_eq!(updates.len(), 3, "one ref update per direct ref");
3599
3600 assert!(
3603 updates.iter().all(|u| u.checkpoint.is_none()),
3604 "mirror mode signals via checkpoint: None on every ref update",
3605 );
3606
3607 let by_name: HashMap<&str, &GitRefUpdateTransfer> =
3608 updates.iter().map(|u| (u.name.as_str(), u)).collect();
3609
3610 let main = by_name["refs/heads/main"];
3611 assert_eq!(main.kind, GrpcGitRefKind::Branch as i32);
3612 assert_eq!(
3613 proto_oid_bytes(&main.target_oid),
3614 Some(main_commit.as_bytes())
3615 );
3616 assert!(main.peeled_oid.is_none(), "commit refs are not peeled");
3617
3618 let feature = by_name["refs/heads/feature"];
3619 assert_eq!(feature.kind, GrpcGitRefKind::Branch as i32);
3620 assert_eq!(
3621 proto_oid_bytes(&feature.target_oid),
3622 Some(feature_commit.as_bytes())
3623 );
3624
3625 let tag_update = by_name["refs/tags/v1"];
3626 assert_eq!(tag_update.kind, GrpcGitRefKind::Tag as i32);
3627 assert_eq!(
3628 proto_oid_bytes(&tag_update.target_oid),
3629 Some(tag_oid.as_bytes())
3630 );
3631 assert_eq!(
3632 proto_oid_bytes(&tag_update.peeled_oid),
3633 Some(tagged_commit.as_bytes()),
3634 "annotated tag ref is peeled to its underlying commit",
3635 );
3636
3637 let plan_pack = plan.pack.as_ref().expect("full pack for fresh server");
3641 let mut pack_roots: Vec<GitObjectId> = plan_pack.roots.clone();
3642 pack_roots.sort_by_key(|oid| oid.to_hex());
3643 let mut expected_roots = vec![main_commit, feature_commit, tag_oid];
3644 expected_roots.sort_by_key(|oid| oid.to_hex());
3645 assert_eq!(
3646 pack_roots, expected_roots,
3647 "pack roots == resolved ref targets",
3648 );
3649
3650 let (tx, mut rx) = mpsc::channel(64);
3653 stream_git_pack_messages_blocking(
3654 tx,
3655 plan_pack.clone(),
3656 64 * 1024,
3657 Progress::null(),
3658 "test-push-op".to_string(),
3659 )
3660 .expect("stream mirror pack");
3661 let mut pack_bytes = Vec::new();
3662 while let Some(message) = rx.blocking_recv() {
3663 if let Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
3664 body: Some(git_lane_transfer::Body::Pack(pack)),
3665 })) = message.frame
3666 {
3667 pack_bytes.extend_from_slice(&pack.pack_chunk);
3668 }
3669 }
3670 assert_eq!(
3671 pack_bytes.len() as u64,
3672 plan_pack.pack_size,
3673 "streamed pack size must match plan",
3674 );
3675 let indexed = sley::plumbing::sley_odb::index_raw_pack(&pack_bytes, git.object_format())
3676 .expect("mirror pack indexes");
3677 let packed: HashSet<Vec<u8>> = indexed
3678 .objects
3679 .iter()
3680 .map(|obj| obj.oid.as_bytes().to_vec())
3681 .collect();
3682 for oid in [main_commit, feature_commit, tagged_commit, tag_oid] {
3683 assert!(
3684 packed.contains(oid.as_bytes()),
3685 "pack must contain {}",
3686 oid.to_hex()
3687 );
3688 }
3689 }
3690
3691 #[test]
3694 fn git_mirror_plan_skips_symbolic_refs() {
3695 let dir = TempDir::new().expect("tempdir");
3696 let mut git = sley::Repository::init(dir.path()).expect("init git");
3697 let main_commit = write_commit(&git, "main");
3698 set_ref(&mut git, "refs/heads/main", main_commit);
3699 {
3702 let refs = git.references();
3703 let mut tx = refs.transaction();
3704 tx.update_to(
3705 "HEAD".to_string(),
3706 ReferenceTarget::Symbolic("refs/heads/main".to_string()),
3707 RefPrecondition::Any,
3708 None,
3709 );
3710 tx.commit().expect("set HEAD");
3711 }
3712
3713 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3714 .expect("build mirror plan");
3715 let updates = mirror_ref_updates(&plan);
3716 assert_eq!(updates.len(), 1, "only the direct ref is pushed");
3717 assert_eq!(updates[0].name, "refs/heads/main");
3718 }
3719
3720 #[test]
3723 fn git_mirror_plan_applies_per_ref_remote_expectations() {
3724 let dir = TempDir::new().expect("tempdir");
3725 let mut git = sley::Repository::init(dir.path()).expect("init git");
3726 let main_commit = write_commit(&git, "main");
3727 let feature_commit = write_commit(&git, "feature");
3728 set_ref(&mut git, "refs/heads/main", main_commit);
3729 set_ref(&mut git, "refs/heads/feature", feature_commit);
3730
3731 let remote_oid = "89abcdef012345670123456789abcdef01234567";
3732 let mut expectations = HashMap::new();
3733 expectations.insert(
3734 "refs/heads/main".to_string(),
3735 GitRefRemoteExpectation::Value(hex::decode(remote_oid).expect("hex")),
3736 );
3737 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &expectations)
3740 .expect("build mirror plan");
3741 let updates = mirror_ref_updates(&plan);
3742 let by_name: HashMap<&str, &GitRefUpdateTransfer> =
3743 updates.iter().map(|u| (u.name.as_str(), u)).collect();
3744
3745 let main = by_name["refs/heads/main"];
3746 assert!(!main.expected_missing);
3747 assert_eq!(
3748 hex::encode(proto_oid_bytes(&main.expected_target_oid).expect("expected oid")),
3749 remote_oid
3750 );
3751
3752 let feature = by_name["refs/heads/feature"];
3753 assert!(
3754 feature.expected_missing,
3755 "ref absent from ListRefs is expected-missing (create)",
3756 );
3757 }
3758
3759 #[test]
3763 fn git_mirror_plan_includes_pull_request_refs() {
3764 let dir = TempDir::new().expect("tempdir");
3765 let mut git = sley::Repository::init(dir.path()).expect("init git");
3766 let pr_commit = write_commit(&git, "pr");
3767 set_ref(&mut git, "refs/pull/42/head", pr_commit);
3768
3769 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3770 .expect("build mirror plan");
3771 let updates = mirror_ref_updates(&plan);
3772 let pull = updates
3773 .iter()
3774 .find(|update| update.name == "refs/pull/42/head")
3775 .expect("refs/pull/* ref must be mirrored");
3776 assert_eq!(
3777 pull.kind,
3778 GrpcGitRefKind::Other as i32,
3779 "refs/pull/* classifies as Other",
3780 );
3781 assert_eq!(
3782 proto_oid_bytes(&pull.target_oid),
3783 Some(pr_commit.as_bytes())
3784 );
3785 assert!(
3786 pull.checkpoint.is_none(),
3787 "mirror ref updates are checkpoint-less",
3788 );
3789 assert!(pull.peeled_oid.is_none(), "a commit ref is not peeled");
3790 }
3791
3792 #[test]
3797 fn git_mirror_plan_excludes_local_only_refs_and_keeps_content() {
3798 let dir = TempDir::new().expect("tempdir");
3799 let mut git = sley::Repository::init(dir.path()).expect("init git");
3800
3801 let main_commit = write_commit(&git, "main");
3802 let notes_commit = write_commit(&git, "notes");
3803 let stash_commit = write_commit(&git, "stash");
3804 let remote_commit = write_commit(&git, "remote");
3805 let original_commit = write_commit(&git, "original");
3806 let replace_commit = write_commit(&git, "replace");
3807
3808 set_ref(&mut git, "refs/heads/main", main_commit);
3810 set_ref(&mut git, "refs/notes/heddle", notes_commit);
3811
3812 set_ref(&mut git, "refs/stash", stash_commit);
3814 set_ref(&mut git, "refs/remotes/origin/main", remote_commit);
3815 set_ref(&mut git, "refs/original/refs/heads/main", original_commit);
3816 set_ref(&mut git, "refs/replace/deadbeef", replace_commit);
3817
3818 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3819 .expect("build mirror plan");
3820 let names: Vec<&str> = plan
3821 .ref_updates
3822 .iter()
3823 .map(|update| update.name.as_str())
3824 .collect();
3825
3826 assert!(
3827 names.contains(&"refs/heads/main"),
3828 "content branch is mirrored: {names:?}",
3829 );
3830 assert!(
3831 names.contains(&"refs/notes/heddle"),
3832 "heddle state-note ref is mirrored: {names:?}",
3833 );
3834 for excluded in [
3835 "refs/stash",
3836 "refs/remotes/origin/main",
3837 "refs/original/refs/heads/main",
3838 "refs/replace/deadbeef",
3839 ] {
3840 assert!(
3841 !names.contains(&excluded),
3842 "local-only ref {excluded} must NOT be mirrored: {names:?}",
3843 );
3844 }
3845 assert_eq!(
3846 names.len(),
3847 2,
3848 "exactly the two content refs ship: {names:?}"
3849 );
3850 }
3851
3852 #[test]
3857 fn git_mirror_plan_ignores_dangling_local_only_ref() {
3858 let dir = TempDir::new().expect("tempdir");
3859 let mut git = sley::Repository::init(dir.path()).expect("init git");
3860
3861 let main_commit = write_commit(&git, "main");
3862 set_ref(&mut git, "refs/heads/main", main_commit);
3863
3864 let missing = GitObjectId::from_hex(
3866 sley::ObjectFormat::Sha1,
3867 "0123456789abcdef0123456789abcdef01234567",
3868 )
3869 .expect("oid");
3870 set_ref(&mut git, "refs/original/refs/heads/main", missing);
3871
3872 let plan = build_git_mirror_plan_from_sley(&git, 64 * 1024, &HashMap::new())
3873 .expect("dangling local-only ref must not fail the mirror plan");
3874 let names: Vec<&str> = plan
3875 .ref_updates
3876 .iter()
3877 .map(|update| update.name.as_str())
3878 .collect();
3879 assert_eq!(
3880 names,
3881 vec!["refs/heads/main"],
3882 "only content ships: {names:?}"
3883 );
3884 }
3885
3886 #[test]
3887 fn git_ref_expectation_marks_missing_when_remote_has_no_git_revision() {
3888 let commit_oid = GitObjectId::from_hex(
3889 sley::ObjectFormat::Sha1,
3890 "0123456789abcdef0123456789abcdef01234567",
3891 )
3892 .expect("oid");
3893 let message = sample_ref_update_message(commit_oid);
3894
3895 let expectation = parse_git_ref_expectation("").expect("missing expectation");
3896 let update = git_ref_update_from_message(&message);
3897 let mut update = update.clone();
3898 apply_git_ref_expectation_value(&mut update, &expectation);
3899 assert!(update.expected_missing);
3900 assert!(update.expected_target_oid.is_none());
3901 }
3902
3903 #[test]
3904 fn git_ref_expectation_uses_remote_git_revision_oid() {
3905 let commit_oid = GitObjectId::from_hex(
3906 sley::ObjectFormat::Sha1,
3907 "0123456789abcdef0123456789abcdef01234567",
3908 )
3909 .expect("oid");
3910 let remote_oid = "89abcdef012345670123456789abcdef01234567";
3911 let message = sample_ref_update_message(commit_oid);
3912
3913 let expectation =
3914 parse_git_ref_expectation(&format!("git:{remote_oid}")).expect("git expectation");
3915 let update = git_ref_update_from_message(&message);
3916 let mut update = update.clone();
3917 apply_git_ref_expectation_value(&mut update, &expectation);
3918 assert!(!update.expected_missing);
3919 assert_eq!(
3920 hex::encode(proto_oid_bytes(&update.expected_target_oid).expect("expected oid")),
3921 remote_oid
3922 );
3923 }
3924
3925 #[test]
3926 fn git_pack_stream_writer_emits_ordered_chunks() {
3927 let (tx, mut rx) = mpsc::channel(4);
3928 let mut writer = GitPackPushMessageWriter::new(
3929 tx,
3930 "git-pack:test".to_string(),
3931 vec![0x42; 20],
3932 10,
3933 4,
3934 Progress::null(),
3935 "test-push-op".to_string(),
3936 );
3937 writer.write_all(b"abcdefghij").expect("write pack bytes");
3938 writer.finish().expect("finish pack stream");
3939
3940 let mut chunks = Vec::new();
3941 while let Some(message) = rx.blocking_recv() {
3942 assert_eq!(message.client_operation_id, "test-push-op");
3943 let Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
3944 body: Some(git_lane_transfer::Body::Pack(pack)),
3945 })) = message.frame
3946 else {
3947 panic!("expected git pack chunk");
3948 };
3949 chunks.push(pack);
3950 }
3951
3952 assert_eq!(chunks.len(), 3);
3953 assert_eq!(chunks[0].offset, 0);
3954 assert_eq!(chunks[0].chunk_index, 0);
3955 assert!(!chunks[0].is_final_chunk);
3956 assert_eq!(chunks[0].pack_chunk.as_slice(), b"abcd");
3957 assert_eq!(chunks[1].offset, 4);
3958 assert_eq!(chunks[1].chunk_index, 1);
3959 assert!(!chunks[1].is_final_chunk);
3960 assert_eq!(chunks[1].pack_chunk.as_slice(), b"efgh");
3961 assert_eq!(chunks[2].offset, 8);
3962 assert_eq!(chunks[2].chunk_index, 2);
3963 assert!(chunks[2].is_final_chunk);
3964 assert_eq!(chunks[2].pack_chunk.as_slice(), b"ij");
3965 assert!(
3966 chunks
3967 .iter()
3968 .all(|chunk| chunk.pack_id.as_slice() == &[0x42; 20][..])
3969 );
3970 }
3971
3972 #[derive(Default)]
3975 struct PhaseCapturingSink {
3976 phases: std::sync::Mutex<Vec<String>>,
3977 }
3978
3979 impl PhaseCapturingSink {
3980 fn phases(&self) -> Vec<String> {
3981 self.phases.lock().unwrap().clone()
3982 }
3983 }
3984
3985 impl objects::Sink for PhaseCapturingSink {
3986 fn render(&self, snap: objects::ProgressSnapshot) {
3987 self.phases.lock().unwrap().push(snap.phase);
3988 }
3989 }
3990
3991 #[test]
3995 fn git_pack_stream_reports_upload_progress() {
3996 let dir = TempDir::new().expect("tempdir");
3997 let git = sley::Repository::init(dir.path()).expect("init git");
3998 let commit_oid = write_commit(&git, "progress");
3999 let pack = build_git_lane_multi_root_pack_plan(&git, vec![commit_oid], Vec::new(), 64)
4001 .expect("build pack plan")
4002 .expect("non-empty pack plan");
4003
4004 let sink = std::sync::Arc::new(PhaseCapturingSink::default());
4005 struct Forward(std::sync::Arc<PhaseCapturingSink>);
4006 impl objects::Sink for Forward {
4007 fn render(&self, snap: objects::ProgressSnapshot) {
4008 self.0.render(snap);
4009 }
4010 }
4011 let progress = Progress::with_sink(Box::new(Forward(std::sync::Arc::clone(&sink))));
4012
4013 let (tx, mut rx) = mpsc::channel(1024);
4014 stream_git_pack_messages_blocking(
4015 tx,
4016 pack.clone(),
4017 64,
4018 progress.clone(),
4019 "test-push-op".to_string(),
4020 )
4021 .expect("stream pack");
4022 while rx.blocking_recv().is_some() {}
4023
4024 let phases = sink.phases();
4025 assert!(
4026 phases.iter().any(|phase| phase.contains("uploading")),
4027 "pack streamer must drive an 'uploading' progress phase; saw {phases:?}",
4028 );
4029 let last_uploading = phases
4030 .iter()
4031 .rev()
4032 .find(|phase| phase.contains("uploading"))
4033 .cloned();
4034 assert!(
4035 last_uploading
4036 .as_deref()
4037 .is_some_and(|phase| phase.contains(&pack.pack_size.to_string())),
4038 "final uploading phase must show the full pack size ({} bytes); saw {last_uploading:?}",
4039 pack.pack_size,
4040 );
4041 }
4042
4043 #[test]
4044 fn git_pack_pull_install_state_streams_pack_into_sley_store() {
4045 let source_dir = TempDir::new().expect("source tempdir");
4046 let source = sley::Repository::init(source_dir.path()).expect("init source git");
4047 let blob_oid = source.write_blob(b"streamed pull pack\n").expect("blob");
4048 let pack = sley::plumbing::sley_odb::build_reachable_pack(
4049 source.objects().as_ref(),
4050 source.object_format(),
4051 [blob_oid],
4052 &HashSet::new(),
4053 )
4054 .expect("build pack")
4055 .expect("reachable pack");
4056
4057 let dest_dir = TempDir::new().expect("dest tempdir");
4058 let dest = sley::Repository::init(dest_dir.path()).expect("init dest git");
4059 let mut state = GitPackPullInstallState::default();
4060 let chunk_size = 7usize;
4061 let mut offset = 0u64;
4062 for (chunk_index, chunk) in pack.pack.chunks(chunk_size).enumerate() {
4063 let next_offset = offset + chunk.len() as u64;
4064 state
4065 .receive_chunk(
4066 &dest,
4067 GitPackTransfer {
4068 transfer_id: "git-pack:test".to_string(),
4069 offset,
4070 chunk_index: chunk_index as u32,
4071 is_final_chunk: next_offset == pack.pack.len() as u64,
4072 pack_size: pack.pack.len() as u64,
4073 pack_chunk: chunk.to_vec(),
4074 pack_id: pack.checksum.as_bytes().to_vec(),
4075 },
4076 )
4077 .expect("receive chunk");
4078 offset = next_offset;
4079 }
4080
4081 state.ensure_idle().expect("stream should finish");
4082 let object = dest.read_object(&blob_oid).expect("read installed object");
4083 assert_eq!(object.body.as_slice(), b"streamed pull pack\n");
4084 }
4085
4086 #[test]
4087 fn accept_git_lane_pull_transfer_errors_for_non_overlay_repo() {
4088 let (_dir, repo) = temp_repo();
4089 assert_ne!(
4090 repo.capability(),
4091 RepositoryCapability::GitOverlay,
4092 "init_default repo must be non-GitOverlay for this guard test",
4093 );
4094 let mut git_repo = None;
4095 let mut state = GitPackPullInstallState::default();
4096 let transfer = GitLaneTransfer {
4097 body: Some(git_lane_transfer::Body::Pack(GitPackTransfer {
4098 transfer_id: "git-pack:test".to_string(),
4099 offset: 0,
4100 chunk_index: 0,
4101 is_final_chunk: true,
4102 pack_size: 0,
4103 pack_chunk: Vec::new(),
4104 pack_id: Vec::new(),
4105 })),
4106 };
4107 let err = accept_git_lane_pull_transfer(&repo, &mut git_repo, &mut state, transfer)
4108 .expect_err("git-lane pull to a non-GitOverlay repo must fail loud, not silently drop");
4109 assert!(
4110 matches!(err, ProtocolError::InvalidState(_)),
4111 "expected InvalidState protocol error, got {err:?}",
4112 );
4113 }
4114
4115 fn git_ref_update_from_message(message: &PushClientFrame) -> &GitRefUpdateTransfer {
4116 let Some(push_client_frame::Frame::GitLane(GitLaneTransfer {
4117 body: Some(git_lane_transfer::Body::RefUpdate(update)),
4118 })) = message.frame.as_ref()
4119 else {
4120 panic!("expected git ref update message");
4121 };
4122 update
4123 }
4124
4125 fn loose_tree_path(repo: &Repository, hash: &ContentHash) -> std::path::PathBuf {
4126 let hex = hash.to_hex();
4127 let (prefix, rest) = hex.split_at(2);
4128 repo.heddle_dir()
4129 .join("objects")
4130 .join("trees")
4131 .join(prefix)
4132 .join(rest)
4133 }
4134
4135 fn sample_redaction(blob: ContentHash) -> Redaction {
4136 Redaction {
4137 redacted_blob: blob,
4138 state: StateId::from_bytes([1u8; 32]),
4139 path: "config/secrets.toml".into(),
4140 reason: "leaked credential".into(),
4141 redactor: Principal {
4142 name: "Grace Hopper".into(),
4143 email: "grace@example.com".into(),
4144 },
4145 redacted_at: Utc.with_ymd_and_hms(2026, 5, 10, 14, 33, 0).unwrap(),
4146 signature: None,
4147 purged_at: None,
4148 supersedes: None,
4149 }
4150 }
4151
4152 fn sample_state_visibility(state: StateId) -> StateVisibility {
4153 StateVisibility {
4154 state,
4155 tier: VisibilityTier::Restricted {
4156 scope_label: "security-embargo".into(),
4157 },
4158 embargo_until: None,
4159 declarer: Principal {
4160 name: "Grace Hopper".into(),
4161 email: "grace@example.com".into(),
4162 },
4163 declared_at: Utc.with_ymd_and_hms(2026, 6, 1, 12, 0, 0).unwrap(),
4164 signature: None,
4165 supersedes: None,
4166 }
4167 }
4168
4169 #[test]
4170 fn non_packable_object_types_are_in_out_of_pack_transfer_partition() {
4171 for obj_type in wire::native_pack_excluded_object_types() {
4172 assert!(
4173 wire::TransferPartitions::<ObjectInfo>::is_sidecar_object_type(*obj_type),
4174 "{obj_type:?} is excluded from native packs but missing from the out-of-pack transfer partition"
4175 );
4176 }
4177 }
4178
4179 fn state_attachment_info(state: StateId) -> ObjectInfo {
4180 ObjectInfo {
4181 id: ObjectId::StateAttachment {
4182 state,
4183 id: objects::object::StateAttachmentId::from_hash(ContentHash::from_bytes(
4184 [5u8; 32],
4185 )),
4186 kind: objects::object::StateAttachmentKind::SemanticIndex,
4187 },
4188 obj_type: ObjectType::StateAttachment,
4189 size: 0,
4190 delta_base: None,
4191 }
4192 }
4193
4194 #[test]
4198 fn state_attachment_pushes_via_sidecar_lane_but_pulls_in_pack() {
4199 let state = StateId::from_bytes([9u8; 32]);
4200 let attachment = state_attachment_info(state);
4201
4202 let plan = RepositoryTransferPlan::from_object_infos(
4205 vec![
4206 state_info(state),
4207 state_visibility_info(state),
4208 attachment.clone(),
4209 ],
4210 GitLaneTransferIntent::HeddleObjectsOnly,
4211 );
4212 assert!(
4213 plan.partitions
4214 .packable_objects
4215 .iter()
4216 .any(|info| info.obj_type == ObjectType::StateAttachment),
4217 "pull plan must keep the attachment record in the pack partition"
4218 );
4219
4220 let (pack_objects, push_sidecar_objects): (Vec<_>, Vec<_>) = plan
4224 .partitions
4225 .packable_objects
4226 .into_iter()
4227 .partition(|info| info.obj_type.packable_for_push());
4228 assert!(
4229 pack_objects.iter().all(|info| info.obj_type != ObjectType::StateAttachment),
4230 "attachment record must never ride the push pack"
4231 );
4232 assert!(
4233 pack_objects.iter().any(|info| info.obj_type == ObjectType::State),
4234 "ordinary content-addressed objects still ride the push pack"
4235 );
4236 assert!(
4237 push_sidecar_objects
4238 .iter()
4239 .any(|info| info.obj_type == ObjectType::StateAttachment),
4240 "attachment record must be routed to the sidecar lane on push"
4241 );
4242
4243 assert!(!attachment.obj_type.packable_for_push());
4245 assert!(attachment.obj_type.packable_for_pull());
4246 }
4247
4248 #[test]
4252 fn state_attachment_push_message_emits_transfer_frame() {
4253 use grpc::heddle::api::v1alpha1::StateAttachmentKind as ProtoStateAttachmentKind;
4254
4255 let (dir, repo) = temp_repo();
4256 std::fs::write(dir.path().join("lib.rs"), "pub fn f() {}\n").unwrap();
4257 let state = repo.snapshot(Some("seed".to_string()), None).unwrap();
4258
4259 let (node, node_digest) = objects::object::SemanticTreeNode::new(Vec::new());
4261 let node_hash = repo
4262 .store()
4263 .put_blob(&Blob::new(node.encode().unwrap()))
4264 .expect("put semantic tree node");
4265 let root =
4266 objects::object::SemanticIndexRoot::new(1, Default::default(), node_hash, node_digest);
4267 let root_hash = repo
4268 .store()
4269 .put_blob(&Blob::new(root.encode().unwrap()))
4270 .expect("put semantic index root");
4271 let attachment = objects::object::StateAttachment {
4272 state_id: state.state_id,
4273 body: objects::object::StateAttachmentBody::SemanticIndex(root_hash),
4274 attribution: Attribution::human(Principal::new("H3 Tester", "h3@example.com")),
4275 created_at: Utc::now(),
4276 supersedes: None,
4277 };
4278 repo.put_state_attachment(&attachment).unwrap();
4279
4280 let info = ObjectInfo {
4281 id: ObjectId::StateAttachment {
4282 state: state.state_id,
4283 id: attachment.id(),
4284 kind: attachment.body.kind(),
4285 },
4286 obj_type: ObjectType::StateAttachment,
4287 size: 0,
4288 delta_base: None,
4289 };
4290
4291 let frame = sidecar_push_message(&repo, info, "op-1").expect("emit attachment sidecar frame");
4292 assert_eq!(frame.client_operation_id, "op-1");
4293 let Some(push_client_frame::Frame::StateAttachment(transfer)) = frame.frame else {
4294 panic!("expected a StateAttachmentTransfer frame, got {:?}", frame.frame);
4295 };
4296 assert_eq!(
4297 transfer.state_id,
4298 crate::grpc_hosted::helpers::proto_state_id(state.state_id),
4299 "frame must carry the attachment's state id",
4300 );
4301 assert_eq!(
4302 transfer.attachment_id,
4303 attachment.id().as_hash().to_hex(),
4304 "frame must carry the attachment id",
4305 );
4306 assert_eq!(
4307 transfer.attachment_kind,
4308 ProtoStateAttachmentKind::SemanticIndex as i32,
4309 "frame must carry the descriptor kind",
4310 );
4311 assert!(
4312 !transfer.attachment_object.is_empty(),
4313 "frame must carry the serialized attachment record",
4314 );
4315 let canonical = wire::load_object_data(
4318 repo.store(),
4319 &ObjectId::StateAttachment {
4320 state: state.state_id,
4321 id: attachment.id(),
4322 kind: attachment.body.kind(),
4323 },
4324 ObjectType::StateAttachment,
4325 )
4326 .expect("load canonical attachment record");
4327 assert_eq!(transfer.attachment_object, canonical.data);
4328
4329 assert!(ObjectType::StateAttachment.packable_for_pull());
4331 assert!(!ObjectType::StateAttachment.packable_for_push());
4332 }
4333
4334 #[test]
4335 fn native_pack_required_tracks_packable_pull_wants() {
4336 let blob = sample_blob();
4337 let state = StateId::from_bytes([9u8; 32]);
4338
4339 let sidecar_only = RepositoryTransferPlan::from_object_infos(
4340 vec![state_visibility_info(state)],
4341 GitLaneTransferIntent::HeddleObjectsOnly,
4342 );
4343 assert!(!native_pack_required_for_pull(false, &sidecar_only));
4344
4345 let redaction_only = RepositoryTransferPlan::from_object_infos(
4346 vec![redaction_info(blob)],
4347 GitLaneTransferIntent::HeddleObjectsOnly,
4348 );
4349 assert!(!native_pack_required_for_pull(false, &redaction_only));
4350
4351 let packable = RepositoryTransferPlan::from_object_infos(
4352 vec![ObjectInfo {
4353 id: ObjectId::Hash(blob),
4354 obj_type: ObjectType::Blob,
4355 size: 0,
4356 delta_base: None,
4357 }],
4358 GitLaneTransferIntent::HeddleObjectsOnly,
4359 );
4360 assert!(native_pack_required_for_pull(false, &packable));
4361
4362 let state_with_sidecar = RepositoryTransferPlan::from_object_infos(
4363 vec![state_info(state), state_visibility_info(state)],
4364 GitLaneTransferIntent::HeddleObjectsOnly,
4365 );
4366 assert!(native_pack_required_for_pull(false, &state_with_sidecar));
4367 let empty = RepositoryTransferPlan::from_object_infos(
4368 Vec::<ObjectInfo>::new(),
4369 GitLaneTransferIntent::HeddleObjectsOnly,
4370 );
4371 assert!(native_pack_required_for_pull(true, &empty));
4372 }
4373
4374 #[test]
4375 fn plan_pull_wants_accumulates_state_and_visibility_for_same_state_id() {
4376 let (_dir, repo) = temp_repo();
4377 let state = StateId::from_bytes([9u8; 32]);
4378 let plan = plan_pull_wants(
4379 &repo,
4380 &state,
4381 false,
4382 vec![
4383 object_descriptor_with_status(
4384 &state_info(state),
4385 ObjectAvailabilityStatus::Missing,
4386 "missing state",
4387 ),
4388 object_descriptor_with_status(
4389 &state_visibility_info(state),
4390 ObjectAvailabilityStatus::Missing,
4391 "missing state visibility",
4392 ),
4393 ],
4394 false,
4395 )
4396 .expect("plan pull wants");
4397
4398 let wanted = plan
4399 .wanted_types
4400 .get(&PackObjectId::StateId(state))
4401 .expect("same StateId want entry");
4402 assert_eq!(
4403 wanted.as_slice(),
4404 &[ObjectType::State, ObjectType::StateVisibility]
4405 );
4406 assert!(native_pack_required_for_pull(
4407 plan.want_full_closure,
4408 &plan.transfer_plan
4409 ));
4410 }
4411
4412 #[derive(Clone)]
4413 struct SidecarOnlyPullService {
4414 state: StateId,
4415 state_visibility_blob: Vec<u8>,
4416 }
4417
4418 #[tonic::async_trait]
4419 impl RepoSyncService for SidecarOnlyPullService {
4420 async fn list_refs(
4421 &self,
4422 _request: tonic::Request<ListRefsRequest>,
4423 ) -> Result<Response<ListRefsResponse>, Status> {
4424 Ok(Response::new(ListRefsResponse::default()))
4425 }
4426
4427 async fn update_ref(
4428 &self,
4429 _request: tonic::Request<UpdateRefRequest>,
4430 ) -> Result<Response<UpdateRefResponse>, Status> {
4431 Ok(Response::new(UpdateRefResponse::default()))
4432 }
4433
4434 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushServerFrame, Status>>;
4435
4436 async fn push(
4437 &self,
4438 _request: tonic::Request<tonic::Streaming<PushClientFrame>>,
4439 ) -> Result<Response<Self::PushStream>, Status> {
4440 let (_tx, rx) = mpsc::channel(1);
4441 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4442 rx,
4443 )))
4444 }
4445
4446 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullServerFrame, Status>>;
4447
4448 async fn pull(
4449 &self,
4450 request: tonic::Request<tonic::Streaming<PullClientFrame>>,
4451 ) -> Result<Response<Self::PullStream>, Status> {
4452 let state = self.state;
4453 let state_visibility_blob = self.state_visibility_blob.clone();
4454 let (tx, rx) = mpsc::channel(4);
4455
4456 tokio::spawn(async move {
4457 let mut inbound = request.into_inner();
4458 if !matches!(
4459 inbound.message().await,
4460 Ok(Some(PullClientFrame {
4461 frame: Some(pull_client_frame::Frame::Open(_)),
4462 }))
4463 ) {
4464 let _ = tx
4465 .send(Err(Status::invalid_argument("expected signed pull open")))
4466 .await;
4467 return;
4468 }
4469 match inbound.message().await {
4470 Ok(Some(PullClientFrame {
4471 frame: Some(pull_client_frame::Frame::Request(_)),
4472 })) => {}
4473 other => {
4474 let _ = tx
4475 .send(Err(Status::invalid_argument(format!(
4476 "expected pull request, got {other:?}"
4477 ))))
4478 .await;
4479 return;
4480 }
4481 }
4482
4483 let descriptor = object_descriptor_with_status(
4484 &state_visibility_info(state),
4485 ObjectAvailabilityStatus::Missing,
4486 "missing state visibility",
4487 );
4488 let ready = PullServerFrame {
4489 frame: Some(pull_server_frame::Frame::Ready(PullReady {
4490 remote_state: proto_state_id(state),
4491 objects_to_fetch: vec![descriptor],
4492 transfer: None,
4493 partial_fetch_status: PartialFetchStatus::Disabled as i32,
4494 missing_objects: Vec::new(),
4495 full_closure_available: false,
4496 object_count: 1,
4497 remote_revision_address: RevisionAddress::heddle(state).to_string(),
4498 })),
4499 };
4500 if tx.send(Ok(ready)).await.is_err() {
4501 return;
4502 }
4503
4504 match inbound.message().await {
4505 Ok(Some(PullClientFrame {
4506 frame: Some(pull_client_frame::Frame::Want(want)),
4507 })) if !want.want_full_closure
4508 && want.objects.len() == 1
4509 && want.objects[0].object_type == "state_visibility" => {}
4510 other => {
4511 let _ = tx
4512 .send(Err(Status::invalid_argument(format!(
4513 "expected sidecar-only want, got {other:?}"
4514 ))))
4515 .await;
4516 return;
4517 }
4518 }
4519
4520 let transfer = PullServerFrame {
4521 frame: Some(pull_server_frame::Frame::StateVisibility(
4522 StateVisibilityTransfer {
4523 state_id: proto_state_id(state),
4524 state_visibility_blob,
4525 },
4526 )),
4527 };
4528 if tx.send(Ok(transfer)).await.is_err() {
4529 return;
4530 }
4531
4532 let complete = PullServerFrame {
4533 frame: Some(pull_server_frame::Frame::Complete(GrpcPullComplete {
4534 success: true,
4535 new_state: proto_state_id(state),
4536 error: String::new(),
4537 transfer: Some(TransferCheckpoint {
4538 transfer_id: "sidecar-only-test".to_string(),
4539 transport_mode: TransportMode::NativePack as i32,
4540 resume_offset: 0,
4541 chunk_index: 0,
4542 checkpoint: b"heddle-markers-v1\n".to_vec(),
4543 is_complete: true,
4544 }),
4545 new_revision_address: RevisionAddress::heddle(state).to_string(),
4546 })),
4547 };
4548 let _ = tx.send(Ok(complete)).await;
4549 });
4550
4551 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4552 rx,
4553 )))
4554 }
4555 }
4556
4557 async fn connect_sidecar_only_service(
4558 service: SidecarOnlyPullService,
4559 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
4560 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
4561 Ok(listener) => listener,
4562 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
4563 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
4564 return None;
4565 }
4566 Err(err) => panic!("bind test server: {err}"),
4567 };
4568 let addr = listener.local_addr().expect("local addr");
4569 let incoming = futures::stream::unfold(listener, |listener| async {
4570 match listener.accept().await {
4571 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
4572 Err(err) => Some((Err(err), listener)),
4573 }
4574 });
4575
4576 let handle = tokio::spawn(async move {
4577 Server::builder()
4578 .add_service(RepoSyncServiceServer::new(service))
4579 .serve_with_incoming(incoming)
4580 .await
4581 .expect("serve sidecar-only test service");
4582 });
4583
4584 let client = HostedGrpcClient::connect(addr, &signed_test_config())
4585 .await
4586 .expect("connect client");
4587 Some((client, handle))
4588 }
4589
4590 #[tokio::test]
4591 async fn state_visibility_sidecar_only_pull_completes_without_native_pack() {
4592 let (_dir, repo) = temp_repo();
4593 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4594 let state = State::new_snapshot(
4595 tree_hash,
4596 vec![],
4597 Attribution::human(Principal {
4598 name: "Grace Hopper".into(),
4599 email: "grace@example.com".into(),
4600 }),
4601 );
4602 let state_id = state.state_id;
4603 repo.store().put_state(&state).expect("put state");
4604 assert!(
4605 repo.get_state_visibility_bytes_for_state(&state_id)
4606 .expect("load local sidecar")
4607 .is_none(),
4608 "test starts with state present and StateVisibility sidecar absent"
4609 );
4610
4611 let state_visibility_blob =
4612 StateVisibilityBlob::new(vec![sample_state_visibility(state_id)])
4613 .encode()
4614 .expect("encode state visibility blob");
4615 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4616 state: state_id,
4617 state_visibility_blob,
4618 })
4619 .await
4620 else {
4621 return;
4622 };
4623
4624 let exchange = tokio::time::timeout(
4625 Duration::from_secs(5),
4626 client.pull_exchange(
4627 &repo,
4628 "owner/repo",
4629 "main",
4630 PullOptions {
4631 local_thread: None,
4632 depth: None,
4633 target_state: Some(state_id),
4634 materialization: PullMaterialization::Full,
4635 },
4636 ),
4637 )
4638 .await
4639 .expect("sidecar-only pull must not hang waiting for native pack")
4640 .expect("sidecar-only pull succeeds");
4641 server.abort();
4642
4643 assert!(exchange.result.success);
4644 assert_eq!(exchange.object_count, 0);
4645 assert_eq!(exchange.profile.pack_bytes_received, 0);
4646 assert_eq!(exchange.profile.object_mix.state_visibilities, 1);
4647 assert!(
4648 repo.get_state_visibility_for_state(&state_id)
4649 .expect("load accepted sidecar")
4650 .has_record(),
4651 "pull must accept the out-of-pack StateVisibility sidecar"
4652 );
4653 }
4654
4655 #[tokio::test]
4659 async fn legitimate_large_sidecar_blob_decodes_and_installs() {
4660 let (_dir, repo) = temp_repo();
4661 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4662 let state = State::new_snapshot(
4663 tree_hash,
4664 vec![],
4665 Attribution::human(Principal {
4666 name: "Grace Hopper".into(),
4667 email: "grace@example.com".into(),
4668 }),
4669 );
4670 let state_id = state.state_id;
4671 repo.store().put_state(&state).expect("put state");
4672
4673 let mut record = sample_state_visibility(state_id);
4676 record.tier = VisibilityTier::Restricted {
4677 scope_label: "x".repeat(8 * 1024 * 1024),
4678 };
4679 let state_visibility_blob = StateVisibilityBlob::new(vec![record])
4680 .encode()
4681 .expect("encode large state visibility blob");
4682 assert!(
4683 state_visibility_blob.len() > 4 * 1024 * 1024,
4684 "blob must exceed tonic's 4 MiB default to exercise the raised decode limit"
4685 );
4686 assert!(
4687 (state_visibility_blob.len() as u64) <= wire::MAX_RECEIVED_STATE_VISIBILITY_BLOB_SIZE,
4688 "blob must stay within the legitimate sidecar receive cap"
4689 );
4690
4691 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4692 state: state_id,
4693 state_visibility_blob,
4694 })
4695 .await
4696 else {
4697 return;
4698 };
4699
4700 let exchange = tokio::time::timeout(
4701 Duration::from_secs(30),
4702 client.pull_exchange(
4703 &repo,
4704 "owner/repo",
4705 "main",
4706 PullOptions {
4707 local_thread: None,
4708 depth: None,
4709 target_state: Some(state_id),
4710 materialization: PullMaterialization::Full,
4711 },
4712 ),
4713 )
4714 .await
4715 .expect("large-sidecar pull must not hang")
4716 .expect("large but legitimate sidecar pull succeeds");
4717 server.abort();
4718
4719 assert!(exchange.result.success);
4720 assert!(
4721 repo.get_state_visibility_for_state(&state_id)
4722 .expect("load accepted sidecar")
4723 .has_record(),
4724 "pull must accept a legitimately-large StateVisibility sidecar"
4725 );
4726 }
4727
4728 #[tokio::test]
4732 async fn oversized_sidecar_blob_rejected_at_grpc_decode_boundary() {
4733 let (_dir, repo) = temp_repo();
4734 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
4735 let state = State::new_snapshot(
4736 tree_hash,
4737 vec![],
4738 Attribution::human(Principal {
4739 name: "Grace Hopper".into(),
4740 email: "grace@example.com".into(),
4741 }),
4742 );
4743 let state_id = state.state_id;
4744 repo.store().put_state(&state).expect("put state");
4745
4746 let oversized = vec![0u8; wire::MAX_PULL_DECODE_MESSAGE_SIZE + 1];
4749 let Some((mut client, server)) = connect_sidecar_only_service(SidecarOnlyPullService {
4750 state: state_id,
4751 state_visibility_blob: oversized,
4752 })
4753 .await
4754 else {
4755 return;
4756 };
4757
4758 let result = tokio::time::timeout(
4759 Duration::from_secs(30),
4760 client.pull_exchange(
4761 &repo,
4762 "owner/repo",
4763 "main",
4764 PullOptions {
4765 local_thread: None,
4766 depth: None,
4767 target_state: Some(state_id),
4768 materialization: PullMaterialization::Full,
4769 },
4770 ),
4771 )
4772 .await
4773 .expect("oversized-sidecar pull must not hang");
4774 server.abort();
4775
4776 let err = match result {
4777 Err(err) => err,
4778 Ok(_) => panic!("oversized sidecar PullClientFrame must be rejected at decode"),
4779 };
4780 let message = err.to_string();
4781 assert!(
4782 !message.contains("exceeds receive size limit"),
4783 "rejection must come from the decode-size limit, before the post-decode check: {message}"
4784 );
4785 assert!(
4786 repo.get_state_visibility_for_state(&state_id)
4787 .expect("load sidecar")
4788 .latest()
4789 .expect("resolve visibility")
4790 .is_none(),
4791 "an oversized sidecar must never be installed"
4792 );
4793 }
4794
4795 #[derive(Clone)]
4796 struct StateAndVisibilityPullService {
4797 state: StateId,
4798 pack_bundle: wire::NativePackBundle,
4799 state_visibility_blob: Vec<u8>,
4800 }
4801
4802 #[tonic::async_trait]
4803 impl RepoSyncService for StateAndVisibilityPullService {
4804 async fn list_refs(
4805 &self,
4806 _request: tonic::Request<ListRefsRequest>,
4807 ) -> Result<Response<ListRefsResponse>, Status> {
4808 Ok(Response::new(ListRefsResponse::default()))
4809 }
4810
4811 async fn update_ref(
4812 &self,
4813 _request: tonic::Request<UpdateRefRequest>,
4814 ) -> Result<Response<UpdateRefResponse>, Status> {
4815 Ok(Response::new(UpdateRefResponse::default()))
4816 }
4817
4818 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushServerFrame, Status>>;
4819
4820 async fn push(
4821 &self,
4822 _request: tonic::Request<tonic::Streaming<PushClientFrame>>,
4823 ) -> Result<Response<Self::PushStream>, Status> {
4824 let (_tx, rx) = mpsc::channel(1);
4825 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4826 rx,
4827 )))
4828 }
4829
4830 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullServerFrame, Status>>;
4831
4832 async fn pull(
4833 &self,
4834 request: tonic::Request<tonic::Streaming<PullClientFrame>>,
4835 ) -> Result<Response<Self::PullStream>, Status> {
4836 let state = self.state;
4837 let pack_bundle = self.pack_bundle.clone();
4838 let state_visibility_blob = self.state_visibility_blob.clone();
4839 let (tx, rx) = mpsc::channel(8);
4840
4841 tokio::spawn(async move {
4842 let mut inbound = request.into_inner();
4843 if !matches!(
4844 inbound.message().await,
4845 Ok(Some(PullClientFrame {
4846 frame: Some(pull_client_frame::Frame::Open(_)),
4847 }))
4848 ) {
4849 let _ = tx
4850 .send(Err(Status::invalid_argument("expected signed pull open")))
4851 .await;
4852 return;
4853 }
4854 match inbound.message().await {
4855 Ok(Some(PullClientFrame {
4856 frame: Some(pull_client_frame::Frame::Request(_)),
4857 })) => {}
4858 other => {
4859 let _ = tx
4860 .send(Err(Status::invalid_argument(format!(
4861 "expected pull request, got {other:?}"
4862 ))))
4863 .await;
4864 return;
4865 }
4866 }
4867
4868 let ready = PullServerFrame {
4869 frame: Some(pull_server_frame::Frame::Ready(PullReady {
4870 remote_state: proto_state_id(state),
4871 objects_to_fetch: vec![
4872 object_descriptor_with_status(
4873 &state_info(state),
4874 ObjectAvailabilityStatus::Missing,
4875 "missing state",
4876 ),
4877 object_descriptor_with_status(
4878 &state_visibility_info(state),
4879 ObjectAvailabilityStatus::Missing,
4880 "missing state visibility",
4881 ),
4882 ],
4883 transfer: None,
4884 partial_fetch_status: PartialFetchStatus::Disabled as i32,
4885 missing_objects: Vec::new(),
4886 full_closure_available: false,
4887 object_count: 2,
4888 remote_revision_address: RevisionAddress::heddle(state).to_string(),
4889 })),
4890 };
4891 if tx.send(Ok(ready)).await.is_err() {
4892 return;
4893 }
4894
4895 match inbound.message().await {
4896 Ok(Some(PullClientFrame {
4897 frame: Some(pull_client_frame::Frame::Want(want)),
4898 })) if !want.want_full_closure
4899 && want.objects.len() == 2
4900 && want
4901 .objects
4902 .iter()
4903 .any(|object| object.object_type == "state")
4904 && want
4905 .objects
4906 .iter()
4907 .any(|object| object.object_type == "state_visibility") => {}
4908 other => {
4909 let _ = tx
4910 .send(Err(Status::invalid_argument(format!(
4911 "expected state + sidecar wants, got {other:?}"
4912 ))))
4913 .await;
4914 return;
4915 }
4916 }
4917
4918 for message in
4919 encode_pull_native_pack_messages(&pack_bundle, "state-and-visibility-test", 16)
4920 {
4921 if tx.send(Ok(message)).await.is_err() {
4922 return;
4923 }
4924 }
4925
4926 let transfer = PullServerFrame {
4927 frame: Some(pull_server_frame::Frame::StateVisibility(
4928 StateVisibilityTransfer {
4929 state_id: proto_state_id(state),
4930 state_visibility_blob,
4931 },
4932 )),
4933 };
4934 if tx.send(Ok(transfer)).await.is_err() {
4935 return;
4936 }
4937
4938 let complete = PullServerFrame {
4939 frame: Some(pull_server_frame::Frame::Complete(GrpcPullComplete {
4940 success: true,
4941 new_state: proto_state_id(state),
4942 error: String::new(),
4943 transfer: Some(TransferCheckpoint {
4944 transfer_id: "state-and-visibility-test".to_string(),
4945 transport_mode: TransportMode::NativePack as i32,
4946 resume_offset: 0,
4947 chunk_index: 0,
4948 checkpoint: b"heddle-markers-v1\n".to_vec(),
4949 is_complete: true,
4950 }),
4951 new_revision_address: RevisionAddress::heddle(state).to_string(),
4952 })),
4953 };
4954 let _ = tx.send(Ok(complete)).await;
4955 });
4956
4957 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
4958 rx,
4959 )))
4960 }
4961 }
4962
4963 fn encode_pull_native_pack_messages(
4964 bundle: &wire::NativePackBundle,
4965 transfer_id: &str,
4966 chunk_size: usize,
4967 ) -> Vec<PullServerFrame> {
4968 let mut messages = Vec::new();
4969 let chunk_size = chunk_size.max(1);
4970
4971 let pack_total_chunks = wire::chunk_count(bundle.pack_data.len(), chunk_size);
4972 for chunk_index in 0..pack_total_chunks.max(1) {
4973 let Some((start, len)) =
4974 wire::chunk_bounds(bundle.pack_data.len(), chunk_size, chunk_index)
4975 else {
4976 break;
4977 };
4978 messages.push(PullServerFrame {
4979 frame: Some(pull_server_frame::Frame::Pack(PackChunk {
4980 stream_kind: PackStreamKind::Pack as i32,
4981 data: bundle.pack_data[start..start + len].to_vec(),
4982 transfer: Some(TransferCheckpoint {
4983 transfer_id: transfer_id.to_string(),
4984 transport_mode: TransportMode::NativePack as i32,
4985 resume_offset: start as u64,
4986 chunk_index: chunk_index as u32,
4987 checkpoint: Vec::new(),
4988 is_complete: chunk_index + 1 == pack_total_chunks,
4989 }),
4990 chunk_length: len as u32,
4991 is_final_chunk: chunk_index + 1 == pack_total_chunks,
4992 })),
4993 });
4994 }
4995
4996 let index_total_chunks = wire::chunk_count(bundle.index_data.len(), chunk_size);
4997 for chunk_index in 0..index_total_chunks.max(1) {
4998 let Some((start, len)) =
4999 wire::chunk_bounds(bundle.index_data.len(), chunk_size, chunk_index)
5000 else {
5001 break;
5002 };
5003 messages.push(PullServerFrame {
5004 frame: Some(pull_server_frame::Frame::Pack(PackChunk {
5005 stream_kind: PackStreamKind::Index as i32,
5006 data: bundle.index_data[start..start + len].to_vec(),
5007 transfer: Some(TransferCheckpoint {
5008 transfer_id: transfer_id.to_string(),
5009 transport_mode: TransportMode::NativePack as i32,
5010 resume_offset: start as u64,
5011 chunk_index: chunk_index as u32,
5012 checkpoint: Vec::new(),
5013 is_complete: chunk_index + 1 == index_total_chunks,
5014 }),
5015 chunk_length: len as u32,
5016 is_final_chunk: chunk_index + 1 == index_total_chunks,
5017 })),
5018 });
5019 }
5020
5021 messages
5022 }
5023
5024 async fn connect_state_and_visibility_service(
5025 service: StateAndVisibilityPullService,
5026 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
5027 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
5028 Ok(listener) => listener,
5029 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
5030 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
5031 return None;
5032 }
5033 Err(err) => panic!("bind test server: {err}"),
5034 };
5035 let addr = listener.local_addr().expect("local addr");
5036 let incoming = futures::stream::unfold(listener, |listener| async {
5037 match listener.accept().await {
5038 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
5039 Err(err) => Some((Err(err), listener)),
5040 }
5041 });
5042
5043 let handle = tokio::spawn(async move {
5044 Server::builder()
5045 .add_service(RepoSyncServiceServer::new(service))
5046 .serve_with_incoming(incoming)
5047 .await
5048 .expect("serve state-and-visibility test service");
5049 });
5050
5051 let client = HostedGrpcClient::connect(addr, &signed_test_config())
5052 .await
5053 .expect("connect client");
5054 Some((client, handle))
5055 }
5056
5057 #[tokio::test]
5058 async fn state_and_visibility_same_state_id_pull_requests_pack_and_sidecar() {
5059 let (_source_dir, source_repo) = temp_repo();
5060 let (_target_dir, target_repo) = temp_repo();
5061 let tree_hash = source_repo
5062 .store()
5063 .put_tree(&Tree::new())
5064 .expect("put tree");
5065 let state = State::new_snapshot(
5066 tree_hash,
5067 vec![],
5068 Attribution::human(Principal {
5069 name: "Grace Hopper".into(),
5070 email: "grace@example.com".into(),
5071 }),
5072 );
5073 let state_id = state.state_id;
5074 source_repo
5075 .store()
5076 .put_state(&state)
5077 .expect("put source state");
5078 let state_visibility_blob =
5079 StateVisibilityBlob::new(vec![sample_state_visibility(state_id)])
5080 .encode()
5081 .expect("encode state visibility blob");
5082 source_repo
5083 .accept_wire_state_visibility(state_id, &state_visibility_blob)
5084 .expect("put source state visibility");
5085 let pack_bundle = wire::build_native_pack(source_repo.store(), &[state_info(state_id)])
5086 .expect("build state pack");
5087
5088 assert!(
5089 target_repo
5090 .store()
5091 .get_state(&state_id)
5092 .expect("load target state")
5093 .is_none(),
5094 "test starts with state absent"
5095 );
5096 assert!(
5097 target_repo
5098 .get_state_visibility_bytes_for_state(&state_id)
5099 .expect("load target sidecar")
5100 .is_none(),
5101 "test starts with StateVisibility sidecar absent"
5102 );
5103
5104 let Some((mut client, server)) =
5105 connect_state_and_visibility_service(StateAndVisibilityPullService {
5106 state: state_id,
5107 pack_bundle,
5108 state_visibility_blob,
5109 })
5110 .await
5111 else {
5112 return;
5113 };
5114
5115 let exchange = tokio::time::timeout(
5116 Duration::from_secs(5),
5117 client.pull_exchange(
5118 &target_repo,
5119 "owner/repo",
5120 "main",
5121 PullOptions {
5122 local_thread: None,
5123 depth: None,
5124 target_state: Some(state_id),
5125 materialization: PullMaterialization::Full,
5126 },
5127 ),
5128 )
5129 .await
5130 .expect("state + sidecar pull must not hang waiting for native pack")
5131 .expect("state + sidecar pull succeeds");
5132 server.abort();
5133
5134 assert!(exchange.result.success);
5135 assert_eq!(exchange.object_count, 1);
5136 assert!(exchange.profile.pack_bytes_received > 0);
5137 assert_eq!(exchange.profile.object_mix.states, 1);
5138 assert_eq!(exchange.profile.object_mix.state_visibilities, 1);
5139 assert!(
5140 target_repo
5141 .store()
5142 .get_state(&state_id)
5143 .expect("load installed state")
5144 .is_some(),
5145 "native pack must install the State"
5146 );
5147 assert!(
5148 target_repo
5149 .get_state_visibility_for_state(&state_id)
5150 .expect("load accepted sidecar")
5151 .has_record(),
5152 "pull must accept the out-of-pack StateVisibility sidecar"
5153 );
5154 }
5155
5156 #[test]
5157 fn missing_blobs_in_tree_treats_absent_tree_as_empty() {
5158 let (_dir, repo) = temp_repo();
5159 let absent_tree = ContentHash::from_bytes([99u8; 32]);
5160
5161 let missing = wire::missing_blobs_in_tree(repo.store(), absent_tree)
5162 .expect("absent tree is not an error");
5163
5164 assert!(missing.is_empty());
5165 }
5166
5167 #[test]
5168 fn missing_blobs_in_tree_reports_only_genuinely_missing_blobs() {
5169 let (_dir, repo) = temp_repo();
5170 let present_blob = Blob::from("already local");
5171 let present_hash = repo.store().put_blob(&present_blob).expect("put blob");
5172 let missing_hash = ContentHash::from_bytes([42u8; 32]);
5173 let tree = Tree::from_entries(vec![
5174 TreeEntry::file("local.txt", present_hash, false).expect("present entry"),
5175 TreeEntry::file("remote.txt", missing_hash, false).expect("missing entry"),
5176 ]);
5177 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
5178
5179 let missing =
5180 wire::missing_blobs_in_tree(repo.store(), tree_hash).expect("collect missing blobs");
5181
5182 assert_eq!(missing, vec![missing_hash]);
5183 }
5184
5185 #[test]
5186 fn missing_blobs_in_tree_reports_corrupt_tree_read() {
5187 let (_dir, repo) = temp_repo();
5188 let tree_hash = repo.store().put_tree(&Tree::new()).expect("put tree");
5189 std::fs::write(loose_tree_path(&repo, &tree_hash), [0xc1]).expect("corrupt tree");
5190 repo.store().clear_recent_caches();
5191
5192 let err = wire::missing_blobs_in_tree(repo.store(), tree_hash)
5193 .expect_err("corrupt tree must fail");
5194
5195 assert!(matches!(err, ProtocolError::InvalidState(_)));
5196 assert!(
5197 err.to_string().contains(&format!(
5198 "load tree {} while collecting lazy hydration missing blobs",
5199 tree_hash.to_hex()
5200 )),
5201 "unexpected error: {err}"
5202 );
5203 }
5204
5205 #[test]
5206 fn redaction_push_message_uses_hex_keyed_sidecar_payload() {
5207 let (_dir, repo) = temp_repo();
5208 let blob = sample_blob();
5209 repo.put_redaction(sample_redaction(blob))
5210 .expect("put redaction");
5211 let expected_bytes = repo
5212 .store()
5213 .get_redactions_bytes_for_blob(&blob)
5214 .expect("load sidecar")
5215 .expect("sidecar exists");
5216
5217 let message =
5218 redaction_push_message(&repo, redaction_info(blob), "test-push-op").expect("message");
5219 assert_eq!(message.client_operation_id, "test-push-op");
5220
5221 let Some(push_client_frame::Frame::Redaction(transfer)) = message.frame else {
5222 panic!("expected redaction transfer");
5223 };
5224 assert_eq!(transfer.blob_hash, blob.to_hex());
5225 assert_eq!(transfer.redactions_blob, expected_bytes);
5226 }
5227
5228 #[test]
5229 fn redaction_push_message_reports_missing_sidecar_with_blob_hex() {
5230 let (_dir, repo) = temp_repo();
5231 let blob = sample_blob();
5232
5233 let err = redaction_push_message(&repo, redaction_info(blob), "test-push-op")
5234 .expect_err("missing sidecar");
5235
5236 assert!(matches!(err, ProtocolError::InvalidState(_)));
5237 assert!(
5238 err.to_string().contains(&format!(
5239 "server wants redaction for blob {} but sender has no sidecar",
5240 blob.to_hex()
5241 )),
5242 "unexpected error: {err}"
5243 );
5244 }
5245
5246 #[test]
5247 fn redaction_push_message_reports_sidecar_load_error_with_blob_hex() {
5248 let (_dir, repo) = temp_repo();
5249 let blob = sample_blob();
5250 let redaction_path = repo
5251 .heddle_dir()
5252 .join("redactions")
5253 .join(format!("{}.bin", blob.to_hex()));
5254 std::fs::create_dir_all(&redaction_path).expect("directory at redaction path");
5255
5256 let err = redaction_push_message(&repo, redaction_info(blob), "test-push-op")
5257 .expect_err("load error");
5258
5259 assert!(matches!(err, ProtocolError::InvalidState(_)));
5260 assert!(
5261 err.to_string()
5262 .contains(&format!("load redactions sidecar for {}:", blob.to_hex())),
5263 "unexpected error: {err}"
5264 );
5265 }
5266
5267 #[test]
5268 fn state_visibility_push_message_uses_state_keyed_sidecar_payload() {
5269 let (_dir, repo) = temp_repo();
5270 let state = StateId::from_bytes([17u8; 32]);
5271 repo.put_state_visibility(sample_state_visibility(state))
5272 .expect("put state visibility");
5273 let expected_bytes = repo
5274 .get_state_visibility_bytes_for_state(&state)
5275 .expect("load sidecar")
5276 .expect("sidecar exists");
5277
5278 let message =
5279 state_visibility_push_message(&repo, state_visibility_info(state), "test-push-op")
5280 .expect("message");
5281 assert_eq!(message.client_operation_id, "test-push-op");
5282
5283 let Some(push_client_frame::Frame::StateVisibility(transfer)) = message.frame else {
5284 panic!("expected state visibility transfer");
5285 };
5286 assert_eq!(
5287 transfer
5288 .state_id
5289 .as_ref()
5290 .map(|state| state.value.as_slice()),
5291 Some(state.as_bytes().as_slice())
5292 );
5293 assert_eq!(transfer.state_visibility_blob, expected_bytes);
5294 }
5295
5296 fn sample_attribution() -> Attribution {
5299 Attribution::human(Principal {
5300 name: "Grace Hopper".into(),
5301 email: "grace@example.com".into(),
5302 })
5303 }
5304
5305 fn put_state_with_blob(repo: &Repository, contents: &str, parents: Vec<StateId>) -> StateId {
5308 let blob = Blob::from(contents);
5309 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
5310 let tree = Tree::from_entries(vec![
5311 TreeEntry::file(format!("{contents}.txt"), blob_hash, false).expect("tree entry"),
5312 ]);
5313 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
5314 let state = State::new_snapshot(tree_hash, parents, sample_attribution());
5315 let state_id = state.state_id;
5316 repo.store().put_state(&state).expect("put state");
5317 state_id
5318 }
5319
5320 #[test]
5321 fn locally_complete_pull_head_advertises_fully_present_thread_head() {
5322 let (_dir, repo) = temp_repo();
5323 let head = put_state_with_blob(&repo, "alpha", vec![]);
5324 repo.refs()
5325 .set_thread(&ThreadName::from("main"), &head)
5326 .expect("set thread");
5327
5328 let advertised =
5329 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
5330 assert_eq!(
5331 advertised,
5332 Some(head),
5333 "a thread head whose full closure is present locally must be advertised"
5334 );
5335 }
5336
5337 #[test]
5338 fn locally_complete_pull_head_skips_unknown_thread() {
5339 let (_dir, repo) = temp_repo_unseeded();
5343 let advertised =
5344 locally_complete_pull_head(&repo, "nonexistent", None).expect("completeness check");
5345 assert_eq!(advertised, None);
5346 }
5347
5348 #[test]
5349 fn locally_complete_pull_head_skips_when_target_state_override_in_play() {
5350 let (_dir, repo) = temp_repo();
5351 let head = put_state_with_blob(&repo, "alpha", vec![]);
5352 repo.refs()
5353 .set_thread(&ThreadName::from("main"), &head)
5354 .expect("set thread");
5355
5356 let advertised =
5359 locally_complete_pull_head(&repo, "main", Some(head)).expect("completeness check");
5360 assert_eq!(advertised, None);
5361 }
5362
5363 #[test]
5364 fn locally_complete_pull_head_skips_repo_with_missing_blobs() {
5365 let (_dir, repo) = temp_repo();
5370 let head = put_state_with_blob(&repo, "alpha", vec![]);
5371 repo.refs()
5372 .set_thread(&ThreadName::from("main"), &head)
5373 .expect("set thread");
5374 repo.record_missing_blob(ContentHash::from_bytes([88u8; 32]))
5375 .expect("record missing blob");
5376
5377 let advertised =
5378 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
5379 assert_eq!(
5380 advertised, None,
5381 "a repo carrying missing blobs must never advertise a head"
5382 );
5383 }
5384
5385 #[test]
5386 fn locally_complete_pull_head_skips_head_with_incomplete_closure() {
5387 let (_dir, repo) = temp_repo();
5392 let absent_parent = StateId::from_bytes([21; 32]);
5393 let blob = Blob::from("orphan");
5394 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
5395 let tree = Tree::from_entries(vec![
5396 TreeEntry::file("orphan.txt", blob_hash, false).expect("tree entry"),
5397 ]);
5398 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
5399 let child = State::new_snapshot(tree_hash, vec![absent_parent], sample_attribution());
5401 let child_id = child.state_id;
5402 repo.store().put_state(&child).expect("put child state");
5403 repo.refs()
5404 .set_thread(&ThreadName::from("main"), &child_id)
5405 .expect("set thread");
5406
5407 let advertised =
5408 locally_complete_pull_head(&repo, "main", None).expect("completeness check");
5409 assert_eq!(
5410 advertised, None,
5411 "a head whose closure has an absent parent state must not be advertised"
5412 );
5413 }
5414
5415 #[test]
5416 fn locally_complete_local_thread_head_advertises_fully_present_thread_head() {
5417 let (_dir, repo) = temp_repo();
5420 let head = put_state_with_blob(&repo, "alpha", vec![]);
5421 repo.refs()
5422 .set_thread(&ThreadName::from("feature"), &head)
5423 .expect("set thread");
5424
5425 let advertised =
5426 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
5427 assert_eq!(
5428 advertised,
5429 Some(head),
5430 "an explicit local-thread head whose full closure is present must be advertised"
5431 );
5432 }
5433
5434 #[test]
5435 fn locally_complete_local_thread_head_skips_unknown_thread() {
5436 let (_dir, repo) = temp_repo_unseeded();
5439 let advertised = locally_complete_local_thread_head(&repo, "nonexistent", None)
5440 .expect("completeness check");
5441 assert_eq!(advertised, None);
5442 }
5443
5444 #[test]
5445 fn locally_complete_local_thread_head_skips_when_target_state_override_in_play() {
5446 let (_dir, repo) = temp_repo();
5447 let head = put_state_with_blob(&repo, "alpha", vec![]);
5448 repo.refs()
5449 .set_thread(&ThreadName::from("feature"), &head)
5450 .expect("set thread");
5451
5452 let advertised = locally_complete_local_thread_head(&repo, "feature", Some(head))
5455 .expect("completeness check");
5456 assert_eq!(advertised, None);
5457 }
5458
5459 #[test]
5460 fn locally_complete_local_thread_head_skips_repo_with_missing_blobs() {
5461 let (_dir, repo) = temp_repo();
5464 let head = put_state_with_blob(&repo, "alpha", vec![]);
5465 repo.refs()
5466 .set_thread(&ThreadName::from("feature"), &head)
5467 .expect("set thread");
5468 repo.record_missing_blob(ContentHash::from_bytes([88u8; 32]))
5469 .expect("record missing blob");
5470
5471 let advertised =
5472 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
5473 assert_eq!(
5474 advertised, None,
5475 "a repo carrying missing blobs must never advertise an explicit local-thread head"
5476 );
5477 }
5478
5479 #[test]
5480 fn locally_complete_local_thread_head_skips_head_with_incomplete_closure() {
5481 let (_dir, repo) = temp_repo();
5486 let absent_parent = StateId::from_bytes([22; 32]);
5487 let blob = Blob::from("orphan");
5488 let blob_hash = repo.store().put_blob(&blob).expect("put blob");
5489 let tree = Tree::from_entries(vec![
5490 TreeEntry::file("orphan.txt", blob_hash, false).expect("tree entry"),
5491 ]);
5492 let tree_hash = repo.store().put_tree(&tree).expect("put tree");
5493 let child = State::new_snapshot(tree_hash, vec![absent_parent], sample_attribution());
5494 let child_id = child.state_id;
5495 repo.store().put_state(&child).expect("put child state");
5496 repo.refs()
5497 .set_thread(&ThreadName::from("feature"), &child_id)
5498 .expect("set thread");
5499
5500 let advertised =
5501 locally_complete_local_thread_head(&repo, "feature", None).expect("completeness check");
5502 assert_eq!(
5503 advertised, None,
5504 "an explicit local-thread head whose closure has an absent parent must not be advertised"
5505 );
5506 }
5507
5508 #[derive(Clone)]
5514 struct DeltaAwarePullService {
5515 remote_state: StateId,
5516 full_closure: Vec<ObjectInfo>,
5522 delta_objects: Vec<ObjectInfo>,
5523 known_parent: StateId,
5524 full_pack: wire::NativePackBundle,
5525 delta_pack: wire::NativePackBundle,
5526 captured_exclude: std::sync::Arc<std::sync::Mutex<Option<Vec<ProtoStateId>>>>,
5527 }
5528
5529 #[tonic::async_trait]
5530 impl RepoSyncService for DeltaAwarePullService {
5531 async fn list_refs(
5532 &self,
5533 _request: tonic::Request<ListRefsRequest>,
5534 ) -> Result<Response<ListRefsResponse>, Status> {
5535 Ok(Response::new(ListRefsResponse::default()))
5536 }
5537
5538 async fn update_ref(
5539 &self,
5540 _request: tonic::Request<UpdateRefRequest>,
5541 ) -> Result<Response<UpdateRefResponse>, Status> {
5542 Ok(Response::new(UpdateRefResponse::default()))
5543 }
5544
5545 type PushStream = tokio_stream::wrappers::ReceiverStream<Result<PushServerFrame, Status>>;
5546
5547 async fn push(
5548 &self,
5549 _request: tonic::Request<tonic::Streaming<PushClientFrame>>,
5550 ) -> Result<Response<Self::PushStream>, Status> {
5551 let (_tx, rx) = mpsc::channel(1);
5552 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
5553 rx,
5554 )))
5555 }
5556
5557 type PullStream = tokio_stream::wrappers::ReceiverStream<Result<PullServerFrame, Status>>;
5558
5559 async fn pull(
5560 &self,
5561 request: tonic::Request<tonic::Streaming<PullClientFrame>>,
5562 ) -> Result<Response<Self::PullStream>, Status> {
5563 let svc = self.clone();
5564 let (tx, rx) = mpsc::channel(16);
5565
5566 tokio::spawn(async move {
5567 let mut inbound = request.into_inner();
5568 if !matches!(
5569 inbound.message().await,
5570 Ok(Some(PullClientFrame {
5571 frame: Some(pull_client_frame::Frame::Open(_)),
5572 }))
5573 ) {
5574 let _ = tx
5575 .send(Err(Status::invalid_argument("expected signed pull open")))
5576 .await;
5577 return;
5578 }
5579 let exclude = match inbound.message().await {
5580 Ok(Some(PullClientFrame {
5581 frame: Some(pull_client_frame::Frame::Request(req)),
5582 })) => req.exclude_states,
5583 other => {
5584 let _ = tx
5585 .send(Err(Status::invalid_argument(format!(
5586 "expected pull request, got {other:?}"
5587 ))))
5588 .await;
5589 return;
5590 }
5591 };
5592 *svc.captured_exclude.lock().unwrap() = Some(exclude.clone());
5593
5594 let remote_proto = proto_state_id(svc.remote_state).expect("valid remote state");
5595 let parent_proto = proto_state_id(svc.known_parent).expect("valid parent state");
5596
5597 let (objects, pack, short_circuit) = if exclude.contains(&remote_proto) {
5599 (Vec::new(), None, true)
5601 } else if exclude.contains(&parent_proto) {
5602 (
5604 svc.delta_objects.clone(),
5605 Some(svc.delta_pack.clone()),
5606 false,
5607 )
5608 } else {
5609 (svc.full_closure.clone(), Some(svc.full_pack.clone()), false)
5611 };
5612
5613 let descriptors: Vec<_> = objects
5614 .iter()
5615 .map(|info| {
5616 object_descriptor_with_status(
5617 info,
5618 ObjectAvailabilityStatus::Missing,
5619 "requested by client",
5620 )
5621 })
5622 .collect();
5623
5624 let ready = PullServerFrame {
5625 frame: Some(pull_server_frame::Frame::Ready(PullReady {
5626 remote_state: proto_state_id(svc.remote_state),
5627 objects_to_fetch: descriptors,
5628 transfer: None,
5629 partial_fetch_status: PartialFetchStatus::Disabled as i32,
5630 missing_objects: Vec::new(),
5631 full_closure_available: false,
5632 object_count: objects.len() as u32,
5633 remote_revision_address: RevisionAddress::heddle(svc.remote_state)
5634 .to_string(),
5635 })),
5636 };
5637 if tx.send(Ok(ready)).await.is_err() {
5638 return;
5639 }
5640
5641 match inbound.message().await {
5643 Ok(Some(PullClientFrame {
5644 frame: Some(pull_client_frame::Frame::Want(_)),
5645 })) => {}
5646 other => {
5647 let _ = tx
5648 .send(Err(Status::invalid_argument(format!(
5649 "expected want, got {other:?}"
5650 ))))
5651 .await;
5652 return;
5653 }
5654 }
5655
5656 if let Some(pack) = pack
5657 && !short_circuit
5658 {
5659 for message in encode_pull_native_pack_messages(&pack, "delta-aware-test", 64) {
5660 if tx.send(Ok(message)).await.is_err() {
5661 return;
5662 }
5663 }
5664 }
5665
5666 let complete = PullServerFrame {
5667 frame: Some(pull_server_frame::Frame::Complete(GrpcPullComplete {
5668 success: true,
5669 new_state: proto_state_id(svc.remote_state),
5670 error: String::new(),
5671 transfer: Some(TransferCheckpoint {
5672 transfer_id: "delta-aware-test".to_string(),
5673 transport_mode: TransportMode::NativePack as i32,
5674 resume_offset: 0,
5675 chunk_index: 0,
5676 checkpoint: b"heddle-markers-v1\n".to_vec(),
5677 is_complete: true,
5678 }),
5679 new_revision_address: RevisionAddress::heddle(svc.remote_state).to_string(),
5680 })),
5681 };
5682 let _ = tx.send(Ok(complete)).await;
5683 });
5684
5685 Ok(Response::new(tokio_stream::wrappers::ReceiverStream::new(
5686 rx,
5687 )))
5688 }
5689 }
5690
5691 async fn connect_delta_aware_service(
5692 service: DeltaAwarePullService,
5693 ) -> Option<(HostedGrpcClient, tokio::task::JoinHandle<()>)> {
5694 let listener = match tokio::net::TcpListener::bind(("127.0.0.1", 0)).await {
5695 Ok(listener) => listener,
5696 Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
5697 eprintln!("skipping hosted sync local gRPC test: TCP bind denied: {err}");
5698 return None;
5699 }
5700 Err(err) => panic!("bind test server: {err}"),
5701 };
5702 let addr = listener.local_addr().expect("local addr");
5703 let incoming = futures::stream::unfold(listener, |listener| async {
5704 match listener.accept().await {
5705 Ok((stream, _addr)) => Some((Ok::<_, std::io::Error>(stream), listener)),
5706 Err(err) => Some((Err(err), listener)),
5707 }
5708 });
5709 let handle = tokio::spawn(async move {
5710 Server::builder()
5711 .add_service(RepoSyncServiceServer::new(service))
5712 .serve_with_incoming(incoming)
5713 .await
5714 .expect("serve delta-aware test service");
5715 });
5716 let client = HostedGrpcClient::connect(addr, &signed_test_config())
5717 .await
5718 .expect("connect client");
5719 Some((client, handle))
5720 }
5721
5722 #[tokio::test]
5723 async fn warm_bare_pull_advertises_head_and_fires_zero_delta_short_circuit() {
5724 let (_dir, repo) = temp_repo();
5728 let head = put_state_with_blob(&repo, "tip", vec![]);
5729 repo.refs()
5730 .set_thread(&ThreadName::from("main"), &head)
5731 .expect("set thread");
5732
5733 let full_closure =
5734 wire::enumerate_state_closure(repo.store(), head).expect("enumerate closure");
5735 let full_pack =
5736 wire::build_native_pack(repo.store(), &full_closure).expect("build full pack");
5737 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5738
5739 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5740 remote_state: head,
5741 full_closure,
5742 delta_objects: Vec::new(),
5743 known_parent: StateId::from_bytes([23; 32]),
5744 full_pack: full_pack.clone(),
5745 delta_pack: full_pack,
5746 captured_exclude: captured.clone(),
5747 })
5748 .await
5749 else {
5750 return;
5751 };
5752
5753 let exchange = tokio::time::timeout(
5754 Duration::from_secs(5),
5755 client.pull_exchange(
5756 &repo,
5757 "owner/repo",
5758 "main",
5759 PullOptions {
5760 local_thread: None,
5761 depth: None,
5762 target_state: None,
5763 materialization: PullMaterialization::Full,
5764 },
5765 ),
5766 )
5767 .await
5768 .expect("warm bare pull must not hang")
5769 .expect("warm bare pull succeeds");
5770 server.abort();
5771
5772 let advertised = captured.lock().unwrap().clone().expect("request captured");
5773 assert_eq!(
5774 advertised,
5775 vec![proto_state_id(head).expect("valid state")],
5776 "a bare pull at the tip must advertise that tip in exclude_states"
5777 );
5778 assert!(exchange.result.success);
5779 assert_eq!(
5780 exchange.object_count, 0,
5781 "the zero-delta short-circuit must transfer no objects, not the full closure"
5782 );
5783 }
5784
5785 #[tokio::test]
5786 async fn behind_client_bare_pull_receives_exactly_the_missing_delta() {
5787 let (_src_dir, src_repo) = temp_repo_unseeded();
5794 let parent = put_state_with_blob(&src_repo, "base", vec![]);
5795 let child = put_state_with_blob(&src_repo, "feature", vec![parent]);
5796 let child_closure =
5797 wire::enumerate_state_closure(src_repo.store(), child).expect("child closure");
5798
5799 let (_dir, repo) = temp_repo_unseeded();
5803 let parent_closure =
5804 wire::enumerate_state_closure(src_repo.store(), parent).expect("parent closure");
5805 let parent_pack =
5806 wire::build_native_pack(src_repo.store(), &parent_closure).expect("parent pack");
5807 wire::install_received_pack(
5808 repo.store(),
5809 &parent_pack.pack_data,
5810 &parent_pack.index_data,
5811 )
5812 .expect("install parent closure into client");
5813 repo.refs()
5814 .set_thread(&ThreadName::from("main"), &parent)
5815 .expect("set thread to parent");
5816 wire::enumerate_state_closure(repo.store(), parent).expect("client holds parent closure");
5818 assert!(
5820 repo.store()
5821 .get_state(&child)
5822 .expect("probe child")
5823 .is_none(),
5824 "client must start without the child state"
5825 );
5826 let parent_clone = parent;
5827 let full_pack =
5828 wire::build_native_pack(src_repo.store(), &child_closure).expect("full pack");
5829 let delta_objects = wire::enumerate_state_closure_with_options(
5832 src_repo.store(),
5833 child,
5834 wire::StateClosureOptions {
5835 depth: None,
5836 exclude_states: vec![parent_clone],
5837 },
5838 )
5839 .expect("delta closure");
5840 assert!(
5841 !delta_objects.is_empty() && delta_objects.len() < child_closure.len(),
5842 "delta must be a strict, non-empty subset of the full closure"
5843 );
5844 let delta_pack =
5845 wire::build_native_pack(src_repo.store(), &delta_objects).expect("delta pack");
5846
5847 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5848 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5849 remote_state: child,
5850 full_closure: child_closure.clone(),
5851 delta_objects: delta_objects.clone(),
5852 known_parent: parent_clone,
5853 full_pack,
5854 delta_pack,
5855 captured_exclude: captured.clone(),
5856 })
5857 .await
5858 else {
5859 return;
5860 };
5861
5862 let exchange = tokio::time::timeout(
5863 Duration::from_secs(5),
5864 client.pull_exchange(
5865 &repo,
5866 "owner/repo",
5867 "main",
5868 PullOptions {
5869 local_thread: None,
5870 depth: None,
5871 target_state: None,
5872 materialization: PullMaterialization::Full,
5873 },
5874 ),
5875 )
5876 .await
5877 .expect("behind-client pull must not hang")
5878 .expect("behind-client pull succeeds");
5879 server.abort();
5880
5881 let advertised = captured.lock().unwrap().clone().expect("request captured");
5882 assert_eq!(
5883 advertised,
5884 vec![proto_state_id(parent).expect("valid state")],
5885 "a behind client must advertise the parent head it holds"
5886 );
5887 assert!(exchange.result.success);
5888 let reassembled = wire::enumerate_state_closure(repo.store(), child)
5892 .expect("the client must hold the complete child closure after a delta pull");
5893 assert_eq!(
5894 reassembled.len(),
5895 child_closure.len(),
5896 "delta pull must leave the client with the full child closure, no gaps"
5897 );
5898 }
5899
5900 #[tokio::test]
5901 async fn fresh_bare_pull_advertises_nothing_and_gets_full_closure() {
5902 let (_dir, repo) = temp_repo_unseeded();
5906 let (_src_dir, src_repo) = temp_repo_unseeded();
5907 let remote = put_state_with_blob(&src_repo, "seed", vec![]);
5908 let full_closure =
5909 wire::enumerate_state_closure(src_repo.store(), remote).expect("closure");
5910 let full_pack =
5911 wire::build_native_pack(src_repo.store(), &full_closure).expect("full pack");
5912
5913 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
5914 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
5915 remote_state: remote,
5916 full_closure: full_closure.clone(),
5917 delta_objects: Vec::new(),
5918 known_parent: StateId::from_bytes([24; 32]),
5919 full_pack: full_pack.clone(),
5920 delta_pack: full_pack,
5921 captured_exclude: captured.clone(),
5922 })
5923 .await
5924 else {
5925 return;
5926 };
5927
5928 let exchange = tokio::time::timeout(
5929 Duration::from_secs(5),
5930 client.pull_exchange(
5931 &repo,
5932 "owner/repo",
5933 "main",
5934 PullOptions {
5935 local_thread: None,
5936 depth: None,
5937 target_state: None,
5938 materialization: PullMaterialization::Full,
5939 },
5940 ),
5941 )
5942 .await
5943 .expect("fresh bare pull must not hang")
5944 .expect("fresh bare pull succeeds");
5945 server.abort();
5946
5947 let advertised = captured.lock().unwrap().clone().expect("request captured");
5948 assert!(
5949 advertised.is_empty(),
5950 "a fresh repo must advertise no exclude_states"
5951 );
5952 assert!(exchange.result.success);
5953 assert_eq!(
5954 exchange.object_count,
5955 full_closure.len(),
5956 "a fresh pull must receive the full closure, nothing wrongly excluded"
5957 );
5958 assert!(
5959 wire::enumerate_state_closure(repo.store(), remote).is_ok(),
5960 "fresh pull must install the complete closure"
5961 );
5962 }
5963
5964 #[tokio::test]
5965 async fn explicit_local_thread_incomplete_closure_does_not_advertise_and_repairs() {
5966 let (_src_dir, src_repo) = temp_repo_unseeded();
5976 let parent = put_state_with_blob(&src_repo, "base", vec![]);
5977 let child = put_state_with_blob(&src_repo, "feature", vec![parent]);
5978 let child_closure =
5979 wire::enumerate_state_closure(src_repo.store(), child).expect("child closure");
5980 let full_pack =
5981 wire::build_native_pack(src_repo.store(), &child_closure).expect("full pack");
5982
5983 let (_dir, repo) = temp_repo_unseeded();
5988 let child_only = wire::enumerate_state_closure_with_options(
5989 src_repo.store(),
5990 child,
5991 wire::StateClosureOptions {
5992 depth: None,
5993 exclude_states: vec![parent],
5994 },
5995 )
5996 .expect("child-only objects");
5997 let child_only_pack =
5998 wire::build_native_pack(src_repo.store(), &child_only).expect("child-only pack");
5999 wire::install_received_pack(
6000 repo.store(),
6001 &child_only_pack.pack_data,
6002 &child_only_pack.index_data,
6003 )
6004 .expect("install child-only objects");
6005 repo.refs()
6006 .set_thread(&ThreadName::from("feature"), &child)
6007 .expect("set local thread to child");
6008 assert!(
6011 matches!(
6012 wire::enumerate_state_closure(repo.store(), child),
6013 Err(ProtocolError::ObjectNotFound(_))
6014 ),
6015 "the explicit thread head's closure must start incomplete"
6016 );
6017
6018 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
6019 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
6020 remote_state: child,
6021 full_closure: child_closure.clone(),
6022 delta_objects: Vec::new(),
6023 known_parent: parent,
6024 full_pack: full_pack.clone(),
6025 delta_pack: full_pack,
6026 captured_exclude: captured.clone(),
6027 })
6028 .await
6029 else {
6030 return;
6031 };
6032
6033 let exchange = tokio::time::timeout(
6034 Duration::from_secs(5),
6035 client.pull_exchange(
6036 &repo,
6037 "owner/repo",
6038 "main",
6039 PullOptions {
6040 local_thread: Some("feature"),
6041 depth: None,
6042 target_state: None,
6043 materialization: PullMaterialization::Full,
6044 },
6045 ),
6046 )
6047 .await
6048 .expect("explicit-thread pull must not hang")
6049 .expect("explicit-thread pull succeeds");
6050 server.abort();
6051
6052 let advertised = captured.lock().unwrap().clone().expect("request captured");
6053 assert!(
6054 advertised.is_empty(),
6055 "an explicit --local-thread with an incomplete closure must advertise nothing, \
6056 falling back to a full pull (got {advertised:?})"
6057 );
6058 assert!(exchange.result.success);
6059 let reassembled = wire::enumerate_state_closure(repo.store(), child)
6061 .expect("the client must hold the complete child closure after the fallback full pull");
6062 assert_eq!(
6063 reassembled.len(),
6064 child_closure.len(),
6065 "the full-pull fallback must leave the client with the complete closure, no gaps"
6066 );
6067 }
6068
6069 #[tokio::test]
6070 async fn explicit_local_thread_complete_closure_advertises_and_fires_short_circuit() {
6071 let (_dir, repo) = temp_repo();
6076 let head = put_state_with_blob(&repo, "tip", vec![]);
6077 repo.refs()
6078 .set_thread(&ThreadName::from("feature"), &head)
6079 .expect("set local thread");
6080 wire::enumerate_state_closure(repo.store(), head)
6082 .expect("client holds the complete thread closure");
6083
6084 let full_closure =
6085 wire::enumerate_state_closure(repo.store(), head).expect("enumerate closure");
6086 let full_pack =
6087 wire::build_native_pack(repo.store(), &full_closure).expect("build full pack");
6088 let captured = std::sync::Arc::new(std::sync::Mutex::new(None));
6089
6090 let Some((mut client, server)) = connect_delta_aware_service(DeltaAwarePullService {
6091 remote_state: head,
6092 full_closure,
6093 delta_objects: Vec::new(),
6094 known_parent: StateId::from_bytes([25; 32]),
6095 full_pack: full_pack.clone(),
6096 delta_pack: full_pack,
6097 captured_exclude: captured.clone(),
6098 })
6099 .await
6100 else {
6101 return;
6102 };
6103
6104 let exchange = tokio::time::timeout(
6105 Duration::from_secs(5),
6106 client.pull_exchange(
6107 &repo,
6108 "owner/repo",
6109 "main",
6110 PullOptions {
6111 local_thread: Some("feature"),
6112 depth: None,
6113 target_state: None,
6114 materialization: PullMaterialization::Full,
6115 },
6116 ),
6117 )
6118 .await
6119 .expect("explicit-thread pull must not hang")
6120 .expect("explicit-thread pull succeeds");
6121 server.abort();
6122
6123 let advertised = captured.lock().unwrap().clone().expect("request captured");
6124 assert_eq!(
6125 advertised,
6126 vec![proto_state_id(head).expect("valid state")],
6127 "an explicit --local-thread with a complete closure must advertise its head"
6128 );
6129 assert!(exchange.result.success);
6130 assert_eq!(
6131 exchange.object_count, 0,
6132 "advertising the complete head must fire the zero-delta short-circuit, not a full pull"
6133 );
6134 }
6135}