tape_git_remote/
index.rs

1//! The ref index: the one mutable thing in an otherwise append-only store
2//!
3//! This module is the crate's public description of what a repository looks
4//! like on a tape. Tooling that publishes a repository through the SDK
5//! directly, or lists a published repository's refs without invoking Git, uses
6//! these types so that it stays in agreement with the remote helper.
7//!
8//! Packs are content-addressed and immutable, so refs need somewhere to live
9//! that can change. That is a *named* object, and a named write appends a new
10//! version whose `hash(name)` key resolves to the newest one, which is exactly
11//! mutable-pointer semantics for free.
12//!
13//! The encoding is kept deliberately small. Under 825 bytes a write is a single
14//! inline transaction that is readable almost immediately. Past that it becomes
15//! an erasure-coded track that has to be uploaded and certified before anyone can
16//! read it back. Staying inline for as long as possible is the difference between
17//! a push that takes half a second and one that takes several.
18
19use std::collections::BTreeMap;
20
21use anyhow::Result;
22use serde::{Deserialize, Serialize};
23
24use tape_crypto::hash::hash;
25
26/// Object name the index is stored under
27///
28/// Namespaced so it cannot collide with a bucket that also serves a website out
29/// of its named objects.
30pub const INDEX_NAME: &str = "git/refs.json";
31
32/// Content type recorded on the ref index object
33pub const INDEX_CONTENT_TYPE: &str = "application/json";
34
35/// Current index encoding version
36pub const INDEX_VERSION: u64 = 1;
37
38/// Hex characters kept from a pack's sha256
39///
40/// The SDK verifies every track against its on-chain commitment, but this digest
41/// is what lets an *untrusted* gateway serve pack bytes. The index is proven
42/// against the chain, so its digests are trustworthy statements about the packs.
43/// That makes the width security-relevant rather than merely anti-corruption,
44/// hence 128 bits. The 32 characters saved per pack still help keep the index
45/// inline.
46const DIGEST_CHARS: usize = 32;
47
48/// Truncated sha256 of a pack, as recorded in the index
49pub fn digest(bytes: &[u8]) -> String {
50    let mut hex = hex::encode(hash(bytes).to_bytes());
51    hex.truncate(DIGEST_CHARS);
52    hex
53}
54
55#[derive(Clone, Debug, Deserialize, Serialize)]
56pub struct PackEntry {
57    /// Track number on the bucket's tape
58    ///
59    /// Recorded so reads go straight to `track_pda(tape, track)`. Resolving by
60    /// content hash would make the node scan every track on the tape per lookup.
61    pub track: u64,
62
63    pub size: u64,
64
65    /// Truncated sha256 of the pack bytes
66    ///
67    /// `sha256` is accepted as an alias so indexes written before the field was
68    /// renamed still load.
69    #[serde(alias = "sha256")]
70    pub digest: String,
71
72    /// Whether the track is a stream manifest rather than a direct blob.
73    #[serde(default, skip_serializing_if = "is_false")]
74    pub stream: bool,
75}
76
77fn is_false(value: &bool) -> bool {
78    !*value
79}
80
81impl PackEntry {
82    /// Whether `bytes` is the pack this entry points at
83    ///
84    /// Compares only as many characters as the entry stored, so a full-length
85    /// digest written by an older version still matches.
86    pub fn matches(&self, bytes: &[u8]) -> bool {
87        let full = hex::encode(hash(bytes).to_bytes());
88        let width = self.digest.len().min(full.len());
89
90        self.digest[..width] == full[..width]
91    }
92}
93
94#[derive(Clone, Debug, Deserialize, Serialize)]
95pub struct Index {
96    pub version: u64,
97
98    /// Ref HEAD points at, so `git clone` knows which branch to check out
99    #[serde(default, skip_serializing_if = "Option::is_none")]
100    pub head: Option<String>,
101
102    /// Full refname to object id
103    #[serde(default)]
104    pub refs: BTreeMap<String, String>,
105
106    /// Packs in push order, replayed in this order on fetch
107    #[serde(default)]
108    pub packs: Vec<PackEntry>,
109
110    /// Track number of the index version this one was derived from
111    ///
112    /// Recorded so a reader can tell whether two versions were written from the
113    /// same base, which is the only signal available that a push raced.
114    #[serde(default, skip_serializing_if = "Option::is_none")]
115    pub parent: Option<u64>,
116}
117
118impl Default for Index {
119    fn default() -> Self {
120        Self {
121            version: INDEX_VERSION,
122            head: None,
123            refs: BTreeMap::new(),
124            packs: Vec::new(),
125            parent: None,
126        }
127    }
128}
129
130impl Index {
131    pub fn decode(bytes: &[u8]) -> Result<Self> {
132        Ok(serde_json::from_slice(bytes)?)
133    }
134
135    pub fn encode(&self) -> Result<Vec<u8>> {
136        Ok(serde_json::to_vec(self)?)
137    }
138
139    /// Every object id the remote already has
140    ///
141    /// Used as the `--not` basis so a push only carries objects the remote is
142    /// missing.
143    pub fn tips(&self) -> Vec<String> {
144        let mut tips = Vec::with_capacity(self.refs.len());
145        for object_id in self.refs.values() {
146            tips.push(object_id.clone());
147        }
148        tips
149    }
150
151    /// Whether this index already lists the given pack
152    pub fn has_pack(&self, track: u64) -> bool {
153        for entry in &self.packs {
154            if entry.track == track {
155                return true;
156            }
157        }
158        false
159    }
160
161    /// Take on every pack from another version, keeping push order
162    ///
163    /// Packs are immutable and purely additive, so their union is always safe, and
164    /// dropping one would orphan objects somebody else's refs depend on.
165    pub fn absorb_packs(&mut self, other: &Index) {
166        for entry in &other.packs {
167            if !self.has_pack(entry.track) {
168                self.packs.push(entry.clone());
169            }
170        }
171        self.packs.sort_by_key(|entry| entry.track);
172    }
173}
174
175#[cfg(test)]
176mod tests {
177    use super::*;
178
179    fn entry(track: u64, bytes: &[u8]) -> PackEntry {
180        PackEntry {
181            track,
182            size: bytes.len() as u64,
183            digest: digest(bytes),
184            stream: false,
185        }
186    }
187
188    // a digest matches the bytes it was made from and nothing else
189    #[test]
190    fn digest_matching() {
191        let pack = entry(1, b"pack contents");
192
193        assert!(pack.matches(b"pack contents"));
194        assert!(!pack.matches(b"pack contentt"));
195    }
196
197    // a full-length digest from an older writer still verifies
198    #[test]
199    fn legacy_digest() {
200        let mut pack = entry(1, b"pack contents");
201        pack.digest = hex::encode(hash(b"pack contents").to_bytes());
202
203        assert!(pack.matches(b"pack contents"));
204        assert!(!pack.matches(b"something else"));
205    }
206
207    // the older field name still deserializes
208    #[test]
209    fn sha256_alias() {
210        let json = br#"{"version":1,"packs":[{"track":3,"size":9,"sha256":"abcdef"}]}"#;
211
212        let index = Index::decode(json).expect("index should decode");
213
214        assert_eq!(index.packs[0].digest, "abcdef");
215    }
216
217    // a round trip preserves refs, head and packs
218    #[test]
219    fn round_trip() {
220        let mut index = Index {
221            head: Some("refs/heads/main".to_string()),
222            ..Default::default()
223        };
224        index
225            .refs
226            .insert("refs/heads/main".to_string(), "a".repeat(40));
227        index.packs.push(entry(7, b"pack"));
228
229        let decoded = Index::decode(&index.encode().expect("encode")).expect("decode");
230
231        assert_eq!(decoded.head.as_deref(), Some("refs/heads/main"));
232        assert_eq!(decoded.refs.len(), 1);
233        assert_eq!(decoded.packs[0].track, 7);
234    }
235
236    // absorbing another version keeps both pack sets, ordered, without duplicates
237    #[test]
238    fn absorb_packs() {
239        let mut ours = Index::default();
240        ours.packs.push(entry(4, b"ours"));
241        let mut theirs = Index::default();
242        theirs.packs.push(entry(2, b"theirs"));
243        theirs.packs.push(entry(4, b"ours"));
244
245        ours.absorb_packs(&theirs);
246
247        let mut tracks = Vec::new();
248        for pack in &ours.packs {
249            tracks.push(pack.track);
250        }
251        assert_eq!(tracks, vec![2, 4]);
252    }
253
254    // a handful of refs and packs still fits inside one inline write
255    #[test]
256    fn stays_inline() {
257        const INLINE_LIMIT: usize = 825;
258        let mut index = Index {
259            head: Some("refs/heads/main".to_string()),
260            ..Default::default()
261        };
262        for name in ["main", "develop", "release", "feature-one", "feature-two"] {
263            index
264                .refs
265                .insert(format!("refs/heads/{name}"), "a".repeat(40));
266        }
267        for track in 0..5 {
268            index.packs.push(entry(track, b"pack"));
269        }
270
271        let encoded = index.encode().expect("encode");
272
273        assert!(
274            encoded.len() < INLINE_LIMIT,
275            "index grew to {} bytes",
276            encoded.len()
277        );
278    }
279}
tape_git_remote/index.rs

tape_git_remote/
index.rs
