jujube_lib/
local_store.rs

1// Copyright 2020 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::fmt::Debug;
16use std::fs;
17use std::fs::File;
18use std::io::Write;
19use std::io::{ErrorKind, Read};
20use std::path::PathBuf;
21
22use blake2::{Blake2b, Digest};
23use protobuf::{Message, ProtobufError};
24use tempfile::{NamedTempFile, PersistError};
25
26use crate::repo_path::{DirRepoPath, FileRepoPath};
27use crate::store::{
28    ChangeId, Commit, CommitId, Conflict, ConflictId, ConflictPart, FileId, MillisSinceEpoch,
29    Signature, Store, StoreError, StoreResult, SymlinkId, Timestamp, Tree, TreeId, TreeValue,
30};
31
32impl From<std::io::Error> for StoreError {
33    fn from(err: std::io::Error) -> Self {
34        StoreError::Other(err.to_string())
35    }
36}
37
38impl From<PersistError> for StoreError {
39    fn from(err: PersistError) -> Self {
40        StoreError::Other(err.to_string())
41    }
42}
43
44impl From<ProtobufError> for StoreError {
45    fn from(err: ProtobufError) -> Self {
46        StoreError::Other(err.to_string())
47    }
48}
49
50#[derive(Debug)]
51pub struct LocalStore {
52    path: PathBuf,
53    empty_tree_id: TreeId,
54}
55
56impl LocalStore {
57    pub fn init(store_path: PathBuf) -> Self {
58        fs::create_dir(store_path.join("commits")).unwrap();
59        fs::create_dir(store_path.join("trees")).unwrap();
60        fs::create_dir(store_path.join("files")).unwrap();
61        fs::create_dir(store_path.join("symlinks")).unwrap();
62        fs::create_dir(store_path.join("conflicts")).unwrap();
63        let store = Self::load(store_path);
64        let empty_tree_id = store
65            .write_tree(&DirRepoPath::root(), &Tree::default())
66            .unwrap();
67        assert_eq!(empty_tree_id, store.empty_tree_id);
68        store
69    }
70
71    pub fn load(store_path: PathBuf) -> Self {
72        let empty_tree_id = TreeId(hex::decode("786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419d25e1031afee585313896444934eb04b903a685b1448b755d56f701afe9be2ce").unwrap());
73        LocalStore {
74            path: store_path,
75            empty_tree_id,
76        }
77    }
78
79    fn file_path(&self, id: &FileId) -> PathBuf {
80        self.path.join("files").join(id.hex())
81    }
82
83    fn symlink_path(&self, id: &SymlinkId) -> PathBuf {
84        self.path.join("symlinks").join(id.hex())
85    }
86
87    fn tree_path(&self, id: &TreeId) -> PathBuf {
88        self.path.join("trees").join(id.hex())
89    }
90
91    fn commit_path(&self, id: &CommitId) -> PathBuf {
92        self.path.join("commits").join(id.hex())
93    }
94
95    fn conflict_path(&self, id: &ConflictId) -> PathBuf {
96        self.path.join("conflicts").join(id.hex())
97    }
98}
99
100fn not_found_to_store_error(err: std::io::Error) -> StoreError {
101    if err.kind() == ErrorKind::NotFound {
102        StoreError::NotFound
103    } else {
104        StoreError::from(err)
105    }
106}
107
108impl Store for LocalStore {
109    fn hash_length(&self) -> usize {
110        64
111    }
112
113    fn git_repo(&self) -> Option<git2::Repository> {
114        None
115    }
116
117    fn read_file(&self, _path: &FileRepoPath, id: &FileId) -> StoreResult<Box<dyn Read>> {
118        let path = self.file_path(&id);
119        let file = File::open(path).map_err(not_found_to_store_error)?;
120        Ok(Box::new(zstd::Decoder::new(file)?))
121    }
122
123    fn write_file(&self, _path: &FileRepoPath, contents: &mut dyn Read) -> StoreResult<FileId> {
124        let temp_file = NamedTempFile::new_in(&self.path)?;
125        let mut encoder = zstd::Encoder::new(temp_file.as_file(), 0)?;
126        let mut hasher = Blake2b::new();
127        loop {
128            let mut buff: Vec<u8> = Vec::with_capacity(1 << 14);
129            let bytes_read;
130            unsafe {
131                buff.set_len(1 << 14);
132                bytes_read = contents.read(&mut buff)?;
133                buff.set_len(bytes_read);
134            }
135            if bytes_read == 0 {
136                break;
137            }
138            encoder.write_all(&buff)?;
139            hasher.update(&buff);
140        }
141        encoder.finish()?;
142        let id = FileId(hasher.finalize().to_vec());
143
144        temp_file.persist(self.file_path(&id))?;
145        Ok(id)
146    }
147
148    fn read_symlink(&self, _path: &FileRepoPath, id: &SymlinkId) -> Result<String, StoreError> {
149        let path = self.symlink_path(&id);
150        let mut file = File::open(path).map_err(not_found_to_store_error)?;
151        let mut target = String::new();
152        file.read_to_string(&mut target).unwrap();
153        Ok(target)
154    }
155
156    fn write_symlink(&self, _path: &FileRepoPath, target: &str) -> Result<SymlinkId, StoreError> {
157        let mut temp_file = NamedTempFile::new_in(&self.path)?;
158        temp_file.write_all(target.as_bytes()).unwrap();
159        let mut hasher = Blake2b::new();
160        hasher.update(&target.as_bytes());
161        let id = SymlinkId(hasher.finalize().to_vec());
162
163        temp_file.persist(self.symlink_path(&id))?;
164        Ok(id)
165    }
166
167    fn empty_tree_id(&self) -> &TreeId {
168        &self.empty_tree_id
169    }
170
171    fn read_tree(&self, _path: &DirRepoPath, id: &TreeId) -> StoreResult<Tree> {
172        let path = self.tree_path(&id);
173        let mut file = File::open(path).map_err(not_found_to_store_error)?;
174
175        let proto: crate::protos::store::Tree = protobuf::parse_from_reader(&mut file)?;
176        Ok(tree_from_proto(&proto))
177    }
178
179    fn write_tree(&self, _path: &DirRepoPath, tree: &Tree) -> StoreResult<TreeId> {
180        let temp_file = NamedTempFile::new_in(&self.path)?;
181
182        let proto = tree_to_proto(tree);
183        let mut proto_bytes: Vec<u8> = Vec::new();
184        proto.write_to_writer(&mut proto_bytes)?;
185
186        temp_file.as_file().write_all(&proto_bytes)?;
187
188        let id = TreeId(Blake2b::digest(&proto_bytes).to_vec());
189
190        temp_file.persist(self.tree_path(&id))?;
191        Ok(id)
192    }
193
194    fn read_commit(&self, id: &CommitId) -> StoreResult<Commit> {
195        let path = self.commit_path(&id);
196        let mut file = File::open(path).map_err(not_found_to_store_error)?;
197
198        let proto: crate::protos::store::Commit = protobuf::parse_from_reader(&mut file)?;
199        Ok(commit_from_proto(&proto))
200    }
201
202    fn write_commit(&self, commit: &Commit) -> StoreResult<CommitId> {
203        let temp_file = NamedTempFile::new_in(&self.path)?;
204
205        let proto = commit_to_proto(commit);
206        let mut proto_bytes: Vec<u8> = Vec::new();
207        proto.write_to_writer(&mut proto_bytes)?;
208
209        temp_file.as_file().write_all(&proto_bytes)?;
210
211        let id = CommitId(Blake2b::digest(&proto_bytes).to_vec());
212
213        temp_file.persist(self.commit_path(&id))?;
214        Ok(id)
215    }
216
217    fn read_conflict(&self, id: &ConflictId) -> StoreResult<Conflict> {
218        let path = self.conflict_path(&id);
219        let mut file = File::open(path).map_err(not_found_to_store_error)?;
220
221        let proto: crate::protos::store::Conflict = protobuf::parse_from_reader(&mut file)?;
222        Ok(conflict_from_proto(&proto))
223    }
224
225    fn write_conflict(&self, conflict: &Conflict) -> StoreResult<ConflictId> {
226        let temp_file = NamedTempFile::new_in(&self.path)?;
227
228        let proto = conflict_to_proto(conflict);
229        let mut proto_bytes: Vec<u8> = Vec::new();
230        proto.write_to_writer(&mut proto_bytes)?;
231
232        temp_file.as_file().write_all(&proto_bytes)?;
233
234        let id = ConflictId(Blake2b::digest(&proto_bytes).to_vec());
235
236        temp_file.persist(self.conflict_path(&id))?;
237        Ok(id)
238    }
239}
240
241pub fn commit_to_proto(commit: &Commit) -> crate::protos::store::Commit {
242    let mut proto = crate::protos::store::Commit::new();
243    for parent in &commit.parents {
244        proto.parents.push(parent.0.clone());
245    }
246    for predecessor in &commit.predecessors {
247        proto.predecessors.push(predecessor.0.clone());
248    }
249    proto.set_root_tree(commit.root_tree.0.clone());
250    proto.set_change_id(commit.change_id.0.clone());
251    proto.set_description(commit.description.clone());
252    proto.set_author(signature_to_proto(&commit.author));
253    proto.set_committer(signature_to_proto(&commit.committer));
254    proto.set_is_open(commit.is_open);
255    proto.set_is_pruned(commit.is_pruned);
256    proto
257}
258
259fn commit_from_proto(proto: &crate::protos::store::Commit) -> Commit {
260    let commit_id_from_proto = |parent: &Vec<u8>| CommitId(parent.clone());
261    let parents = proto.parents.iter().map(commit_id_from_proto).collect();
262    let predecessors = proto
263        .predecessors
264        .iter()
265        .map(commit_id_from_proto)
266        .collect();
267    let root_tree = TreeId(proto.root_tree.to_vec());
268    let change_id = ChangeId(proto.change_id.to_vec());
269    Commit {
270        parents,
271        predecessors,
272        root_tree,
273        change_id,
274        description: proto.description.clone(),
275        author: signature_from_proto(proto.author.get_ref()),
276        committer: signature_from_proto(proto.committer.get_ref()),
277        is_open: proto.is_open,
278        is_pruned: proto.is_pruned,
279    }
280}
281
282fn tree_to_proto(tree: &Tree) -> crate::protos::store::Tree {
283    let mut proto = crate::protos::store::Tree::new();
284    for entry in tree.entries() {
285        let mut proto_entry = crate::protos::store::Tree_Entry::new();
286        proto_entry.set_name(entry.name().to_owned());
287        proto_entry.set_value(tree_value_to_proto(entry.value()));
288        proto.entries.push(proto_entry);
289    }
290    proto
291}
292
293fn tree_from_proto(proto: &crate::protos::store::Tree) -> Tree {
294    let mut tree = Tree::default();
295    for proto_entry in proto.entries.iter() {
296        let value = tree_value_from_proto(proto_entry.value.as_ref().unwrap());
297        tree.set(proto_entry.name.to_string(), value);
298    }
299    tree
300}
301
302fn tree_value_to_proto(value: &TreeValue) -> crate::protos::store::TreeValue {
303    let mut proto = crate::protos::store::TreeValue::new();
304    match value {
305        TreeValue::Normal { id, executable } => {
306            let mut file = crate::protos::store::TreeValue_NormalFile::new();
307            file.set_id(id.0.clone());
308            file.set_executable(*executable);
309            proto.set_normal_file(file);
310        }
311        TreeValue::Symlink(id) => {
312            proto.set_symlink_id(id.0.clone());
313        }
314        TreeValue::GitSubmodule(_id) => {
315            panic!("cannot store git submodules");
316        }
317        TreeValue::Tree(id) => {
318            proto.set_tree_id(id.0.clone());
319        }
320        TreeValue::Conflict(id) => {
321            proto.set_conflict_id(id.0.clone());
322        }
323    };
324    proto
325}
326
327fn tree_value_from_proto(proto: &crate::protos::store::TreeValue) -> TreeValue {
328    match proto.value.as_ref().unwrap() {
329        crate::protos::store::TreeValue_oneof_value::tree_id(id) => {
330            TreeValue::Tree(TreeId(id.clone()))
331        }
332        crate::protos::store::TreeValue_oneof_value::normal_file(
333            crate::protos::store::TreeValue_NormalFile { id, executable, .. },
334        ) => TreeValue::Normal {
335            id: FileId(id.clone()),
336            executable: *executable,
337        },
338        crate::protos::store::TreeValue_oneof_value::symlink_id(id) => {
339            TreeValue::Symlink(SymlinkId(id.clone()))
340        }
341        crate::protos::store::TreeValue_oneof_value::conflict_id(id) => {
342            TreeValue::Conflict(ConflictId(id.clone()))
343        }
344    }
345}
346
347fn signature_to_proto(signature: &Signature) -> crate::protos::store::Commit_Signature {
348    let mut proto = crate::protos::store::Commit_Signature::new();
349    proto.set_name(signature.name.clone());
350    proto.set_email(signature.email.clone());
351    let mut timestamp_proto = crate::protos::store::Commit_Timestamp::new();
352    timestamp_proto.set_millis_since_epoch(signature.timestamp.timestamp.0);
353    timestamp_proto.set_tz_offset(signature.timestamp.tz_offset);
354    proto.set_timestamp(timestamp_proto);
355    proto
356}
357
358fn signature_from_proto(proto: &crate::protos::store::Commit_Signature) -> Signature {
359    let timestamp = proto.get_timestamp();
360    Signature {
361        name: proto.name.clone(),
362        email: proto.email.clone(),
363        timestamp: Timestamp {
364            timestamp: MillisSinceEpoch(timestamp.millis_since_epoch),
365            tz_offset: timestamp.tz_offset,
366        },
367    }
368}
369
370fn conflict_to_proto(conflict: &Conflict) -> crate::protos::store::Conflict {
371    let mut proto = crate::protos::store::Conflict::new();
372    for part in &conflict.adds {
373        proto.adds.push(conflict_part_to_proto(part));
374    }
375    for part in &conflict.removes {
376        proto.removes.push(conflict_part_to_proto(part));
377    }
378    proto
379}
380
381fn conflict_from_proto(proto: &crate::protos::store::Conflict) -> Conflict {
382    let mut conflict = Conflict::default();
383    for part in &proto.removes {
384        conflict.removes.push(conflict_part_from_proto(part))
385    }
386    for part in &proto.adds {
387        conflict.adds.push(conflict_part_from_proto(part))
388    }
389    conflict
390}
391
392fn conflict_part_from_proto(proto: &crate::protos::store::Conflict_Part) -> ConflictPart {
393    ConflictPart {
394        value: tree_value_from_proto(proto.content.as_ref().unwrap()),
395    }
396}
397
398fn conflict_part_to_proto(part: &ConflictPart) -> crate::protos::store::Conflict_Part {
399    let mut proto = crate::protos::store::Conflict_Part::new();
400    proto.set_content(tree_value_to_proto(&part.value));
401    proto
402}
jujube_lib/local_store.rs

jujube_lib/
local_store.rs
