jj_lib/
simple_backend.rs

1// Copyright 2020 The Jujutsu Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![allow(missing_docs)]
16
17use std::any::Any;
18use std::fmt::Debug;
19use std::fs;
20use std::fs::File;
21use std::io::Read;
22use std::io::Write as _;
23use std::path::Path;
24use std::path::PathBuf;
25use std::time::SystemTime;
26
27use async_trait::async_trait;
28use blake2::Blake2b512;
29use blake2::Digest as _;
30use futures::stream;
31use futures::stream::BoxStream;
32use pollster::FutureExt as _;
33use prost::Message as _;
34use tempfile::NamedTempFile;
35
36use crate::backend::make_root_commit;
37use crate::backend::Backend;
38use crate::backend::BackendError;
39use crate::backend::BackendResult;
40use crate::backend::ChangeId;
41use crate::backend::Commit;
42use crate::backend::CommitId;
43use crate::backend::Conflict;
44use crate::backend::ConflictId;
45use crate::backend::ConflictTerm;
46use crate::backend::CopyRecord;
47use crate::backend::FileId;
48use crate::backend::MergedTreeId;
49use crate::backend::MillisSinceEpoch;
50use crate::backend::SecureSig;
51use crate::backend::Signature;
52use crate::backend::SigningFn;
53use crate::backend::SymlinkId;
54use crate::backend::Timestamp;
55use crate::backend::Tree;
56use crate::backend::TreeId;
57use crate::backend::TreeValue;
58use crate::content_hash::blake2b_hash;
59use crate::file_util::persist_content_addressed_temp_file;
60use crate::index::Index;
61use crate::merge::MergeBuilder;
62use crate::object_id::ObjectId;
63use crate::repo_path::RepoPath;
64use crate::repo_path::RepoPathBuf;
65use crate::repo_path::RepoPathComponentBuf;
66
67const COMMIT_ID_LENGTH: usize = 64;
68const CHANGE_ID_LENGTH: usize = 16;
69
70fn map_not_found_err(err: std::io::Error, id: &impl ObjectId) -> BackendError {
71    if err.kind() == std::io::ErrorKind::NotFound {
72        BackendError::ObjectNotFound {
73            object_type: id.object_type(),
74            hash: id.hex(),
75            source: Box::new(err),
76        }
77    } else {
78        BackendError::ReadObject {
79            object_type: id.object_type(),
80            hash: id.hex(),
81            source: Box::new(err),
82        }
83    }
84}
85
86fn to_other_err(err: impl Into<Box<dyn std::error::Error + Send + Sync>>) -> BackendError {
87    BackendError::Other(err.into())
88}
89
90#[derive(Debug)]
91pub struct SimpleBackend {
92    path: PathBuf,
93    root_commit_id: CommitId,
94    root_change_id: ChangeId,
95    empty_tree_id: TreeId,
96}
97
98impl SimpleBackend {
99    pub fn name() -> &'static str {
100        "Simple"
101    }
102
103    pub fn init(store_path: &Path) -> Self {
104        fs::create_dir(store_path.join("commits")).unwrap();
105        fs::create_dir(store_path.join("trees")).unwrap();
106        fs::create_dir(store_path.join("files")).unwrap();
107        fs::create_dir(store_path.join("symlinks")).unwrap();
108        fs::create_dir(store_path.join("conflicts")).unwrap();
109        let backend = Self::load(store_path);
110        let empty_tree_id = backend
111            .write_tree(RepoPath::root(), &Tree::default())
112            .block_on()
113            .unwrap();
114        assert_eq!(empty_tree_id, backend.empty_tree_id);
115        backend
116    }
117
118    pub fn load(store_path: &Path) -> Self {
119        let root_commit_id = CommitId::from_bytes(&[0; COMMIT_ID_LENGTH]);
120        let root_change_id = ChangeId::from_bytes(&[0; CHANGE_ID_LENGTH]);
121        let empty_tree_id = TreeId::from_hex(
122            "482ae5a29fbe856c7272f2071b8b0f0359ee2d89ff392b8a900643fbd0836eccd067b8bf41909e206c90d45d6e7d8b6686b93ecaee5fe1a9060d87b672101310",
123        );
124        SimpleBackend {
125            path: store_path.to_path_buf(),
126            root_commit_id,
127            root_change_id,
128            empty_tree_id,
129        }
130    }
131
132    fn file_path(&self, id: &FileId) -> PathBuf {
133        self.path.join("files").join(id.hex())
134    }
135
136    fn symlink_path(&self, id: &SymlinkId) -> PathBuf {
137        self.path.join("symlinks").join(id.hex())
138    }
139
140    fn tree_path(&self, id: &TreeId) -> PathBuf {
141        self.path.join("trees").join(id.hex())
142    }
143
144    fn commit_path(&self, id: &CommitId) -> PathBuf {
145        self.path.join("commits").join(id.hex())
146    }
147
148    fn conflict_path(&self, id: &ConflictId) -> PathBuf {
149        self.path.join("conflicts").join(id.hex())
150    }
151}
152
153#[async_trait]
154impl Backend for SimpleBackend {
155    fn as_any(&self) -> &dyn Any {
156        self
157    }
158
159    fn name(&self) -> &str {
160        Self::name()
161    }
162
163    fn commit_id_length(&self) -> usize {
164        COMMIT_ID_LENGTH
165    }
166
167    fn change_id_length(&self) -> usize {
168        CHANGE_ID_LENGTH
169    }
170
171    fn root_commit_id(&self) -> &CommitId {
172        &self.root_commit_id
173    }
174
175    fn root_change_id(&self) -> &ChangeId {
176        &self.root_change_id
177    }
178
179    fn empty_tree_id(&self) -> &TreeId {
180        &self.empty_tree_id
181    }
182
183    fn concurrency(&self) -> usize {
184        1
185    }
186
187    async fn read_file(&self, _path: &RepoPath, id: &FileId) -> BackendResult<Box<dyn Read>> {
188        let path = self.file_path(id);
189        let file = File::open(path).map_err(|err| map_not_found_err(err, id))?;
190        Ok(Box::new(file))
191    }
192
193    async fn write_file(
194        &self,
195        _path: &RepoPath,
196        contents: &mut (dyn Read + Send),
197    ) -> BackendResult<FileId> {
198        // TODO: Write temporary file in the destination directory (#5712)
199        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
200        let mut file = temp_file.as_file();
201        let mut hasher = Blake2b512::new();
202        let mut buff: Vec<u8> = vec![0; 1 << 14];
203        loop {
204            let bytes_read = contents.read(&mut buff).map_err(to_other_err)?;
205            if bytes_read == 0 {
206                break;
207            }
208            let bytes = &buff[..bytes_read];
209            file.write_all(bytes).map_err(to_other_err)?;
210            hasher.update(bytes);
211        }
212        file.flush().map_err(to_other_err)?;
213        let id = FileId::new(hasher.finalize().to_vec());
214
215        persist_content_addressed_temp_file(temp_file, self.file_path(&id))
216            .map_err(to_other_err)?;
217        Ok(id)
218    }
219
220    async fn read_symlink(&self, _path: &RepoPath, id: &SymlinkId) -> BackendResult<String> {
221        let path = self.symlink_path(id);
222        let target = fs::read_to_string(path).map_err(|err| map_not_found_err(err, id))?;
223        Ok(target)
224    }
225
226    async fn write_symlink(&self, _path: &RepoPath, target: &str) -> BackendResult<SymlinkId> {
227        // TODO: Write temporary file in the destination directory (#5712)
228        let mut temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
229        temp_file
230            .write_all(target.as_bytes())
231            .map_err(to_other_err)?;
232        let mut hasher = Blake2b512::new();
233        hasher.update(target.as_bytes());
234        let id = SymlinkId::new(hasher.finalize().to_vec());
235
236        persist_content_addressed_temp_file(temp_file, self.symlink_path(&id))
237            .map_err(to_other_err)?;
238        Ok(id)
239    }
240
241    async fn read_tree(&self, _path: &RepoPath, id: &TreeId) -> BackendResult<Tree> {
242        let path = self.tree_path(id);
243        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
244
245        let proto = crate::protos::simple_store::Tree::decode(&*buf).map_err(to_other_err)?;
246        Ok(tree_from_proto(proto))
247    }
248
249    async fn write_tree(&self, _path: &RepoPath, tree: &Tree) -> BackendResult<TreeId> {
250        // TODO: Write temporary file in the destination directory (#5712)
251        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
252
253        let proto = tree_to_proto(tree);
254        temp_file
255            .as_file()
256            .write_all(&proto.encode_to_vec())
257            .map_err(to_other_err)?;
258
259        let id = TreeId::new(blake2b_hash(tree).to_vec());
260
261        persist_content_addressed_temp_file(temp_file, self.tree_path(&id))
262            .map_err(to_other_err)?;
263        Ok(id)
264    }
265
266    fn read_conflict(&self, _path: &RepoPath, id: &ConflictId) -> BackendResult<Conflict> {
267        let path = self.conflict_path(id);
268        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
269
270        let proto = crate::protos::simple_store::Conflict::decode(&*buf).map_err(to_other_err)?;
271        Ok(conflict_from_proto(proto))
272    }
273
274    fn write_conflict(&self, _path: &RepoPath, conflict: &Conflict) -> BackendResult<ConflictId> {
275        // TODO: Write temporary file in the destination directory (#5712)
276        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
277
278        let proto = conflict_to_proto(conflict);
279        temp_file
280            .as_file()
281            .write_all(&proto.encode_to_vec())
282            .map_err(to_other_err)?;
283
284        let id = ConflictId::new(blake2b_hash(conflict).to_vec());
285
286        persist_content_addressed_temp_file(temp_file, self.conflict_path(&id))
287            .map_err(to_other_err)?;
288        Ok(id)
289    }
290
291    async fn read_commit(&self, id: &CommitId) -> BackendResult<Commit> {
292        if *id == self.root_commit_id {
293            return Ok(make_root_commit(
294                self.root_change_id().clone(),
295                self.empty_tree_id.clone(),
296            ));
297        }
298
299        let path = self.commit_path(id);
300        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
301
302        let proto = crate::protos::simple_store::Commit::decode(&*buf).map_err(to_other_err)?;
303        Ok(commit_from_proto(proto))
304    }
305
306    async fn write_commit(
307        &self,
308        mut commit: Commit,
309        sign_with: Option<&mut SigningFn>,
310    ) -> BackendResult<(CommitId, Commit)> {
311        assert!(commit.secure_sig.is_none(), "commit.secure_sig was set");
312
313        if commit.parents.is_empty() {
314            return Err(BackendError::Other(
315                "Cannot write a commit with no parents".into(),
316            ));
317        }
318        // TODO: Write temporary file in the destination directory (#5712)
319        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
320
321        let mut proto = commit_to_proto(&commit);
322        if let Some(sign) = sign_with {
323            let data = proto.encode_to_vec();
324            let sig = sign(&data).map_err(to_other_err)?;
325            proto.secure_sig = Some(sig.clone());
326            commit.secure_sig = Some(SecureSig { data, sig });
327        }
328
329        temp_file
330            .as_file()
331            .write_all(&proto.encode_to_vec())
332            .map_err(to_other_err)?;
333
334        let id = CommitId::new(blake2b_hash(&commit).to_vec());
335
336        persist_content_addressed_temp_file(temp_file, self.commit_path(&id))
337            .map_err(to_other_err)?;
338        Ok((id, commit))
339    }
340
341    fn get_copy_records(
342        &self,
343        _paths: Option<&[RepoPathBuf]>,
344        _root: &CommitId,
345        _head: &CommitId,
346    ) -> BackendResult<BoxStream<BackendResult<CopyRecord>>> {
347        Ok(Box::pin(stream::empty()))
348    }
349
350    fn gc(&self, _index: &dyn Index, _keep_newer: SystemTime) -> BackendResult<()> {
351        Ok(())
352    }
353}
354
355#[allow(clippy::assigning_clones)]
356pub fn commit_to_proto(commit: &Commit) -> crate::protos::simple_store::Commit {
357    let mut proto = crate::protos::simple_store::Commit::default();
358    for parent in &commit.parents {
359        proto.parents.push(parent.to_bytes());
360    }
361    for predecessor in &commit.predecessors {
362        proto.predecessors.push(predecessor.to_bytes());
363    }
364    match &commit.root_tree {
365        MergedTreeId::Legacy(tree_id) => {
366            proto.root_tree = vec![tree_id.to_bytes()];
367        }
368        MergedTreeId::Merge(tree_ids) => {
369            proto.uses_tree_conflict_format = true;
370            proto.root_tree = tree_ids.iter().map(|id| id.to_bytes()).collect();
371        }
372    }
373    proto.change_id = commit.change_id.to_bytes();
374    proto.description = commit.description.clone();
375    proto.author = Some(signature_to_proto(&commit.author));
376    proto.committer = Some(signature_to_proto(&commit.committer));
377    proto
378}
379
380fn commit_from_proto(mut proto: crate::protos::simple_store::Commit) -> Commit {
381    // Note how .take() sets the secure_sig field to None before we encode the data.
382    // Needs to be done first since proto is partially moved a bunch below
383    let secure_sig = proto.secure_sig.take().map(|sig| SecureSig {
384        data: proto.encode_to_vec(),
385        sig,
386    });
387
388    let parents = proto.parents.into_iter().map(CommitId::new).collect();
389    let predecessors = proto.predecessors.into_iter().map(CommitId::new).collect();
390    let root_tree = if proto.uses_tree_conflict_format {
391        let merge_builder: MergeBuilder<_> = proto.root_tree.into_iter().map(TreeId::new).collect();
392        MergedTreeId::Merge(merge_builder.build())
393    } else {
394        assert_eq!(proto.root_tree.len(), 1);
395        MergedTreeId::Legacy(TreeId::new(proto.root_tree[0].clone()))
396    };
397    let change_id = ChangeId::new(proto.change_id);
398    Commit {
399        parents,
400        predecessors,
401        root_tree,
402        change_id,
403        description: proto.description,
404        author: signature_from_proto(proto.author.unwrap_or_default()),
405        committer: signature_from_proto(proto.committer.unwrap_or_default()),
406        secure_sig,
407    }
408}
409
410fn tree_to_proto(tree: &Tree) -> crate::protos::simple_store::Tree {
411    let mut proto = crate::protos::simple_store::Tree::default();
412    for entry in tree.entries() {
413        proto
414            .entries
415            .push(crate::protos::simple_store::tree::Entry {
416                name: entry.name().as_internal_str().to_owned(),
417                value: Some(tree_value_to_proto(entry.value())),
418            });
419    }
420    proto
421}
422
423fn tree_from_proto(proto: crate::protos::simple_store::Tree) -> Tree {
424    let mut tree = Tree::default();
425    for proto_entry in proto.entries {
426        let value = tree_value_from_proto(proto_entry.value.unwrap());
427        tree.set(RepoPathComponentBuf::new(proto_entry.name).unwrap(), value);
428    }
429    tree
430}
431
432fn tree_value_to_proto(value: &TreeValue) -> crate::protos::simple_store::TreeValue {
433    let mut proto = crate::protos::simple_store::TreeValue::default();
434    match value {
435        TreeValue::File { id, executable } => {
436            proto.value = Some(crate::protos::simple_store::tree_value::Value::File(
437                crate::protos::simple_store::tree_value::File {
438                    id: id.to_bytes(),
439                    executable: *executable,
440                },
441            ));
442        }
443        TreeValue::Symlink(id) => {
444            proto.value = Some(crate::protos::simple_store::tree_value::Value::SymlinkId(
445                id.to_bytes(),
446            ));
447        }
448        TreeValue::GitSubmodule(_id) => {
449            panic!("cannot store git submodules");
450        }
451        TreeValue::Tree(id) => {
452            proto.value = Some(crate::protos::simple_store::tree_value::Value::TreeId(
453                id.to_bytes(),
454            ));
455        }
456        TreeValue::Conflict(id) => {
457            proto.value = Some(crate::protos::simple_store::tree_value::Value::ConflictId(
458                id.to_bytes(),
459            ));
460        }
461    }
462    proto
463}
464
465fn tree_value_from_proto(proto: crate::protos::simple_store::TreeValue) -> TreeValue {
466    match proto.value.unwrap() {
467        crate::protos::simple_store::tree_value::Value::TreeId(id) => {
468            TreeValue::Tree(TreeId::new(id))
469        }
470        crate::protos::simple_store::tree_value::Value::File(
471            crate::protos::simple_store::tree_value::File { id, executable, .. },
472        ) => TreeValue::File {
473            id: FileId::new(id),
474            executable,
475        },
476        crate::protos::simple_store::tree_value::Value::SymlinkId(id) => {
477            TreeValue::Symlink(SymlinkId::new(id))
478        }
479        crate::protos::simple_store::tree_value::Value::ConflictId(id) => {
480            TreeValue::Conflict(ConflictId::new(id))
481        }
482    }
483}
484
485fn signature_to_proto(signature: &Signature) -> crate::protos::simple_store::commit::Signature {
486    crate::protos::simple_store::commit::Signature {
487        name: signature.name.clone(),
488        email: signature.email.clone(),
489        timestamp: Some(crate::protos::simple_store::commit::Timestamp {
490            millis_since_epoch: signature.timestamp.timestamp.0,
491            tz_offset: signature.timestamp.tz_offset,
492        }),
493    }
494}
495
496fn signature_from_proto(proto: crate::protos::simple_store::commit::Signature) -> Signature {
497    let timestamp = proto.timestamp.unwrap_or_default();
498    Signature {
499        name: proto.name,
500        email: proto.email,
501        timestamp: Timestamp {
502            timestamp: MillisSinceEpoch(timestamp.millis_since_epoch),
503            tz_offset: timestamp.tz_offset,
504        },
505    }
506}
507
508fn conflict_to_proto(conflict: &Conflict) -> crate::protos::simple_store::Conflict {
509    let mut proto = crate::protos::simple_store::Conflict::default();
510    for term in &conflict.removes {
511        proto.removes.push(conflict_term_to_proto(term));
512    }
513    for term in &conflict.adds {
514        proto.adds.push(conflict_term_to_proto(term));
515    }
516    proto
517}
518
519fn conflict_from_proto(proto: crate::protos::simple_store::Conflict) -> Conflict {
520    let removes = proto
521        .removes
522        .into_iter()
523        .map(conflict_term_from_proto)
524        .collect();
525    let adds = proto
526        .adds
527        .into_iter()
528        .map(conflict_term_from_proto)
529        .collect();
530    Conflict { removes, adds }
531}
532
533fn conflict_term_from_proto(proto: crate::protos::simple_store::conflict::Term) -> ConflictTerm {
534    ConflictTerm {
535        value: tree_value_from_proto(proto.content.unwrap()),
536    }
537}
538
539fn conflict_term_to_proto(part: &ConflictTerm) -> crate::protos::simple_store::conflict::Term {
540    crate::protos::simple_store::conflict::Term {
541        content: Some(tree_value_to_proto(&part.value)),
542    }
543}
544
545#[cfg(test)]
546mod tests {
547    use assert_matches::assert_matches;
548    use pollster::FutureExt as _;
549
550    use super::*;
551    use crate::tests::new_temp_dir;
552
553    /// Test that parents get written correctly
554    #[test]
555    fn write_commit_parents() {
556        let temp_dir = new_temp_dir();
557        let store_path = temp_dir.path();
558
559        let backend = SimpleBackend::init(store_path);
560        let mut commit = Commit {
561            parents: vec![],
562            predecessors: vec![],
563            root_tree: MergedTreeId::resolved(backend.empty_tree_id().clone()),
564            change_id: ChangeId::from_hex("abc123"),
565            description: "".to_string(),
566            author: create_signature(),
567            committer: create_signature(),
568            secure_sig: None,
569        };
570
571        let write_commit = |commit: Commit| -> BackendResult<(CommitId, Commit)> {
572            backend.write_commit(commit, None).block_on()
573        };
574
575        // No parents
576        commit.parents = vec![];
577        assert_matches!(
578            write_commit(commit.clone()),
579            Err(BackendError::Other(err)) if err.to_string().contains("no parents")
580        );
581
582        // Only root commit as parent
583        commit.parents = vec![backend.root_commit_id().clone()];
584        let first_id = write_commit(commit.clone()).unwrap().0;
585        let first_commit = backend.read_commit(&first_id).block_on().unwrap();
586        assert_eq!(first_commit, commit);
587
588        // Only non-root commit as parent
589        commit.parents = vec![first_id.clone()];
590        let second_id = write_commit(commit.clone()).unwrap().0;
591        let second_commit = backend.read_commit(&second_id).block_on().unwrap();
592        assert_eq!(second_commit, commit);
593
594        // Merge commit
595        commit.parents = vec![first_id.clone(), second_id.clone()];
596        let merge_id = write_commit(commit.clone()).unwrap().0;
597        let merge_commit = backend.read_commit(&merge_id).block_on().unwrap();
598        assert_eq!(merge_commit, commit);
599
600        // Merge commit with root as one parent
601        commit.parents = vec![first_id, backend.root_commit_id().clone()];
602        let root_merge_id = write_commit(commit.clone()).unwrap().0;
603        let root_merge_commit = backend.read_commit(&root_merge_id).block_on().unwrap();
604        assert_eq!(root_merge_commit, commit);
605    }
606
607    fn create_signature() -> Signature {
608        Signature {
609            name: "Someone".to_string(),
610            email: "someone@example.com".to_string(),
611            timestamp: Timestamp {
612                timestamp: MillisSinceEpoch(0),
613                tz_offset: 0,
614            },
615        }
616    }
617}
jj_lib/simple_backend.rs

jj_lib/
simple_backend.rs
