jj_lib/
simple_backend.rs

1// Copyright 2020 The Jujutsu Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![allow(missing_docs)]
16
17use std::any::Any;
18use std::fmt::Debug;
19use std::fs;
20use std::fs::File;
21use std::io::Cursor;
22use std::io::Read as _;
23use std::io::Write as _;
24use std::path::Path;
25use std::path::PathBuf;
26use std::pin::Pin;
27use std::time::SystemTime;
28
29use async_trait::async_trait;
30use blake2::Blake2b512;
31use blake2::Digest as _;
32use futures::stream;
33use futures::stream::BoxStream;
34use pollster::FutureExt as _;
35use prost::Message as _;
36use tempfile::NamedTempFile;
37use tokio::io::AsyncRead;
38use tokio::io::AsyncReadExt as _;
39
40use crate::backend::make_root_commit;
41use crate::backend::Backend;
42use crate::backend::BackendError;
43use crate::backend::BackendResult;
44use crate::backend::ChangeId;
45use crate::backend::Commit;
46use crate::backend::CommitId;
47use crate::backend::Conflict;
48use crate::backend::ConflictId;
49use crate::backend::ConflictTerm;
50use crate::backend::CopyHistory;
51use crate::backend::CopyId;
52use crate::backend::CopyRecord;
53use crate::backend::FileId;
54use crate::backend::MergedTreeId;
55use crate::backend::MillisSinceEpoch;
56use crate::backend::SecureSig;
57use crate::backend::Signature;
58use crate::backend::SigningFn;
59use crate::backend::SymlinkId;
60use crate::backend::Timestamp;
61use crate::backend::Tree;
62use crate::backend::TreeId;
63use crate::backend::TreeValue;
64use crate::content_hash::blake2b_hash;
65use crate::file_util::persist_content_addressed_temp_file;
66use crate::index::Index;
67use crate::merge::MergeBuilder;
68use crate::object_id::ObjectId;
69use crate::repo_path::RepoPath;
70use crate::repo_path::RepoPathBuf;
71use crate::repo_path::RepoPathComponentBuf;
72
73const COMMIT_ID_LENGTH: usize = 64;
74const CHANGE_ID_LENGTH: usize = 16;
75
76fn map_not_found_err(err: std::io::Error, id: &impl ObjectId) -> BackendError {
77    if err.kind() == std::io::ErrorKind::NotFound {
78        BackendError::ObjectNotFound {
79            object_type: id.object_type(),
80            hash: id.hex(),
81            source: Box::new(err),
82        }
83    } else {
84        BackendError::ReadObject {
85            object_type: id.object_type(),
86            hash: id.hex(),
87            source: Box::new(err),
88        }
89    }
90}
91
92fn to_other_err(err: impl Into<Box<dyn std::error::Error + Send + Sync>>) -> BackendError {
93    BackendError::Other(err.into())
94}
95
96#[derive(Debug)]
97pub struct SimpleBackend {
98    path: PathBuf,
99    root_commit_id: CommitId,
100    root_change_id: ChangeId,
101    empty_tree_id: TreeId,
102}
103
104impl SimpleBackend {
105    pub fn name() -> &'static str {
106        "Simple"
107    }
108
109    pub fn init(store_path: &Path) -> Self {
110        fs::create_dir(store_path.join("commits")).unwrap();
111        fs::create_dir(store_path.join("trees")).unwrap();
112        fs::create_dir(store_path.join("files")).unwrap();
113        fs::create_dir(store_path.join("symlinks")).unwrap();
114        fs::create_dir(store_path.join("conflicts")).unwrap();
115        let backend = Self::load(store_path);
116        let empty_tree_id = backend
117            .write_tree(RepoPath::root(), &Tree::default())
118            .block_on()
119            .unwrap();
120        assert_eq!(empty_tree_id, backend.empty_tree_id);
121        backend
122    }
123
124    pub fn load(store_path: &Path) -> Self {
125        let root_commit_id = CommitId::from_bytes(&[0; COMMIT_ID_LENGTH]);
126        let root_change_id = ChangeId::from_bytes(&[0; CHANGE_ID_LENGTH]);
127        let empty_tree_id = TreeId::from_hex(
128            "482ae5a29fbe856c7272f2071b8b0f0359ee2d89ff392b8a900643fbd0836eccd067b8bf41909e206c90d45d6e7d8b6686b93ecaee5fe1a9060d87b672101310",
129        );
130        SimpleBackend {
131            path: store_path.to_path_buf(),
132            root_commit_id,
133            root_change_id,
134            empty_tree_id,
135        }
136    }
137
138    fn file_path(&self, id: &FileId) -> PathBuf {
139        self.path.join("files").join(id.hex())
140    }
141
142    fn symlink_path(&self, id: &SymlinkId) -> PathBuf {
143        self.path.join("symlinks").join(id.hex())
144    }
145
146    fn tree_path(&self, id: &TreeId) -> PathBuf {
147        self.path.join("trees").join(id.hex())
148    }
149
150    fn commit_path(&self, id: &CommitId) -> PathBuf {
151        self.path.join("commits").join(id.hex())
152    }
153
154    fn conflict_path(&self, id: &ConflictId) -> PathBuf {
155        self.path.join("conflicts").join(id.hex())
156    }
157}
158
159#[async_trait]
160impl Backend for SimpleBackend {
161    fn as_any(&self) -> &dyn Any {
162        self
163    }
164
165    fn name(&self) -> &str {
166        Self::name()
167    }
168
169    fn commit_id_length(&self) -> usize {
170        COMMIT_ID_LENGTH
171    }
172
173    fn change_id_length(&self) -> usize {
174        CHANGE_ID_LENGTH
175    }
176
177    fn root_commit_id(&self) -> &CommitId {
178        &self.root_commit_id
179    }
180
181    fn root_change_id(&self) -> &ChangeId {
182        &self.root_change_id
183    }
184
185    fn empty_tree_id(&self) -> &TreeId {
186        &self.empty_tree_id
187    }
188
189    fn concurrency(&self) -> usize {
190        1
191    }
192
193    async fn read_file(
194        &self,
195        path: &RepoPath,
196        id: &FileId,
197    ) -> BackendResult<Pin<Box<dyn AsyncRead + Send>>> {
198        let disk_path = self.file_path(id);
199        let mut file = File::open(disk_path).map_err(|err| map_not_found_err(err, id))?;
200        let mut buf = vec![];
201        file.read_to_end(&mut buf)
202            .map_err(|err| BackendError::ReadFile {
203                path: path.to_owned(),
204                id: id.clone(),
205                source: err.into(),
206            })?;
207        Ok(Box::pin(Cursor::new(buf)))
208    }
209
210    async fn write_file(
211        &self,
212        _path: &RepoPath,
213        contents: &mut (dyn AsyncRead + Send + Unpin),
214    ) -> BackendResult<FileId> {
215        // TODO: Write temporary file in the destination directory (#5712)
216        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
217        let mut file = temp_file.as_file();
218        let mut hasher = Blake2b512::new();
219        let mut buff: Vec<u8> = vec![0; 1 << 14];
220        loop {
221            let bytes_read = contents.read(&mut buff).await.map_err(to_other_err)?;
222            if bytes_read == 0 {
223                break;
224            }
225            let bytes = &buff[..bytes_read];
226            file.write_all(bytes).map_err(to_other_err)?;
227            hasher.update(bytes);
228        }
229        file.flush().map_err(to_other_err)?;
230        let id = FileId::new(hasher.finalize().to_vec());
231
232        persist_content_addressed_temp_file(temp_file, self.file_path(&id))
233            .map_err(to_other_err)?;
234        Ok(id)
235    }
236
237    async fn read_symlink(&self, _path: &RepoPath, id: &SymlinkId) -> BackendResult<String> {
238        let path = self.symlink_path(id);
239        let target = fs::read_to_string(path).map_err(|err| map_not_found_err(err, id))?;
240        Ok(target)
241    }
242
243    async fn write_symlink(&self, _path: &RepoPath, target: &str) -> BackendResult<SymlinkId> {
244        // TODO: Write temporary file in the destination directory (#5712)
245        let mut temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
246        temp_file
247            .write_all(target.as_bytes())
248            .map_err(to_other_err)?;
249        let mut hasher = Blake2b512::new();
250        hasher.update(target.as_bytes());
251        let id = SymlinkId::new(hasher.finalize().to_vec());
252
253        persist_content_addressed_temp_file(temp_file, self.symlink_path(&id))
254            .map_err(to_other_err)?;
255        Ok(id)
256    }
257
258    async fn read_copy(&self, _id: &CopyId) -> BackendResult<CopyHistory> {
259        Err(BackendError::Unsupported(
260            "The simple backend doesn't support copies".to_string(),
261        ))
262    }
263
264    async fn write_copy(&self, _contents: &CopyHistory) -> BackendResult<CopyId> {
265        Err(BackendError::Unsupported(
266            "The simple backend doesn't support copies".to_string(),
267        ))
268    }
269
270    async fn get_related_copies(&self, _copy_id: &CopyId) -> BackendResult<Vec<CopyHistory>> {
271        Err(BackendError::Unsupported(
272            "The simple backend doesn't support copies".to_string(),
273        ))
274    }
275
276    async fn read_tree(&self, _path: &RepoPath, id: &TreeId) -> BackendResult<Tree> {
277        let path = self.tree_path(id);
278        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
279
280        let proto = crate::protos::simple_store::Tree::decode(&*buf).map_err(to_other_err)?;
281        Ok(tree_from_proto(proto))
282    }
283
284    async fn write_tree(&self, _path: &RepoPath, tree: &Tree) -> BackendResult<TreeId> {
285        // TODO: Write temporary file in the destination directory (#5712)
286        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
287
288        let proto = tree_to_proto(tree);
289        temp_file
290            .as_file()
291            .write_all(&proto.encode_to_vec())
292            .map_err(to_other_err)?;
293
294        let id = TreeId::new(blake2b_hash(tree).to_vec());
295
296        persist_content_addressed_temp_file(temp_file, self.tree_path(&id))
297            .map_err(to_other_err)?;
298        Ok(id)
299    }
300
301    fn read_conflict(&self, _path: &RepoPath, id: &ConflictId) -> BackendResult<Conflict> {
302        let path = self.conflict_path(id);
303        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
304
305        let proto = crate::protos::simple_store::Conflict::decode(&*buf).map_err(to_other_err)?;
306        Ok(conflict_from_proto(proto))
307    }
308
309    fn write_conflict(&self, _path: &RepoPath, conflict: &Conflict) -> BackendResult<ConflictId> {
310        // TODO: Write temporary file in the destination directory (#5712)
311        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
312
313        let proto = conflict_to_proto(conflict);
314        temp_file
315            .as_file()
316            .write_all(&proto.encode_to_vec())
317            .map_err(to_other_err)?;
318
319        let id = ConflictId::new(blake2b_hash(conflict).to_vec());
320
321        persist_content_addressed_temp_file(temp_file, self.conflict_path(&id))
322            .map_err(to_other_err)?;
323        Ok(id)
324    }
325
326    async fn read_commit(&self, id: &CommitId) -> BackendResult<Commit> {
327        if *id == self.root_commit_id {
328            return Ok(make_root_commit(
329                self.root_change_id().clone(),
330                self.empty_tree_id.clone(),
331            ));
332        }
333
334        let path = self.commit_path(id);
335        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
336
337        let proto = crate::protos::simple_store::Commit::decode(&*buf).map_err(to_other_err)?;
338        Ok(commit_from_proto(proto))
339    }
340
341    async fn write_commit(
342        &self,
343        mut commit: Commit,
344        sign_with: Option<&mut SigningFn>,
345    ) -> BackendResult<(CommitId, Commit)> {
346        assert!(commit.secure_sig.is_none(), "commit.secure_sig was set");
347
348        if commit.parents.is_empty() {
349            return Err(BackendError::Other(
350                "Cannot write a commit with no parents".into(),
351            ));
352        }
353        // TODO: Write temporary file in the destination directory (#5712)
354        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
355
356        let mut proto = commit_to_proto(&commit);
357        if let Some(sign) = sign_with {
358            let data = proto.encode_to_vec();
359            let sig = sign(&data).map_err(to_other_err)?;
360            proto.secure_sig = Some(sig.clone());
361            commit.secure_sig = Some(SecureSig { data, sig });
362        }
363
364        temp_file
365            .as_file()
366            .write_all(&proto.encode_to_vec())
367            .map_err(to_other_err)?;
368
369        let id = CommitId::new(blake2b_hash(&commit).to_vec());
370
371        persist_content_addressed_temp_file(temp_file, self.commit_path(&id))
372            .map_err(to_other_err)?;
373        Ok((id, commit))
374    }
375
376    fn get_copy_records(
377        &self,
378        _paths: Option<&[RepoPathBuf]>,
379        _root: &CommitId,
380        _head: &CommitId,
381    ) -> BackendResult<BoxStream<'_, BackendResult<CopyRecord>>> {
382        Ok(Box::pin(stream::empty()))
383    }
384
385    fn gc(&self, _index: &dyn Index, _keep_newer: SystemTime) -> BackendResult<()> {
386        Ok(())
387    }
388}
389
390#[allow(clippy::assigning_clones)]
391pub fn commit_to_proto(commit: &Commit) -> crate::protos::simple_store::Commit {
392    let mut proto = crate::protos::simple_store::Commit::default();
393    for parent in &commit.parents {
394        proto.parents.push(parent.to_bytes());
395    }
396    for predecessor in &commit.predecessors {
397        proto.predecessors.push(predecessor.to_bytes());
398    }
399    match &commit.root_tree {
400        MergedTreeId::Legacy(tree_id) => {
401            proto.root_tree = vec![tree_id.to_bytes()];
402        }
403        MergedTreeId::Merge(tree_ids) => {
404            proto.uses_tree_conflict_format = true;
405            proto.root_tree = tree_ids.iter().map(|id| id.to_bytes()).collect();
406        }
407    }
408    proto.change_id = commit.change_id.to_bytes();
409    proto.description = commit.description.clone();
410    proto.author = Some(signature_to_proto(&commit.author));
411    proto.committer = Some(signature_to_proto(&commit.committer));
412    proto
413}
414
415fn commit_from_proto(mut proto: crate::protos::simple_store::Commit) -> Commit {
416    // Note how .take() sets the secure_sig field to None before we encode the data.
417    // Needs to be done first since proto is partially moved a bunch below
418    let secure_sig = proto.secure_sig.take().map(|sig| SecureSig {
419        data: proto.encode_to_vec(),
420        sig,
421    });
422
423    let parents = proto.parents.into_iter().map(CommitId::new).collect();
424    let predecessors = proto.predecessors.into_iter().map(CommitId::new).collect();
425    let root_tree = if proto.uses_tree_conflict_format {
426        let merge_builder: MergeBuilder<_> = proto.root_tree.into_iter().map(TreeId::new).collect();
427        MergedTreeId::Merge(merge_builder.build())
428    } else {
429        assert_eq!(proto.root_tree.len(), 1);
430        MergedTreeId::Legacy(TreeId::new(proto.root_tree[0].clone()))
431    };
432    let change_id = ChangeId::new(proto.change_id);
433    Commit {
434        parents,
435        predecessors,
436        root_tree,
437        change_id,
438        description: proto.description,
439        author: signature_from_proto(proto.author.unwrap_or_default()),
440        committer: signature_from_proto(proto.committer.unwrap_or_default()),
441        secure_sig,
442    }
443}
444
445fn tree_to_proto(tree: &Tree) -> crate::protos::simple_store::Tree {
446    let mut proto = crate::protos::simple_store::Tree::default();
447    for entry in tree.entries() {
448        proto
449            .entries
450            .push(crate::protos::simple_store::tree::Entry {
451                name: entry.name().as_internal_str().to_owned(),
452                value: Some(tree_value_to_proto(entry.value())),
453            });
454    }
455    proto
456}
457
458fn tree_from_proto(proto: crate::protos::simple_store::Tree) -> Tree {
459    let mut tree = Tree::default();
460    for proto_entry in proto.entries {
461        let value = tree_value_from_proto(proto_entry.value.unwrap());
462        tree.set(RepoPathComponentBuf::new(proto_entry.name).unwrap(), value);
463    }
464    tree
465}
466
467fn tree_value_to_proto(value: &TreeValue) -> crate::protos::simple_store::TreeValue {
468    let mut proto = crate::protos::simple_store::TreeValue::default();
469    match value {
470        TreeValue::File {
471            id,
472            executable,
473            copy_id,
474        } => {
475            proto.value = Some(crate::protos::simple_store::tree_value::Value::File(
476                crate::protos::simple_store::tree_value::File {
477                    id: id.to_bytes(),
478                    executable: *executable,
479                    copy_id: copy_id.to_bytes(),
480                },
481            ));
482        }
483        TreeValue::Symlink(id) => {
484            proto.value = Some(crate::protos::simple_store::tree_value::Value::SymlinkId(
485                id.to_bytes(),
486            ));
487        }
488        TreeValue::GitSubmodule(_id) => {
489            panic!("cannot store git submodules");
490        }
491        TreeValue::Tree(id) => {
492            proto.value = Some(crate::protos::simple_store::tree_value::Value::TreeId(
493                id.to_bytes(),
494            ));
495        }
496        TreeValue::Conflict(id) => {
497            proto.value = Some(crate::protos::simple_store::tree_value::Value::ConflictId(
498                id.to_bytes(),
499            ));
500        }
501    }
502    proto
503}
504
505fn tree_value_from_proto(proto: crate::protos::simple_store::TreeValue) -> TreeValue {
506    match proto.value.unwrap() {
507        crate::protos::simple_store::tree_value::Value::TreeId(id) => {
508            TreeValue::Tree(TreeId::new(id))
509        }
510        crate::protos::simple_store::tree_value::Value::File(
511            crate::protos::simple_store::tree_value::File {
512                id,
513                executable,
514                copy_id,
515            },
516        ) => TreeValue::File {
517            id: FileId::new(id),
518            executable,
519            copy_id: CopyId::new(copy_id),
520        },
521        crate::protos::simple_store::tree_value::Value::SymlinkId(id) => {
522            TreeValue::Symlink(SymlinkId::new(id))
523        }
524        crate::protos::simple_store::tree_value::Value::ConflictId(id) => {
525            TreeValue::Conflict(ConflictId::new(id))
526        }
527    }
528}
529
530fn signature_to_proto(signature: &Signature) -> crate::protos::simple_store::commit::Signature {
531    crate::protos::simple_store::commit::Signature {
532        name: signature.name.clone(),
533        email: signature.email.clone(),
534        timestamp: Some(crate::protos::simple_store::commit::Timestamp {
535            millis_since_epoch: signature.timestamp.timestamp.0,
536            tz_offset: signature.timestamp.tz_offset,
537        }),
538    }
539}
540
541fn signature_from_proto(proto: crate::protos::simple_store::commit::Signature) -> Signature {
542    let timestamp = proto.timestamp.unwrap_or_default();
543    Signature {
544        name: proto.name,
545        email: proto.email,
546        timestamp: Timestamp {
547            timestamp: MillisSinceEpoch(timestamp.millis_since_epoch),
548            tz_offset: timestamp.tz_offset,
549        },
550    }
551}
552
553fn conflict_to_proto(conflict: &Conflict) -> crate::protos::simple_store::Conflict {
554    let mut proto = crate::protos::simple_store::Conflict::default();
555    for term in &conflict.removes {
556        proto.removes.push(conflict_term_to_proto(term));
557    }
558    for term in &conflict.adds {
559        proto.adds.push(conflict_term_to_proto(term));
560    }
561    proto
562}
563
564fn conflict_from_proto(proto: crate::protos::simple_store::Conflict) -> Conflict {
565    let removes = proto
566        .removes
567        .into_iter()
568        .map(conflict_term_from_proto)
569        .collect();
570    let adds = proto
571        .adds
572        .into_iter()
573        .map(conflict_term_from_proto)
574        .collect();
575    Conflict { removes, adds }
576}
577
578fn conflict_term_from_proto(proto: crate::protos::simple_store::conflict::Term) -> ConflictTerm {
579    ConflictTerm {
580        value: tree_value_from_proto(proto.content.unwrap()),
581    }
582}
583
584fn conflict_term_to_proto(part: &ConflictTerm) -> crate::protos::simple_store::conflict::Term {
585    crate::protos::simple_store::conflict::Term {
586        content: Some(tree_value_to_proto(&part.value)),
587    }
588}
589
590#[cfg(test)]
591mod tests {
592    use assert_matches::assert_matches;
593    use pollster::FutureExt as _;
594
595    use super::*;
596    use crate::tests::new_temp_dir;
597
598    /// Test that parents get written correctly
599    #[test]
600    fn write_commit_parents() {
601        let temp_dir = new_temp_dir();
602        let store_path = temp_dir.path();
603
604        let backend = SimpleBackend::init(store_path);
605        let mut commit = Commit {
606            parents: vec![],
607            predecessors: vec![],
608            root_tree: MergedTreeId::resolved(backend.empty_tree_id().clone()),
609            change_id: ChangeId::from_hex("abc123"),
610            description: "".to_string(),
611            author: create_signature(),
612            committer: create_signature(),
613            secure_sig: None,
614        };
615
616        let write_commit = |commit: Commit| -> BackendResult<(CommitId, Commit)> {
617            backend.write_commit(commit, None).block_on()
618        };
619
620        // No parents
621        commit.parents = vec![];
622        assert_matches!(
623            write_commit(commit.clone()),
624            Err(BackendError::Other(err)) if err.to_string().contains("no parents")
625        );
626
627        // Only root commit as parent
628        commit.parents = vec![backend.root_commit_id().clone()];
629        let first_id = write_commit(commit.clone()).unwrap().0;
630        let first_commit = backend.read_commit(&first_id).block_on().unwrap();
631        assert_eq!(first_commit, commit);
632
633        // Only non-root commit as parent
634        commit.parents = vec![first_id.clone()];
635        let second_id = write_commit(commit.clone()).unwrap().0;
636        let second_commit = backend.read_commit(&second_id).block_on().unwrap();
637        assert_eq!(second_commit, commit);
638
639        // Merge commit
640        commit.parents = vec![first_id.clone(), second_id.clone()];
641        let merge_id = write_commit(commit.clone()).unwrap().0;
642        let merge_commit = backend.read_commit(&merge_id).block_on().unwrap();
643        assert_eq!(merge_commit, commit);
644
645        // Merge commit with root as one parent
646        commit.parents = vec![first_id, backend.root_commit_id().clone()];
647        let root_merge_id = write_commit(commit.clone()).unwrap().0;
648        let root_merge_commit = backend.read_commit(&root_merge_id).block_on().unwrap();
649        assert_eq!(root_merge_commit, commit);
650    }
651
652    fn create_signature() -> Signature {
653        Signature {
654            name: "Someone".to_string(),
655            email: "someone@example.com".to_string(),
656            timestamp: Timestamp {
657                timestamp: MillisSinceEpoch(0),
658                tz_offset: 0,
659            },
660        }
661    }
662}
jj_lib/simple_backend.rs

jj_lib/
simple_backend.rs
