jj_lib/
local_backend.rs

1// Copyright 2020 The Jujutsu Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![allow(missing_docs)]
16
17use std::any::Any;
18use std::fmt::Debug;
19use std::fs;
20use std::fs::File;
21use std::io::Read;
22use std::io::Write;
23use std::path::Path;
24use std::path::PathBuf;
25use std::time::SystemTime;
26
27use async_trait::async_trait;
28use blake2::Blake2b512;
29use blake2::Digest;
30use futures::stream;
31use futures::stream::BoxStream;
32use pollster::FutureExt;
33use prost::Message;
34use tempfile::NamedTempFile;
35
36use crate::backend::make_root_commit;
37use crate::backend::Backend;
38use crate::backend::BackendError;
39use crate::backend::BackendResult;
40use crate::backend::ChangeId;
41use crate::backend::Commit;
42use crate::backend::CommitId;
43use crate::backend::Conflict;
44use crate::backend::ConflictId;
45use crate::backend::ConflictTerm;
46use crate::backend::CopyRecord;
47use crate::backend::FileId;
48use crate::backend::MergedTreeId;
49use crate::backend::MillisSinceEpoch;
50use crate::backend::SecureSig;
51use crate::backend::Signature;
52use crate::backend::SigningFn;
53use crate::backend::SymlinkId;
54use crate::backend::Timestamp;
55use crate::backend::Tree;
56use crate::backend::TreeId;
57use crate::backend::TreeValue;
58use crate::content_hash::blake2b_hash;
59use crate::file_util::persist_content_addressed_temp_file;
60use crate::index::Index;
61use crate::merge::MergeBuilder;
62use crate::object_id::ObjectId;
63use crate::repo_path::RepoPath;
64use crate::repo_path::RepoPathBuf;
65use crate::repo_path::RepoPathComponentBuf;
66
67const COMMIT_ID_LENGTH: usize = 64;
68const CHANGE_ID_LENGTH: usize = 16;
69
70fn map_not_found_err(err: std::io::Error, id: &impl ObjectId) -> BackendError {
71    if err.kind() == std::io::ErrorKind::NotFound {
72        BackendError::ObjectNotFound {
73            object_type: id.object_type(),
74            hash: id.hex(),
75            source: Box::new(err),
76        }
77    } else {
78        BackendError::ReadObject {
79            object_type: id.object_type(),
80            hash: id.hex(),
81            source: Box::new(err),
82        }
83    }
84}
85
86fn to_other_err(err: impl Into<Box<dyn std::error::Error + Send + Sync>>) -> BackendError {
87    BackendError::Other(err.into())
88}
89
90#[derive(Debug)]
91pub struct LocalBackend {
92    path: PathBuf,
93    root_commit_id: CommitId,
94    root_change_id: ChangeId,
95    empty_tree_id: TreeId,
96}
97
98impl LocalBackend {
99    pub fn name() -> &'static str {
100        "local"
101    }
102
103    pub fn init(store_path: &Path) -> Self {
104        fs::create_dir(store_path.join("commits")).unwrap();
105        fs::create_dir(store_path.join("trees")).unwrap();
106        fs::create_dir(store_path.join("files")).unwrap();
107        fs::create_dir(store_path.join("symlinks")).unwrap();
108        fs::create_dir(store_path.join("conflicts")).unwrap();
109        let backend = Self::load(store_path);
110        let empty_tree_id = backend
111            .write_tree(RepoPath::root(), &Tree::default())
112            .block_on()
113            .unwrap();
114        assert_eq!(empty_tree_id, backend.empty_tree_id);
115        backend
116    }
117
118    pub fn load(store_path: &Path) -> Self {
119        let root_commit_id = CommitId::from_bytes(&[0; COMMIT_ID_LENGTH]);
120        let root_change_id = ChangeId::from_bytes(&[0; CHANGE_ID_LENGTH]);
121        let empty_tree_id = TreeId::from_hex(
122            "482ae5a29fbe856c7272f2071b8b0f0359ee2d89ff392b8a900643fbd0836eccd067b8bf41909e206c90d45d6e7d8b6686b93ecaee5fe1a9060d87b672101310",
123        );
124        LocalBackend {
125            path: store_path.to_path_buf(),
126            root_commit_id,
127            root_change_id,
128            empty_tree_id,
129        }
130    }
131
132    fn file_path(&self, id: &FileId) -> PathBuf {
133        self.path.join("files").join(id.hex())
134    }
135
136    fn symlink_path(&self, id: &SymlinkId) -> PathBuf {
137        self.path.join("symlinks").join(id.hex())
138    }
139
140    fn tree_path(&self, id: &TreeId) -> PathBuf {
141        self.path.join("trees").join(id.hex())
142    }
143
144    fn commit_path(&self, id: &CommitId) -> PathBuf {
145        self.path.join("commits").join(id.hex())
146    }
147
148    fn conflict_path(&self, id: &ConflictId) -> PathBuf {
149        self.path.join("conflicts").join(id.hex())
150    }
151}
152
153#[async_trait]
154impl Backend for LocalBackend {
155    fn as_any(&self) -> &dyn Any {
156        self
157    }
158
159    fn name(&self) -> &str {
160        Self::name()
161    }
162
163    fn commit_id_length(&self) -> usize {
164        COMMIT_ID_LENGTH
165    }
166
167    fn change_id_length(&self) -> usize {
168        CHANGE_ID_LENGTH
169    }
170
171    fn root_commit_id(&self) -> &CommitId {
172        &self.root_commit_id
173    }
174
175    fn root_change_id(&self) -> &ChangeId {
176        &self.root_change_id
177    }
178
179    fn empty_tree_id(&self) -> &TreeId {
180        &self.empty_tree_id
181    }
182
183    fn concurrency(&self) -> usize {
184        1
185    }
186
187    async fn read_file(&self, _path: &RepoPath, id: &FileId) -> BackendResult<Box<dyn Read>> {
188        let path = self.file_path(id);
189        let file = File::open(path).map_err(|err| map_not_found_err(err, id))?;
190        Ok(Box::new(file))
191    }
192
193    async fn write_file(
194        &self,
195        _path: &RepoPath,
196        contents: &mut (dyn Read + Send),
197    ) -> BackendResult<FileId> {
198        // TODO: Write temporary file in the destination directory (#5712)
199        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
200        let mut file = temp_file.as_file();
201        let mut hasher = Blake2b512::new();
202        let mut buff: Vec<u8> = vec![0; 1 << 14];
203        loop {
204            let bytes_read = contents.read(&mut buff).map_err(to_other_err)?;
205            if bytes_read == 0 {
206                break;
207            }
208            let bytes = &buff[..bytes_read];
209            file.write_all(bytes).map_err(to_other_err)?;
210            hasher.update(bytes);
211        }
212        file.flush().map_err(to_other_err)?;
213        let id = FileId::new(hasher.finalize().to_vec());
214
215        persist_content_addressed_temp_file(temp_file, self.file_path(&id))
216            .map_err(to_other_err)?;
217        Ok(id)
218    }
219
220    async fn read_symlink(&self, _path: &RepoPath, id: &SymlinkId) -> BackendResult<String> {
221        let path = self.symlink_path(id);
222        let target = fs::read_to_string(path).map_err(|err| map_not_found_err(err, id))?;
223        Ok(target)
224    }
225
226    async fn write_symlink(&self, _path: &RepoPath, target: &str) -> BackendResult<SymlinkId> {
227        // TODO: Write temporary file in the destination directory (#5712)
228        let mut temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
229        temp_file
230            .write_all(target.as_bytes())
231            .map_err(to_other_err)?;
232        let mut hasher = Blake2b512::new();
233        hasher.update(target.as_bytes());
234        let id = SymlinkId::new(hasher.finalize().to_vec());
235
236        persist_content_addressed_temp_file(temp_file, self.symlink_path(&id))
237            .map_err(to_other_err)?;
238        Ok(id)
239    }
240
241    async fn read_tree(&self, _path: &RepoPath, id: &TreeId) -> BackendResult<Tree> {
242        let path = self.tree_path(id);
243        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
244
245        let proto = crate::protos::local_store::Tree::decode(&*buf).map_err(to_other_err)?;
246        Ok(tree_from_proto(proto))
247    }
248
249    async fn write_tree(&self, _path: &RepoPath, tree: &Tree) -> BackendResult<TreeId> {
250        // TODO: Write temporary file in the destination directory (#5712)
251        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
252
253        let proto = tree_to_proto(tree);
254        temp_file
255            .as_file()
256            .write_all(&proto.encode_to_vec())
257            .map_err(to_other_err)?;
258
259        let id = TreeId::new(blake2b_hash(tree).to_vec());
260
261        persist_content_addressed_temp_file(temp_file, self.tree_path(&id))
262            .map_err(to_other_err)?;
263        Ok(id)
264    }
265
266    fn read_conflict(&self, _path: &RepoPath, id: &ConflictId) -> BackendResult<Conflict> {
267        let path = self.conflict_path(id);
268        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
269
270        let proto = crate::protos::local_store::Conflict::decode(&*buf).map_err(to_other_err)?;
271        Ok(conflict_from_proto(proto))
272    }
273
274    fn write_conflict(&self, _path: &RepoPath, conflict: &Conflict) -> BackendResult<ConflictId> {
275        // TODO: Write temporary file in the destination directory (#5712)
276        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
277
278        let proto = conflict_to_proto(conflict);
279        temp_file
280            .as_file()
281            .write_all(&proto.encode_to_vec())
282            .map_err(to_other_err)?;
283
284        let id = ConflictId::new(blake2b_hash(conflict).to_vec());
285
286        persist_content_addressed_temp_file(temp_file, self.conflict_path(&id))
287            .map_err(to_other_err)?;
288        Ok(id)
289    }
290
291    async fn read_commit(&self, id: &CommitId) -> BackendResult<Commit> {
292        if *id == self.root_commit_id {
293            return Ok(make_root_commit(
294                self.root_change_id().clone(),
295                self.empty_tree_id.clone(),
296            ));
297        }
298
299        let path = self.commit_path(id);
300        let buf = fs::read(path).map_err(|err| map_not_found_err(err, id))?;
301
302        let proto = crate::protos::local_store::Commit::decode(&*buf).map_err(to_other_err)?;
303        Ok(commit_from_proto(proto))
304    }
305
306    async fn write_commit(
307        &self,
308        mut commit: Commit,
309        sign_with: Option<&mut SigningFn>,
310    ) -> BackendResult<(CommitId, Commit)> {
311        assert!(commit.secure_sig.is_none(), "commit.secure_sig was set");
312
313        if commit.parents.is_empty() {
314            return Err(BackendError::Other(
315                "Cannot write a commit with no parents".into(),
316            ));
317        }
318        // TODO: Write temporary file in the destination directory (#5712)
319        let temp_file = NamedTempFile::new_in(&self.path).map_err(to_other_err)?;
320
321        let mut proto = commit_to_proto(&commit);
322        if let Some(sign) = sign_with {
323            let data = proto.encode_to_vec();
324            let sig = sign(&data).map_err(to_other_err)?;
325            proto.secure_sig = Some(sig.clone());
326            commit.secure_sig = Some(SecureSig { data, sig });
327        }
328
329        temp_file
330            .as_file()
331            .write_all(&proto.encode_to_vec())
332            .map_err(to_other_err)?;
333
334        let id = CommitId::new(blake2b_hash(&commit).to_vec());
335
336        persist_content_addressed_temp_file(temp_file, self.commit_path(&id))
337            .map_err(to_other_err)?;
338        Ok((id, commit))
339    }
340
341    fn get_copy_records(
342        &self,
343        _paths: Option<&[RepoPathBuf]>,
344        _root: &CommitId,
345        _head: &CommitId,
346    ) -> BackendResult<BoxStream<BackendResult<CopyRecord>>> {
347        Ok(Box::pin(stream::empty()))
348    }
349
350    fn gc(&self, _index: &dyn Index, _keep_newer: SystemTime) -> BackendResult<()> {
351        Ok(())
352    }
353}
354
355#[allow(unknown_lints)] // XXX FIXME (aseipp): nightly bogons; re-test this occasionally
356#[allow(clippy::assigning_clones)]
357pub fn commit_to_proto(commit: &Commit) -> crate::protos::local_store::Commit {
358    let mut proto = crate::protos::local_store::Commit::default();
359    for parent in &commit.parents {
360        proto.parents.push(parent.to_bytes());
361    }
362    for predecessor in &commit.predecessors {
363        proto.predecessors.push(predecessor.to_bytes());
364    }
365    match &commit.root_tree {
366        MergedTreeId::Legacy(tree_id) => {
367            proto.root_tree = vec![tree_id.to_bytes()];
368        }
369        MergedTreeId::Merge(tree_ids) => {
370            proto.uses_tree_conflict_format = true;
371            proto.root_tree = tree_ids.iter().map(|id| id.to_bytes()).collect();
372        }
373    }
374    proto.change_id = commit.change_id.to_bytes();
375    proto.description = commit.description.clone();
376    proto.author = Some(signature_to_proto(&commit.author));
377    proto.committer = Some(signature_to_proto(&commit.committer));
378    proto
379}
380
381fn commit_from_proto(mut proto: crate::protos::local_store::Commit) -> Commit {
382    // Note how .take() sets the secure_sig field to None before we encode the data.
383    // Needs to be done first since proto is partially moved a bunch below
384    let secure_sig = proto.secure_sig.take().map(|sig| SecureSig {
385        data: proto.encode_to_vec(),
386        sig,
387    });
388
389    let parents = proto.parents.into_iter().map(CommitId::new).collect();
390    let predecessors = proto.predecessors.into_iter().map(CommitId::new).collect();
391    let root_tree = if proto.uses_tree_conflict_format {
392        let merge_builder: MergeBuilder<_> = proto.root_tree.into_iter().map(TreeId::new).collect();
393        MergedTreeId::Merge(merge_builder.build())
394    } else {
395        assert_eq!(proto.root_tree.len(), 1);
396        MergedTreeId::Legacy(TreeId::new(proto.root_tree[0].clone()))
397    };
398    let change_id = ChangeId::new(proto.change_id);
399    Commit {
400        parents,
401        predecessors,
402        root_tree,
403        change_id,
404        description: proto.description,
405        author: signature_from_proto(proto.author.unwrap_or_default()),
406        committer: signature_from_proto(proto.committer.unwrap_or_default()),
407        secure_sig,
408    }
409}
410
411fn tree_to_proto(tree: &Tree) -> crate::protos::local_store::Tree {
412    let mut proto = crate::protos::local_store::Tree::default();
413    for entry in tree.entries() {
414        proto.entries.push(crate::protos::local_store::tree::Entry {
415            name: entry.name().as_internal_str().to_owned(),
416            value: Some(tree_value_to_proto(entry.value())),
417        });
418    }
419    proto
420}
421
422fn tree_from_proto(proto: crate::protos::local_store::Tree) -> Tree {
423    let mut tree = Tree::default();
424    for proto_entry in proto.entries {
425        let value = tree_value_from_proto(proto_entry.value.unwrap());
426        tree.set(RepoPathComponentBuf::from(proto_entry.name), value);
427    }
428    tree
429}
430
431fn tree_value_to_proto(value: &TreeValue) -> crate::protos::local_store::TreeValue {
432    let mut proto = crate::protos::local_store::TreeValue::default();
433    match value {
434        TreeValue::File { id, executable } => {
435            proto.value = Some(crate::protos::local_store::tree_value::Value::File(
436                crate::protos::local_store::tree_value::File {
437                    id: id.to_bytes(),
438                    executable: *executable,
439                },
440            ));
441        }
442        TreeValue::Symlink(id) => {
443            proto.value = Some(crate::protos::local_store::tree_value::Value::SymlinkId(
444                id.to_bytes(),
445            ));
446        }
447        TreeValue::GitSubmodule(_id) => {
448            panic!("cannot store git submodules");
449        }
450        TreeValue::Tree(id) => {
451            proto.value = Some(crate::protos::local_store::tree_value::Value::TreeId(
452                id.to_bytes(),
453            ));
454        }
455        TreeValue::Conflict(id) => {
456            proto.value = Some(crate::protos::local_store::tree_value::Value::ConflictId(
457                id.to_bytes(),
458            ));
459        }
460    }
461    proto
462}
463
464fn tree_value_from_proto(proto: crate::protos::local_store::TreeValue) -> TreeValue {
465    match proto.value.unwrap() {
466        crate::protos::local_store::tree_value::Value::TreeId(id) => {
467            TreeValue::Tree(TreeId::new(id))
468        }
469        crate::protos::local_store::tree_value::Value::File(
470            crate::protos::local_store::tree_value::File { id, executable, .. },
471        ) => TreeValue::File {
472            id: FileId::new(id),
473            executable,
474        },
475        crate::protos::local_store::tree_value::Value::SymlinkId(id) => {
476            TreeValue::Symlink(SymlinkId::new(id))
477        }
478        crate::protos::local_store::tree_value::Value::ConflictId(id) => {
479            TreeValue::Conflict(ConflictId::new(id))
480        }
481    }
482}
483
484fn signature_to_proto(signature: &Signature) -> crate::protos::local_store::commit::Signature {
485    crate::protos::local_store::commit::Signature {
486        name: signature.name.clone(),
487        email: signature.email.clone(),
488        timestamp: Some(crate::protos::local_store::commit::Timestamp {
489            millis_since_epoch: signature.timestamp.timestamp.0,
490            tz_offset: signature.timestamp.tz_offset,
491        }),
492    }
493}
494
495fn signature_from_proto(proto: crate::protos::local_store::commit::Signature) -> Signature {
496    let timestamp = proto.timestamp.unwrap_or_default();
497    Signature {
498        name: proto.name,
499        email: proto.email,
500        timestamp: Timestamp {
501            timestamp: MillisSinceEpoch(timestamp.millis_since_epoch),
502            tz_offset: timestamp.tz_offset,
503        },
504    }
505}
506
507fn conflict_to_proto(conflict: &Conflict) -> crate::protos::local_store::Conflict {
508    let mut proto = crate::protos::local_store::Conflict::default();
509    for term in &conflict.removes {
510        proto.removes.push(conflict_term_to_proto(term));
511    }
512    for term in &conflict.adds {
513        proto.adds.push(conflict_term_to_proto(term));
514    }
515    proto
516}
517
518fn conflict_from_proto(proto: crate::protos::local_store::Conflict) -> Conflict {
519    let removes = proto
520        .removes
521        .into_iter()
522        .map(conflict_term_from_proto)
523        .collect();
524    let adds = proto
525        .adds
526        .into_iter()
527        .map(conflict_term_from_proto)
528        .collect();
529    Conflict { removes, adds }
530}
531
532fn conflict_term_from_proto(proto: crate::protos::local_store::conflict::Term) -> ConflictTerm {
533    ConflictTerm {
534        value: tree_value_from_proto(proto.content.unwrap()),
535    }
536}
537
538fn conflict_term_to_proto(part: &ConflictTerm) -> crate::protos::local_store::conflict::Term {
539    crate::protos::local_store::conflict::Term {
540        content: Some(tree_value_to_proto(&part.value)),
541    }
542}
543
544#[cfg(test)]
545mod tests {
546    use assert_matches::assert_matches;
547    use pollster::FutureExt;
548
549    use super::*;
550    use crate::tests::new_temp_dir;
551
552    /// Test that parents get written correctly
553    #[test]
554    fn write_commit_parents() {
555        let temp_dir = new_temp_dir();
556        let store_path = temp_dir.path();
557
558        let backend = LocalBackend::init(store_path);
559        let mut commit = Commit {
560            parents: vec![],
561            predecessors: vec![],
562            root_tree: MergedTreeId::resolved(backend.empty_tree_id().clone()),
563            change_id: ChangeId::from_hex("abc123"),
564            description: "".to_string(),
565            author: create_signature(),
566            committer: create_signature(),
567            secure_sig: None,
568        };
569
570        let write_commit = |commit: Commit| -> BackendResult<(CommitId, Commit)> {
571            backend.write_commit(commit, None).block_on()
572        };
573
574        // No parents
575        commit.parents = vec![];
576        assert_matches!(
577            write_commit(commit.clone()),
578            Err(BackendError::Other(err)) if err.to_string().contains("no parents")
579        );
580
581        // Only root commit as parent
582        commit.parents = vec![backend.root_commit_id().clone()];
583        let first_id = write_commit(commit.clone()).unwrap().0;
584        let first_commit = backend.read_commit(&first_id).block_on().unwrap();
585        assert_eq!(first_commit, commit);
586
587        // Only non-root commit as parent
588        commit.parents = vec![first_id.clone()];
589        let second_id = write_commit(commit.clone()).unwrap().0;
590        let second_commit = backend.read_commit(&second_id).block_on().unwrap();
591        assert_eq!(second_commit, commit);
592
593        // Merge commit
594        commit.parents = vec![first_id.clone(), second_id.clone()];
595        let merge_id = write_commit(commit.clone()).unwrap().0;
596        let merge_commit = backend.read_commit(&merge_id).block_on().unwrap();
597        assert_eq!(merge_commit, commit);
598
599        // Merge commit with root as one parent
600        commit.parents = vec![first_id, backend.root_commit_id().clone()];
601        let root_merge_id = write_commit(commit.clone()).unwrap().0;
602        let root_merge_commit = backend.read_commit(&root_merge_id).block_on().unwrap();
603        assert_eq!(root_merge_commit, commit);
604    }
605
606    fn create_signature() -> Signature {
607        Signature {
608            name: "Someone".to_string(),
609            email: "someone@example.com".to_string(),
610            timestamp: Timestamp {
611                timestamp: MillisSinceEpoch(0),
612                tz_offset: 0,
613            },
614        }
615    }
616}
jj_lib/local_backend.rs

jj_lib/
local_backend.rs
