agd_core/document/
index.rs

1//! `WorkspaceIndex`: every agenda file under the roots, parsed, plus the
2//! `:ID:`s of the other org files there; resolving an item ref to its heading;
3//! and refreshing one changed path.
4//!
5//! A ref is the heading's `:ID:` when it is unique, else
6//! `file:line#sha256:digest`. An ID that occurs more than once anywhere under
7//! the roots is refused for writes (`id_conflict`). A non-UTF-8 file or a
8//! symlink out of the roots is skipped with a reason instead of failing the
9//! whole index.
10
11use std::collections::{BTreeMap, BTreeSet};
12use std::path::{Path, PathBuf};
13
14use crate::{AgdError, ErrorCode, Item, RootSet, Settings};
15
16use super::headline::{HeadlineSpan, location_ref};
17use super::org_document::OrgDocument;
18
19#[derive(Clone, Debug)]
20pub struct WorkspaceIndex {
21    roots: RootSet,
22    settings: Settings,
23    documents: BTreeMap<PathBuf, OrgDocument>,
24    ids: BTreeMap<String, Vec<(PathBuf, usize)>>,
25    // `:ID:` values found in org files under the roots that are not agenda
26    // files, so an ID shared with an archive or notes file is still ambiguous.
27    outside_ids: BTreeMap<String, Vec<PathBuf>>,
28    skipped: BTreeMap<PathBuf, String>,
29}
30
31fn scan_property_ids(path: &Path) -> Vec<String> {
32    let Ok(bytes) = std::fs::read(path) else {
33        return Vec::new();
34    };
35    String::from_utf8_lossy(&bytes)
36        .lines()
37        .filter_map(|line| {
38            let line = line.trim();
39            let key = line.get(..4)?;
40            if !key.eq_ignore_ascii_case(":ID:") {
41                return None;
42            }
43            let value = line[4..].trim();
44            (!value.is_empty()).then(|| value.to_string())
45        })
46        .collect()
47}
48
49impl WorkspaceIndex {
50    pub fn load(settings: Settings) -> Result<Self, AgdError> {
51        let roots = RootSet::new(&settings.roots, &settings.agenda_files)?;
52        let agenda_files = roots.agenda_files()?;
53        let skipped = agenda_files
54            .skipped
55            .iter()
56            .map(|(path, reason)| (path.clone(), reason.describe(path)))
57            .collect();
58        let mut index = Self {
59            roots,
60            settings,
61            documents: BTreeMap::new(),
62            ids: BTreeMap::new(),
63            outside_ids: BTreeMap::new(),
64            skipped,
65        };
66        for path in agenda_files.files {
67            index.add_agenda_file(path)?;
68        }
69        for path in index.roots.all_org_files()?.files {
70            if !index.documents.contains_key(&path) && !index.skipped.contains_key(&path) {
71                index.add_outside_ids(&path);
72            }
73        }
74        index.rebuild_ids();
75        Ok(index)
76    }
77
78    /// Reads and parses one agenda file. A non-UTF-8 file is skipped with
79    /// its reason instead of failing, so one unreadable file never hides the
80    /// rest of the agenda.
81    fn add_agenda_file(&mut self, path: PathBuf) -> Result<(), AgdError> {
82        let bytes = std::fs::read(&path).map_err(|error| {
83            AgdError::with_source(
84                ErrorCode::IoFailed,
85                format!("cannot read {}: {error}", path.display()),
86                error,
87            )
88        })?;
89        match OrgDocument::parse(path.clone(), bytes, &self.settings) {
90            Ok(document) => {
91                self.documents.insert(path, document);
92            }
93            Err(error) if error.code() == ErrorCode::InvalidUtf8 => {
94                self.skipped.insert(path, error.to_string());
95            }
96            Err(error) => return Err(error),
97        }
98        Ok(())
99    }
100
101    /// Records the `:ID:`s of an org file under the roots that is not an
102    /// agenda file.
103    fn add_outside_ids(&mut self, path: &Path) {
104        for id in scan_property_ids(path) {
105            self.outside_ids
106                .entry(id)
107                .or_default()
108                .push(path.to_path_buf());
109        }
110    }
111
112    /// Agenda files left out of the index, with the reason: non-UTF-8
113    /// content, or a symlink that resolves outside the roots.
114    pub fn skipped_files(&self) -> impl Iterator<Item = (&Path, &str)> {
115        self.skipped
116            .iter()
117            .map(|(path, reason)| (path.as_path(), reason.as_str()))
118    }
119
120    pub fn roots(&self) -> &RootSet {
121        &self.roots
122    }
123
124    pub fn settings(&self) -> &Settings {
125        &self.settings
126    }
127
128    /// Re-reads one changed path, as a file watcher reports it. An agenda
129    /// file, including one created after the index loaded, is parsed again;
130    /// another `.org` file under the roots only updates the IDs that make
131    /// targets ambiguous; a removed file or directory leaves the index; a
132    /// symlink resolving outside the roots is skipped. Other paths are
133    /// ignored.
134    pub fn refresh_path(&mut self, path: &Path) -> Result<(), AgdError> {
135        if !path.exists() {
136            let removed = crate::path::resolve_existing_prefix(path)?;
137            self.forget_paths_under(&removed);
138            self.rebuild_ids();
139            return Ok(());
140        }
141        if path.extension().is_none_or(|extension| extension != "org") {
142            return Ok(());
143        }
144        let path = match self.roots.resolve_read(path) {
145            Ok(path) => path,
146            Err(error) if error.code() == ErrorCode::InvalidPath => {
147                let reason = crate::SkipReason::OutsideRoots.describe(path);
148                self.skipped.insert(path.to_path_buf(), reason);
149                return Ok(());
150            }
151            Err(error) => return Err(error),
152        };
153        self.forget_paths_under(&path);
154        if self.roots.is_agenda_file(&path)? {
155            self.add_agenda_file(path)?;
156        } else {
157            self.add_outside_ids(&path);
158        }
159        self.rebuild_ids();
160        Ok(())
161    }
162
163    /// Drops every document, skip and outside ID at or below `path`.
164    fn forget_paths_under(&mut self, path: &Path) {
165        self.documents.retain(|known, _| !known.starts_with(path));
166        self.skipped.retain(|known, _| !known.starts_with(path));
167        for locations in self.outside_ids.values_mut() {
168            locations.retain(|location| !location.starts_with(path));
169        }
170        self.outside_ids
171            .retain(|_, locations| !locations.is_empty());
172    }
173
174    /// Recounts `:ID:` values, then gives every item whose ID is not unique
175    /// under the roots a file/line/digest ref, so two items never share a
176    /// ref; resolving it still refuses the write as `id_conflict`.
177    fn rebuild_ids(&mut self) {
178        self.ids.clear();
179        for (path, document) in &self.documents {
180            for (index, headline) in document.headlines().iter().enumerate() {
181                if let Some(id) = &headline.item.id {
182                    self.ids
183                        .entry(id.clone())
184                        .or_default()
185                        .push((path.clone(), index));
186                }
187            }
188        }
189        let duplicated: BTreeSet<String> =
190            self.duplicate_ids().map(|(id, _)| id.to_string()).collect();
191        for document in self.documents.values_mut() {
192            for headline in &mut document.headlines {
193                headline.item.item_ref = match &headline.item.id {
194                    Some(id) if !duplicated.contains(id) => id.clone(),
195                    _ => location_ref(
196                        Path::new(&headline.item.file),
197                        headline.item.line,
198                        &headline.digest,
199                    ),
200                };
201            }
202        }
203    }
204
205    pub fn documents(&self) -> impl Iterator<Item = &OrgDocument> {
206        self.documents.values()
207    }
208
209    pub fn items(&self) -> impl Iterator<Item = &Item> {
210        self.documents
211            .values()
212            .flat_map(|document| document.headlines().iter().map(|headline| &headline.item))
213    }
214
215    pub fn duplicate_ids(&self) -> impl Iterator<Item = (&str, &[(PathBuf, usize)])> {
216        self.ids
217            .iter()
218            .filter(|(id, _)| self.id_count(id) > 1)
219            .map(|(id, locations)| (id.as_str(), locations.as_slice()))
220    }
221
222    /// Whether any heading under the roots, agenda file or not, has this
223    /// `:ID:`.
224    pub fn id_in_use(&self, id: &str) -> bool {
225        self.ids.contains_key(id) || self.outside_ids.contains_key(id)
226    }
227
228    /// How many headings under the roots, agenda files or not, have this
229    /// `:ID:`.
230    fn id_count(&self, id: &str) -> usize {
231        self.ids.get(id).map_or(0, Vec::len) + self.outside_ids.get(id).map_or(0, Vec::len)
232    }
233
234    fn refuse_duplicate_id(&self, id: Option<&str>) -> Result<(), AgdError> {
235        let Some(id) = id else {
236            return Ok(());
237        };
238        let count = self.id_count(id);
239        if count > 1 {
240            return Err(AgdError::new(
241                ErrorCode::IdConflict,
242                format!("ID '{id}' occurs {count} times under the configured roots"),
243            ));
244        }
245        Ok(())
246    }
247
248    pub fn document(&self, path: &Path) -> Option<&OrgDocument> {
249        self.documents.get(path)
250    }
251
252    pub fn resolve(&self, item_ref: &str) -> Result<(&OrgDocument, &HeadlineSpan), AgdError> {
253        if !item_ref.contains("#sha256:") {
254            self.refuse_duplicate_id(Some(item_ref))?;
255            return match self.ids.get(item_ref).and_then(|matches| matches.first()) {
256                Some((path, index)) => {
257                    let document = self.documents.get(path).expect("indexed document");
258                    Ok((document, &document.headlines()[*index]))
259                }
260                None if self.outside_ids.contains_key(item_ref) => Err(AgdError::new(
261                    ErrorCode::RefConflict,
262                    format!("ID '{item_ref}' is in a file outside the agenda files"),
263                )),
264                None => Err(AgdError::new(
265                    ErrorCode::RefConflict,
266                    format!("no item has ID '{item_ref}'"),
267                )),
268            };
269        }
270
271        let (location, digest) = item_ref.rsplit_once("#sha256:").ok_or_else(|| {
272            AgdError::new(
273                ErrorCode::InvalidRef,
274                format!("invalid item ref: {item_ref}"),
275            )
276        })?;
277        if digest.len() != 32 || !digest.bytes().all(|byte| byte.is_ascii_hexdigit()) {
278            return Err(AgdError::new(
279                ErrorCode::InvalidRef,
280                format!("invalid SHA-256 digest in item ref: {item_ref}"),
281            ));
282        }
283        let (file, line) = location.rsplit_once(':').ok_or_else(|| {
284            AgdError::new(
285                ErrorCode::InvalidRef,
286                format!("item ref has no line hint: {item_ref}"),
287            )
288        })?;
289        let line = line.parse::<usize>().map_err(|_| {
290            AgdError::new(
291                ErrorCode::InvalidRef,
292                format!("invalid line hint in item ref: {item_ref}"),
293            )
294        })?;
295        let requested = Path::new(file);
296        self.roots.resolve_read(requested)?;
297        if let Some(reason) = self.skipped.get(requested) {
298            return Err(AgdError::new(ErrorCode::InvalidUtf8, reason.clone()));
299        }
300        let document = self
301            .documents
302            .get(requested)
303            .or_else(|| {
304                requested
305                    .canonicalize()
306                    .ok()
307                    .and_then(|canonical| self.documents.get(&canonical))
308            })
309            .ok_or_else(|| {
310                AgdError::new(
311                    ErrorCode::RefConflict,
312                    format!(
313                        "referenced file is not an agenda file: {}",
314                        requested.display()
315                    ),
316                )
317            })?;
318        if let Some(headline) = document.headline_at_line(line)
319            && headline.digest == digest
320        {
321            self.refuse_duplicate_id(headline.item.id.as_deref())?;
322            return Ok((document, headline));
323        }
324        let matches: Vec<&HeadlineSpan> = document
325            .headlines()
326            .iter()
327            .filter(|headline| headline.digest == digest)
328            .collect();
329        match matches.as_slice() {
330            [headline] => {
331                self.refuse_duplicate_id(headline.item.id.as_deref())?;
332                Ok((document, headline))
333            }
334            [] => {
335                let mut error = AgdError::new(
336                    ErrorCode::RefConflict,
337                    format!("headline bytes no longer match ref: {item_ref}"),
338                );
339                if let Some(current) = document.headline_at_line(line) {
340                    error = error.with_current(current.item.clone());
341                }
342                Err(error)
343            }
344            _ => Err(AgdError::new(
345                ErrorCode::RefConflict,
346                format!("headline digest is ambiguous in {}", requested.display()),
347            )),
348        }
349    }
350}
351
352#[cfg(test)]
353mod tests {
354    use super::*;
355    use crate::config::test_support::settings;
356
357    #[test]
358    fn digest_ref_relocates_uniquely_and_reports_stale_current() {
359        let temp = tempfile::tempdir().unwrap();
360        let path = temp.path().join("tasks.org");
361        std::fs::write(&path, "* TODO One\n* TODO Two\n").unwrap();
362        let index = WorkspaceIndex::load(settings(temp.path())).unwrap();
363        let item_ref = index.items().nth(1).unwrap().item_ref.clone();
364        assert_eq!(index.resolve(&item_ref).unwrap().1.item.title, "Two");
365
366        std::fs::write(&path, "intro\n* TODO One\n* TODO Two\n").unwrap();
367        let relocated = WorkspaceIndex::load(settings(temp.path())).unwrap();
368        assert_eq!(relocated.resolve(&item_ref).unwrap().1.item.line, 3);
369
370        std::fs::write(&path, "intro\n* TODO One\n* TODO Changed\n").unwrap();
371        let stale = WorkspaceIndex::load(settings(temp.path())).unwrap();
372        let error = stale.resolve(&item_ref).unwrap_err();
373        assert_eq!(error.code().as_str(), "ref_conflict");
374    }
375
376    #[test]
377    fn non_utf8_agenda_file_is_skipped_for_reads_and_refused_for_writes() {
378        let temp = tempfile::tempdir().unwrap();
379        std::fs::write(temp.path().join("good.org"), "* TODO Readable\n").unwrap();
380        let bad = temp.path().join("bad.org");
381        std::fs::write(&bad, b"* TODO Latin-1 \xe9\n").unwrap();
382        let index = WorkspaceIndex::load(settings(temp.path())).unwrap();
383        assert_eq!(index.items().count(), 1);
384        let skipped: Vec<_> = index.skipped_files().collect();
385        assert_eq!(skipped.len(), 1);
386        assert!(skipped[0].0.ends_with("bad.org"));
387        let canonical = bad.canonicalize().unwrap();
388        let bad_ref = format!(
389            "{}:1#sha256:00000000000000000000000000000000",
390            canonical.display()
391        );
392        assert_eq!(
393            index.resolve(&bad_ref).unwrap_err().code().as_str(),
394            "invalid_utf8"
395        );
396    }
397
398    #[test]
399    fn ids_in_non_agenda_files_make_targets_ambiguous() {
400        let temp = tempfile::tempdir().unwrap();
401        std::fs::write(
402            temp.path().join("tasks.org"),
403            "* TODO One\n:PROPERTIES:\n:ID: shared\n:END:\n",
404        )
405        .unwrap();
406        std::fs::create_dir(temp.path().join("archive")).unwrap();
407        std::fs::write(
408            temp.path().join("archive/notes.org"),
409            "* Notes\n:PROPERTIES:\n:id:   shared\n:END:\n",
410        )
411        .unwrap();
412        let mut settings = settings(temp.path());
413        settings.agenda_files = vec!["tasks.org".to_string()];
414        let index = WorkspaceIndex::load(settings).unwrap();
415        assert_eq!(
416            index.resolve("shared").unwrap_err().code().as_str(),
417            "id_conflict"
418        );
419        let item_ref = index.items().next().unwrap().item_ref.clone();
420        assert_eq!(
421            index.resolve(&item_ref).unwrap_err().code().as_str(),
422            "id_conflict"
423        );
424        assert_eq!(index.duplicate_ids().count(), 1);
425    }
426
427    #[cfg(unix)]
428    #[test]
429    fn symlinks_leaving_the_roots_are_skipped_not_fatal() {
430        let temp = tempfile::tempdir().unwrap();
431        std::fs::write(temp.path().join("tasks.org"), "* TODO Kept\n").unwrap();
432        let outside = tempfile::tempdir().unwrap();
433        std::fs::write(outside.path().join("secret.org"), "* TODO Secret\n").unwrap();
434        std::os::unix::fs::symlink(
435            outside.path().join("secret.org"),
436            temp.path().join("escape.org"),
437        )
438        .unwrap();
439        let index = WorkspaceIndex::load(settings(temp.path())).unwrap();
440        let titles: Vec<&str> = index.items().map(|item| item.title.as_str()).collect();
441        assert_eq!(titles, ["Kept"]);
442        let skipped: Vec<_> = index.skipped_files().collect();
443        assert_eq!(skipped.len(), 1);
444        assert!(skipped[0].0.ends_with("escape.org"));
445        assert!(
446            skipped[0].1.contains("outside the roots"),
447            "{}",
448            skipped[0].1
449        );
450    }
451
452    #[test]
453    fn files_created_after_loading_join_the_index_by_agenda_pattern() {
454        let temp = tempfile::tempdir().unwrap();
455        std::fs::create_dir(temp.path().join("archive")).unwrap();
456        let mut settings = settings(temp.path());
457        settings.agenda_files = vec!["inbox.org".to_string()];
458        let mut index = WorkspaceIndex::load(settings).unwrap();
459        assert_eq!(index.items().count(), 0);
460
461        let inbox = temp.path().join("inbox.org");
462        std::fs::write(&inbox, "* TODO First task\n").unwrap();
463        index.refresh_path(&inbox).unwrap();
464        let titles: Vec<&str> = index.items().map(|item| item.title.as_str()).collect();
465        assert_eq!(titles, ["First task"]);
466
467        // Not an agenda file: its ID still makes the agenda's copy ambiguous.
468        std::fs::write(
469            &inbox,
470            "* TODO First task\n:PROPERTIES:\n:ID: shared\n:END:\n",
471        )
472        .unwrap();
473        index.refresh_path(&inbox).unwrap();
474        let notes = temp.path().join("archive/notes.org");
475        std::fs::write(&notes, "* Notes\n:PROPERTIES:\n:ID: shared\n:END:\n").unwrap();
476        index.refresh_path(&notes).unwrap();
477        assert_eq!(index.items().count(), 1);
478        assert_eq!(
479            index.resolve("shared").unwrap_err().code().as_str(),
480            "id_conflict"
481        );
482
483        std::fs::remove_file(&notes).unwrap();
484        index.refresh_path(&notes).unwrap();
485        assert_eq!(index.resolve("shared").unwrap().1.item.title, "First task");
486        std::fs::remove_file(&inbox).unwrap();
487        index.refresh_path(&inbox).unwrap();
488        assert_eq!(index.items().count(), 0);
489    }
490
491    #[test]
492    fn refs_to_missing_files_outside_the_roots_are_outside_roots() {
493        let temp = tempfile::tempdir().unwrap();
494        std::fs::write(temp.path().join("tasks.org"), "* TODO One\n").unwrap();
495        let index = WorkspaceIndex::load(settings(temp.path())).unwrap();
496        let outside = tempfile::tempdir().unwrap();
497        let error = index
498            .resolve(&format!(
499                "{}:1#sha256:00000000000000000000000000000000",
500                outside.path().join("no/such/file").display()
501            ))
502            .unwrap_err();
503        assert_eq!(error.code().as_str(), "invalid_path");
504    }
505
506    #[test]
507    fn items_sharing_an_id_get_distinct_location_refs_that_refuse_writes() {
508        let temp = tempfile::tempdir().unwrap();
509        let tasks = temp.path().join("tasks.org");
510        std::fs::write(
511            &tasks,
512            "* TODO One\n:PROPERTIES:\n:ID: dup-id-1\n:END:\n* TODO Two\n:PROPERTIES:\n:ID: dup-id-1\n:END:\n",
513        )
514        .unwrap();
515        let mut index = WorkspaceIndex::load(settings(temp.path())).unwrap();
516        let refs: Vec<String> = index.items().map(|item| item.item_ref.clone()).collect();
517        assert_eq!(refs.len(), 2);
518        assert_ne!(refs[0], refs[1]);
519        // Neither the location refs nor the ID itself selects one of them.
520        for item_ref in refs.iter().map(String::as_str).chain(["dup-id-1"]) {
521            assert_eq!(
522                index.resolve(item_ref).unwrap_err().code().as_str(),
523                "id_conflict"
524            );
525        }
526        assert!(refs.iter().all(|item_ref| item_ref.contains("#sha256:")));
527
528        // Once the ID is unique again, the item's ref is its ID.
529        std::fs::write(
530            &tasks,
531            "* TODO One\n:PROPERTIES:\n:ID: dup-id-1\n:END:\n* TODO Two\n",
532        )
533        .unwrap();
534        index.refresh_path(&tasks).unwrap();
535        assert_eq!(index.items().next().unwrap().item_ref, "dup-id-1");
536    }
537}
agd_core/document/index.rs

agd_core/document/
index.rs
