agora_agentkit/govlog/
data_read.rs

1//! Reading part of a data set's file (0.103): a [`LineRange`], and the
2//! [`DataCitation`] that names a file at a commit, with or without one.
3//!
4//! A range is a selective read, not a page: there is no cursor and no
5//! "next" (agora `memory/feedback_no_paging.md`). A reader who means to
6//! read a whole file reads it whole; a range is for a part, or for a file
7//! too large for one read ([`MAX_WHOLE_READ_BYTES`](super::MAX_WHOLE_READ_BYTES)).
8
9use serde::{Deserialize, Serialize};
10
11use super::{DataFilePath, GitCommitId, GovernanceLogId};
12use crate::ids::GovernanceLogPrefix;
13
14/// Lines `start` to `end` of a file, 1-based and inclusive; `end` absent
15/// means to the end of the file.
16///
17/// Written `"110-300"`, or `"110-"` to the end. Read leniently, as models
18/// and people write ranges: `"110..300"`, `"110–300"`, `"L110-L300"`,
19/// `"lines 110-300"`, `"-300"` (from the first line), `"110"` (that line
20/// alone), or a bare number. Validated: lines are numbered from 1, and a
21/// range cannot end before it starts.
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
23pub struct LineRange {
24    start: u64,
25    end: Option<u64>,
26}
27
28/// A string that is not a [`LineRange`]: the message says why and what the
29/// form is
30#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
31#[error("lines={asked:?}: {why} {}", LineRange::FORM)]
32pub struct LineRangeError {
33    pub asked: String,
34    pub why: &'static str,
35}
36
37/// Where a [`LineRange`] falls in a text: the byte span to serve, and the
38/// range as served (its end clamped to the file's last line)
39#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40pub struct LineSpan {
41    pub served: LineRange,
42    pub start_byte: usize,
43    pub end_byte: usize,
44}
45
46/// A [`LineRange`] that starts past a file's last line
47#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
48#[error(
49    "lines={asked} starts past the end of the file, which has {total_lines} \
50     line(s)"
51)]
52pub struct PastEnd {
53    pub asked: LineRange,
54    pub total_lines: u64,
55}
56
57impl LineRange {
58    /// How a range is written, for error messages
59    pub const FORM: &'static str = "Write a range of lines as \"<from>-<to>\" \
60         (1-based, both included, e.g. \"110-300\"), or \"<from>-\" to read to \
61         the end of the file.";
62
63    /// Lines `start..=end` (`end` absent: to the end), or why not
64    pub fn new(start: u64, end: Option<u64>) -> Result<Self, &'static str> {
65        if start == 0 {
66            return Err("lines are numbered from 1.");
67        }
68        if end.is_some_and(|end| end < start) {
69            return Err("the range ends before it starts.");
70        }
71        Ok(Self { start, end })
72    }
73
74    /// The first line, from 1
75    pub fn start(&self) -> u64 {
76        self.start
77    }
78
79    /// The last line, or `None` for the end of the file
80    pub fn end(&self) -> Option<u64> {
81        self.end
82    }
83
84    /// The span of `text` this range covers. A line is what ends at a
85    /// `\n`, or the text's end; an end past the last line is the last line
86    pub fn span(&self, text: &str) -> Result<LineSpan, PastEnd> {
87        let total_lines = count_lines(text);
88        if self.start > total_lines {
89            return Err(PastEnd {
90                asked: *self,
91                total_lines,
92            });
93        }
94        let end = self.end.map_or(total_lines, |e| e.min(total_lines));
95        // Byte offset of each line's start, as needed
96        let mut start_byte = 0;
97        let mut end_byte = text.len();
98        for (line, (at, _)) in (1..).zip(text.match_indices('\n')) {
99            if line + 1 == self.start {
100                start_byte = at + 1;
101            }
102            if line == end {
103                end_byte = at + 1;
104                break;
105            }
106        }
107        Ok(LineSpan {
108            served: Self {
109                start: self.start,
110                end: Some(end),
111            },
112            start_byte,
113            end_byte,
114        })
115    }
116}
117
118/// How many lines `text` has: one per `\n`, and one more for text after
119/// the last `\n`. An empty text has none
120pub fn count_lines(text: &str) -> u64 {
121    text.split_inclusive('\n').count() as u64
122}
123
124impl std::fmt::Display for LineRange {
125    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
126        match self.end {
127            Some(end) => write!(f, "{}-{end}", self.start),
128            None => write!(f, "{}-", self.start),
129        }
130    }
131}
132
133impl std::str::FromStr for LineRange {
134    type Err = LineRangeError;
135
136    fn from_str(asked: &str) -> Result<Self, Self::Err> {
137        let error = |why| LineRangeError {
138            asked: asked.to_string(),
139            why,
140        };
141        let mut s = asked.trim().trim_start_matches('#');
142        for word in ["lines", "line"] {
143            if let Some(head) = s.get(..word.len())
144                && head.eq_ignore_ascii_case(word)
145            {
146                s = s[word.len()..].trim_start();
147                s = s.strip_prefix('=').unwrap_or(s).trim_start();
148                break;
149            }
150        }
151        // `L110` as a code host writes it
152        let number = |p: &str| -> Result<Option<u64>, LineRangeError> {
153            let p = p.trim();
154            let p = p.strip_prefix(['L', 'l']).unwrap_or(p);
155            if p.is_empty() {
156                return Ok(None);
157            }
158            if !p.chars().all(|c| c.is_ascii_digit()) {
159                return Err(error("that is not a range of line numbers."));
160            }
161            p.parse()
162                .map(Some)
163                .map_err(|_| error("that line number is too large."))
164        };
165        let (from, to, ranged) =
166            ["...", "..", "-", "\u{2013}", "\u{2014}", " to "]
167                .iter()
168                .find_map(|sep| s.split_once(sep).map(|(a, b)| (a, b, true)))
169                .unwrap_or((s, s, false));
170        let start = number(from)?;
171        let end = number(to)?;
172        let (start, end) = match (start, end, ranged) {
173            (None, _, false) => return Err(error("no line numbers given.")),
174            (Some(n), _, false) => (n, Some(n)),
175            (None, None, true) => return Err(error("no line numbers given.")),
176            (None, end, true) => (1, end),
177            (Some(start), end, true) => (start, end),
178        };
179        Self::new(start, end).map_err(error)
180    }
181}
182
183impl Serialize for LineRange {
184    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
185        s.collect_str(self)
186    }
187}
188
189/// A string, or a bare number for one line
190impl<'de> Deserialize<'de> for LineRange {
191    fn deserialize<D: serde::Deserializer<'de>>(
192        d: D,
193    ) -> Result<Self, D::Error> {
194        #[derive(Deserialize)]
195        #[serde(untagged)]
196        enum Raw {
197            Text(String),
198            Line(u64),
199        }
200        match Raw::deserialize(d)? {
201            Raw::Text(s) => s.parse().map_err(serde::de::Error::custom),
202            Raw::Line(n) => Self::new(n, Some(n)).map_err(|why| {
203                serde::de::Error::custom(LineRangeError {
204                    asked: n.to_string(),
205                    why,
206                })
207            }),
208        }
209    }
210}
211
212// A plain string with no `pattern`, inline: the agora CLAUDE.md rules for
213// tool schemas (no `$ref`; no `pattern`, which a strict decoder would
214// enforce by inventing values). Parsing checks it.
215#[cfg(feature = "schemars")]
216impl schemars::JsonSchema for LineRange {
217    fn inline_schema() -> bool {
218        true
219    }
220
221    fn schema_name() -> std::borrow::Cow<'static, str> {
222        std::borrow::Cow::Borrowed("LineRange")
223    }
224
225    fn schema_id() -> std::borrow::Cow<'static, str> {
226        std::borrow::Cow::Borrowed(concat!(module_path!(), "::LineRange"))
227    }
228
229    fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
230        schemars::json_schema!({
231            "type": "string",
232            "description": "Lines of a file, 1-based and inclusive: \
233                            \"110-300\", or \"110-\" to the end of the file",
234        })
235    }
236}
237
238/// The first 7 to 40 hex digits of a git commit id, lowercase: how a
239/// [`DataCitation`] names the commit it was read at
240#[derive(Debug, Clone, PartialEq, Eq, Hash)]
241pub struct GitCommitPrefix(String);
242
243impl GitCommitPrefix {
244    /// The fewest hex digits accepted, as `git` abbreviates by default
245    pub const MIN_LEN: usize = 7;
246    /// A whole SHA-1 commit id
247    pub const MAX_LEN: usize = 40;
248
249    pub fn as_str(&self) -> &str {
250        &self.0
251    }
252
253    /// `true` when `commit` starts with this prefix
254    pub fn matches(&self, commit: &GitCommitId) -> bool {
255        commit.to_hex().starts_with(&self.0)
256    }
257}
258
259impl std::str::FromStr for GitCommitPrefix {
260    type Err = DataCitationError;
261
262    fn from_str(s: &str) -> Result<Self, Self::Err> {
263        let s = s.trim();
264        let ok = (Self::MIN_LEN..=Self::MAX_LEN).contains(&s.len())
265            && s.chars().all(|c| c.is_ascii_hexdigit());
266        ok.then(|| Self(s.to_ascii_lowercase()))
267            .ok_or_else(|| DataCitationError::Commit(s.to_string()))
268    }
269}
270
271impl std::fmt::Display for GitCommitPrefix {
272    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
273        f.write_str(&self.0)
274    }
275}
276
277impl From<GitCommitId> for GitCommitPrefix {
278    fn from(commit: GitCommitId) -> Self {
279        Self(commit.to_hex())
280    }
281}
282
283/// A data set's file, cited at the commit it was read at, and optionally
284/// lines of it: `DATA-2026-0001:<commit>:<path>:110-300`, or without the
285/// range for the whole file (0.103).
286///
287/// `get_content` takes one as its `id` and reads that file (and those
288/// lines). The commit is a prefix of at least seven hex digits of the
289/// commit the data set pins. A server reads the set as revised (its latest
290/// version) when the commit is the latest version's, and as originally
291/// signed when it is the original's, so a citation made before any revision
292/// keeps reading what it cited. One made at an intermediate revision's
293/// commit stops resolving once a later revision pins another commit.
294///
295/// The path runs from the second `:` to the end, or to the last `:` when
296/// what follows that is a range written `<from>-<to>` or `<from>-`, so a
297/// path may hold a `:`.
298#[derive(Debug, Clone, PartialEq, Eq, Hash)]
299pub struct DataCitation {
300    /// A `DATA-…` id
301    pub set: GovernanceLogId,
302    pub commit: GitCommitPrefix,
303    pub path: DataFilePath,
304    pub lines: Option<LineRange>,
305}
306
307/// A string shaped like a [`DataCitation`] (a `DATA-…` id and a `:`) that
308/// is not one
309#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
310pub enum DataCitationError {
311    #[error("{0:?} is not a data set's id (DATA-YYYY-NNNN). {form}", form = DataCitation::FORM)]
312    NotDataSet(String),
313    #[error(
314        "{0:?} is not a commit: give at least {min} of the commit's hex digits. {form}",
315        min = GitCommitPrefix::MIN_LEN,
316        form = DataCitation::FORM
317    )]
318    Commit(String),
319    #[error("no file path after the commit. {form}", form = DataCitation::FORM)]
320    NoPath,
321    #[error("{0}")]
322    Path(#[from] super::DataFilePathError),
323}
324
325impl DataCitation {
326    /// How a citation is written, for error messages
327    pub const FORM: &'static str = "A data set's file is cited as \
328         DATA-YYYY-NNNN:<commit>:<path>, or \
329         DATA-YYYY-NNNN:<commit>:<path>:<from>-<to> for some of its lines; \
330         to read the data set itself, give its id alone.";
331
332    /// `true` when `s` starts with a `DATA-…` id and a `:`, so it is meant
333    /// as a citation and its error is the one to show
334    pub fn is_citation_shaped(s: &str) -> bool {
335        s.trim_start().split_once(':').is_some_and(|(head, _)| {
336            head.parse::<GovernanceLogId>()
337                .is_ok_and(|id| id.prefix() == GovernanceLogPrefix::Data)
338        })
339    }
340}
341
342impl std::str::FromStr for DataCitation {
343    type Err = DataCitationError;
344
345    fn from_str(s: &str) -> Result<Self, Self::Err> {
346        let s = s.trim();
347        let (head, rest) = s
348            .split_once(':')
349            .ok_or_else(|| DataCitationError::NotDataSet(s.to_string()))?;
350        let set: GovernanceLogId = head
351            .parse()
352            .ok()
353            .filter(|id: &GovernanceLogId| {
354                id.prefix() == GovernanceLogPrefix::Data
355            })
356            .ok_or_else(|| DataCitationError::NotDataSet(head.to_string()))?;
357        let (commit, rest) = match rest.split_once(':') {
358            Some((commit, rest)) => (commit, rest),
359            None => {
360                // Say what is wrong with the commit first
361                rest.parse::<GitCommitPrefix>()?;
362                return Err(DataCitationError::NoPath);
363            }
364        };
365        let commit: GitCommitPrefix = commit.parse()?;
366        // The range in a citation is strictly `<from>-<to>` or `<from>-`,
367        // as `Display` writes it, so a path ending in `:12` is never read
368        // as a range of another file (the review of 0.103)
369        let canonical = |r: &str| {
370            r.split_once('-').is_some_and(|(a, b)| {
371                !a.is_empty()
372                    && a.bytes().all(|c| c.is_ascii_digit())
373                    && b.bytes().all(|c| c.is_ascii_digit())
374            })
375        };
376        let (path, lines) = match rest.rsplit_once(':') {
377            Some((path, range)) if !path.is_empty() && canonical(range) => {
378                match range.parse::<LineRange>() {
379                    Ok(lines) => (path, Some(lines)),
380                    Err(_) => (rest, None),
381                }
382            }
383            _ => (rest, None),
384        };
385        let path = path.trim();
386        let path = path.strip_prefix("./").unwrap_or(path);
387        let path = path.trim_start_matches('/');
388        if path.is_empty() {
389            return Err(DataCitationError::NoPath);
390        }
391        Ok(Self {
392            set,
393            commit,
394            path: path.parse()?,
395            lines,
396        })
397    }
398}
399
400impl std::fmt::Display for DataCitation {
401    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
402        write!(f, "{}:{}:{}", self.set, self.commit, self.path)?;
403        if let Some(lines) = self.lines {
404            write!(f, ":{lines}")?;
405        }
406        Ok(())
407    }
408}
409
410#[cfg(test)]
411mod tests {
412    use super::*;
413
414    fn range(s: &str) -> LineRange {
415        s.parse().unwrap_or_else(|e| panic!("{s:?}: {e}"))
416    }
417
418    #[test]
419    fn ranges_parse_leniently_and_validate() {
420        for (written, start, end) in [
421            ("110-300", 110, Some(300)),
422            (" 110 - 300 ", 110, Some(300)),
423            ("110..300", 110, Some(300)),
424            ("110...300", 110, Some(300)),
425            ("110\u{2013}300", 110, Some(300)),
426            ("L110-L300", 110, Some(300)),
427            ("#L110-L300", 110, Some(300)),
428            ("lines 110-300", 110, Some(300)),
429            ("Lines=110-300", 110, Some(300)),
430            ("110 to 300", 110, Some(300)),
431            ("110-", 110, None),
432            ("110..", 110, None),
433            ("-300", 1, Some(300)),
434            ("110", 110, Some(110)),
435            ("7-7", 7, Some(7)),
436        ] {
437            let r = range(written);
438            assert_eq!((r.start(), r.end()), (start, end), "{written:?}");
439        }
440        assert_eq!(range("110-300").to_string(), "110-300");
441        assert_eq!(range("110-").to_string(), "110-");
442        for (bad, why) in [
443            ("0-10", "numbered from 1"),
444            ("300-110", "ends before it starts"),
445            ("", "no line numbers"),
446            ("-", "no line numbers"),
447            ("abc", "not a range"),
448            ("1-2-3", "not a range"),
449            ("1.5-3", "not a range"),
450            ("99999999999999999999999", "too large"),
451        ] {
452            let err = bad.parse::<LineRange>().unwrap_err();
453            let text = err.to_string();
454            assert!(text.contains(why), "{bad:?}: {text}");
455            assert!(text.contains("\"110-300\""), "says the form: {text}");
456        }
457    }
458
459    #[test]
460    fn ranges_round_trip_as_strings_and_take_a_bare_number() {
461        let r: LineRange = serde_json::from_str("\"110-300\"").unwrap();
462        assert_eq!(serde_json::to_value(r).unwrap(), "110-300");
463        let one: LineRange = serde_json::from_str("42").unwrap();
464        assert_eq!(one.to_string(), "42-42");
465        assert!(serde_json::from_str::<LineRange>("0").is_err());
466        assert!(serde_json::from_str::<LineRange>("\"9-1\"").is_err());
467    }
468
469    #[cfg(feature = "schemars")]
470    #[test]
471    fn a_range_is_a_plain_string_in_a_schema() {
472        let schema = schemars::schema_for!(LineRange);
473        let text = serde_json::to_string(&schema).unwrap();
474        assert!(!text.contains("$ref") && !text.contains("$defs"), "{text}");
475        assert!(!text.contains("pattern"), "{text}");
476        assert_eq!(schema.get("type").unwrap(), "string");
477    }
478
479    #[test]
480    fn spans_cover_whole_lines_and_clamp_the_end() {
481        let text = "one\ntwo\nthree\nfour";
482        assert_eq!(count_lines(text), 4);
483        assert_eq!(count_lines("a\n"), 1);
484        assert_eq!(count_lines(""), 0);
485        let span = |r: &str| {
486            let s = range(r).span(text).unwrap();
487            (&text[s.start_byte..s.end_byte], s.served.to_string())
488        };
489        assert_eq!(span("1-1"), ("one\n", "1-1".into()));
490        assert_eq!(span("2-3"), ("two\nthree\n", "2-3".into()));
491        assert_eq!(span("3-"), ("three\nfour", "3-4".into()));
492        assert_eq!(span("4-99"), ("four", "4-4".into()));
493        assert_eq!(span("1-"), (text, "1-4".into()));
494        let past = range("5-9").span(text).unwrap_err();
495        assert_eq!(past.total_lines, 4);
496        assert!(past.to_string().contains("has 4 line(s)"), "{past}");
497        assert!(range("1-").span("").is_err());
498        // A trailing newline ends the last line; it does not start one
499        assert_eq!(range("2-").span("a\nb\n").unwrap().end_byte, 4);
500        assert!(range("3-").span("a\nb\n").is_err());
501    }
502
503    #[test]
504    fn citations_parse_and_print() {
505        let c: DataCitation =
506            "DATA-2026-0001:89fe7c6:votes/ballots.csv:110-300"
507                .parse()
508                .unwrap();
509        assert_eq!(c.set.as_str(), "DATA-2026-0001");
510        assert_eq!(c.commit.as_str(), "89fe7c6");
511        assert_eq!(c.path.as_str(), "votes/ballots.csv");
512        assert_eq!(c.lines, Some(range("110-300")));
513        assert_eq!(
514            c.to_string(),
515            "DATA-2026-0001:89fe7c6:votes/ballots.csv:110-300"
516        );
517
518        let whole: DataCitation =
519            " data-2026-1:89FE7C6:votes/ballots.csv ".parse().unwrap();
520        assert_eq!(whole.set.as_str(), "DATA-2026-0001");
521        assert_eq!(whole.commit.as_str(), "89fe7c6");
522        assert_eq!(whole.lines, None);
523        assert_eq!(
524            whole.to_string(),
525            "DATA-2026-0001:89fe7c6:votes/ballots.csv"
526        );
527
528        // A path may hold a `:`; what follows the last one is a range only
529        // when it reads as one
530        let colon: DataCitation =
531            "DATA-2026-0001:89fe7c6:a:b.txt".parse().unwrap();
532        assert_eq!(colon.path.as_str(), "a:b.txt");
533        assert_eq!(colon.lines, None);
534        // Only the canonical `<from>-<to>` is a range: `log:12` is a file
535        let file: DataCitation =
536            "DATA-2026-0001:89fe7c6:log:12".parse().unwrap();
537        assert_eq!((file.path.as_str(), file.lines), ("log:12", None));
538        assert_eq!(file.to_string(), "DATA-2026-0001:89fe7c6:log:12");
539        let both: DataCitation =
540            "DATA-2026-0001:89fe7c6:a:b.txt:5-".parse().unwrap();
541        assert_eq!(both.path.as_str(), "a:b.txt");
542        assert_eq!(both.lines, Some(range("5-")));
543        let lead: DataCitation =
544            "DATA-2026-0001:89fe7c6:./votes/x.md".parse().unwrap();
545        assert_eq!(lead.path.as_str(), "votes/x.md");
546    }
547
548    #[test]
549    fn citations_refuse_what_they_cannot_name() {
550        let err = |s: &str| s.parse::<DataCitation>().unwrap_err();
551        assert!(matches!(
552            err("GOV-2026-0006:89fe7c6:x"),
553            DataCitationError::NotDataSet(_)
554        ));
555        assert!(matches!(
556            err("DATA-2026-0001:89fe7c:x"),
557            DataCitationError::Commit(_)
558        ));
559        assert!(matches!(
560            err("DATA-2026-0001:not-hex!:x"),
561            DataCitationError::Commit(_)
562        ));
563        assert!(matches!(
564            err(&format!("DATA-2026-0001:{}:x", "a".repeat(41))),
565            DataCitationError::Commit(_)
566        ));
567        assert!(matches!(
568            err("DATA-2026-0001:89fe7c6"),
569            DataCitationError::NoPath
570        ));
571        assert!(matches!(
572            err("DATA-2026-0001:89fe7c6:"),
573            DataCitationError::NoPath
574        ));
575        assert!(matches!(
576            err("DATA-2026-0001: the votes"),
577            DataCitationError::Commit(_)
578        ));
579        assert!(matches!(
580            err("DATA-2026-0001:89fe7c6:../secrets"),
581            DataCitationError::Path(_)
582        ));
583        let text = err("DATA-2026-0001:89fe7c6").to_string();
584        assert!(text.contains("DATA-YYYY-NNNN:<commit>:<path>"), "{text}");
585        assert!(text.contains("give its id alone"), "{text}");
586
587        assert!(DataCitation::is_citation_shaped("DATA-2026-0001:x"));
588        assert!(DataCitation::is_citation_shaped(" data-2026-1:"));
589        assert!(!DataCitation::is_citation_shaped("GOV-2026-0006: yes"));
590        assert!(!DataCitation::is_citation_shaped("DATA-2026-0001"));
591    }
592
593    #[test]
594    fn a_prefix_matches_its_commit() {
595        let commit: GitCommitId =
596            "89fe7c6aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".parse().unwrap();
597        let prefix: GitCommitPrefix = "89FE7C6".parse().unwrap();
598        assert!(prefix.matches(&commit));
599        assert!(
600            !"89fe7c7"
601                .parse::<GitCommitPrefix>()
602                .unwrap()
603                .matches(&commit)
604        );
605        assert!(GitCommitPrefix::from(commit).matches(&commit));
606    }
607}
agora_agentkit/govlog/data_read.rs

agora_agentkit/govlog/
data_read.rs
