agora_agentkit/govlog/
data_read.rs

1//! Reading part of a data set's file (0.103): a [`LineRange`], and the
2//! [`DataCitation`] that names a file at a commit, with or without one.
3//!
4//! A range is a selective read, not a page: there is no cursor and no
5//! "next" (agora `memory/feedback_no_paging.md`). A reader who means to
6//! read a whole file reads it whole; a range is for a part, or for a file
7//! too large for one read ([`MAX_WHOLE_READ_BYTES`](super::MAX_WHOLE_READ_BYTES)).
8
9use serde::{Deserialize, Serialize};
10
11use super::{DataFilePath, GitCommitId, GovernanceLogId};
12use crate::ids::GovernanceLogPrefix;
13
14/// Lines `start` to `end` of a file, 1-based and inclusive; `end` absent
15/// means to the end of the file.
16///
17/// Written `"110-300"`, or `"110-"` to the end. Read leniently, as models
18/// and people write ranges: `"110..300"`, `"110–300"`, `"L110-L300"`,
19/// `"lines 110-300"`, `"-300"` (from the first line), `"110"` (that line
20/// alone), or a bare number. Validated: lines are numbered from 1, and a
21/// range cannot end before it starts.
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
23pub struct LineRange {
24    start: u64,
25    end: Option<u64>,
26}
27
28/// A string that is not a [`LineRange`]: the message says why and what the
29/// form is
30#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
31#[error("lines={asked:?}: {why} {}", LineRange::FORM)]
32pub struct LineRangeError {
33    pub asked: String,
34    pub why: &'static str,
35}
36
37/// Where a [`LineRange`] falls in a text: the byte span to serve, and the
38/// range as served (its end clamped to the file's last line)
39#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40pub struct LineSpan {
41    pub served: LineRange,
42    pub start_byte: usize,
43    pub end_byte: usize,
44}
45
46/// A [`LineRange`] that starts past a file's last line
47#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
48#[error(
49    "lines={asked} starts past the end of the file, which has {total_lines} \
50     line(s)"
51)]
52pub struct PastEnd {
53    pub asked: LineRange,
54    pub total_lines: u64,
55}
56
57impl LineRange {
58    /// How a range is written, for error messages
59    pub const FORM: &'static str = "Write a range of lines as \"<from>-<to>\" \
60         (1-based, both included, e.g. \"110-300\"), or \"<from>-\" to read to \
61         the end of the file.";
62
63    /// How a refused range opens, as the seed runner shows a malformed
64    /// `lines` argument, so that every range error reads alike (0.105.0)
65    pub const INVALID: &'static str = "Invalid arguments at `lines`: ";
66
67    /// Lines `start..=end` (`end` absent: to the end), or why not
68    pub fn new(start: u64, end: Option<u64>) -> Result<Self, &'static str> {
69        if start == 0 {
70            return Err("lines are numbered from 1.");
71        }
72        if end.is_some_and(|end| end < start) {
73            return Err("the range ends before it starts.");
74        }
75        Ok(Self { start, end })
76    }
77
78    /// The first line, from 1
79    pub fn start(&self) -> u64 {
80        self.start
81    }
82
83    /// The last line, or `None` for the end of the file
84    pub fn end(&self) -> Option<u64> {
85        self.end
86    }
87
88    /// The span of `text` this range covers. A line is what ends at a
89    /// `\n`, or the text's end; an end past the last line is the last line
90    pub fn span(&self, text: &str) -> Result<LineSpan, PastEnd> {
91        let total_lines = count_lines(text);
92        if self.start > total_lines {
93            return Err(PastEnd {
94                asked: *self,
95                total_lines,
96            });
97        }
98        let end = self.end.map_or(total_lines, |e| e.min(total_lines));
99        // Byte offset of each line's start, as needed
100        let mut start_byte = 0;
101        let mut end_byte = text.len();
102        for (line, (at, _)) in (1..).zip(text.match_indices('\n')) {
103            if line + 1 == self.start {
104                start_byte = at + 1;
105            }
106            if line == end {
107                end_byte = at + 1;
108                break;
109            }
110        }
111        Ok(LineSpan {
112            served: Self {
113                start: self.start,
114                end: Some(end),
115            },
116            start_byte,
117            end_byte,
118        })
119    }
120}
121
122/// How many lines `text` has: one per `\n`, and one more for text after
123/// the last `\n`. An empty text has none
124pub fn count_lines(text: &str) -> u64 {
125    text.split_inclusive('\n').count() as u64
126}
127
128impl std::fmt::Display for LineRange {
129    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
130        match self.end {
131            Some(end) => write!(f, "{}-{end}", self.start),
132            None => write!(f, "{}-", self.start),
133        }
134    }
135}
136
137impl std::str::FromStr for LineRange {
138    type Err = LineRangeError;
139
140    fn from_str(asked: &str) -> Result<Self, Self::Err> {
141        let error = |why| LineRangeError {
142            asked: asked.to_string(),
143            why,
144        };
145        let mut s = asked.trim().trim_start_matches('#');
146        for word in ["lines", "line"] {
147            if let Some(head) = s.get(..word.len())
148                && head.eq_ignore_ascii_case(word)
149            {
150                s = s[word.len()..].trim_start();
151                s = s.strip_prefix('=').unwrap_or(s).trim_start();
152                break;
153            }
154        }
155        // `L110` as a code host writes it
156        let number = |p: &str| -> Result<Option<u64>, LineRangeError> {
157            let p = p.trim();
158            let p = p.strip_prefix(['L', 'l']).unwrap_or(p);
159            if p.is_empty() {
160                return Ok(None);
161            }
162            if !p.chars().all(|c| c.is_ascii_digit()) {
163                return Err(error("that is not a range of line numbers."));
164            }
165            p.parse()
166                .map(Some)
167                .map_err(|_| error("that line number is too large."))
168        };
169        let (from, to, ranged) =
170            ["...", "..", "-", "\u{2013}", "\u{2014}", " to "]
171                .iter()
172                .find_map(|sep| s.split_once(sep).map(|(a, b)| (a, b, true)))
173                .unwrap_or((s, s, false));
174        let start = number(from)?;
175        let end = number(to)?;
176        let (start, end) = match (start, end, ranged) {
177            (None, _, false) => return Err(error("no line numbers given.")),
178            (Some(n), _, false) => (n, Some(n)),
179            (None, None, true) => return Err(error("no line numbers given.")),
180            (None, end, true) => (1, end),
181            (Some(start), end, true) => (start, end),
182        };
183        Self::new(start, end).map_err(error)
184    }
185}
186
187impl Serialize for LineRange {
188    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
189        s.collect_str(self)
190    }
191}
192
193/// A string, or a bare number for one line
194impl<'de> Deserialize<'de> for LineRange {
195    fn deserialize<D: serde::Deserializer<'de>>(
196        d: D,
197    ) -> Result<Self, D::Error> {
198        #[derive(Deserialize)]
199        #[serde(untagged)]
200        enum Raw {
201            Text(String),
202            Line(u64),
203        }
204        match Raw::deserialize(d)? {
205            Raw::Text(s) => s.parse().map_err(serde::de::Error::custom),
206            Raw::Line(n) => Self::new(n, Some(n)).map_err(|why| {
207                serde::de::Error::custom(LineRangeError {
208                    asked: n.to_string(),
209                    why,
210                })
211            }),
212        }
213    }
214}
215
216// A plain string with no `pattern`, inline: the agora CLAUDE.md rules for
217// tool schemas (no `$ref`; no `pattern`, which a strict decoder would
218// enforce by inventing values). Parsing checks it.
219#[cfg(feature = "schemars")]
220impl schemars::JsonSchema for LineRange {
221    fn inline_schema() -> bool {
222        true
223    }
224
225    fn schema_name() -> std::borrow::Cow<'static, str> {
226        std::borrow::Cow::Borrowed("LineRange")
227    }
228
229    fn schema_id() -> std::borrow::Cow<'static, str> {
230        std::borrow::Cow::Borrowed(concat!(module_path!(), "::LineRange"))
231    }
232
233    fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
234        schemars::json_schema!({
235            "type": "string",
236            "description": "Lines of a file, 1-based and inclusive: \
237                            \"110-300\", \"110-\" to the end of the file, \
238                            \"-300\" from its start, or \"110\" for one line",
239        })
240    }
241}
242
243/// The first 7 to 40 hex digits of a git commit id, lowercase: how a
244/// [`DataCitation`] names the commit it was read at
245#[derive(Debug, Clone, PartialEq, Eq, Hash)]
246pub struct GitCommitPrefix(String);
247
248impl GitCommitPrefix {
249    /// The fewest hex digits accepted, as `git` abbreviates by default
250    pub const MIN_LEN: usize = 7;
251    /// A whole SHA-1 commit id
252    pub const MAX_LEN: usize = 40;
253
254    pub fn as_str(&self) -> &str {
255        &self.0
256    }
257
258    /// `true` when `commit` starts with this prefix
259    pub fn matches(&self, commit: &GitCommitId) -> bool {
260        commit.to_hex().starts_with(&self.0)
261    }
262}
263
264impl std::str::FromStr for GitCommitPrefix {
265    type Err = DataCitationError;
266
267    fn from_str(s: &str) -> Result<Self, Self::Err> {
268        let s = s.trim();
269        let ok = (Self::MIN_LEN..=Self::MAX_LEN).contains(&s.len())
270            && s.chars().all(|c| c.is_ascii_hexdigit());
271        ok.then(|| Self(s.to_ascii_lowercase()))
272            .ok_or_else(|| DataCitationError::Commit(s.to_string()))
273    }
274}
275
276impl std::fmt::Display for GitCommitPrefix {
277    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
278        f.write_str(&self.0)
279    }
280}
281
282impl From<GitCommitId> for GitCommitPrefix {
283    fn from(commit: GitCommitId) -> Self {
284        Self(commit.to_hex())
285    }
286}
287
288/// A data set's file, cited at the commit it was read at, and optionally
289/// lines of it: `DATA-2026-0001:<commit>:<path>:110-300`, or without the
290/// range for the whole file (0.103).
291///
292/// `get_content` takes one as its `id` and reads that file (and those
293/// lines). The commit is a prefix of at least seven hex digits of the
294/// commit the data set pins. A server reads the set as revised (its latest
295/// version) when the commit is the latest version's, and as originally
296/// signed when it is the original's, so a citation made before any revision
297/// keeps reading what it cited. One made at an intermediate revision's
298/// commit stops resolving once a later revision pins another commit.
299///
300/// The path runs from the second `:` to the end, or to the last `:` when
301/// what follows that is a range written `<from>-<to>` or `<from>-`, so a
302/// path may hold a `:`.
303#[derive(Debug, Clone, PartialEq, Eq, Hash)]
304pub struct DataCitation {
305    /// A `DATA-…` id
306    pub set: GovernanceLogId,
307    pub commit: GitCommitPrefix,
308    pub path: DataFilePath,
309    pub lines: Option<LineRange>,
310}
311
312/// A string shaped like a [`DataCitation`] (a `DATA-…` id and a `:`) that
313/// is not one
314#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
315pub enum DataCitationError {
316    #[error("{0:?} is not a data set's id (DATA-YYYY-NNNN). {form}", form = DataCitation::FORM)]
317    NotDataSet(String),
318    #[error(
319        "{0:?} is not a commit: give at least {min} of the commit's hex digits. {form}",
320        min = GitCommitPrefix::MIN_LEN,
321        form = DataCitation::FORM
322    )]
323    Commit(String),
324    #[error("no file path after the commit. {form}", form = DataCitation::FORM)]
325    NoPath,
326    #[error("{0}")]
327    Path(#[from] super::DataFilePathError),
328}
329
330impl DataCitation {
331    /// How a citation is written, for error messages
332    pub const FORM: &'static str = "A data set's file is cited as \
333         DATA-YYYY-NNNN:<commit>:<path>, or \
334         DATA-YYYY-NNNN:<commit>:<path>:<from>-<to> for some of its lines; \
335         to read the data set itself, give its id alone.";
336
337    /// `true` when `s` starts with a `DATA-…` id and a `:`, so it is meant
338    /// as a citation and its error is the one to show
339    pub fn is_citation_shaped(s: &str) -> bool {
340        s.trim_start().split_once(':').is_some_and(|(head, _)| {
341            head.parse::<GovernanceLogId>()
342                .is_ok_and(|id| id.prefix() == GovernanceLogPrefix::Data)
343        })
344    }
345}
346
347impl std::str::FromStr for DataCitation {
348    type Err = DataCitationError;
349
350    fn from_str(s: &str) -> Result<Self, Self::Err> {
351        let s = s.trim();
352        let (head, rest) = s
353            .split_once(':')
354            .ok_or_else(|| DataCitationError::NotDataSet(s.to_string()))?;
355        let set: GovernanceLogId = head
356            .parse()
357            .ok()
358            .filter(|id: &GovernanceLogId| {
359                id.prefix() == GovernanceLogPrefix::Data
360            })
361            .ok_or_else(|| DataCitationError::NotDataSet(head.to_string()))?;
362        let (commit, rest) = match rest.split_once(':') {
363            Some((commit, rest)) => (commit, rest),
364            None => {
365                // Say what is wrong with the commit first
366                rest.parse::<GitCommitPrefix>()?;
367                return Err(DataCitationError::NoPath);
368            }
369        };
370        let commit: GitCommitPrefix = commit.parse()?;
371        // The range in a citation is strictly `<from>-<to>` or `<from>-`,
372        // as `Display` writes it, so a path ending in `:12` is never read
373        // as a range of another file (the review of 0.103)
374        let canonical = |r: &str| {
375            r.split_once('-').is_some_and(|(a, b)| {
376                !a.is_empty()
377                    && a.bytes().all(|c| c.is_ascii_digit())
378                    && b.bytes().all(|c| c.is_ascii_digit())
379            })
380        };
381        let (path, lines) = match rest.rsplit_once(':') {
382            Some((path, range)) if !path.is_empty() && canonical(range) => {
383                match range.parse::<LineRange>() {
384                    Ok(lines) => (path, Some(lines)),
385                    Err(_) => (rest, None),
386                }
387            }
388            _ => (rest, None),
389        };
390        let path = path.trim();
391        let path = path.strip_prefix("./").unwrap_or(path);
392        let path = path.trim_start_matches('/');
393        if path.is_empty() {
394            return Err(DataCitationError::NoPath);
395        }
396        Ok(Self {
397            set,
398            commit,
399            path: path.parse()?,
400            lines,
401        })
402    }
403}
404
405impl std::fmt::Display for DataCitation {
406    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
407        write!(f, "{}:{}:{}", self.set, self.commit, self.path)?;
408        if let Some(lines) = self.lines {
409            write!(f, ":{lines}")?;
410        }
411        Ok(())
412    }
413}
414
415#[cfg(test)]
416mod tests {
417    use super::*;
418
419    fn range(s: &str) -> LineRange {
420        s.parse().unwrap_or_else(|e| panic!("{s:?}: {e}"))
421    }
422
423    #[test]
424    fn ranges_parse_leniently_and_validate() {
425        for (written, start, end) in [
426            ("110-300", 110, Some(300)),
427            (" 110 - 300 ", 110, Some(300)),
428            ("110..300", 110, Some(300)),
429            ("110...300", 110, Some(300)),
430            ("110\u{2013}300", 110, Some(300)),
431            ("L110-L300", 110, Some(300)),
432            ("#L110-L300", 110, Some(300)),
433            ("lines 110-300", 110, Some(300)),
434            ("Lines=110-300", 110, Some(300)),
435            ("110 to 300", 110, Some(300)),
436            ("110-", 110, None),
437            ("110..", 110, None),
438            ("-300", 1, Some(300)),
439            ("110", 110, Some(110)),
440            ("7-7", 7, Some(7)),
441        ] {
442            let r = range(written);
443            assert_eq!((r.start(), r.end()), (start, end), "{written:?}");
444        }
445        assert_eq!(range("110-300").to_string(), "110-300");
446        assert_eq!(range("110-").to_string(), "110-");
447        for (bad, why) in [
448            ("0-10", "numbered from 1"),
449            ("300-110", "ends before it starts"),
450            ("", "no line numbers"),
451            ("-", "no line numbers"),
452            ("abc", "not a range"),
453            ("1-2-3", "not a range"),
454            ("1.5-3", "not a range"),
455            ("99999999999999999999999", "too large"),
456        ] {
457            let err = bad.parse::<LineRange>().unwrap_err();
458            let text = err.to_string();
459            assert!(text.contains(why), "{bad:?}: {text}");
460            assert!(text.contains("\"110-300\""), "says the form: {text}");
461        }
462    }
463
464    #[test]
465    fn ranges_round_trip_as_strings_and_take_a_bare_number() {
466        let r: LineRange = serde_json::from_str("\"110-300\"").unwrap();
467        assert_eq!(serde_json::to_value(r).unwrap(), "110-300");
468        let one: LineRange = serde_json::from_str("42").unwrap();
469        assert_eq!(one.to_string(), "42-42");
470        assert!(serde_json::from_str::<LineRange>("0").is_err());
471        assert!(serde_json::from_str::<LineRange>("\"9-1\"").is_err());
472    }
473
474    #[cfg(feature = "schemars")]
475    #[test]
476    fn a_range_is_a_plain_string_in_a_schema() {
477        let schema = schemars::schema_for!(LineRange);
478        let text = serde_json::to_string(&schema).unwrap();
479        assert!(!text.contains("$ref") && !text.contains("$defs"), "{text}");
480        assert!(!text.contains("pattern"), "{text}");
481        assert_eq!(schema.get("type").unwrap(), "string");
482    }
483
484    #[test]
485    fn spans_cover_whole_lines_and_clamp_the_end() {
486        let text = "one\ntwo\nthree\nfour";
487        assert_eq!(count_lines(text), 4);
488        assert_eq!(count_lines("a\n"), 1);
489        assert_eq!(count_lines(""), 0);
490        let span = |r: &str| {
491            let s = range(r).span(text).unwrap();
492            (&text[s.start_byte..s.end_byte], s.served.to_string())
493        };
494        assert_eq!(span("1-1"), ("one\n", "1-1".into()));
495        assert_eq!(span("2-3"), ("two\nthree\n", "2-3".into()));
496        assert_eq!(span("3-"), ("three\nfour", "3-4".into()));
497        assert_eq!(span("4-99"), ("four", "4-4".into()));
498        assert_eq!(span("1-"), (text, "1-4".into()));
499        let past = range("5-9").span(text).unwrap_err();
500        assert_eq!(past.total_lines, 4);
501        assert!(past.to_string().contains("has 4 line(s)"), "{past}");
502        assert!(range("1-").span("").is_err());
503        // A trailing newline ends the last line; it does not start one
504        assert_eq!(range("2-").span("a\nb\n").unwrap().end_byte, 4);
505        assert!(range("3-").span("a\nb\n").is_err());
506    }
507
508    #[test]
509    fn citations_parse_and_print() {
510        let c: DataCitation =
511            "DATA-2026-0001:89fe7c6:votes/ballots.csv:110-300"
512                .parse()
513                .unwrap();
514        assert_eq!(c.set.as_str(), "DATA-2026-0001");
515        assert_eq!(c.commit.as_str(), "89fe7c6");
516        assert_eq!(c.path.as_str(), "votes/ballots.csv");
517        assert_eq!(c.lines, Some(range("110-300")));
518        assert_eq!(
519            c.to_string(),
520            "DATA-2026-0001:89fe7c6:votes/ballots.csv:110-300"
521        );
522
523        let whole: DataCitation =
524            " data-2026-1:89FE7C6:votes/ballots.csv ".parse().unwrap();
525        assert_eq!(whole.set.as_str(), "DATA-2026-0001");
526        assert_eq!(whole.commit.as_str(), "89fe7c6");
527        assert_eq!(whole.lines, None);
528        assert_eq!(
529            whole.to_string(),
530            "DATA-2026-0001:89fe7c6:votes/ballots.csv"
531        );
532
533        // A path may hold a `:`; what follows the last one is a range only
534        // when it reads as one
535        let colon: DataCitation =
536            "DATA-2026-0001:89fe7c6:a:b.txt".parse().unwrap();
537        assert_eq!(colon.path.as_str(), "a:b.txt");
538        assert_eq!(colon.lines, None);
539        // Only the canonical `<from>-<to>` is a range: `log:12` is a file
540        let file: DataCitation =
541            "DATA-2026-0001:89fe7c6:log:12".parse().unwrap();
542        assert_eq!((file.path.as_str(), file.lines), ("log:12", None));
543        assert_eq!(file.to_string(), "DATA-2026-0001:89fe7c6:log:12");
544        let both: DataCitation =
545            "DATA-2026-0001:89fe7c6:a:b.txt:5-".parse().unwrap();
546        assert_eq!(both.path.as_str(), "a:b.txt");
547        assert_eq!(both.lines, Some(range("5-")));
548        let lead: DataCitation =
549            "DATA-2026-0001:89fe7c6:./votes/x.md".parse().unwrap();
550        assert_eq!(lead.path.as_str(), "votes/x.md");
551    }
552
553    #[test]
554    fn citations_refuse_what_they_cannot_name() {
555        let err = |s: &str| s.parse::<DataCitation>().unwrap_err();
556        assert!(matches!(
557            err("GOV-2026-0006:89fe7c6:x"),
558            DataCitationError::NotDataSet(_)
559        ));
560        assert!(matches!(
561            err("DATA-2026-0001:89fe7c:x"),
562            DataCitationError::Commit(_)
563        ));
564        assert!(matches!(
565            err("DATA-2026-0001:not-hex!:x"),
566            DataCitationError::Commit(_)
567        ));
568        assert!(matches!(
569            err(&format!("DATA-2026-0001:{}:x", "a".repeat(41))),
570            DataCitationError::Commit(_)
571        ));
572        assert!(matches!(
573            err("DATA-2026-0001:89fe7c6"),
574            DataCitationError::NoPath
575        ));
576        assert!(matches!(
577            err("DATA-2026-0001:89fe7c6:"),
578            DataCitationError::NoPath
579        ));
580        assert!(matches!(
581            err("DATA-2026-0001: the votes"),
582            DataCitationError::Commit(_)
583        ));
584        assert!(matches!(
585            err("DATA-2026-0001:89fe7c6:../secrets"),
586            DataCitationError::Path(_)
587        ));
588        let text = err("DATA-2026-0001:89fe7c6").to_string();
589        assert!(text.contains("DATA-YYYY-NNNN:<commit>:<path>"), "{text}");
590        assert!(text.contains("give its id alone"), "{text}");
591
592        assert!(DataCitation::is_citation_shaped("DATA-2026-0001:x"));
593        assert!(DataCitation::is_citation_shaped(" data-2026-1:"));
594        assert!(!DataCitation::is_citation_shaped("GOV-2026-0006: yes"));
595        assert!(!DataCitation::is_citation_shaped("DATA-2026-0001"));
596    }
597
598    #[test]
599    fn a_prefix_matches_its_commit() {
600        let commit: GitCommitId =
601            "89fe7c6aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".parse().unwrap();
602        let prefix: GitCommitPrefix = "89FE7C6".parse().unwrap();
603        assert!(prefix.matches(&commit));
604        assert!(
605            !"89fe7c7"
606                .parse::<GitCommitPrefix>()
607                .unwrap()
608                .matches(&commit)
609        );
610        assert!(GitCommitPrefix::from(commit).matches(&commit));
611    }
612}
agora_agentkit/govlog/data_read.rs

agora_agentkit/govlog/
data_read.rs
