Skip to main content

rig_tools/
read.rs

1//! The `read` tool.
2
3use std::collections::{HashMap, VecDeque};
4use std::fmt::Write as _;
5use std::path::{Path, PathBuf};
6use std::sync::{Arc, Mutex, PoisonError};
7
8use rig_core::message::{ImageMediaType, ToolResultContent};
9use rig_core::tool::{
10    ContentHash, FileRead, FilesRead, KeptEnd, LinesShown, OutputCut, Scope, Tool, ToolContext,
11    ToolExecutionError, ToolOutput, Touches, args_schema, content_hash,
12};
13use schemars::JsonSchema;
14use serde::Deserialize;
15
16use crate::attach::{self, NoImage, is_image_path};
17use crate::workspace::{Local, Workspace, text_error};
18use crate::{
19    CUT_MARK, MAX_BYTES, MAX_FILE_BYTES, MAX_LINE_CHARS, MAX_LINES, blocking, clip, shell_path,
20};
21
22/// Reads a text file's lines as they are, or a PNG, JPEG, GIF or WebP
23/// image as a line saying what it is and the image. Publishes the file it
24/// read ([`FilesRead`]), by its [resolved](Workspace) path, with the hash
25/// of the bytes it read and, for a text file, the lines it showed, and,
26/// when lines after those were left out, [`OutputCut`].
27#[derive(Clone, Debug)]
28pub struct Read {
29    /// Where the file is read.
30    pub workspace: Arc<dyn Workspace>,
31    /// The directory a relative path is read from; `None` is the
32    /// workspace's working directory.
33    pub base: Option<PathBuf>,
34    /// Whether a path that resolves outside `base` is refused, before
35    /// anything is read.
36    pub confine: bool,
37    /// Where each file read is noted, for an [`Edit`](crate::Edit) given
38    /// the same to tell whether a file changed since.
39    pub seen: Seen,
40}
41
42impl Default for Read {
43    /// Reads this machine's files, from this process's working directory.
44    fn default() -> Self {
45        Self {
46            workspace: Arc::new(Local),
47            base: None,
48            confine: false,
49            seen: Seen::default(),
50        }
51    }
52}
53
54/// Most bytes of file text a [`Seen`] keeps, for the files seen last.
55const SEEN_TEXT_BYTES: usize = 8 * 1024 * 1024;
56
57/// The hash ([`content_hash`]) of each file's text as [`Read`] last showed
58/// it or [`Edit`](crate::Edit) or [`Write`](crate::Write) last left it, by
59/// resolved path; shared by its clones. With it, an edit whose `old_text`
60/// is not found says whether the file changed since, and so needs reading
61/// again.
62///
63/// It also keeps the whole text of the files last seen whole, up to 8 MB
64/// of them, so an edit starts from it and only writes, if the file still
65/// holds that text ([`Workspace::write_if`]).
66#[derive(Clone, Debug, Default)]
67pub struct Seen(Arc<Mutex<Files>>);
68
69/// What a [`Seen`] holds.
70#[derive(Debug, Default)]
71struct Files {
72    hashes: HashMap<PathBuf, ContentHash>,
73    /// The texts kept, oldest first, each with its hash.
74    texts: VecDeque<(PathBuf, ContentHash, Arc<str>)>,
75    /// The bytes of `texts`.
76    bytes: usize,
77}
78
79impl Files {
80    /// Drops the text kept of `path`, if one is.
81    fn drop_text(&mut self, path: &Path) {
82        let Self { texts, bytes, .. } = self;
83        texts.retain(|(kept, _, text)| {
84            let same = kept == path;
85            if same {
86                *bytes -= text.len();
87            }
88            !same
89        });
90    }
91}
92
93impl Seen {
94    /// Notes that the file `path` held the text with `hash`.
95    pub fn saw(&self, path: &Path, hash: ContentHash) {
96        let mut seen = self.0.lock().unwrap_or_else(PoisonError::into_inner);
97        seen.hashes.insert(path.to_path_buf(), hash);
98    }
99
100    /// Notes that the file `path` held `text`, all of it: its hash, and the
101    /// text itself, which a text seen longer ago may make room for.
102    pub fn saw_whole(&self, path: &Path, text: &str) {
103        let hash = content_hash(text.as_bytes());
104        let mut seen = self.0.lock().unwrap_or_else(PoisonError::into_inner);
105        seen.hashes.insert(path.to_path_buf(), hash.clone());
106        seen.drop_text(path);
107        let Files { texts, bytes, .. } = &mut *seen;
108        if text.len() > SEEN_TEXT_BYTES {
109            return;
110        }
111        while *bytes + text.len() > SEEN_TEXT_BYTES
112            && let Some((_, _, old)) = texts.pop_front()
113        {
114            *bytes -= old.len();
115        }
116        *bytes += text.len();
117        texts.push_back((path.to_path_buf(), hash, Arc::from(text)));
118    }
119
120    /// Forgets what `path` held, as after a write of it that may or may
121    /// not have happened, so the next edit starts from the file itself.
122    pub fn forget(&self, path: &Path) {
123        let mut seen = self.0.lock().unwrap_or_else(PoisonError::into_inner);
124        seen.hashes.remove(path);
125        seen.drop_text(path);
126    }
127
128    /// The hash of the text `path` last held when it was seen, if it was.
129    pub fn hash(&self, path: &Path) -> Option<ContentHash> {
130        let seen = self.0.lock().unwrap_or_else(PoisonError::into_inner);
131        seen.hashes.get(path).cloned()
132    }
133
134    /// The whole text `path` held when it was last seen, and its hash, if
135    /// it was seen whole then and the text is still kept.
136    pub fn text(&self, path: &Path) -> Option<(ContentHash, Arc<str>)> {
137        let seen = self.0.lock().unwrap_or_else(PoisonError::into_inner);
138        let last = seen.hashes.get(path)?;
139        seen.texts
140            .iter()
141            .find(|(kept, hash, _)| kept == path && hash == last)
142            .map(|(_, hash, text)| (hash.clone(), text.clone()))
143    }
144}
145
146/// Arguments of [`Read`].
147#[derive(Clone, Debug, PartialEq, Eq, Deserialize, JsonSchema)]
148#[serde(deny_unknown_fields)]
149pub struct ReadArgs {
150    /// The file to read.
151    pub path: String,
152    /// The first line to read, from 1.
153    pub offset: Option<usize>,
154    /// How many lines to read.
155    pub limit: Option<usize>,
156}
157
158impl ReadArgs {
159    /// Read `path` from its start.
160    pub fn new(path: impl Into<String>) -> Self {
161        Self {
162            path: path.into(),
163            offset: None,
164            limit: None,
165        }
166    }
167
168    /// The same read, of at most `limit` lines from line `offset` (from 1).
169    pub fn lines(mut self, offset: usize, limit: usize) -> Self {
170        self.offset = Some(offset);
171        self.limit = Some(limit);
172        self
173    }
174
175    /// What [`Read`] shows of `text`, the file's whole text, for this read:
176    /// its lines from `offset` (default 1), at most `limit` of them (default
177    /// and most [`MAX_LINES`]) and [`MAX_BYTES`], as [`show_lines`] shows them.
178    ///
179    /// ```
180    /// use rig_tools::ReadArgs;
181    ///
182    /// let shown = ReadArgs::new("a.txt").lines(2, 1).show("one\ntwo\nthree\n");
183    /// assert_eq!(shown, "two\n[1 more lines; read on with offset 3]\n");
184    /// ```
185    pub fn show(&self, text: &str) -> String {
186        self.shown(text).0
187    }
188
189    /// [`Self::show`], and which lines it shows: what [`Read`] publishes
190    /// in its [`FileRead`], for code that shows a file as `read` would and
191    /// records what it showed.
192    ///
193    /// ```
194    /// use rig_tools::ReadArgs;
195    ///
196    /// let (_, lines) = ReadArgs::new("a.txt").lines(2, 1).shown("one\ntwo\nthree\n");
197    /// assert_eq!((lines.first, lines.count, lines.total), (2, 1, 3));
198    /// ```
199    pub fn shown(&self, text: &str) -> (String, LinesShown) {
200        let (first, limit) = self.span();
201        lines_from(text, first, limit, MAX_BYTES)
202    }
203
204    /// The first line this read shows, from 1, and how many lines at most.
205    fn span(&self) -> (usize, usize) {
206        let first = self.offset.unwrap_or(1).max(1);
207        (first, self.limit.unwrap_or(MAX_LINES).min(MAX_LINES))
208    }
209}
210
211impl Tool for Read {
212    const NAME: &'static str = "read";
213    type Args = ReadArgs;
214    type Output = ToolOutput;
215    type Error = ToolExecutionError;
216
217    fn description(&self) -> String {
218        format!(
219            "Read a regular text file of at most {} MB, or a PNG, JPEG, GIF or WebP image of at \
220             most {} KB, which comes back as the image. Returns a text file's lines as they are, \
221             at most {MAX_LINES} lines or {} KB (lines too long for that are cut shorter, and a \
222             note says so and how to read on), then the offset to read on from if it stopped \
223             early; use offset (from 1) and limit for part of a file, and `rg -n` in `shell` to \
224             find a line. Use `read` to look at a file, not `cat`, `head` or `sed` in `shell` \
225             (but for the command a cut line's note gives), and read several files at once by \
226             calling `read` several times in one reply.",
227            MAX_FILE_BYTES / (1024 * 1024),
228            attach::MOST_BYTES / 1024,
229            MAX_BYTES / 1024
230        )
231    }
232
233    fn parameters(&self) -> serde_json::Value {
234        args_schema::<ReadArgs>()
235    }
236
237    fn annotations(&self) -> rig_core::completion::ToolAnnotations {
238        rig_core::completion::ToolAnnotations::default()
239            .with_read_only(true)
240            .with_open_world(false)
241    }
242
243    fn footprint(&self, args: &ReadArgs) -> Touches {
244        Touches::Reads(Scope::Paths(vec![
245            self.workspace.resolve(self.base.as_deref(), &args.path),
246        ]))
247    }
248
249    async fn call(
250        &self,
251        context: &mut ToolContext,
252        args: ReadArgs,
253    ) -> Result<ToolOutput, ToolExecutionError> {
254        let path = self
255            .workspace
256            .place(self.base.as_deref(), self.confine, &args.path)?;
257        let shown = path.to_string_lossy().into_owned();
258        // The name asked counts too, for a link named as an image.
259        let image = is_image_path(&path) || is_image_path(Path::new(&args.path));
260        let workspace = self.workspace.clone();
261        let seen = self.seen.clone();
262        let (output, read) = blocking(move || {
263            if image {
264                match attach::image(workspace.as_ref(), &path) {
265                    Ok(image) => {
266                        let read = FileRead::new(&shown, content_hash(&image.bytes));
267                        let output = ToolOutput::content(vec![
268                            ToolResultContent::text(format!(
269                                "{shown}: a {} image of {} KB",
270                                name(&image.media_type),
271                                image.bytes.len().div_ceil(1024)
272                            )),
273                            ToolResultContent::image_base64(
274                                image.base64(),
275                                Some(image.media_type),
276                                None,
277                            ),
278                        ])?;
279                        return Ok((output, read));
280                    }
281                    Err(large @ NoImage::TooLarge(_)) => {
282                        return Err(ToolExecutionError::invalid_args(format!(
283                            "{shown}: {large}"
284                        )));
285                    }
286                    Err(NoImage::Read(error)) => return Err(text_error(&shown, error)),
287                    // Named as an image but not one: read as any other file.
288                    Err(NoImage::Other) => {}
289                }
290            }
291            let (first, limit) = args.span();
292            let excerpt = workspace
293                .excerpt(&path, first, limit)
294                .map_err(|error| text_error(&shown, error))?;
295            let asked = excerpt.text.lines().take(limit).collect();
296            let (output, lines) = show(asked, first, excerpt.total, MAX_BYTES, Some(&shown));
297            let hash = excerpt.hash;
298            // A read that got the whole file leaves its text for an edit.
299            let whole = excerpt
300                .whole
301                .as_deref()
302                .or((first == 1 && excerpt.total <= limit).then_some(excerpt.text.as_str()))
303                .filter(|whole| content_hash(whole.as_bytes()) == hash);
304            match whole {
305                Some(whole) => seen.saw_whole(&path, whole),
306                None => seen.saw(&path, hash.clone()),
307            }
308            let read = FileRead::new(shown, hash).with_lines(lines);
309            Ok::<_, ToolExecutionError>((ToolOutput::text(output), read))
310        })
311        .await?;
312        let cut = read.lines.and_then(cut_of);
313        context.insert_result(FilesRead(vec![read]))?;
314        if let Some(cut) = cut {
315            context.insert_result(cut)?;
316        }
317        Ok(output)
318    }
319}
320
321/// How a read that showed `lines` was cut: from its first line on, the
322/// lines after those it showed, which its last line names.
323fn cut_of(lines: LinesShown) -> Option<OutputCut> {
324    let from_first = lines.total.checked_sub(lines.first.saturating_sub(1))?;
325    (lines.count < from_first).then_some(OutputCut {
326        kept: KeptEnd::Start,
327        kept_lines: lines.count,
328        total_lines: from_first,
329        spill: None,
330    })
331}
332
333/// `text`'s lines from line `first` (counted from 1) as [`Read`] shows
334/// them: as they are, at most `limit` lines and about `max_bytes` bytes,
335/// and a last line saying how many lines were left out and the offset to
336/// read on from, or that `first` is past the end. Lines longer than a
337/// width are cut, marked `[…]`, and counted in a note: 2000 characters, or
338/// less when that lets the asked lines fit `max_bytes`, as in a file of
339/// long lines, but never below 100 characters.
340pub fn show_lines(text: &str, first: usize, limit: usize, max_bytes: usize) -> String {
341    lines_from(text, first, limit, max_bytes).0
342}
343
344/// The narrowest a line is cut to so that more lines fit.
345const MIN_LINE_CHARS: usize = 100;
346/// The bytes of a cut line the command in a read's note shows at once:
347/// a fifth of what `shell` returns, in lines it keeps whole.
348const CHUNK_BYTES: usize = 10_000;
349
350/// [`show_lines`], and which lines it shows.
351fn lines_from(text: &str, first: usize, limit: usize, max_bytes: usize) -> (String, LinesShown) {
352    let first = first.max(1);
353    let asked = text.lines().skip(first - 1).take(limit).collect();
354    show(asked, first, text.lines().count(), max_bytes, None)
355}
356
357/// [`lines_from`] given the lines `asked`, from line `first` (from 1) of
358/// a file of `total` lines. With the file's `path`, the note on the lines
359/// cut says how to read on in the first of them.
360fn show(
361    asked: Vec<&str>,
362    first: usize,
363    total: usize,
364    max_bytes: usize,
365    path: Option<&str>,
366) -> (String, LinesShown) {
367    let width = width(&asked, max_bytes);
368    let mut out = String::new();
369    let mut shown = 0;
370    let mut cut = 0;
371    // The first line cut, its number, and the bytes shown of it.
372    let mut first_cut = None;
373    for (at, line) in asked.iter().enumerate() {
374        let kept = clip(line, width);
375        let clipped = kept.len() < line.len();
376        let mark = if clipped { CUT_MARK.len() } else { 0 };
377        // The first line always shows, so a read always makes progress.
378        if shown > 0 && out.len() + kept.len() + mark + 1 > max_bytes {
379            break;
380        }
381        out.push_str(kept);
382        if clipped {
383            out.push_str(CUT_MARK);
384            cut += 1;
385            first_cut = first_cut.or(Some((first + at, kept.len(), line.len())));
386        }
387        out.push('\n');
388        shown += 1;
389    }
390    let last = first - 1 + shown;
391    if cut > 0 {
392        let _ = write!(
393            out,
394            "[{cut} {} cut at {width} characters{}",
395            if cut == 1 { "line was" } else { "lines were" },
396            if width < MAX_LINE_CHARS {
397                format!(
398                    " to fit {} KB; a smaller limit shows more of each",
399                    max_bytes / 1024
400                )
401            } else {
402                String::new()
403            }
404        );
405        if let (Some(path), Some((number, kept, bytes))) = (path.and_then(shell_path), first_cut) {
406            out.push_str(&read_on(&path, cut, number, kept, bytes));
407        }
408        out.push_str("]\n");
409    }
410    if last < total {
411        let full = shown < asked.len();
412        let _ = writeln!(
413            out,
414            "[{} more lines; read on with offset {}{}]",
415            total - last,
416            last + 1,
417            if full {
418                format!(", as this read reached its {} KB", max_bytes / 1024)
419            } else {
420                String::new()
421            }
422        );
423    }
424    if total == 0 {
425        out.push_str("[empty file]\n");
426    } else if first > total {
427        out.push_str(&format!(
428            "[offset {first} is past the end: the file has {total} lines]\n"
429        ));
430    }
431    (
432        out,
433        LinesShown {
434            first,
435            count: shown,
436            total,
437        },
438    )
439}
440
441/// What an image of `media_type` is called.
442fn name(media_type: &ImageMediaType) -> &'static str {
443    match media_type {
444        ImageMediaType::PNG => "PNG",
445        ImageMediaType::JPEG => "JPEG",
446        ImageMediaType::GIF => "GIF",
447        ImageMediaType::WEBP => "WebP",
448        ImageMediaType::HEIC => "HEIC",
449        ImageMediaType::HEIF => "HEIF",
450        ImageMediaType::SVG => "SVG",
451    }
452}
453
454/// How to read on in line `number` of the file at `path`, quoted for
455/// `sh`, cut after its first `kept` of `bytes` bytes, the first of the
456/// `cut` lines cut: a command that prints the next bytes of it in lines
457/// `shell` keeps whole.
458fn read_on(path: &str, cut: usize, number: usize, kept: usize, bytes: usize) -> String {
459    let which = match cut {
460        1 => format!("line {number}"),
461        _ => format!("the first, line {number},"),
462    };
463    let from = kept + 1;
464    let to = bytes.min(kept + CHUNK_BYTES);
465    let next = match to < bytes {
466        true => format!(
467            "the next {CHUNK_BYTES} bytes, and `cut -b {}-{}` the ones after",
468            to + 1,
469            bytes.min(to + CHUNK_BYTES)
470        ),
471        false => format!("the remaining {} bytes", to - kept),
472    };
473    format!(
474        "; {which} shows {kept} of its {bytes} bytes: in `shell`, `sed -n '{number}p;{number}q' \
475         {path} | cut -b {from}-{to} | LC_ALL=C fold -b -w {MAX_LINE_CHARS}` shows {next}"
476    )
477}
478
479/// The width the lines of `asked` are cut at to be shown in `max_bytes`:
480/// [`MAX_LINE_CHARS`] when they fit so, else the widest of at least
481/// [`MIN_LINE_CHARS`] at which they do. When even that is too wide, the
482/// lines are cut at [`MIN_LINE_CHARS`] only if most of them are longer, so
483/// a file of ordinary lines with a few long ones keeps its lines whole and
484/// is read on in pieces.
485fn width(asked: &[&str], max_bytes: usize) -> usize {
486    let size = |width: usize| -> usize {
487        asked
488            .iter()
489            .map(|line| {
490                let kept = clip(line, width);
491                let mark = if kept.len() < line.len() {
492                    CUT_MARK.len()
493                } else {
494                    0
495                };
496                kept.len() + mark + 1
497            })
498            .sum()
499    };
500    if size(MAX_LINE_CHARS) <= max_bytes {
501        return MAX_LINE_CHARS;
502    }
503    if size(MIN_LINE_CHARS) > max_bytes {
504        let long = asked
505            .iter()
506            .filter(|line| line.chars().nth(MIN_LINE_CHARS).is_some())
507            .count();
508        return if long * 2 > asked.len() {
509            MIN_LINE_CHARS
510        } else {
511            MAX_LINE_CHARS
512        };
513    }
514    // The widest that fits: `low` always does, `high` never.
515    let (mut low, mut high) = (MIN_LINE_CHARS, MAX_LINE_CHARS);
516    while high - low > 1 {
517        let middle = low + (high - low) / 2;
518        if size(middle) <= max_bytes {
519            low = middle;
520        } else {
521            high = middle;
522        }
523    }
524    low
525}
526
527#[cfg(test)]
528mod tests;