Skip to main content

rig_tools/
search.rs

1//! The `search` tool, and [`find`], the search it runs, as data.
2
3use std::fmt;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6
7use regex::{Regex, RegexBuilder};
8use rig_core::completion::ToolAnnotations;
9use rig_core::tool::{
10    ContextValue, FileRead, FilesRead, Scope, Tool, ToolContext, ToolExecutionError, Touches,
11    args_schema, content_hash,
12};
13use schemars::JsonSchema;
14use serde::{Deserialize, Serialize, Serializer};
15
16use crate::fs::{listed, shown};
17use crate::workspace::{Listing, Local, Workspace};
18use crate::{MAX_FILE_BYTES, blocking, clip};
19
20/// Most matches a search returns.
21pub const MAX_MATCHES: usize = 200;
22/// Most files a search for file names ([`SearchArgs::files`]) returns.
23pub const MAX_FILES: usize = 1000;
24const MAX_LINE: usize = 300;
25
26/// Searches file contents with a regular expression, honouring
27/// `.gitignore`, or lists the files with a match, or every file. Shows each
28/// path as `read` takes it from the same base: below the base when the
29/// searched path is relative or not given, absolute when it is absolute or
30/// starts with `~`.
31/// Publishes what it found ([`SearchHits`]) and the files it found matches
32/// in ([`FilesRead`]), by their [resolved](Workspace) paths.
33#[derive(Clone, Debug)]
34pub struct Search {
35    /// Where the files are searched.
36    pub workspace: Arc<dyn Workspace>,
37    /// The directory searched, and a relative path is searched from;
38    /// `None` is the workspace's working directory. Hits of a relative
39    /// path are shown, and published, by their paths below it.
40    pub base: Option<PathBuf>,
41    /// Whether a path that resolves outside `base` is refused, before
42    /// anything is searched.
43    pub confine: bool,
44}
45
46impl Default for Search {
47    /// Searches this machine's files, from this process's working directory.
48    fn default() -> Self {
49        Self {
50            workspace: Arc::new(Local),
51            base: None,
52            confine: false,
53        }
54    }
55}
56
57/// Arguments of [`Search`].
58#[derive(Clone, Debug, PartialEq, Eq, Deserialize, JsonSchema)]
59#[serde(deny_unknown_fields)]
60pub struct SearchArgs {
61    /// The regular expression; with `files`, empty lists every file.
62    pub pattern: String,
63    /// The directory or file to search; default the current directory.
64    pub path: Option<String>,
65    /// Only files matching this glob, such as `*.rs`.
66    pub glob: Option<String>,
67    /// Also hidden files and directories (named with a leading `.`), such
68    /// as `.github/`; files git ignores stay out.
69    #[serde(default)]
70    pub hidden: bool,
71    /// Only the paths of the files with a match, one per line.
72    #[serde(default)]
73    pub files: bool,
74}
75
76impl SearchArgs {
77    /// Search the current directory for `pattern`.
78    pub fn new(pattern: impl Into<String>) -> Self {
79        Self {
80            pattern: pattern.into(),
81            path: None,
82            glob: None,
83            hidden: false,
84            files: false,
85        }
86    }
87
88    /// The same search, returning only the paths of the files with a
89    /// match; with an empty pattern, every file.
90    pub fn files(mut self) -> Self {
91        self.files = true;
92        self
93    }
94
95    /// The same search, in `path`.
96    pub fn in_path(mut self, path: impl Into<String>) -> Self {
97        self.path = Some(path.into());
98        self
99    }
100
101    /// The same search, only in files matching `glob`.
102    pub fn glob(mut self, glob: impl Into<String>) -> Self {
103        self.glob = Some(glob.into());
104        self
105    }
106
107    /// The same search, in hidden files too.
108    pub fn hidden(mut self) -> Self {
109        self.hidden = true;
110        self
111    }
112
113    /// Which files it searches.
114    pub fn listing(&self) -> Listing {
115        Listing::from_options(self.hidden, self.glob.as_deref())
116    }
117}
118
119/// One line a search matched.
120#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
121pub struct Hit {
122    /// The file, as the walk from the searched path reached it, without a
123    /// leading `./` unless it starts with `~`: `./~/notes.md` is a folder
124    /// named `~`, which `~/notes.md` is not.
125    #[serde(serialize_with = "lossy")]
126    pub path: PathBuf,
127    /// The line's number, from 1.
128    pub line: usize,
129    /// The line, cut at 300 characters.
130    pub text: String,
131}
132
133/// `path:line: text`, as [`Search`] shows a hit.
134impl fmt::Display for Hit {
135    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
136        write!(f, "{}:{}: {}", self.path.display(), self.line, self.text)
137    }
138}
139
140/// `path` as text, any bytes that are not UTF-8 replaced.
141fn lossy<S: Serializer>(path: &Path, serializer: S) -> Result<S::Ok, S::Error> {
142    serializer.serialize_str(&path.to_string_lossy())
143}
144
145/// What a search found.
146#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
147pub struct Found {
148    /// The matched lines, file by file in walk order; a search for file
149    /// names ([`SearchArgs::files`]) has none.
150    pub hits: Vec<Hit>,
151    /// The files with a match, of a search for file names, in walk order,
152    /// as the walk from the searched path reached them, without a leading
153    /// `./` unless one starts with `~`, as [`Hit::path`].
154    #[serde(default, skip_serializing_if = "Vec::is_empty")]
155    pub names: Vec<PathBuf>,
156    /// Whether it stopped at its [`MAX_MATCHES`] matches, or
157    /// [`MAX_FILES`] files, so there may be more.
158    pub stopped: bool,
159    /// The files with hits, each once, in walk order, by their
160    /// [resolved](Workspace) paths, with the hash of the bytes
161    /// searched.
162    #[serde(default)]
163    pub files: Vec<FileRead>,
164}
165
166/// What a [`Search`] call found, as it published it beside the text.
167#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
168#[serde(transparent)]
169pub struct SearchHits(pub Found);
170
171impl ContextValue for SearchHits {
172    const KEY: &'static str = "search_hits";
173}
174
175impl Tool for Search {
176    const NAME: &'static str = "search";
177    type Args = SearchArgs;
178    type Output = String;
179    type Error = ToolExecutionError;
180
181    fn description(&self) -> String {
182        format!(
183            "Search file contents with a regular expression (Rust regex syntax). Skips files \
184             ignored by git, target directories and, unless hidden is set or the glob names \
185             one, hidden files. Returns `path:line: text`, at most {MAX_MATCHES} matches; with \
186             files, only the paths of the files with a match, one per line, at most \
187             {MAX_FILES}, and with files and an empty pattern every file. Each path is one \
188             `read` takes: relative to the current directory when path is relative or not \
189             given, absolute when path is absolute or starts with `~`. Use it to find code, not `grep` or `rg` in \
190             `shell`, and with files and an empty pattern to list files, not `ls` or `find`."
191        )
192    }
193
194    fn parameters(&self) -> serde_json::Value {
195        args_schema::<SearchArgs>()
196    }
197
198    fn annotations(&self) -> ToolAnnotations {
199        ToolAnnotations::default()
200            .with_read_only(true)
201            .with_open_world(false)
202    }
203
204    fn footprint(&self, args: &SearchArgs) -> Touches {
205        Touches::Reads(Scope::Paths(vec![
206            self.workspace
207                .resolve(self.base.as_deref(), args.path.as_deref().unwrap_or(".")),
208        ]))
209    }
210
211    async fn call(
212        &self,
213        context: &mut ToolContext,
214        mut args: SearchArgs,
215    ) -> Result<String, ToolExecutionError> {
216        let path = args.path.as_deref().unwrap_or(".");
217        self.workspace
218            .place(self.base.as_deref(), self.confine, path)?;
219        let joined = self.workspace.join(
220            self.base.as_deref().unwrap_or(Path::new("")),
221            Path::new(path),
222        );
223        let given = Path::new(path);
224        let base = (given.is_relative() && !given.starts_with("~"))
225            .then(|| self.base.clone())
226            .flatten();
227        args.path = Some(joined.to_string_lossy().into_owned());
228        let workspace = self.workspace.clone();
229        let mut found = blocking(move || find(&*workspace, &args)).await?;
230        if let Some(base) = &base {
231            let paths = found.hits.iter_mut().map(|hit| &mut hit.path);
232            for path in paths.chain(&mut found.names) {
233                if let Ok(below) = path.strip_prefix(base) {
234                    *path = shown(below);
235                }
236            }
237        }
238        if !found.files.is_empty() {
239            context.insert_result(FilesRead(found.files.clone()))?;
240        }
241        let text = render(&found);
242        context.insert_result(SearchHits(found))?;
243        Ok(text)
244    }
245}
246
247/// `found` as [`Search`] shows it: a `path:line: text` line per hit, or a
248/// path per file of a search for file names, and a last line when it
249/// stopped at [`MAX_MATCHES`] or [`MAX_FILES`].
250pub fn render(found: &Found) -> String {
251    let (mut lines, stop): (Vec<String>, _) = match found.names.is_empty() {
252        true => (
253            found.hits.iter().map(Hit::to_string).collect(),
254            format!("[stopped at {MAX_MATCHES} matches]"),
255        ),
256        false => (
257            found
258                .names
259                .iter()
260                .map(|name| name.display().to_string())
261                .collect(),
262            format!("[stopped at {MAX_FILES} files; narrow the search with path or glob]"),
263        ),
264    };
265    if lines.is_empty() {
266        return "No matches.".to_owned();
267    }
268    if found.stopped {
269        lines.push(stop);
270    }
271    lines.join("\n")
272}
273
274/// Searches as [`Search`] does, in `workspace`, and returns the matches as
275/// data: the [files](Workspace::files) under the searched path that its
276/// [listing](SearchArgs::listing) keeps, each
277/// of at most [`MAX_FILE_BYTES`] and read as UTF-8; a search for file
278/// names with an empty pattern reads none and keeps them all. Blocks: run
279/// it on a thread of its own ([`blocking`]).
280///
281/// # Errors
282///
283/// When the pattern or the glob is malformed, the path does not exist, or
284/// the walk fails.
285pub fn find(workspace: &dyn Workspace, args: &SearchArgs) -> Result<Found, ToolExecutionError> {
286    let pattern = Regex::new(&args.pattern)
287        .map_err(|error| ToolExecutionError::invalid_args(format!("bad pattern: {error}")))?;
288    let whole = whole_text(&args.pattern);
289    let root = args.path.as_deref().unwrap_or(".");
290    let files = listed(
291        workspace,
292        Path::new(root),
293        args.glob.as_deref(),
294        args.listing(),
295    )?;
296    let mut found = Found::default();
297    if args.files && args.pattern.is_empty() {
298        for file in files {
299            found.names.push(walked(&file).to_path_buf());
300            if found.names.len() == MAX_FILES {
301                found.stopped = true;
302                break;
303            }
304        }
305        return Ok(found);
306    }
307    let read = files.chunks(100).flat_map(|chunk| {
308        let read = workspace.read_files(chunk, MAX_FILE_BYTES);
309        chunk.iter().zip(read)
310    });
311    for (file, bytes) in read {
312        let Some(text) = bytes.and_then(|bytes| String::from_utf8(bytes).ok()) else {
313            continue;
314        };
315        if let Some(whole) = &whole
316            && !lone_cr(&text)
317            && !whole.is_match(&text)
318        {
319            continue;
320        }
321        let path = walked(file);
322        let mut lines = text
323            .lines()
324            .enumerate()
325            .filter(|(_, line)| pattern.is_match(line))
326            .peekable();
327        if lines.peek().is_none() {
328            continue;
329        }
330        found.files.push(FileRead::new(
331            workspace.canonical(path).to_string_lossy(),
332            content_hash(text.as_bytes()),
333        ));
334        if args.files {
335            found.names.push(path.to_path_buf());
336            if found.names.len() == MAX_FILES {
337                found.stopped = true;
338                return Ok(found);
339            }
340            continue;
341        }
342        for (number, line) in lines {
343            found.hits.push(Hit {
344                path: path.to_path_buf(),
345                line: number + 1,
346                text: clip(line, MAX_LINE).to_owned(),
347            });
348            if found.hits.len() == MAX_MATCHES {
349                found.stopped = true;
350                return Ok(found);
351            }
352        }
353    }
354    Ok(found)
355}
356
357/// `pattern` over a whole text, matching it wherever it matches one of its
358/// lines, so a file it does not match has no line to search: `^` and `$`
359/// match at each line end, `\n` or `\r\n`. `None` when the pattern anchors
360/// to the ends of the text (`\A`, `\z`) or may turn a flag off, such as
361/// `(?-m)`, so its lines' ends would not count.
362fn whole_text(pattern: &str) -> Option<Regex> {
363    let groups = pattern.split("(?").skip(1);
364    let mut flags = groups.filter_map(|group| group.split([')', ':']).next());
365    if pattern.contains("\\A") || pattern.contains("\\z") || flags.any(|it| it.contains('-')) {
366        return None;
367    }
368    RegexBuilder::new(pattern)
369        .multi_line(true)
370        .crlf(true)
371        .build()
372        .ok()
373}
374
375/// Whether `text` has a `\r` that does not end a line: [`whole_text`]'s
376/// `.` does not match it, though a line's does.
377fn lone_cr(text: &str) -> bool {
378    text.contains('\r') && text.matches('\r').count() != text.matches("\r\n").count()
379}
380
381/// `path` as the walk reached it, without a leading `./` unless it is
382/// [`shown`] with one.
383fn walked(path: &Path) -> &Path {
384    match path.strip_prefix("./") {
385        Ok(rest) if !rest.starts_with("~") => rest,
386        _ => path,
387    }
388}
389
390#[cfg(test)]
391mod tests;