intelli_shell/service/
tldr.rs

1use color_eyre::{
2    Report,
3    eyre::{Context, OptionExt, eyre},
4};
5use futures_util::{FutureExt, StreamExt, TryStreamExt, stream};
6use git2::{
7    FetchOptions, ProxyOptions, RemoteCallbacks, Repository,
8    build::{CheckoutBuilder, RepoBuilder},
9};
10use tokio::{fs::File, sync::mpsc};
11use tokio_util::sync::CancellationToken;
12use tracing::instrument;
13use walkdir::WalkDir;
14
15use super::{IntelliShellService, import::parse_import_items};
16use crate::{
17    errors::{Result, UserFacingError},
18    model::{ImportStats, SOURCE_TLDR},
19};
20
21/// Progress events for the `tldr fetch` operation
22#[derive(Debug)]
23pub enum TldrFetchProgress {
24    /// Indicates the status of the tldr git repository
25    Repository(RepoStatus),
26    /// Indicates that the tldr command files are being located
27    LocatingFiles,
28    /// Indicates that the tldr command files have been located
29    FilesLocated(u64),
30    /// Indicates the start of the file processing stage
31    ProcessingStart(u64),
32    /// Indicates that a single file is being processed
33    ProcessingFile(String),
34    /// Indicates that a single file has been processed
35    FileProcessed(String),
36}
37
38/// The status of the tldr git repository
39#[derive(Debug)]
40pub enum RepoStatus {
41    /// Cloning the repository for the first time
42    Cloning,
43    /// The repository has been successfully cloned
44    DoneCloning,
45    /// Fetching latest changes
46    Fetching,
47    /// The repository is already up-to-date
48    UpToDate,
49    /// Updating the local repository
50    Updating,
51    /// The repository has been successfully updated
52    DoneUpdating,
53}
54
55impl IntelliShellService {
56    /// Removes tldr commands matching the given criteria.
57    ///
58    /// Returns the number of commands removed
59    #[instrument(skip_all)]
60    pub async fn clear_tldr_commands(&self, category: Option<String>) -> Result<u64> {
61        self.storage.delete_tldr_commands(category).await
62    }
63
64    /// Fetches and imports tldr commands matching the given criteria
65    #[instrument(skip_all)]
66    pub async fn fetch_tldr_commands(
67        &self,
68        category: Option<String>,
69        commands: Vec<String>,
70        progress: mpsc::Sender<TldrFetchProgress>,
71        cancellation_token: CancellationToken,
72    ) -> Result<ImportStats> {
73        // Check for cancellation at the beginning
74        if cancellation_token.is_cancelled() {
75            tracing::info!("TLDR fetch cancelled before starting");
76            return Err(UserFacingError::Cancelled.into());
77        }
78
79        // Setup repository
80        self.setup_tldr_repo(progress.clone(), cancellation_token.clone())
81            .await?;
82
83        // Determine which categories to import
84        let categories = if let Some(cat) = category {
85            vec![cat]
86        } else {
87            vec![
88                "common".to_owned(),
89                #[cfg(target_os = "windows")]
90                "windows".to_owned(),
91                #[cfg(target_os = "android")]
92                "android".to_owned(),
93                #[cfg(target_os = "macos")]
94                "osx".to_owned(),
95                #[cfg(target_os = "freebsd")]
96                "freebsd".to_owned(),
97                #[cfg(target_os = "openbsd")]
98                "openbsd".to_owned(),
99                #[cfg(target_os = "netbsd")]
100                "netbsd".to_owned(),
101                #[cfg(any(
102                    target_os = "linux",
103                    target_os = "freebsd",
104                    target_os = "openbsd",
105                    target_os = "netbsd",
106                    target_os = "dragonfly",
107                ))]
108                "linux".to_owned(),
109            ]
110        };
111
112        // Construct the path to the tldr pages directory
113        let pages_path = self.tldr_repo_path.join("pages");
114
115        tracing::info!("Locating files for categories: {}", categories.join(", "));
116        progress.send(TldrFetchProgress::LocatingFiles).await.ok();
117
118        // Iterate over directory entries
119        let mut command_files = Vec::new();
120        let mut iter = WalkDir::new(&pages_path).max_depth(2).into_iter();
121        while let Some(result) = iter.next() {
122            // Check for cancellation within the file discovery loop
123            if cancellation_token.is_cancelled() {
124                tracing::info!("TLDR fetch cancelled during file discovery");
125                return Err(UserFacingError::Cancelled.into());
126            }
127
128            let entry = result.wrap_err("Couldn't read tldr repository files")?;
129            let path = entry.path();
130
131            // Skip base path
132            if path == pages_path {
133                continue;
134            }
135
136            // Skip non-included categories
137            let file_name = entry.file_name().to_str().ok_or_eyre("Non valid file name")?;
138            if entry.file_type().is_dir() {
139                if !categories.iter().any(|c| c == file_name) {
140                    tracing::trace!("Skipped directory: {file_name}");
141                    iter.skip_current_dir();
142                    continue;
143                } else {
144                    // The directory entry itself must be skipped as well, we only care about files
145                    continue;
146                }
147            }
148
149            // We only care about markdown files
150            let Some(file_name_no_ext) = file_name.strip_suffix(".md") else {
151                tracing::warn!("Unexpected file found: {}", path.display());
152                continue;
153            };
154
155            // Skip non-included commands
156            if !commands.is_empty() {
157                if !commands.iter().any(|c| c == file_name_no_ext) {
158                    continue;
159                } else {
160                    tracing::trace!("Included command: {file_name_no_ext}");
161                }
162            }
163
164            // Retrieve the category
165            let category = path
166                .parent()
167                .and_then(|p| p.file_name())
168                .and_then(|p| p.to_str())
169                .ok_or_eyre("Couldn't read tldr category")?
170                .to_owned();
171
172            // Include the command file
173            command_files.push((path.to_path_buf(), category, file_name_no_ext.to_owned()));
174        }
175
176        progress
177            .send(TldrFetchProgress::FilesLocated(command_files.len() as u64))
178            .await
179            .ok();
180
181        tracing::info!("Found {} files to be processed", command_files.len());
182
183        progress
184            .send(TldrFetchProgress::ProcessingStart(command_files.len() as u64))
185            .await
186            .ok();
187
188        // Create a stream that reads and parses each command file concurrently
189        let items_stream = stream::iter(command_files)
190            .map(move |(path, category, command)| {
191                let progress = progress.clone();
192                async move {
193                    progress
194                        .send(TldrFetchProgress::ProcessingFile(command.clone()))
195                        .await
196                        .ok();
197
198                    // Open and parse the file
199                    let file = File::open(&path)
200                        .await
201                        .wrap_err_with(|| format!("Failed to open tldr file: {}", path.display()))?;
202                    let stream = parse_import_items(file, vec![], category, SOURCE_TLDR);
203
204                    progress.send(TldrFetchProgress::FileProcessed(command)).await.ok();
205                    Ok::<_, Report>(stream)
206                }
207            })
208            .buffered(5)
209            .try_flatten();
210
211        // Import items while the token is not cancelled
212        let stats = self
213            .storage
214            .import_items(
215                items_stream.take_until(cancellation_token.clone().cancelled_owned().fuse()),
216                true,
217                false,
218            )
219            .await?;
220
221        // After processing, check if cancellation was the reason the stream ended
222        if cancellation_token.is_cancelled() {
223            tracing::info!("TLDR fetch cancelled during command processing");
224            return Err(UserFacingError::Cancelled.into());
225        }
226
227        Ok(stats)
228    }
229
230    #[instrument(skip_all)]
231    async fn setup_tldr_repo(
232        &self,
233        progress: mpsc::Sender<TldrFetchProgress>,
234        cancellation_token: CancellationToken,
235    ) -> Result<bool> {
236        const BRANCH: &str = "main";
237        const REPO_URL: &str = "https://github.com/tldr-pages/tldr.git";
238
239        let tldr_repo_path = self.tldr_repo_path.clone();
240
241        tokio::task::spawn_blocking(move || {
242            // Helper to send progress to the channel
243            let send_progress = |status| {
244                // Use blocking_send as we are in a sync context
245                progress.blocking_send(TldrFetchProgress::Repository(status)).ok();
246            };
247            // Setup git callbacks to enable cancellation
248            let mut callbacks = RemoteCallbacks::new();
249            callbacks.transfer_progress(move |_| !cancellation_token.is_cancelled());
250            // Setup git fetch options for a swallow copy with auto proxy config
251            let mut proxy_opts = ProxyOptions::new();
252            proxy_opts.auto();
253            let mut fetch_options = FetchOptions::new();
254            fetch_options.proxy_options(proxy_opts);
255            fetch_options.remote_callbacks(callbacks);
256            fetch_options.depth(1);
257            // Fetch latest repo changes or clone it if it doesn't exist yet
258            if tldr_repo_path.exists() {
259                tracing::info!("Fetching latest tldr changes ...");
260                send_progress(RepoStatus::Fetching);
261
262                // Open the existing repository.
263                let repo = Repository::open(&tldr_repo_path).wrap_err("Failed to open existing tldr repository")?;
264
265                // Get the 'origin' remote
266                let mut remote = repo.find_remote("origin")?;
267
268                // Fetch the latest changes from the remote 'main' branch
269                let refspec = format!("refs/heads/{BRANCH}:refs/remotes/origin/{BRANCH}");
270                if let Err(err) = remote.fetch(&[refspec], Some(&mut fetch_options), None) {
271                    // Check if the error was a user-initiated cancellation
272                    if err.code() == git2::ErrorCode::User && err.class() == git2::ErrorClass::Callback {
273                        return Err(UserFacingError::Cancelled.into());
274                    }
275                    return Err(Report::from(err).wrap_err("Failed to fetch from tldr remote").into());
276                }
277
278                // Get the commit OID from the fetched data (FETCH_HEAD)
279                let fetch_head = repo.find_reference("FETCH_HEAD")?;
280                let fetch_commit_oid = fetch_head
281                    .target()
282                    .ok_or_else(|| eyre!("FETCH_HEAD is not a direct reference"))?;
283
284                // Get the OID of the current commit on the local branch
285                let local_ref_name = format!("refs/heads/{BRANCH}");
286                let local_commit_oid = repo.find_reference(&local_ref_name)?.target();
287
288                // If the commit OIDs are the same, the repo is already up-to-date
289                if Some(fetch_commit_oid) == local_commit_oid {
290                    tracing::info!("Repository is already up-to-date");
291                    send_progress(RepoStatus::UpToDate);
292                    return Ok(false);
293                }
294
295                tracing::info!("Updating to the latest version ...");
296                send_progress(RepoStatus::Updating);
297
298                // Find the local branch reference
299                let mut local_ref = repo.find_reference(&local_ref_name)?;
300                // Update the local branch to point directly to the newly fetched commit
301                let msg = format!("Resetting to latest commit {fetch_commit_oid}");
302                local_ref.set_target(fetch_commit_oid, &msg)?;
303
304                // Point HEAD to the updated local branch
305                repo.set_head(&local_ref_name)?;
306
307                // Checkout the new HEAD to update the files in the working directory
308                let mut checkout_builder = CheckoutBuilder::new();
309                checkout_builder.force();
310                repo.checkout_head(Some(&mut checkout_builder))?;
311
312                tracing::info!("Repository successfully updated");
313                send_progress(RepoStatus::DoneUpdating);
314                Ok(true)
315            } else {
316                tracing::info!("Performing a shallow clone of '{REPO_URL}' ...");
317                send_progress(RepoStatus::Cloning);
318
319                // Clone the repository
320                if let Err(err) = RepoBuilder::new()
321                    .branch(BRANCH)
322                    .fetch_options(fetch_options)
323                    .clone(REPO_URL, &tldr_repo_path)
324                {
325                    if err.code() == git2::ErrorCode::User && err.class() == git2::ErrorClass::Callback {
326                        return Err(UserFacingError::Cancelled.into());
327                    }
328                    return Err(Report::from(err).wrap_err("Failed to clone tldr repository").into());
329                }
330
331                tracing::info!("Repository successfully cloned");
332                send_progress(RepoStatus::DoneCloning);
333                Ok(true)
334            }
335        })
336        .await
337        .wrap_err("tldr repository task failed")?
338    }
339}