intelli_shell/service/
tldr.rs

1use color_eyre::{
2    Report,
3    eyre::{Context, OptionExt, eyre},
4};
5use futures_util::{StreamExt, TryStreamExt, stream};
6use git2::{
7    FetchOptions, Repository,
8    build::{CheckoutBuilder, RepoBuilder},
9};
10use tokio::{fs::File, sync::mpsc};
11use tracing::instrument;
12use walkdir::WalkDir;
13
14use super::{IntelliShellService, import_export::parse_commands};
15use crate::{errors::Result, model::SOURCE_TLDR};
16
17/// Progress events for the `tldr fetch` operation
18#[derive(Debug)]
19pub enum TldrFetchProgress {
20    /// Indicates the status of the tldr git repository
21    Repository(RepoStatus),
22    /// Indicates that the tldr command files are being located
23    LocatingFiles,
24    /// Indicates that the tldr command files have been located
25    FilesLocated(u64),
26    /// Indicates the start of the file processing stage
27    ProcessingStart(u64),
28    /// Indicates that a single file is being processed
29    ProcessingFile(String),
30    /// Indicates that a single file has been processed
31    FileProcessed(String),
32}
33
34/// The status of the tldr git repository
35#[derive(Debug)]
36pub enum RepoStatus {
37    /// Cloning the repository for the first time
38    Cloning,
39    /// The repository has been successfully cloned
40    DoneCloning,
41    /// Fetching latest changes
42    Fetching,
43    /// The repository is already up-to-date
44    UpToDate,
45    /// Updating the local repository
46    Updating,
47    /// The repository has been successfully updated
48    DoneUpdating,
49}
50
51impl IntelliShellService {
52    /// Removes tldr commands matching the given criteria.
53    ///
54    /// Returns the number of commands removed
55    #[instrument(skip_all)]
56    pub async fn clear_tldr_commands(&self, category: Option<String>) -> Result<u64> {
57        self.storage.delete_tldr_commands(category).await
58    }
59
60    /// Fetches and imports tldr commands matching the given criteria.
61    ///
62    /// Returns the number of new commands inserted and potentially updated (because they already existed)
63    #[instrument(skip_all)]
64    pub async fn fetch_tldr_commands(
65        &self,
66        category: Option<String>,
67        commands: Vec<String>,
68        progress: mpsc::Sender<TldrFetchProgress>,
69    ) -> Result<(u64, u64)> {
70        // Setup repository
71        self.setup_tldr_repo(progress.clone()).await?;
72
73        // Determine which categories to import
74        let categories = if let Some(cat) = category {
75            vec![cat]
76        } else {
77            vec![
78                "common".to_owned(),
79                #[cfg(target_os = "windows")]
80                "windows".to_owned(),
81                #[cfg(target_os = "android")]
82                "android".to_owned(),
83                #[cfg(target_os = "macos")]
84                "osx".to_owned(),
85                #[cfg(target_os = "freebsd")]
86                "freebsd".to_owned(),
87                #[cfg(target_os = "openbsd")]
88                "openbsd".to_owned(),
89                #[cfg(target_os = "netbsd")]
90                "netbsd".to_owned(),
91                #[cfg(any(
92                    target_os = "linux",
93                    target_os = "freebsd",
94                    target_os = "openbsd",
95                    target_os = "netbsd",
96                    target_os = "dragonfly",
97                ))]
98                "linux".to_owned(),
99            ]
100        };
101
102        // Construct the path to the tldr pages directory
103        let pages_path = self.tldr_repo_path.join("pages");
104
105        tracing::info!("Locating files for categories: {}", categories.join(", "));
106        progress.send(TldrFetchProgress::LocatingFiles).await.ok();
107
108        // Iterate over directory entries
109        let mut command_files = Vec::new();
110        let mut iter = WalkDir::new(&pages_path).max_depth(2).into_iter();
111        while let Some(result) = iter.next() {
112            let entry = result.wrap_err("Couldn't read tldr repository files")?;
113            let path = entry.path();
114
115            // Skip base path
116            if path == pages_path {
117                continue;
118            }
119
120            // Skip non-included categories
121            let file_name = entry.file_name().to_str().ok_or_eyre("Non valid file name")?;
122            if entry.file_type().is_dir() {
123                if !categories.iter().any(|c| c == file_name) {
124                    tracing::trace!("Skipped directory: {file_name}");
125                    iter.skip_current_dir();
126                    continue;
127                } else {
128                    // The directory entry itself must be skipped as well, we only care about files
129                    continue;
130                }
131            }
132
133            // We only care about markdown files
134            let Some(file_name_no_ext) = file_name.strip_suffix(".md") else {
135                tracing::warn!("Unexpected file found: {}", path.display());
136                continue;
137            };
138
139            // Skip non-included commands
140            if !commands.is_empty() {
141                if !commands.iter().any(|c| c == file_name_no_ext) {
142                    continue;
143                } else {
144                    tracing::trace!("Included command: {file_name_no_ext}");
145                }
146            }
147
148            // Retrieve the category
149            let category = path
150                .parent()
151                .and_then(|p| p.file_name())
152                .and_then(|p| p.to_str())
153                .ok_or_eyre("Couldn't read tldr category")?
154                .to_owned();
155
156            // Include the command file
157            command_files.push((path.to_path_buf(), category, file_name_no_ext.to_owned()));
158        }
159
160        progress
161            .send(TldrFetchProgress::FilesLocated(command_files.len() as u64))
162            .await
163            .ok();
164
165        tracing::info!("Found {} files to be processed", command_files.len());
166
167        progress
168            .send(TldrFetchProgress::ProcessingStart(command_files.len() as u64))
169            .await
170            .ok();
171
172        // Create a stream that reads and parses each command file concurrently
173        let commands_stream = stream::iter(command_files)
174            .map(move |(path, category, command)| {
175                let progress = progress.clone();
176                async move {
177                    progress
178                        .send(TldrFetchProgress::ProcessingFile(command.clone()))
179                        .await
180                        .ok();
181
182                    // Open and parse the file
183                    let file = File::open(&path)
184                        .await
185                        .wrap_err_with(|| format!("Failed to open tldr file: {}", path.display()))?;
186                    let stream = parse_commands(file, vec![], category, SOURCE_TLDR);
187
188                    progress.send(TldrFetchProgress::FileProcessed(command)).await.ok();
189                    Ok::<_, Report>(stream)
190                }
191            })
192            .buffered(5)
193            .try_flatten();
194
195        // Import the commands
196        self.storage.import_commands(commands_stream, true, false).await
197    }
198
199    #[instrument(skip_all)]
200    async fn setup_tldr_repo(&self, progress: mpsc::Sender<TldrFetchProgress>) -> Result<bool> {
201        const BRANCH: &str = "main";
202        const REPO_URL: &str = "https://github.com/tldr-pages/tldr.git";
203
204        let tldr_repo_path = self.tldr_repo_path.clone();
205
206        tokio::task::spawn_blocking(move || {
207            let send_progress = |status| {
208                // Use blocking_send as we are in a sync context
209                progress.blocking_send(TldrFetchProgress::Repository(status)).ok();
210            };
211            if tldr_repo_path.exists() {
212                tracing::info!("Fetching latest tldr changes ...");
213                send_progress(RepoStatus::Fetching);
214
215                // Open the existing repository.
216                let repo = Repository::open(&tldr_repo_path).wrap_err("Failed to open existing tldr repository")?;
217
218                // Get the 'origin' remote
219                let mut remote = repo.find_remote("origin")?;
220
221                // Configure fetch options for a shallow fetch
222                let mut fetch_options = FetchOptions::new();
223                fetch_options.depth(1);
224
225                // Fetch the latest changes from the remote 'main' branch
226                let refspec = format!("refs/heads/{BRANCH}:refs/remotes/origin/{BRANCH}");
227                remote
228                    .fetch(&[refspec], Some(&mut fetch_options), None)
229                    .wrap_err("Failed to fetch from tldr remote")?;
230
231                // Get the commit OID from the fetched data (FETCH_HEAD)
232                let fetch_head = repo.find_reference("FETCH_HEAD")?;
233                let fetch_commit_oid = fetch_head
234                    .target()
235                    .ok_or_else(|| eyre!("FETCH_HEAD is not a direct reference"))?;
236
237                // Get the OID of the current commit on the local branch
238                let local_ref_name = format!("refs/heads/{BRANCH}");
239                let local_commit_oid = repo.find_reference(&local_ref_name)?.target();
240
241                // If the commit OIDs are the same, the repo is already up-to-date
242                if Some(fetch_commit_oid) == local_commit_oid {
243                    tracing::info!("Repository is already up-to-date");
244                    send_progress(RepoStatus::UpToDate);
245                    return Ok(false);
246                }
247
248                tracing::info!("Updating to the latest version ...");
249                send_progress(RepoStatus::Updating);
250
251                // Find the local branch reference
252                let mut local_ref = repo.find_reference(&local_ref_name)?;
253                // Update the local branch to point directly to the newly fetched commit
254                let msg = format!("Resetting to latest commit {fetch_commit_oid}");
255                local_ref.set_target(fetch_commit_oid, &msg)?;
256
257                // Point HEAD to the updated local branch
258                repo.set_head(&local_ref_name)?;
259
260                // Checkout the new HEAD to update the files in the working directory
261                let mut checkout_builder = CheckoutBuilder::new();
262                checkout_builder.force();
263                repo.checkout_head(Some(&mut checkout_builder))?;
264
265                tracing::info!("Repository successfully updated");
266                send_progress(RepoStatus::DoneUpdating);
267                Ok(true)
268            } else {
269                tracing::info!("Performing a shallow clone of '{REPO_URL}' ...");
270                send_progress(RepoStatus::Cloning);
271
272                // Configure fetch options for a shallow fetch
273                let mut fetch_options = FetchOptions::new();
274                fetch_options.depth(1);
275
276                // Clone the repository
277                RepoBuilder::new()
278                    .branch(BRANCH)
279                    .fetch_options(fetch_options)
280                    .clone(REPO_URL, &tldr_repo_path)
281                    .wrap_err("Failed to clone tldr repository")?;
282
283                tracing::info!("Repository successfully cloned");
284                send_progress(RepoStatus::DoneCloning);
285                Ok(true)
286            }
287        })
288        .await
289        .wrap_err("tldr repository task failed")?
290    }
291}
intelli_shell/service/tldr.rs

intelli_shell/service/
tldr.rs
