intelli_shell/service/
tldr.rs

1use color_eyre::{
2    Report,
3    eyre::{Context, OptionExt, eyre},
4};
5use futures_util::{StreamExt, TryStreamExt, stream};
6use git2::{
7    FetchOptions, ProxyOptions, Repository,
8    build::{CheckoutBuilder, RepoBuilder},
9};
10use tokio::{fs::File, sync::mpsc};
11use tracing::instrument;
12use walkdir::WalkDir;
13
14use super::{IntelliShellService, import::parse_import_items};
15use crate::{
16    errors::Result,
17    model::{ImportStats, SOURCE_TLDR},
18};
19
20/// Progress events for the `tldr fetch` operation
21#[derive(Debug)]
22pub enum TldrFetchProgress {
23    /// Indicates the status of the tldr git repository
24    Repository(RepoStatus),
25    /// Indicates that the tldr command files are being located
26    LocatingFiles,
27    /// Indicates that the tldr command files have been located
28    FilesLocated(u64),
29    /// Indicates the start of the file processing stage
30    ProcessingStart(u64),
31    /// Indicates that a single file is being processed
32    ProcessingFile(String),
33    /// Indicates that a single file has been processed
34    FileProcessed(String),
35}
36
37/// The status of the tldr git repository
38#[derive(Debug)]
39pub enum RepoStatus {
40    /// Cloning the repository for the first time
41    Cloning,
42    /// The repository has been successfully cloned
43    DoneCloning,
44    /// Fetching latest changes
45    Fetching,
46    /// The repository is already up-to-date
47    UpToDate,
48    /// Updating the local repository
49    Updating,
50    /// The repository has been successfully updated
51    DoneUpdating,
52}
53
54impl IntelliShellService {
55    /// Removes tldr commands matching the given criteria.
56    ///
57    /// Returns the number of commands removed
58    #[instrument(skip_all)]
59    pub async fn clear_tldr_commands(&self, category: Option<String>) -> Result<u64> {
60        self.storage.delete_tldr_commands(category).await
61    }
62
63    /// Fetches and imports tldr commands matching the given criteria
64    #[instrument(skip_all)]
65    pub async fn fetch_tldr_commands(
66        &self,
67        category: Option<String>,
68        commands: Vec<String>,
69        progress: mpsc::Sender<TldrFetchProgress>,
70    ) -> Result<ImportStats> {
71        // Setup repository
72        self.setup_tldr_repo(progress.clone()).await?;
73
74        // Determine which categories to import
75        let categories = if let Some(cat) = category {
76            vec![cat]
77        } else {
78            vec![
79                "common".to_owned(),
80                #[cfg(target_os = "windows")]
81                "windows".to_owned(),
82                #[cfg(target_os = "android")]
83                "android".to_owned(),
84                #[cfg(target_os = "macos")]
85                "osx".to_owned(),
86                #[cfg(target_os = "freebsd")]
87                "freebsd".to_owned(),
88                #[cfg(target_os = "openbsd")]
89                "openbsd".to_owned(),
90                #[cfg(target_os = "netbsd")]
91                "netbsd".to_owned(),
92                #[cfg(any(
93                    target_os = "linux",
94                    target_os = "freebsd",
95                    target_os = "openbsd",
96                    target_os = "netbsd",
97                    target_os = "dragonfly",
98                ))]
99                "linux".to_owned(),
100            ]
101        };
102
103        // Construct the path to the tldr pages directory
104        let pages_path = self.tldr_repo_path.join("pages");
105
106        tracing::info!("Locating files for categories: {}", categories.join(", "));
107        progress.send(TldrFetchProgress::LocatingFiles).await.ok();
108
109        // Iterate over directory entries
110        let mut command_files = Vec::new();
111        let mut iter = WalkDir::new(&pages_path).max_depth(2).into_iter();
112        while let Some(result) = iter.next() {
113            let entry = result.wrap_err("Couldn't read tldr repository files")?;
114            let path = entry.path();
115
116            // Skip base path
117            if path == pages_path {
118                continue;
119            }
120
121            // Skip non-included categories
122            let file_name = entry.file_name().to_str().ok_or_eyre("Non valid file name")?;
123            if entry.file_type().is_dir() {
124                if !categories.iter().any(|c| c == file_name) {
125                    tracing::trace!("Skipped directory: {file_name}");
126                    iter.skip_current_dir();
127                    continue;
128                } else {
129                    // The directory entry itself must be skipped as well, we only care about files
130                    continue;
131                }
132            }
133
134            // We only care about markdown files
135            let Some(file_name_no_ext) = file_name.strip_suffix(".md") else {
136                tracing::warn!("Unexpected file found: {}", path.display());
137                continue;
138            };
139
140            // Skip non-included commands
141            if !commands.is_empty() {
142                if !commands.iter().any(|c| c == file_name_no_ext) {
143                    continue;
144                } else {
145                    tracing::trace!("Included command: {file_name_no_ext}");
146                }
147            }
148
149            // Retrieve the category
150            let category = path
151                .parent()
152                .and_then(|p| p.file_name())
153                .and_then(|p| p.to_str())
154                .ok_or_eyre("Couldn't read tldr category")?
155                .to_owned();
156
157            // Include the command file
158            command_files.push((path.to_path_buf(), category, file_name_no_ext.to_owned()));
159        }
160
161        progress
162            .send(TldrFetchProgress::FilesLocated(command_files.len() as u64))
163            .await
164            .ok();
165
166        tracing::info!("Found {} files to be processed", command_files.len());
167
168        progress
169            .send(TldrFetchProgress::ProcessingStart(command_files.len() as u64))
170            .await
171            .ok();
172
173        // Create a stream that reads and parses each command file concurrently
174        let items_stream = stream::iter(command_files)
175            .map(move |(path, category, command)| {
176                let progress = progress.clone();
177                async move {
178                    progress
179                        .send(TldrFetchProgress::ProcessingFile(command.clone()))
180                        .await
181                        .ok();
182
183                    // Open and parse the file
184                    let file = File::open(&path)
185                        .await
186                        .wrap_err_with(|| format!("Failed to open tldr file: {}", path.display()))?;
187                    let stream = parse_import_items(file, vec![], category, SOURCE_TLDR);
188
189                    progress.send(TldrFetchProgress::FileProcessed(command)).await.ok();
190                    Ok::<_, Report>(stream)
191                }
192            })
193            .buffered(5)
194            .try_flatten();
195
196        // Import the commands
197        self.storage.import_items(items_stream, true, false).await
198    }
199
200    #[instrument(skip_all)]
201    async fn setup_tldr_repo(&self, progress: mpsc::Sender<TldrFetchProgress>) -> Result<bool> {
202        const BRANCH: &str = "main";
203        const REPO_URL: &str = "https://github.com/tldr-pages/tldr.git";
204
205        let tldr_repo_path = self.tldr_repo_path.clone();
206
207        tokio::task::spawn_blocking(move || {
208            // Helper to send progress to the channel
209            let send_progress = |status| {
210                // Use blocking_send as we are in a sync context
211                progress.blocking_send(TldrFetchProgress::Repository(status)).ok();
212            };
213            // Setup git fetch options for a swallow copy with auto proxy config
214            let mut proxy_opts = ProxyOptions::new();
215            proxy_opts.auto();
216            let mut fetch_options = FetchOptions::new();
217            fetch_options.proxy_options(proxy_opts);
218            fetch_options.depth(1);
219            // Fetch latest repo changes or clone it if it doesn't exist yet
220            if tldr_repo_path.exists() {
221                tracing::info!("Fetching latest tldr changes ...");
222                send_progress(RepoStatus::Fetching);
223
224                // Open the existing repository.
225                let repo = Repository::open(&tldr_repo_path).wrap_err("Failed to open existing tldr repository")?;
226
227                // Get the 'origin' remote
228                let mut remote = repo.find_remote("origin")?;
229
230                // Fetch the latest changes from the remote 'main' branch
231                let refspec = format!("refs/heads/{BRANCH}:refs/remotes/origin/{BRANCH}");
232                remote
233                    .fetch(&[refspec], Some(&mut fetch_options), None)
234                    .wrap_err("Failed to fetch from tldr remote")?;
235
236                // Get the commit OID from the fetched data (FETCH_HEAD)
237                let fetch_head = repo.find_reference("FETCH_HEAD")?;
238                let fetch_commit_oid = fetch_head
239                    .target()
240                    .ok_or_else(|| eyre!("FETCH_HEAD is not a direct reference"))?;
241
242                // Get the OID of the current commit on the local branch
243                let local_ref_name = format!("refs/heads/{BRANCH}");
244                let local_commit_oid = repo.find_reference(&local_ref_name)?.target();
245
246                // If the commit OIDs are the same, the repo is already up-to-date
247                if Some(fetch_commit_oid) == local_commit_oid {
248                    tracing::info!("Repository is already up-to-date");
249                    send_progress(RepoStatus::UpToDate);
250                    return Ok(false);
251                }
252
253                tracing::info!("Updating to the latest version ...");
254                send_progress(RepoStatus::Updating);
255
256                // Find the local branch reference
257                let mut local_ref = repo.find_reference(&local_ref_name)?;
258                // Update the local branch to point directly to the newly fetched commit
259                let msg = format!("Resetting to latest commit {fetch_commit_oid}");
260                local_ref.set_target(fetch_commit_oid, &msg)?;
261
262                // Point HEAD to the updated local branch
263                repo.set_head(&local_ref_name)?;
264
265                // Checkout the new HEAD to update the files in the working directory
266                let mut checkout_builder = CheckoutBuilder::new();
267                checkout_builder.force();
268                repo.checkout_head(Some(&mut checkout_builder))?;
269
270                tracing::info!("Repository successfully updated");
271                send_progress(RepoStatus::DoneUpdating);
272                Ok(true)
273            } else {
274                tracing::info!("Performing a shallow clone of '{REPO_URL}' ...");
275                send_progress(RepoStatus::Cloning);
276
277                // Clone the repository
278                RepoBuilder::new()
279                    .branch(BRANCH)
280                    .fetch_options(fetch_options)
281                    .clone(REPO_URL, &tldr_repo_path)
282                    .wrap_err("Failed to clone tldr repository")?;
283
284                tracing::info!("Repository successfully cloned");
285                send_progress(RepoStatus::DoneCloning);
286                Ok(true)
287            }
288        })
289        .await
290        .wrap_err("tldr repository task failed")?
291    }
292}
intelli_shell/service/tldr.rs

intelli_shell/service/
tldr.rs
