intelli_shell/service/
tldr.rs1use color_eyre::{
2 Report,
3 eyre::{Context, OptionExt, eyre},
4};
5use futures_util::{FutureExt, StreamExt, TryStreamExt, stream};
6use git2::{
7 FetchOptions, ProxyOptions, RemoteCallbacks, Repository,
8 build::{CheckoutBuilder, RepoBuilder},
9};
10use tokio::{fs::File, sync::mpsc};
11use tokio_util::sync::CancellationToken;
12use tracing::instrument;
13use walkdir::WalkDir;
14
15use super::{IntelliShellService, import::parse_import_items};
16use crate::{
17 errors::{Result, UserFacingError},
18 model::{ImportStats, SOURCE_TLDR},
19};
20
21#[derive(Debug)]
23pub enum TldrFetchProgress {
24 Repository(RepoStatus),
26 LocatingFiles,
28 FilesLocated(u64),
30 ProcessingStart(u64),
32 ProcessingFile(String),
34 FileProcessed(String),
36}
37
38#[derive(Debug)]
40pub enum RepoStatus {
41 Cloning,
43 DoneCloning,
45 Fetching,
47 UpToDate,
49 Updating,
51 DoneUpdating,
53}
54
55impl IntelliShellService {
56 #[instrument(skip_all)]
60 pub async fn clear_tldr_commands(&self, category: Option<String>) -> Result<u64> {
61 self.storage.delete_tldr_commands(category).await
62 }
63
64 #[instrument(skip_all)]
66 pub async fn fetch_tldr_commands(
67 &self,
68 category: Option<String>,
69 commands: Vec<String>,
70 progress: mpsc::Sender<TldrFetchProgress>,
71 cancellation_token: CancellationToken,
72 ) -> Result<ImportStats> {
73 if cancellation_token.is_cancelled() {
75 tracing::info!("TLDR fetch cancelled before starting");
76 return Err(UserFacingError::Cancelled.into());
77 }
78
79 self.setup_tldr_repo(progress.clone(), cancellation_token.clone())
81 .await?;
82
83 let categories = if let Some(cat) = category {
85 vec![cat]
86 } else {
87 vec![
88 "common".to_owned(),
89 #[cfg(target_os = "windows")]
90 "windows".to_owned(),
91 #[cfg(target_os = "android")]
92 "android".to_owned(),
93 #[cfg(target_os = "macos")]
94 "osx".to_owned(),
95 #[cfg(target_os = "freebsd")]
96 "freebsd".to_owned(),
97 #[cfg(target_os = "openbsd")]
98 "openbsd".to_owned(),
99 #[cfg(target_os = "netbsd")]
100 "netbsd".to_owned(),
101 #[cfg(any(
102 target_os = "linux",
103 target_os = "freebsd",
104 target_os = "openbsd",
105 target_os = "netbsd",
106 target_os = "dragonfly",
107 ))]
108 "linux".to_owned(),
109 ]
110 };
111
112 let pages_path = self.tldr_repo_path.join("pages");
114
115 tracing::info!("Locating files for categories: {}", categories.join(", "));
116 progress.send(TldrFetchProgress::LocatingFiles).await.ok();
117
118 let mut command_files = Vec::new();
120 let mut iter = WalkDir::new(&pages_path).max_depth(2).into_iter();
121 while let Some(result) = iter.next() {
122 if cancellation_token.is_cancelled() {
124 tracing::info!("TLDR fetch cancelled during file discovery");
125 return Err(UserFacingError::Cancelled.into());
126 }
127
128 let entry = result.wrap_err("Couldn't read tldr repository files")?;
129 let path = entry.path();
130
131 if path == pages_path {
133 continue;
134 }
135
136 let file_name = entry.file_name().to_str().ok_or_eyre("Non valid file name")?;
138 if entry.file_type().is_dir() {
139 if !categories.iter().any(|c| c == file_name) {
140 tracing::trace!("Skipped directory: {file_name}");
141 iter.skip_current_dir();
142 continue;
143 } else {
144 continue;
146 }
147 }
148
149 let Some(file_name_no_ext) = file_name.strip_suffix(".md") else {
151 tracing::warn!("Unexpected file found: {}", path.display());
152 continue;
153 };
154
155 if !commands.is_empty() {
157 if !commands.iter().any(|c| c == file_name_no_ext) {
158 continue;
159 } else {
160 tracing::trace!("Included command: {file_name_no_ext}");
161 }
162 }
163
164 let category = path
166 .parent()
167 .and_then(|p| p.file_name())
168 .and_then(|p| p.to_str())
169 .ok_or_eyre("Couldn't read tldr category")?
170 .to_owned();
171
172 command_files.push((path.to_path_buf(), category, file_name_no_ext.to_owned()));
174 }
175
176 progress
177 .send(TldrFetchProgress::FilesLocated(command_files.len() as u64))
178 .await
179 .ok();
180
181 tracing::info!("Found {} files to be processed", command_files.len());
182
183 progress
184 .send(TldrFetchProgress::ProcessingStart(command_files.len() as u64))
185 .await
186 .ok();
187
188 let items_stream = stream::iter(command_files)
190 .map(move |(path, category, command)| {
191 let progress = progress.clone();
192 async move {
193 progress
194 .send(TldrFetchProgress::ProcessingFile(command.clone()))
195 .await
196 .ok();
197
198 let file = File::open(&path)
200 .await
201 .wrap_err_with(|| format!("Failed to open tldr file: {}", path.display()))?;
202 let stream = parse_import_items(file, vec![], category, SOURCE_TLDR);
203
204 progress.send(TldrFetchProgress::FileProcessed(command)).await.ok();
205 Ok::<_, Report>(stream)
206 }
207 })
208 .buffered(5)
209 .try_flatten();
210
211 let stats = self
213 .storage
214 .import_items(
215 items_stream.take_until(cancellation_token.clone().cancelled_owned().fuse()),
216 true,
217 false,
218 )
219 .await?;
220
221 if cancellation_token.is_cancelled() {
223 tracing::info!("TLDR fetch cancelled during command processing");
224 return Err(UserFacingError::Cancelled.into());
225 }
226
227 Ok(stats)
228 }
229
230 #[instrument(skip_all)]
231 async fn setup_tldr_repo(
232 &self,
233 progress: mpsc::Sender<TldrFetchProgress>,
234 cancellation_token: CancellationToken,
235 ) -> Result<bool> {
236 const BRANCH: &str = "main";
237 const REPO_URL: &str = "https://github.com/tldr-pages/tldr.git";
238
239 let tldr_repo_path = self.tldr_repo_path.clone();
240
241 tokio::task::spawn_blocking(move || {
242 let send_progress = |status| {
244 progress.blocking_send(TldrFetchProgress::Repository(status)).ok();
246 };
247 let mut callbacks = RemoteCallbacks::new();
249 callbacks.transfer_progress(move |_| !cancellation_token.is_cancelled());
250 let mut proxy_opts = ProxyOptions::new();
252 proxy_opts.auto();
253 let mut fetch_options = FetchOptions::new();
254 fetch_options.proxy_options(proxy_opts);
255 fetch_options.remote_callbacks(callbacks);
256 fetch_options.depth(1);
257 if tldr_repo_path.exists() {
259 tracing::info!("Fetching latest tldr changes ...");
260 send_progress(RepoStatus::Fetching);
261
262 let repo = Repository::open(&tldr_repo_path).wrap_err("Failed to open existing tldr repository")?;
264
265 let mut remote = repo.find_remote("origin")?;
267
268 let refspec = format!("refs/heads/{BRANCH}:refs/remotes/origin/{BRANCH}");
270 if let Err(err) = remote.fetch(&[refspec], Some(&mut fetch_options), None) {
271 if err.code() == git2::ErrorCode::User && err.class() == git2::ErrorClass::Callback {
273 return Err(UserFacingError::Cancelled.into());
274 }
275 return Err(Report::from(err).wrap_err("Failed to fetch from tldr remote").into());
276 }
277
278 let fetch_head = repo.find_reference("FETCH_HEAD")?;
280 let fetch_commit_oid = fetch_head
281 .target()
282 .ok_or_else(|| eyre!("FETCH_HEAD is not a direct reference"))?;
283
284 let local_ref_name = format!("refs/heads/{BRANCH}");
286 let local_commit_oid = repo.find_reference(&local_ref_name)?.target();
287
288 if Some(fetch_commit_oid) == local_commit_oid {
290 tracing::info!("Repository is already up-to-date");
291 send_progress(RepoStatus::UpToDate);
292 return Ok(false);
293 }
294
295 tracing::info!("Updating to the latest version ...");
296 send_progress(RepoStatus::Updating);
297
298 let mut local_ref = repo.find_reference(&local_ref_name)?;
300 let msg = format!("Resetting to latest commit {fetch_commit_oid}");
302 local_ref.set_target(fetch_commit_oid, &msg)?;
303
304 repo.set_head(&local_ref_name)?;
306
307 let mut checkout_builder = CheckoutBuilder::new();
309 checkout_builder.force();
310 repo.checkout_head(Some(&mut checkout_builder))?;
311
312 tracing::info!("Repository successfully updated");
313 send_progress(RepoStatus::DoneUpdating);
314 Ok(true)
315 } else {
316 tracing::info!("Performing a shallow clone of '{REPO_URL}' ...");
317 send_progress(RepoStatus::Cloning);
318
319 if let Err(err) = RepoBuilder::new()
321 .branch(BRANCH)
322 .fetch_options(fetch_options)
323 .clone(REPO_URL, &tldr_repo_path)
324 {
325 if err.code() == git2::ErrorCode::User && err.class() == git2::ErrorClass::Callback {
326 return Err(UserFacingError::Cancelled.into());
327 }
328 return Err(Report::from(err).wrap_err("Failed to clone tldr repository").into());
329 }
330
331 tracing::info!("Repository successfully cloned");
332 send_progress(RepoStatus::DoneCloning);
333 Ok(true)
334 }
335 })
336 .await
337 .wrap_err("tldr repository task failed")?
338 }
339}