intelli_shell/service/
import.rs

1use std::{
2    io::{Cursor, ErrorKind},
3    sync::LazyLock,
4};
5
6use color_eyre::{Report, eyre::Context};
7use futures_util::{TryStreamExt, stream};
8use itertools::Itertools;
9use regex::Regex;
10use reqwest::{
11    Url,
12    header::{self, HeaderName, HeaderValue},
13};
14use tokio::{
15    fs::{self, File},
16    io::{AsyncBufReadExt, AsyncRead, BufReader, Lines},
17};
18use tokio_stream::Stream;
19use tracing::instrument;
20
21use super::IntelliShellService;
22use crate::{
23    cli::{HistorySource, HttpMethod, ImportItemsProcess},
24    config::GistConfig,
25    errors::{AppError, Result, UserFacingError},
26    model::{
27        CATEGORY_USER, Command, ImportExportItem, ImportExportStream, ImportStats, SOURCE_IMPORT, VariableCompletion,
28    },
29    utils::{
30        add_tags_to_description, convert_alt_to_regular,
31        dto::{GIST_README_FILENAME, GIST_README_FILENAME_UPPER, GistDto, ImportExportItemDto},
32        extract_gist_data, github_to_raw, read_history,
33    },
34};
35
36impl IntelliShellService {
37    /// Import commands and completions
38    pub async fn import_items(&self, items: ImportExportStream, overwrite: bool) -> Result<ImportStats> {
39        self.storage.import_items(items, overwrite, false).await
40    }
41
42    /// Returns a list of items to import from a location
43    pub async fn get_items_from_location(
44        &self,
45        args: ImportItemsProcess,
46        gist_config: GistConfig,
47    ) -> Result<ImportExportStream> {
48        let ImportItemsProcess {
49            location,
50            file,
51            http,
52            gist,
53            history,
54            ai,
55            filter,
56            dry_run: _,
57            tags,
58            headers,
59            method,
60        } = args;
61
62        // Make sure the tags starts with a hashtag (#)
63        let tags = tags
64            .into_iter()
65            .filter_map(|mut tag| {
66                tag.chars().next().map(|first_char| {
67                    if first_char == '#' {
68                        tag
69                    } else {
70                        tag.insert(0, '#');
71                        tag
72                    }
73                })
74            })
75            .collect::<Vec<_>>();
76
77        // Retrieve the commands from the location
78        let commands = if let Some(history) = history {
79            self.get_history_items(history, filter, tags, ai).await?
80        } else if file {
81            if location == "-" {
82                self.get_stdin_items(filter, tags, ai).await?
83            } else {
84                self.get_file_items(location, filter, tags, ai).await?
85            }
86        } else if http {
87            self.get_http_items(location, headers, method, filter, tags, ai).await?
88        } else if gist {
89            self.get_gist_items(location, gist_config, filter, tags, ai).await?
90        } else {
91            // Determine which mode based on the location
92            if location == "gist"
93                || location.starts_with("https://gist.github.com")
94                || location.starts_with("https://api.github.com/gists")
95            {
96                self.get_gist_items(location, gist_config, filter, tags, ai).await?
97            } else if location.starts_with("http://") || location.starts_with("https://") {
98                self.get_http_items(location, headers, method, filter, tags, ai).await?
99            } else if location == "-" {
100                self.get_stdin_items(filter, tags, ai).await?
101            } else {
102                self.get_file_items(location, filter, tags, ai).await?
103            }
104        };
105
106        Ok(commands)
107    }
108
109    #[instrument(skip_all)]
110    async fn get_history_items(
111        &self,
112        history: HistorySource,
113        filter: Option<Regex>,
114        tags: Vec<String>,
115        ai: bool,
116    ) -> Result<ImportExportStream> {
117        if let Some(ref filter) = filter {
118            tracing::info!(ai, "Importing commands matching `{filter}` from {history:?} history");
119        } else {
120            tracing::info!(ai, "Importing commands from {history:?} history");
121        }
122        let content = Cursor::new(read_history(history)?);
123        self.extract_and_filter_items(content, filter, tags, ai).await
124    }
125
126    #[instrument(skip_all)]
127    async fn get_stdin_items(&self, filter: Option<Regex>, tags: Vec<String>, ai: bool) -> Result<ImportExportStream> {
128        if let Some(ref filter) = filter {
129            tracing::info!(ai, "Importing commands matching `{filter}` from stdin");
130        } else {
131            tracing::info!(ai, "Importing commands from stdin");
132        }
133        let content = tokio::io::stdin();
134        self.extract_and_filter_items(content, filter, tags, ai).await
135    }
136
137    #[instrument(skip_all)]
138    async fn get_file_items(
139        &self,
140        path: String,
141        filter: Option<Regex>,
142        tags: Vec<String>,
143        ai: bool,
144    ) -> Result<ImportExportStream> {
145        // Otherwise, check the path to import the file
146        match fs::metadata(&path).await {
147            Ok(m) if m.is_file() => (),
148            Ok(_) => return Err(UserFacingError::ImportLocationNotAFile.into()),
149            Err(err) if err.kind() == ErrorKind::NotFound => return Err(UserFacingError::ImportFileNotFound.into()),
150            Err(err) if err.kind() == ErrorKind::PermissionDenied => {
151                return Err(UserFacingError::FileNotAccessible("read").into());
152            }
153            Err(err) => return Err(Report::from(err).into()),
154        }
155        if let Some(ref filter) = filter {
156            tracing::info!(ai, "Importing commands matching `{filter}` from file: {path}");
157        } else {
158            tracing::info!(ai, "Importing commands from file: {path}");
159        }
160        let content = File::open(path).await.wrap_err("Couldn't open the file")?;
161        self.extract_and_filter_items(content, filter, tags, ai).await
162    }
163
164    #[instrument(skip_all)]
165    async fn get_http_items(
166        &self,
167        mut url: String,
168        headers: Vec<(HeaderName, HeaderValue)>,
169        method: HttpMethod,
170        filter: Option<Regex>,
171        tags: Vec<String>,
172        ai: bool,
173    ) -> Result<ImportExportStream> {
174        // If the URL is the stdin placeholder, read a line from it
175        if url == "-" {
176            let mut buffer = String::new();
177            std::io::stdin().read_line(&mut buffer)?;
178            url = buffer.trim_end_matches("\n").to_string();
179            tracing::debug!("Read url from stdin: {url}");
180        }
181
182        // Parse the URL
183        let mut url = Url::parse(&url).map_err(|err| {
184            tracing::error!("Couldn't parse url: {err}");
185            UserFacingError::HttpInvalidUrl
186        })?;
187
188        // Try to convert github regular urls to raw
189        if let Some(raw_url) = github_to_raw(&url) {
190            url = raw_url;
191        }
192
193        let method = method.into();
194        if let Some(ref filter) = filter {
195            tracing::info!(ai, "Importing commands matching `{filter}` from http: {method} {url}");
196        } else {
197            tracing::info!(ai, "Importing commands from http: {method} {url}");
198        }
199
200        // Build the request
201        let client = reqwest::Client::new();
202        let mut req = client.request(method, url);
203
204        // Add headers
205        for (name, value) in headers {
206            tracing::debug!("Appending '{name}' header");
207            req = req.header(name, value);
208        }
209
210        // Send the request
211        let res = req.send().await.map_err(|err| {
212            tracing::error!("{err:?}");
213            UserFacingError::HttpRequestFailed(err.to_string())
214        })?;
215
216        // Check the response status
217        if !res.status().is_success() {
218            let status = res.status();
219            let status_str = status.as_str();
220            let body = res.text().await.unwrap_or_default();
221            if let Some(reason) = status.canonical_reason() {
222                tracing::error!("Got response [{status_str}] {reason}:\n{body}");
223                return Err(
224                    UserFacingError::HttpRequestFailed(format!("received {status_str} {reason} response")).into(),
225                );
226            } else {
227                tracing::error!("Got response [{status_str}]:\n{body}");
228                return Err(UserFacingError::HttpRequestFailed(format!("received {status_str} response")).into());
229            }
230        }
231
232        // Check the response content type
233        let mut json = false;
234        if let Some(content_type) = res.headers().get(header::CONTENT_TYPE) {
235            let Ok(content_type) = content_type.to_str() else {
236                return Err(
237                    UserFacingError::HttpRequestFailed(String::from("couldn't read content-type header")).into(),
238                );
239            };
240            if content_type.starts_with("application/json") {
241                json = true;
242            } else if !content_type.starts_with("text") {
243                return Err(
244                    UserFacingError::HttpRequestFailed(format!("unsupported content-type: {content_type}")).into(),
245                );
246            }
247        }
248
249        if json {
250            // Parse the body as a list of commands
251            let items: Vec<ImportExportItemDto> = match res.json().await {
252                Ok(b) => b,
253                Err(err) if err.is_decode() => {
254                    tracing::error!("Couldn't parse api response: {err}");
255                    return Err(UserFacingError::GistRequestFailed(String::from("couldn't parse api response")).into());
256                }
257                Err(err) => {
258                    tracing::error!("{err:?}");
259                    return Err(UserFacingError::GistRequestFailed(err.to_string()).into());
260                }
261            };
262
263            Ok(Box::pin(stream::iter(
264                items.into_iter().map(ImportExportItem::from).map(Ok),
265            )))
266        } else {
267            let content = Cursor::new(res.text().await.map_err(|err| {
268                tracing::error!("Couldn't read api response: {err}");
269                UserFacingError::HttpRequestFailed(String::from("couldn't read api response"))
270            })?);
271            self.extract_and_filter_items(content, filter, tags, ai).await
272        }
273    }
274
275    #[instrument(skip_all)]
276    async fn get_gist_items(
277        &self,
278        mut gist: String,
279        gist_config: GistConfig,
280        filter: Option<Regex>,
281        tags: Vec<String>,
282        ai: bool,
283    ) -> Result<ImportExportStream> {
284        // If the gist is the stdin placeholder, read a line from it
285        if gist == "-" {
286            let mut buffer = String::new();
287            std::io::stdin().read_line(&mut buffer)?;
288            gist = buffer.trim_end_matches("\n").to_string();
289            tracing::debug!("Read gist from stdin: {gist}");
290        }
291
292        // For raw gists, import as regular http requests
293        if gist.starts_with("https://gist.githubusercontent.com") {
294            return self
295                .get_http_items(gist, Vec::new(), HttpMethod::GET, filter, tags, ai)
296                .await;
297        }
298
299        // Retrieve the gist id and optional sha and file
300        let (gist_id, gist_sha, gist_file) = extract_gist_data(&gist, &gist_config)?;
301
302        // Determine the URL based on the presence of sha
303        let url = if let Some(sha) = gist_sha {
304            format!("https://api.github.com/gists/{gist_id}/{sha}")
305        } else {
306            format!("https://api.github.com/gists/{gist_id}")
307        };
308
309        if let Some(ref filter) = filter {
310            tracing::info!(ai, "Importing commands matching `{filter}` from gist: {url}");
311        } else {
312            tracing::info!(ai, "Importing commands from gist: {url}");
313        }
314
315        // Call the API
316        let client = reqwest::Client::new();
317        let res = client
318            .get(url)
319            .header(header::ACCEPT, "application/vnd.github+json")
320            .header(header::USER_AGENT, "intelli-shell")
321            .header("X-GitHub-Api-Version", "2022-11-28")
322            .send()
323            .await
324            .map_err(|err| {
325                tracing::error!("{err:?}");
326                UserFacingError::GistRequestFailed(err.to_string())
327            })?;
328
329        // Check the response status
330        if !res.status().is_success() {
331            let status = res.status();
332            let status_str = status.as_str();
333            let body = res.text().await.unwrap_or_default();
334            if let Some(reason) = status.canonical_reason() {
335                tracing::error!("Got response [{status_str}] {reason}:\n{body}");
336                return Err(
337                    UserFacingError::GistRequestFailed(format!("received {status_str} {reason} response")).into(),
338                );
339            } else {
340                tracing::error!("Got response [{status_str}]:\n{body}");
341                return Err(UserFacingError::GistRequestFailed(format!("received {status_str} response")).into());
342            }
343        }
344
345        // Parse the body as a json
346        let mut body: GistDto = match res.json().await {
347            Ok(b) => b,
348            Err(err) if err.is_decode() => {
349                tracing::error!("Couldn't parse api response: {err}");
350                return Err(UserFacingError::GistRequestFailed(String::from("couldn't parse api response")).into());
351            }
352            Err(err) => {
353                tracing::error!("{err:?}");
354                return Err(UserFacingError::GistRequestFailed(err.to_string()).into());
355            }
356        };
357
358        let full_content = if let Some(ref gist_file) = gist_file {
359            // If there's a file specified, import just it
360            body.files
361                .remove(gist_file)
362                .ok_or(UserFacingError::GistFileNotFound)?
363                .content
364        } else {
365            // Otherwise import all of the files (except the readme)
366            body.files
367                .into_iter()
368                .filter(|(k, _)| k != GIST_README_FILENAME && k != GIST_README_FILENAME_UPPER)
369                .map(|(_, f)| f.content)
370                .join("\n")
371        };
372
373        let content = Cursor::new(full_content);
374        self.extract_and_filter_items(content, filter, tags, ai).await
375    }
376
377    /// Extract the commands from the given content, prompting ai or parsing it, and then filters them
378    async fn extract_and_filter_items(
379        &self,
380        content: impl AsyncRead + Unpin + Send + 'static,
381        filter: Option<Regex>,
382        tags: Vec<String>,
383        ai: bool,
384    ) -> Result<ImportExportStream> {
385        let stream: ImportExportStream = if ai {
386            let commands = self
387                .prompt_commands_import(content, tags, CATEGORY_USER, SOURCE_IMPORT)
388                .await?;
389            Box::pin(commands.map_ok(ImportExportItem::Command))
390        } else {
391            Box::pin(parse_import_items(content, tags, CATEGORY_USER, SOURCE_IMPORT))
392        };
393
394        if let Some(filter) = filter {
395            Ok(Box::pin(stream.try_filter(move |item| {
396                let pass = match item {
397                    ImportExportItem::Command(c) => c.matches(&filter),
398                    ImportExportItem::Completion(_) => true,
399                };
400                async move { pass }
401            })))
402        } else {
403            Ok(stream)
404        }
405    }
406}
407
408/// Lazily parses a stream of text into a [`Stream`] of [`ImportExportItem`].
409///
410/// This function is the primary entry point for parsing command definitions from a file or any other async source.
411/// It operates in a streaming fashion, meaning it reads the input line-by-line without loading the entire content into
412/// memory, making it highly efficient for large files.
413///
414/// # Format Rules
415///
416/// The parser follows a set of rules to interpret the text content:
417///
418/// - **Completions**: Any line starting with `$` is treated as a completion. It must follow the format `$ (root_cmd)
419///   variable: provider`.
420///
421/// - **Commands**: Any line that is not a blank line or a comment is treated as the start of a command.
422///
423/// - **Multi-line Commands**: A command can span multiple lines if a line ends with a backslash (`\`). The parser will
424///   join these lines into a single command string.
425///
426/// - **Descriptions**: A command can have an optional description, specified in one of two ways:
427///   1. **Preceding Comments**: A block of lines starting with `#`, `//`, `::` or `- ` immediately before a command
428///      will be treated as its multi-line description. The comment markers are stripped and the lines are joined with
429///      newlines. Empty comment lines (e.g., `# `) are preserved as blank lines within the description.
430///   2. **Inline Comments** (legacy): An inline description can be provided on the same line, separated by ` ## `. If
431///      both a preceding and an inline description are present, the _inline_ one takes precedence.
432///
433/// - **Aliases**: An optional alias can be extracted from the description by using the format `[alias:your-alias]`.
434///   - The alias tag must be at the very beginning or very end of the entire description block (including multi-line
435///     descriptions).
436///   - The parser extracts the alias and removes it from the final description. For example, `# [alias:a] my command`
437///     results in the alias `a` and the description `my command`.
438///
439/// - **Comments & Spacing**:
440///   - Lines starting with `#`, `//`, `::`, or `- ` (ignoring leading whitespace) are treated as comments.
441///   - Comment lines found _within_ a multi-line command block are ignored and do not become part of the command or its
442///     description.
443///   - Blank lines (i.e., empty or whitespace-only lines) act as separators for description blocks. The description for
444///     a command is the comment block that immediately precedes it.
445///       - A blank line between a comment block and a command is allowed and does not break the association.
446///       - A blank line between two comment blocks makes them distinct; only the latter block will be considered as a
447///         potential description for a subsequent command.
448///
449/// # Errors
450///
451/// The stream will yield an `Err` if an underlying I/O error occurs while reading from the `content` stream.
452#[instrument(skip_all)]
453pub(super) fn parse_import_items(
454    content: impl AsyncRead + Unpin + Send,
455    tags: Vec<String>,
456    category: impl Into<String>,
457    source: impl Into<String>,
458) -> impl Stream<Item = Result<ImportExportItem>> + Send {
459    /// The state of the parser
460    struct ParserState<R: AsyncRead> {
461        category: String,
462        source: String,
463        tags: Vec<String>,
464        lines: Lines<BufReader<R>>,
465        description_buffer: Vec<String>,
466        description_paused: bool,
467    }
468
469    // The initial state for the stream generator
470    let initial_state = ParserState {
471        category: category.into(),
472        source: source.into(),
473        tags,
474        lines: BufReader::new(content).lines(),
475        description_buffer: Vec::new(),
476        description_paused: false,
477    };
478
479    /// Helper to extract the comment content from a trimmed line
480    fn get_comment_content(trimmed_line: &str) -> Option<&str> {
481        if let Some(stripped) = trimmed_line.strip_prefix('#') {
482            return Some(stripped.trim());
483        }
484        if let Some(stripped) = trimmed_line.strip_prefix("//") {
485            return Some(stripped.trim());
486        }
487        if let Some(stripped) = trimmed_line.strip_prefix("- ") {
488            return Some(stripped.trim());
489        }
490        if let Some(stripped) = trimmed_line.strip_prefix("::") {
491            return Some(stripped.trim());
492        }
493        None
494    }
495
496    // Return the commands stream
497    stream::unfold(initial_state, move |mut state| async move {
498        loop {
499            // Read the next line from the input
500            let line: String = match state.lines.next_line().await {
501                // A line is found
502                Ok(Some(line)) => line,
503                // End of the input stream, so we end our command stream
504                Ok(None) => return None,
505                // An I/O error occurred, yield it
506                Err(err) => return Some((Err(AppError::from(err)), state)),
507            };
508            let trimmed_line = line.trim();
509
510            // If the line is the shebang header, skip it
511            if trimmed_line == "#!intelli-shell" {
512                continue;
513            }
514
515            // Skip some line prefixes
516            if trimmed_line.starts_with(">")
517                || trimmed_line.starts_with("```")
518                || trimmed_line.starts_with("%")
519                || trimmed_line.starts_with(";")
520                || trimmed_line.starts_with("@")
521            {
522                continue;
523            }
524
525            // If the line is a completion, parse it
526            if trimmed_line.starts_with('$') {
527                // Regex for completions, with an optional command part
528                // It matches both `$ (cmd) var: provider` and `$ var: provider`
529                static COMPLETION_RE: LazyLock<Regex> = LazyLock::new(|| {
530                    Regex::new(r"^\$\s*(?:\((?P<cmd>[\w-]+)\)\s*)?(?P<var>[^:|{}]+):\s*(?P<provider>.+)$").unwrap()
531                });
532
533                let item = if let Some(caps) = COMPLETION_RE.captures(trimmed_line) {
534                    let cmd = caps.name("cmd").map_or("", |m| m.as_str()).trim();
535                    let var = caps.name("var").map_or("", |m| m.as_str()).trim();
536                    let provider = caps.name("provider").map_or("", |m| m.as_str()).trim();
537
538                    if var.is_empty() || provider.is_empty() {
539                        Err(UserFacingError::ImportCompletionInvalidFormat(line).into())
540                    } else {
541                        Ok(ImportExportItem::Completion(VariableCompletion::new(
542                            state.source.clone(),
543                            cmd,
544                            var,
545                            provider,
546                        )))
547                    }
548                } else {
549                    Err(UserFacingError::ImportCompletionInvalidFormat(line).into())
550                };
551
552                // In all completion cases, we reset the description buffer and yield the item
553                state.description_buffer.clear();
554                state.description_paused = false;
555                return Some((item, state));
556            }
557
558            // If the line is a comment, accumulate it and continue to the next line
559            if let Some(comment_content) = get_comment_content(trimmed_line) {
560                if state.description_paused {
561                    // If the description was 'paused' by a blank line, a new comment indicates a new description block
562                    state.description_buffer.clear();
563                }
564                state.description_buffer.push(comment_content.to_string());
565                state.description_paused = false;
566                continue;
567            }
568
569            // If the line is blank, it might be a separator between comment blocks or trailing after a description
570            if trimmed_line.is_empty() {
571                // We 'pause' the description accumulation.
572                if !state.description_buffer.is_empty() {
573                    state.description_paused = true;
574                }
575                continue;
576            }
577
578            // Otherwise the line is a command that can potentially span across multiple lines
579            let mut current_trimmed_line = trimmed_line.to_string();
580            let mut command_parts: Vec<String> = Vec::new();
581            let mut inline_description: Option<String> = None;
582
583            // Inner loop to handle multi-line commands
584            loop {
585                // Before processing a line as part of a command
586                if get_comment_content(&current_trimmed_line).is_some() || current_trimmed_line.is_empty() {
587                    // If the line is a comment or a blank line, restart the loop with the next line
588                    if let Some(next_line_res) = state.lines.next_line().await.transpose() {
589                        current_trimmed_line = match next_line_res {
590                            Ok(next_line) => next_line.trim().to_string(),
591                            Err(err) => return Some((Err(AppError::from(err)), state)),
592                        };
593                        continue;
594                    } else {
595                        // End of stream mid-command
596                        break;
597                    }
598                }
599
600                // Check if theres an inline comment after the command
601                let (command_segment, desc) = match current_trimmed_line.split_once(" ## ") {
602                    Some((cmd, desc)) => (cmd, Some(desc.trim().to_string())),
603                    None => (current_trimmed_line.as_str(), None),
604                };
605                if inline_description.is_none() {
606                    inline_description = desc;
607                }
608
609                // If the line ends with the escape char, that means the newline was escaped
610                if let Some(stripped) = command_segment.strip_suffix('\\') {
611                    command_parts.push(stripped.trim().to_string());
612                    // This command spans multiple lines, read the next one and continue with the loop
613                    if let Some(next_line_res) = state.lines.next_line().await.transpose() {
614                        current_trimmed_line = match next_line_res {
615                            Ok(next_line) => next_line.trim().to_string(),
616                            Err(err) => return Some((Err(AppError::from(err)), state)),
617                        };
618                    } else {
619                        // End of stream mid-command
620                        break;
621                    }
622                } else {
623                    // This command consist of a single line, break out of the loop
624                    command_parts.push(command_segment.to_string());
625                    break;
626                }
627            }
628
629            // Setup the cmd
630            let mut full_cmd = command_parts.join(" ");
631            if full_cmd.starts_with('`') && full_cmd.ends_with('`') {
632                full_cmd = full_cmd[1..full_cmd.len() - 1].to_string();
633            }
634            full_cmd = convert_alt_to_regular(&full_cmd);
635            // Setup the description
636            let pre_description = if let Some(inline) = inline_description {
637                inline
638            } else {
639                state.description_buffer.join("\n")
640            };
641            // Extract the alias from the description and clean it up
642            let (alias, mut full_description) = extract_alias(pre_description);
643            // Remove ending colon
644            if let Some(stripped) = full_description.strip_suffix(':') {
645                full_description = stripped.to_owned();
646            }
647            // Include tags if any
648            if !state.tags.is_empty() {
649                full_description = add_tags_to_description(&state.tags, full_description);
650            }
651
652            // Create the command
653            let command = Command::new(state.category.clone(), state.source.clone(), full_cmd)
654                .with_description(Some(full_description))
655                .with_alias(alias);
656
657            // Clear the buffer for the next iteration
658            state.description_buffer.clear();
659            state.description_paused = false;
660
661            // Yield the command and the updated state for the next run
662            return Some((Ok(ImportExportItem::Command(command)), state));
663        }
664    })
665}
666
667/// Extracts an alias `[alias:...]` from the start or end of a description string.
668///
669/// It returns a tuple containing an `Option<String>` for the alias and the cleaned description.
670fn extract_alias(description: String) -> (Option<String>, String) {
671    /// Regex to find an alias at the very start or very end of the string
672    /// Group 2 captures the alias from the start, Group 4 from the end
673    static ALIAS_RE: LazyLock<Regex> =
674        LazyLock::new(|| Regex::new(r"(?s)(?:\A\s*\[alias:([^\]]+)\]\s*)|(?:\s*\[alias:([^\]]+)\]\s*\z)").unwrap());
675
676    let mut alias = None;
677
678    // Use `replace` with a closure to capture the alias while removing the tag
679    let new_description = ALIAS_RE.replace(&description, |caps: &regex::Captures| {
680        alias = caps.get(1).or_else(|| caps.get(2)).map(|m| m.as_str().to_string());
681        // The matched tag is replaced with an empty string
682        ""
683    });
684
685    (alias, new_description.trim().to_string())
686}
687
688#[cfg(test)]
689mod tests {
690    use futures_util::TryStreamExt;
691
692    use super::*;
693
694    const CMD_1: &str = "cmd number 1";
695    const CMD_2: &str = "cmd number 2";
696    const CMD_3: &str = "cmd number 3";
697
698    const ALIAS_1: &str = "a1";
699    const ALIAS_2: &str = "a2";
700    const ALIAS_3: &str = "a3";
701
702    const DESCRIPTION_1: &str = "Line of a description 1";
703    const DESCRIPTION_2: &str = "Line of a description 2";
704    const DESCRIPTION_3: &str = "Line of a description 3";
705
706    const CMD_MULTI_1: &str = "cmd very long";
707    const CMD_MULTI_2: &str = "that is split across";
708    const CMD_MULTI_3: &str = "multiple lines for readability";
709
710    #[tokio::test]
711    async fn test_parse_import_items_empty_input() {
712        let items = parse_import_items("".as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
713            .try_collect::<Vec<_>>()
714            .await
715            .unwrap();
716        assert!(items.is_empty());
717    }
718
719    #[tokio::test]
720    async fn test_parse_import_items_simple() {
721        let input = format!(
722            r"{CMD_1}
723              {CMD_2}
724              {CMD_3}"
725        );
726        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
727            .try_collect::<Vec<_>>()
728            .await
729            .unwrap();
730
731        assert_eq!(items.len(), 3);
732        assert_eq!(get_command(&items[0]).cmd, CMD_1);
733        assert!(get_command(&items[0]).description.is_none());
734        assert_eq!(get_command(&items[1]).cmd, CMD_2);
735        assert!(get_command(&items[1]).description.is_none());
736        assert_eq!(get_command(&items[2]).cmd, CMD_3);
737        assert!(get_command(&items[2]).description.is_none());
738    }
739
740    #[tokio::test]
741    async fn test_parse_import_items_legacy() {
742        let input = format!(
743            r"{CMD_1} ## {DESCRIPTION_1}
744              {CMD_2} ## {DESCRIPTION_2}
745              {CMD_3} ## {DESCRIPTION_3}"
746        );
747        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
748            .try_collect::<Vec<_>>()
749            .await
750            .unwrap();
751
752        assert_eq!(items.len(), 3);
753        assert_eq!(get_command(&items[0]).cmd, CMD_1);
754        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
755        assert_eq!(get_command(&items[1]).cmd, CMD_2);
756        assert_eq!(get_command(&items[1]).description.as_deref(), Some(DESCRIPTION_2));
757        assert_eq!(get_command(&items[2]).cmd, CMD_3);
758        assert_eq!(get_command(&items[2]).description.as_deref(), Some(DESCRIPTION_3));
759    }
760
761    #[tokio::test]
762    async fn test_parse_import_items_sh_style() {
763        let input = format!(
764            r"# {DESCRIPTION_1}
765              {CMD_1}
766
767              # {DESCRIPTION_2}
768              {CMD_2}
769
770              # {DESCRIPTION_3}
771              {CMD_3}"
772        );
773        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
774            .try_collect::<Vec<_>>()
775            .await
776            .unwrap();
777
778        assert_eq!(items.len(), 3);
779        assert_eq!(get_command(&items[0]).cmd, CMD_1);
780        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
781        assert_eq!(get_command(&items[1]).cmd, CMD_2);
782        assert_eq!(get_command(&items[1]).description.as_deref(), Some(DESCRIPTION_2));
783        assert_eq!(get_command(&items[2]).cmd, CMD_3);
784        assert_eq!(get_command(&items[2]).description.as_deref(), Some(DESCRIPTION_3));
785    }
786
787    #[tokio::test]
788    async fn test_parse_import_items_tldr_style() {
789        // https://github.com/tldr-pages/tldr/blob/main/CONTRIBUTING.md#markdown-format
790        let input = format!(
791            r"# command-name
792
793              > Short, snappy description.
794              > Preferably one line; two are acceptable if necessary.
795              > More information: <https://url-to-upstream.tld>.
796
797              - {DESCRIPTION_1}:
798              
799              `{CMD_1}`
800
801              - {DESCRIPTION_2}:
802
803              `{CMD_2}`
804
805              - {DESCRIPTION_3}:
806
807              `{CMD_3}`"
808        );
809        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
810            .try_collect::<Vec<_>>()
811            .await
812            .unwrap();
813
814        assert_eq!(items.len(), 3);
815        assert_eq!(get_command(&items[0]).cmd, CMD_1);
816        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
817        assert_eq!(get_command(&items[1]).cmd, CMD_2);
818        assert_eq!(get_command(&items[1]).description.as_deref(), Some(DESCRIPTION_2));
819        assert_eq!(get_command(&items[2]).cmd, CMD_3);
820        assert_eq!(get_command(&items[2]).description.as_deref(), Some(DESCRIPTION_3));
821    }
822
823    #[tokio::test]
824    async fn test_parse_import_items_discard_orphan_descriptions() {
825        let input = format!(
826            r"# This is a comment without a command
827
828              # {DESCRIPTION_1}
829              {CMD_1}"
830        );
831        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
832            .try_collect::<Vec<_>>()
833            .await
834            .unwrap();
835
836        assert_eq!(items.len(), 1);
837        assert_eq!(get_command(&items[0]).cmd, CMD_1);
838        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
839    }
840
841    #[tokio::test]
842    async fn test_parse_import_items_inline_description_takes_precedence() {
843        let input = format!(
844            r"# {DESCRIPTION_2}
845              {CMD_1} ## {DESCRIPTION_1}"
846        );
847        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
848            .try_collect::<Vec<_>>()
849            .await
850            .unwrap();
851
852        assert_eq!(items.len(), 1);
853        assert_eq!(get_command(&items[0]).cmd, CMD_1);
854        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
855    }
856
857    #[tokio::test]
858    async fn test_parse_import_items_multiline_description() {
859        let input = format!(
860            r"# {DESCRIPTION_1}
861              # 
862              # {DESCRIPTION_2}
863              {CMD_1}"
864        );
865        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
866            .try_collect::<Vec<_>>()
867            .await
868            .unwrap();
869
870        assert_eq!(items.len(), 1);
871        let cmd = get_command(&items[0]);
872        assert_eq!(cmd.cmd, CMD_1);
873        assert_eq!(
874            cmd.description.as_ref(),
875            Some(&format!("{DESCRIPTION_1}\n\n{DESCRIPTION_2}"))
876        );
877    }
878
879    #[tokio::test]
880    async fn test_parse_import_items_multiline() {
881        let input = format!(
882            r"# {DESCRIPTION_1}
883              {CMD_MULTI_1} \
884                  # inner comment, not part of the description or command
885                  {CMD_MULTI_2} \ 
886                  {CMD_MULTI_3}"
887        );
888        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
889            .try_collect::<Vec<_>>()
890            .await
891            .unwrap();
892
893        assert_eq!(items.len(), 1);
894        let cmd = get_command(&items[0]);
895        assert_eq!(cmd.cmd, format!("{CMD_MULTI_1} {CMD_MULTI_2} {CMD_MULTI_3}"));
896        assert_eq!(cmd.description.as_deref(), Some(DESCRIPTION_1));
897    }
898
899    #[tokio::test]
900    async fn test_parse_import_items_with_tags_no_description() {
901        let input = CMD_1;
902        let tags = vec!["#test".to_string(), "#tag2".to_string()];
903        let items = parse_import_items(input.as_bytes(), tags, CATEGORY_USER, SOURCE_IMPORT)
904            .try_collect::<Vec<_>>()
905            .await
906            .unwrap();
907
908        assert_eq!(items.len(), 1);
909        let cmd = get_command(&items[0]);
910        assert_eq!(cmd.cmd, CMD_1);
911        assert_eq!(cmd.description.as_deref(), Some("#test #tag2"));
912    }
913
914    #[tokio::test]
915    async fn test_parse_import_items_with_tags_simple_description() {
916        let input = format!(
917            r"# {DESCRIPTION_1}
918              {CMD_1}
919                    
920              {CMD_2} ## {DESCRIPTION_2}"
921        );
922        let tags = vec!["#test".to_string()];
923        let items = parse_import_items(input.as_bytes(), tags, CATEGORY_USER, SOURCE_IMPORT)
924            .try_collect::<Vec<_>>()
925            .await
926            .unwrap();
927
928        assert_eq!(items.len(), 2);
929        let cmd0 = get_command(&items[0]);
930        assert_eq!(cmd0.cmd, CMD_1);
931        assert_eq!(cmd0.description.as_ref(), Some(&format!("{DESCRIPTION_1} #test")));
932        let cmd1 = get_command(&items[1]);
933        assert_eq!(cmd1.cmd, CMD_2);
934        assert_eq!(cmd1.description.as_ref(), Some(&format!("{DESCRIPTION_2} #test")));
935    }
936
937    #[tokio::test]
938    async fn test_parse_import_items_with_tags_and_multiline_description() {
939        let input = format!(
940            r"# {DESCRIPTION_1}
941              # {DESCRIPTION_2}
942              {CMD_1}"
943        );
944        let tags = vec!["#test".to_string()];
945        let items = parse_import_items(input.as_bytes(), tags, CATEGORY_USER, SOURCE_IMPORT)
946            .try_collect::<Vec<_>>()
947            .await
948            .unwrap();
949
950        assert_eq!(items.len(), 1);
951        let cmd = get_command(&items[0]);
952        assert_eq!(cmd.cmd, CMD_1);
953        assert_eq!(
954            cmd.description.as_ref(),
955            Some(&format!("{DESCRIPTION_1}\n{DESCRIPTION_2}\n#test"))
956        );
957    }
958
959    #[tokio::test]
960    async fn test_parse_import_items_skips_existing_tags() {
961        let input = format!(
962            r"# {DESCRIPTION_1} #test
963              {CMD_1}"
964        );
965        let tags = vec!["#test".to_string(), "#new".to_string()];
966        let items = parse_import_items(input.as_bytes(), tags, CATEGORY_USER, SOURCE_IMPORT)
967            .try_collect::<Vec<_>>()
968            .await
969            .unwrap();
970
971        assert_eq!(items.len(), 1);
972        let cmd = get_command(&items[0]);
973        assert_eq!(cmd.cmd, CMD_1);
974        assert_eq!(cmd.description.as_ref(), Some(&format!("{DESCRIPTION_1} #test #new")));
975    }
976
977    #[tokio::test]
978    async fn test_parse_import_items_with_aliases() {
979        let input = format!(
980            r"# [alias:{ALIAS_1}] {DESCRIPTION_1}
981              {CMD_1}
982
983              # [alias:{ALIAS_2}] 
984              # {DESCRIPTION_2}
985              # {DESCRIPTION_2}
986              {CMD_2}
987
988              # [alias:{ALIAS_3}]
989              {CMD_3}"
990        );
991        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
992            .try_collect::<Vec<_>>()
993            .await
994            .unwrap();
995
996        assert_eq!(items.len(), 3);
997        let cmd0 = get_command(&items[0]);
998        assert_eq!(cmd0.cmd, CMD_1);
999        assert_eq!(cmd0.description.as_deref(), Some(DESCRIPTION_1));
1000        assert_eq!(cmd0.alias.as_deref(), Some(ALIAS_1));
1001
1002        let cmd1 = get_command(&items[1]);
1003        assert_eq!(cmd1.cmd, CMD_2);
1004        assert_eq!(
1005            cmd1.description.as_ref(),
1006            Some(&format!("{DESCRIPTION_2}\n{DESCRIPTION_2}"))
1007        );
1008        assert_eq!(cmd1.alias.as_deref(), Some(ALIAS_2));
1009
1010        let cmd2 = get_command(&items[2]);
1011        assert_eq!(cmd2.cmd, CMD_3);
1012        assert!(cmd2.description.is_none());
1013        assert_eq!(cmd2.alias.as_deref(), Some(ALIAS_3));
1014    }
1015
1016    #[tokio::test]
1017    async fn test_parse_import_items_completions() {
1018        let input = r#"
1019            # A command to ensure both types are handled
1020            ls -l ## list files
1021
1022            # Completions
1023            $(git) branch: git branch --all
1024            $ file: ls -F
1025            $ (az) group: az group list --output tsv
1026            "#;
1027
1028        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
1029            .try_collect::<Vec<_>>()
1030            .await
1031            .unwrap();
1032
1033        assert_eq!(items.len(), 4);
1034
1035        let cmd = get_command(&items[0]);
1036        assert_eq!(cmd.cmd, "ls -l");
1037        assert_eq!(cmd.description.as_deref(), Some("list files"));
1038
1039        if let ImportExportItem::Completion(c) = &items[1] {
1040            assert_eq!(c.flat_root_cmd, "git");
1041            assert_eq!(c.flat_variable, "branch");
1042            assert_eq!(c.suggestions_provider, "git branch --all");
1043        } else {
1044            panic!("Expected a Completion at index 1");
1045        }
1046
1047        if let ImportExportItem::Completion(c) = &items[2] {
1048            assert_eq!(c.flat_root_cmd, ""); // Global
1049            assert_eq!(c.flat_variable, "file");
1050            assert_eq!(c.suggestions_provider, "ls -F");
1051        } else {
1052            panic!("Expected a Completion at index 2");
1053        }
1054
1055        if let ImportExportItem::Completion(c) = &items[3] {
1056            assert_eq!(c.flat_root_cmd, "az");
1057            assert_eq!(c.flat_variable, "group");
1058            assert_eq!(c.suggestions_provider, "az group list --output tsv");
1059        } else {
1060            panic!("Expected a Completion at index 3");
1061        }
1062    }
1063
1064    #[tokio::test]
1065    async fn test_parse_import_items_invalid_completion_format() {
1066        let line = "$ invalid completion format";
1067        let result = parse_import_items(line.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
1068            .try_collect::<Vec<_>>()
1069            .await;
1070
1071        assert!(result.is_err());
1072        if let Err(err) = result {
1073            assert!(
1074                matches!(err, AppError::UserFacing(UserFacingError::ImportCompletionInvalidFormat(s)) if s == line)
1075            );
1076        }
1077    }
1078
1079    /// Test helper to extract a Command from an ImportExportItem, panicking if it's the wrong variant
1080    fn get_command(item: &ImportExportItem) -> &Command {
1081        match item {
1082            ImportExportItem::Command(command) => command,
1083            ImportExportItem::Completion(_) => panic!("Expected ImportExportItem::Command, found completion"),
1084        }
1085    }
1086}