intelli_shell/service/
import.rs

1use std::{
2    io::{Cursor, ErrorKind},
3    sync::LazyLock,
4};
5
6use color_eyre::{Report, eyre::Context};
7use futures_util::{TryStreamExt, stream};
8use itertools::Itertools;
9use regex::Regex;
10use reqwest::{
11    Url,
12    header::{self, HeaderName, HeaderValue},
13};
14use tokio::{
15    fs::{self, File},
16    io::{AsyncBufReadExt, AsyncRead, BufReader, Lines},
17};
18use tokio_stream::Stream;
19use tokio_util::sync::CancellationToken;
20use tracing::instrument;
21
22use super::IntelliShellService;
23use crate::{
24    cli::{HistorySource, HttpMethod, ImportItemsProcess},
25    config::GistConfig,
26    errors::{AppError, Result, UserFacingError},
27    model::{
28        CATEGORY_USER, Command, ImportExportItem, ImportExportStream, ImportStats, SOURCE_IMPORT, VariableCompletion,
29    },
30    utils::{
31        add_tags_to_description, convert_alt_to_regular,
32        dto::{GIST_README_FILENAME, GIST_README_FILENAME_UPPER, GistDto, ImportExportItemDto},
33        extract_gist_data, github_to_raw, read_history,
34    },
35};
36
37impl IntelliShellService {
38    /// Import commands and completions
39    pub async fn import_items(&self, items: ImportExportStream, overwrite: bool) -> Result<ImportStats> {
40        self.storage.import_items(items, overwrite, false).await
41    }
42
43    /// Returns a list of items to import from a location
44    pub async fn get_items_from_location(
45        &self,
46        args: ImportItemsProcess,
47        gist_config: GistConfig,
48        cancellation_token: CancellationToken,
49    ) -> Result<ImportExportStream> {
50        let ImportItemsProcess {
51            location,
52            file,
53            http,
54            gist,
55            history,
56            ai,
57            filter,
58            dry_run: _,
59            tags,
60            headers,
61            method,
62        } = args;
63
64        // Make sure the tags starts with a hashtag (#)
65        let tags = tags
66            .into_iter()
67            .filter_map(|mut tag| {
68                tag.chars().next().map(|first_char| {
69                    if first_char == '#' {
70                        tag
71                    } else {
72                        tag.insert(0, '#');
73                        tag
74                    }
75                })
76            })
77            .collect::<Vec<_>>();
78
79        // Retrieve the commands from the location
80        let commands = if let Some(history) = history {
81            self.get_history_items(history, filter, tags, ai, cancellation_token)
82                .await?
83        } else if file {
84            if location == "-" {
85                self.get_stdin_items(filter, tags, ai, cancellation_token).await?
86            } else {
87                self.get_file_items(location, filter, tags, ai, cancellation_token)
88                    .await?
89            }
90        } else if http {
91            self.get_http_items(location, headers, method, filter, tags, ai, cancellation_token)
92                .await?
93        } else if gist {
94            self.get_gist_items(location, gist_config, filter, tags, ai, cancellation_token)
95                .await?
96        } else {
97            // Determine which mode based on the location
98            if location == "gist"
99                || location.starts_with("https://gist.github.com")
100                || location.starts_with("https://api.github.com/gists")
101            {
102                self.get_gist_items(location, gist_config, filter, tags, ai, cancellation_token)
103                    .await?
104            } else if location.starts_with("http://") || location.starts_with("https://") {
105                self.get_http_items(location, headers, method, filter, tags, ai, cancellation_token)
106                    .await?
107            } else if location == "-" {
108                self.get_stdin_items(filter, tags, ai, cancellation_token).await?
109            } else {
110                self.get_file_items(location, filter, tags, ai, cancellation_token)
111                    .await?
112            }
113        };
114
115        Ok(commands)
116    }
117
118    #[instrument(skip_all)]
119    async fn get_history_items(
120        &self,
121        history: HistorySource,
122        filter: Option<Regex>,
123        tags: Vec<String>,
124        ai: bool,
125        cancellation_token: CancellationToken,
126    ) -> Result<ImportExportStream> {
127        if let Some(ref filter) = filter {
128            tracing::info!(ai, "Importing commands matching `{filter}` from {history:?} history");
129        } else {
130            tracing::info!(ai, "Importing commands from {history:?} history");
131        }
132        let content = Cursor::new(read_history(history)?);
133        self.extract_and_filter_items(content, filter, tags, ai, cancellation_token)
134            .await
135    }
136
137    #[instrument(skip_all)]
138    async fn get_stdin_items(
139        &self,
140        filter: Option<Regex>,
141        tags: Vec<String>,
142        ai: bool,
143        cancellation_token: CancellationToken,
144    ) -> Result<ImportExportStream> {
145        if let Some(ref filter) = filter {
146            tracing::info!(ai, "Importing commands matching `{filter}` from stdin");
147        } else {
148            tracing::info!(ai, "Importing commands from stdin");
149        }
150        let content = tokio::io::stdin();
151        self.extract_and_filter_items(content, filter, tags, ai, cancellation_token)
152            .await
153    }
154
155    #[instrument(skip_all)]
156    async fn get_file_items(
157        &self,
158        path: String,
159        filter: Option<Regex>,
160        tags: Vec<String>,
161        ai: bool,
162        cancellation_token: CancellationToken,
163    ) -> Result<ImportExportStream> {
164        // Otherwise, check the path to import the file
165        match fs::metadata(&path).await {
166            Ok(m) if m.is_file() => (),
167            Ok(_) => return Err(UserFacingError::ImportLocationNotAFile.into()),
168            Err(err) if err.kind() == ErrorKind::NotFound => return Err(UserFacingError::ImportFileNotFound.into()),
169            Err(err) if err.kind() == ErrorKind::PermissionDenied => {
170                return Err(UserFacingError::FileNotAccessible("read").into());
171            }
172            Err(err) => return Err(Report::from(err).into()),
173        }
174        if let Some(ref filter) = filter {
175            tracing::info!(ai, "Importing commands matching `{filter}` from file: {path}");
176        } else {
177            tracing::info!(ai, "Importing commands from file: {path}");
178        }
179        let content = File::open(path).await.wrap_err("Couldn't open the file")?;
180        self.extract_and_filter_items(content, filter, tags, ai, cancellation_token)
181            .await
182    }
183
184    #[instrument(skip_all)]
185    async fn get_http_items(
186        &self,
187        mut url: String,
188        headers: Vec<(HeaderName, HeaderValue)>,
189        method: HttpMethod,
190        filter: Option<Regex>,
191        tags: Vec<String>,
192        ai: bool,
193        cancellation_token: CancellationToken,
194    ) -> Result<ImportExportStream> {
195        // If the URL is the stdin placeholder, read a line from it
196        if url == "-" {
197            let mut buffer = String::new();
198            std::io::stdin().read_line(&mut buffer)?;
199            url = buffer.trim_end_matches("\n").to_string();
200            tracing::debug!("Read url from stdin: {url}");
201        }
202
203        // Parse the URL
204        let mut url = Url::parse(&url).map_err(|err| {
205            tracing::error!("Couldn't parse url: {err}");
206            UserFacingError::HttpInvalidUrl
207        })?;
208
209        // Try to convert github regular urls to raw
210        if let Some(raw_url) = github_to_raw(&url) {
211            url = raw_url;
212        }
213
214        let method = method.into();
215        if let Some(ref filter) = filter {
216            tracing::info!(ai, "Importing commands matching `{filter}` from http: {method} {url}");
217        } else {
218            tracing::info!(ai, "Importing commands from http: {method} {url}");
219        }
220
221        // Build the request
222        let client = reqwest::Client::new();
223        let mut req = client.request(method, url);
224
225        // Add headers
226        for (name, value) in headers {
227            tracing::debug!("Appending '{name}' header");
228            req = req.header(name, value);
229        }
230
231        // Send the request
232        let res = req.send().await.map_err(|err| {
233            tracing::error!("{err:?}");
234            UserFacingError::HttpRequestFailed(err.to_string())
235        })?;
236
237        // Check the response status
238        if !res.status().is_success() {
239            let status = res.status();
240            let status_str = status.as_str();
241            let body = res.text().await.unwrap_or_default();
242            if let Some(reason) = status.canonical_reason() {
243                tracing::error!("Got response [{status_str}] {reason}:\n{body}");
244                return Err(
245                    UserFacingError::HttpRequestFailed(format!("received {status_str} {reason} response")).into(),
246                );
247            } else {
248                tracing::error!("Got response [{status_str}]:\n{body}");
249                return Err(UserFacingError::HttpRequestFailed(format!("received {status_str} response")).into());
250            }
251        }
252
253        // Check the response content type
254        let mut json = false;
255        if let Some(content_type) = res.headers().get(header::CONTENT_TYPE) {
256            let Ok(content_type) = content_type.to_str() else {
257                return Err(
258                    UserFacingError::HttpRequestFailed(String::from("couldn't read content-type header")).into(),
259                );
260            };
261            if content_type.starts_with("application/json") {
262                json = true;
263            } else if !content_type.starts_with("text") {
264                return Err(
265                    UserFacingError::HttpRequestFailed(format!("unsupported content-type: {content_type}")).into(),
266                );
267            }
268        }
269
270        if json {
271            // Parse the body as a list of commands
272            let items: Vec<ImportExportItemDto> = match res.json().await {
273                Ok(b) => b,
274                Err(err) if err.is_decode() => {
275                    tracing::error!("Couldn't parse api response: {err}");
276                    return Err(UserFacingError::GistRequestFailed(String::from("couldn't parse api response")).into());
277                }
278                Err(err) => {
279                    tracing::error!("{err:?}");
280                    return Err(UserFacingError::GistRequestFailed(err.to_string()).into());
281                }
282            };
283
284            Ok(Box::pin(stream::iter(
285                items.into_iter().map(ImportExportItem::from).map(Ok),
286            )))
287        } else {
288            let content = Cursor::new(res.text().await.map_err(|err| {
289                tracing::error!("Couldn't read api response: {err}");
290                UserFacingError::HttpRequestFailed(String::from("couldn't read api response"))
291            })?);
292            self.extract_and_filter_items(content, filter, tags, ai, cancellation_token)
293                .await
294        }
295    }
296
297    #[instrument(skip_all)]
298    async fn get_gist_items(
299        &self,
300        mut gist: String,
301        gist_config: GistConfig,
302        filter: Option<Regex>,
303        tags: Vec<String>,
304        ai: bool,
305        cancellation_token: CancellationToken,
306    ) -> Result<ImportExportStream> {
307        // If the gist is the stdin placeholder, read a line from it
308        if gist == "-" {
309            let mut buffer = String::new();
310            std::io::stdin().read_line(&mut buffer)?;
311            gist = buffer.trim_end_matches("\n").to_string();
312            tracing::debug!("Read gist from stdin: {gist}");
313        }
314
315        // For raw gists, import as regular http requests
316        if gist.starts_with("https://gist.githubusercontent.com") {
317            return self
318                .get_http_items(gist, Vec::new(), HttpMethod::GET, filter, tags, ai, cancellation_token)
319                .await;
320        }
321
322        // Retrieve the gist id and optional sha and file
323        let (gist_id, gist_sha, gist_file) = extract_gist_data(&gist, &gist_config)?;
324
325        // Determine the URL based on the presence of sha
326        let url = if let Some(sha) = gist_sha {
327            format!("https://api.github.com/gists/{gist_id}/{sha}")
328        } else {
329            format!("https://api.github.com/gists/{gist_id}")
330        };
331
332        if let Some(ref filter) = filter {
333            tracing::info!(ai, "Importing commands matching `{filter}` from gist: {url}");
334        } else {
335            tracing::info!(ai, "Importing commands from gist: {url}");
336        }
337
338        // Call the API
339        let client = reqwest::Client::new();
340        let res = client
341            .get(url)
342            .header(header::ACCEPT, "application/vnd.github+json")
343            .header(header::USER_AGENT, "intelli-shell")
344            .header("X-GitHub-Api-Version", "2022-11-28")
345            .send()
346            .await
347            .map_err(|err| {
348                tracing::error!("{err:?}");
349                UserFacingError::GistRequestFailed(err.to_string())
350            })?;
351
352        // Check the response status
353        if !res.status().is_success() {
354            let status = res.status();
355            let status_str = status.as_str();
356            let body = res.text().await.unwrap_or_default();
357            if let Some(reason) = status.canonical_reason() {
358                tracing::error!("Got response [{status_str}] {reason}:\n{body}");
359                return Err(
360                    UserFacingError::GistRequestFailed(format!("received {status_str} {reason} response")).into(),
361                );
362            } else {
363                tracing::error!("Got response [{status_str}]:\n{body}");
364                return Err(UserFacingError::GistRequestFailed(format!("received {status_str} response")).into());
365            }
366        }
367
368        // Parse the body as a json
369        let mut body: GistDto = match res.json().await {
370            Ok(b) => b,
371            Err(err) if err.is_decode() => {
372                tracing::error!("Couldn't parse api response: {err}");
373                return Err(UserFacingError::GistRequestFailed(String::from("couldn't parse api response")).into());
374            }
375            Err(err) => {
376                tracing::error!("{err:?}");
377                return Err(UserFacingError::GistRequestFailed(err.to_string()).into());
378            }
379        };
380
381        let full_content = if let Some(ref gist_file) = gist_file {
382            // If there's a file specified, import just it
383            body.files
384                .remove(gist_file)
385                .ok_or(UserFacingError::GistFileNotFound)?
386                .content
387        } else {
388            // Otherwise import all of the files (except the readme)
389            body.files
390                .into_iter()
391                .filter(|(k, _)| k != GIST_README_FILENAME && k != GIST_README_FILENAME_UPPER)
392                .map(|(_, f)| f.content)
393                .join("\n")
394        };
395
396        let content = Cursor::new(full_content);
397        self.extract_and_filter_items(content, filter, tags, ai, cancellation_token)
398            .await
399    }
400
401    /// Extract the commands from the given content, prompting ai or parsing it, and then filters them
402    async fn extract_and_filter_items(
403        &self,
404        content: impl AsyncRead + Unpin + Send + 'static,
405        filter: Option<Regex>,
406        tags: Vec<String>,
407        ai: bool,
408        cancellation_token: CancellationToken,
409    ) -> Result<ImportExportStream> {
410        let stream: ImportExportStream = if ai {
411            let commands = self
412                .prompt_commands_import(content, tags, CATEGORY_USER, SOURCE_IMPORT, cancellation_token)
413                .await?;
414            Box::pin(commands.map_ok(ImportExportItem::Command))
415        } else {
416            Box::pin(parse_import_items(content, tags, CATEGORY_USER, SOURCE_IMPORT))
417        };
418
419        if let Some(filter) = filter {
420            Ok(Box::pin(stream.try_filter(move |item| {
421                let pass = match item {
422                    ImportExportItem::Command(c) => c.matches(&filter),
423                    ImportExportItem::Completion(_) => true,
424                };
425                async move { pass }
426            })))
427        } else {
428            Ok(stream)
429        }
430    }
431}
432
433/// Lazily parses a stream of text into a [`Stream`] of [`ImportExportItem`].
434///
435/// This function is the primary entry point for parsing command definitions from a file or any other async source.
436/// It operates in a streaming fashion, meaning it reads the input line-by-line without loading the entire content into
437/// memory, making it highly efficient for large files.
438///
439/// # Format Rules
440///
441/// The parser follows a set of rules to interpret the text content:
442///
443/// - **Completions**: Any line starting with `$` is treated as a completion. It must follow the format `$ (root_cmd)
444///   variable: provider`.
445///
446/// - **Commands**: Any line that is not a blank line or a comment is treated as the start of a command.
447///
448/// - **Multi-line Commands**: A command can span multiple lines if a line ends with a backslash (`\`). The parser will
449///   join these lines into a single command string.
450///
451/// - **Descriptions**: A command can have an optional description, specified in one of two ways:
452///   1. **Preceding Comments**: A block of lines starting with `#`, `//`, `::` or `- ` immediately before a command
453///      will be treated as its multi-line description. The comment markers are stripped and the lines are joined with
454///      newlines. Empty comment lines (e.g., `# `) are preserved as blank lines within the description.
455///   2. **Inline Comments** (legacy): An inline description can be provided on the same line, separated by ` ## `. If
456///      both a preceding and an inline description are present, the _inline_ one takes precedence.
457///
458/// - **Aliases**: An optional alias can be extracted from the description by using the format `[alias:your-alias]`.
459///   - The alias tag must be at the very beginning or very end of the entire description block (including multi-line
460///     descriptions).
461///   - The parser extracts the alias and removes it from the final description. For example, `# [alias:a] my command`
462///     results in the alias `a` and the description `my command`.
463///
464/// - **Comments & Spacing**:
465///   - Lines starting with `#`, `//`, `::`, or `- ` (ignoring leading whitespace) are treated as comments.
466///   - Comment lines found _within_ a multi-line command block are ignored and do not become part of the command or its
467///     description.
468///   - Blank lines (i.e., empty or whitespace-only lines) act as separators for description blocks. The description for
469///     a command is the comment block that immediately precedes it.
470///       - A blank line between a comment block and a command is allowed and does not break the association.
471///       - A blank line between two comment blocks makes them distinct; only the latter block will be considered as a
472///         potential description for a subsequent command.
473///
474/// # Errors
475///
476/// The stream will yield an `Err` if an underlying I/O error occurs while reading from the `content` stream.
477#[instrument(skip_all)]
478pub(super) fn parse_import_items(
479    content: impl AsyncRead + Unpin + Send,
480    tags: Vec<String>,
481    category: impl Into<String>,
482    source: impl Into<String>,
483) -> impl Stream<Item = Result<ImportExportItem>> + Send {
484    /// The state of the parser
485    struct ParserState<R: AsyncRead> {
486        category: String,
487        source: String,
488        tags: Vec<String>,
489        lines: Lines<BufReader<R>>,
490        description_buffer: Vec<String>,
491        description_paused: bool,
492    }
493
494    // The initial state for the stream generator
495    let initial_state = ParserState {
496        category: category.into(),
497        source: source.into(),
498        tags,
499        lines: BufReader::new(content).lines(),
500        description_buffer: Vec::new(),
501        description_paused: false,
502    };
503
504    /// Helper to extract the comment content from a trimmed line
505    fn get_comment_content(trimmed_line: &str) -> Option<&str> {
506        if let Some(stripped) = trimmed_line.strip_prefix('#') {
507            return Some(stripped.trim());
508        }
509        if let Some(stripped) = trimmed_line.strip_prefix("//") {
510            return Some(stripped.trim());
511        }
512        if let Some(stripped) = trimmed_line.strip_prefix("- ") {
513            return Some(stripped.trim());
514        }
515        if let Some(stripped) = trimmed_line.strip_prefix("::") {
516            return Some(stripped.trim());
517        }
518        None
519    }
520
521    // Return the commands stream
522    stream::unfold(initial_state, move |mut state| async move {
523        loop {
524            // Read the next line from the input
525            let line: String = match state.lines.next_line().await {
526                // A line is found
527                Ok(Some(line)) => line,
528                // End of the input stream, so we end our command stream
529                Ok(None) => return None,
530                // An I/O error occurred, yield it
531                Err(err) => return Some((Err(AppError::from(err)), state)),
532            };
533            let trimmed_line = line.trim();
534
535            // If the line is the shebang header, skip it
536            if trimmed_line == "#!intelli-shell" {
537                continue;
538            }
539
540            // Skip some line prefixes
541            if trimmed_line.starts_with(">")
542                || trimmed_line.starts_with("```")
543                || trimmed_line.starts_with("%")
544                || trimmed_line.starts_with(";")
545                || trimmed_line.starts_with("@")
546            {
547                continue;
548            }
549
550            // If the line is a completion, parse it
551            if trimmed_line.starts_with('$') {
552                // Regex for completions, with an optional command part
553                // It matches both `$ (cmd) var: provider` and `$ var: provider`
554                static COMPLETION_RE: LazyLock<Regex> = LazyLock::new(|| {
555                    Regex::new(r"^\$\s*(?:\((?P<cmd>[\w./\\:~-]+)\)\s*)?(?P<var>[^:|{}]+):\s*(?P<provider>.+)$")
556                        .unwrap()
557                });
558
559                let item = if let Some(caps) = COMPLETION_RE.captures(trimmed_line) {
560                    let cmd = caps.name("cmd").map_or("", |m| m.as_str()).trim();
561                    let var = caps.name("var").map_or("", |m| m.as_str()).trim();
562                    let provider = caps.name("provider").map_or("", |m| m.as_str()).trim();
563
564                    if var.is_empty() || provider.is_empty() {
565                        Err(UserFacingError::ImportCompletionInvalidFormat(line).into())
566                    } else {
567                        Ok(ImportExportItem::Completion(VariableCompletion::new(
568                            state.source.clone(),
569                            cmd,
570                            var,
571                            provider,
572                        )))
573                    }
574                } else {
575                    Err(UserFacingError::ImportCompletionInvalidFormat(line).into())
576                };
577
578                // In all completion cases, we reset the description buffer and yield the item
579                state.description_buffer.clear();
580                state.description_paused = false;
581                return Some((item, state));
582            }
583
584            // If the line is a comment, accumulate it and continue to the next line
585            if let Some(comment_content) = get_comment_content(trimmed_line) {
586                if state.description_paused {
587                    // If the description was 'paused' by a blank line, a new comment indicates a new description block
588                    state.description_buffer.clear();
589                }
590                state.description_buffer.push(comment_content.to_string());
591                state.description_paused = false;
592                continue;
593            }
594
595            // If the line is blank, it might be a separator between comment blocks or trailing after a description
596            if trimmed_line.is_empty() {
597                // We 'pause' the description accumulation.
598                if !state.description_buffer.is_empty() {
599                    state.description_paused = true;
600                }
601                continue;
602            }
603
604            // Otherwise the line is a command that can potentially span across multiple lines
605            let mut current_trimmed_line = trimmed_line.to_string();
606            let mut command_parts: Vec<String> = Vec::new();
607            let mut inline_description: Option<String> = None;
608
609            // Inner loop to handle multi-line commands
610            loop {
611                // Before processing a line as part of a command
612                if get_comment_content(&current_trimmed_line).is_some() || current_trimmed_line.is_empty() {
613                    // If the line is a comment or a blank line, restart the loop with the next line
614                    if let Some(next_line_res) = state.lines.next_line().await.transpose() {
615                        current_trimmed_line = match next_line_res {
616                            Ok(next_line) => next_line.trim().to_string(),
617                            Err(err) => return Some((Err(AppError::from(err)), state)),
618                        };
619                        continue;
620                    } else {
621                        // End of stream mid-command
622                        break;
623                    }
624                }
625
626                // Check if theres an inline comment after the command
627                let (command_segment, desc) = match current_trimmed_line.split_once(" ## ") {
628                    Some((cmd, desc)) => (cmd, Some(desc.trim().to_string())),
629                    None => (current_trimmed_line.as_str(), None),
630                };
631                if inline_description.is_none() {
632                    inline_description = desc;
633                }
634
635                // If the line ends with the escape char, that means the newline was escaped
636                if let Some(stripped) = command_segment.strip_suffix('\\') {
637                    command_parts.push(stripped.trim().to_string());
638                    // This command spans multiple lines, read the next one and continue with the loop
639                    if let Some(next_line_res) = state.lines.next_line().await.transpose() {
640                        current_trimmed_line = match next_line_res {
641                            Ok(next_line) => next_line.trim().to_string(),
642                            Err(err) => return Some((Err(AppError::from(err)), state)),
643                        };
644                    } else {
645                        // End of stream mid-command
646                        break;
647                    }
648                } else {
649                    // This command consist of a single line, break out of the loop
650                    command_parts.push(command_segment.to_string());
651                    break;
652                }
653            }
654
655            // Setup the cmd
656            let mut full_cmd = command_parts.join(" ");
657            if full_cmd.starts_with('`') && full_cmd.ends_with('`') {
658                full_cmd = full_cmd[1..full_cmd.len() - 1].to_string();
659            }
660            full_cmd = convert_alt_to_regular(&full_cmd);
661            // Setup the description
662            let pre_description = if let Some(inline) = inline_description {
663                inline
664            } else {
665                state.description_buffer.join("\n")
666            };
667            // Extract the alias from the description and clean it up
668            let (alias, mut full_description) = extract_alias(pre_description);
669            // Remove ending colon
670            if let Some(stripped) = full_description.strip_suffix(':') {
671                full_description = stripped.to_owned();
672            }
673            // Include tags if any
674            if !state.tags.is_empty() {
675                full_description = add_tags_to_description(&state.tags, full_description);
676            }
677
678            // Create the command
679            let command = Command::new(state.category.clone(), state.source.clone(), full_cmd)
680                .with_description(Some(full_description))
681                .with_alias(alias);
682
683            // Clear the buffer for the next iteration
684            state.description_buffer.clear();
685            state.description_paused = false;
686
687            // Yield the command and the updated state for the next run
688            return Some((Ok(ImportExportItem::Command(command)), state));
689        }
690    })
691}
692
693/// Extracts an alias `[alias:...]` from the start or end of a description string.
694///
695/// It returns a tuple containing an `Option<String>` for the alias and the cleaned description.
696fn extract_alias(description: String) -> (Option<String>, String) {
697    /// Regex to find an alias at the very start or very end of the string
698    /// Group 2 captures the alias from the start, Group 4 from the end
699    static ALIAS_RE: LazyLock<Regex> =
700        LazyLock::new(|| Regex::new(r"(?s)(?:\A\s*\[alias:([^\]]+)\]\s*)|(?:\s*\[alias:([^\]]+)\]\s*\z)").unwrap());
701
702    let mut alias = None;
703
704    // Use `replace` with a closure to capture the alias while removing the tag
705    let new_description = ALIAS_RE.replace(&description, |caps: &regex::Captures| {
706        alias = caps.get(1).or_else(|| caps.get(2)).map(|m| m.as_str().to_string());
707        // The matched tag is replaced with an empty string
708        ""
709    });
710
711    (alias, new_description.trim().to_string())
712}
713
714#[cfg(test)]
715mod tests {
716    use futures_util::TryStreamExt;
717
718    use super::*;
719
720    const CMD_1: &str = "cmd number 1";
721    const CMD_2: &str = "cmd number 2";
722    const CMD_3: &str = "cmd number 3";
723
724    const ALIAS_1: &str = "a1";
725    const ALIAS_2: &str = "a2";
726    const ALIAS_3: &str = "a3";
727
728    const DESCRIPTION_1: &str = "Line of a description 1";
729    const DESCRIPTION_2: &str = "Line of a description 2";
730    const DESCRIPTION_3: &str = "Line of a description 3";
731
732    const CMD_MULTI_1: &str = "cmd very long";
733    const CMD_MULTI_2: &str = "that is split across";
734    const CMD_MULTI_3: &str = "multiple lines for readability";
735
736    #[tokio::test]
737    async fn test_parse_import_items_empty_input() {
738        let items = parse_import_items("".as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
739            .try_collect::<Vec<_>>()
740            .await
741            .unwrap();
742        assert!(items.is_empty());
743    }
744
745    #[tokio::test]
746    async fn test_parse_import_items_simple() {
747        let input = format!(
748            r"{CMD_1}
749              {CMD_2}
750              {CMD_3}"
751        );
752        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
753            .try_collect::<Vec<_>>()
754            .await
755            .unwrap();
756
757        assert_eq!(items.len(), 3);
758        assert_eq!(get_command(&items[0]).cmd, CMD_1);
759        assert!(get_command(&items[0]).description.is_none());
760        assert_eq!(get_command(&items[1]).cmd, CMD_2);
761        assert!(get_command(&items[1]).description.is_none());
762        assert_eq!(get_command(&items[2]).cmd, CMD_3);
763        assert!(get_command(&items[2]).description.is_none());
764    }
765
766    #[tokio::test]
767    async fn test_parse_import_items_legacy() {
768        let input = format!(
769            r"{CMD_1} ## {DESCRIPTION_1}
770              {CMD_2} ## {DESCRIPTION_2}
771              {CMD_3} ## {DESCRIPTION_3}"
772        );
773        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
774            .try_collect::<Vec<_>>()
775            .await
776            .unwrap();
777
778        assert_eq!(items.len(), 3);
779        assert_eq!(get_command(&items[0]).cmd, CMD_1);
780        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
781        assert_eq!(get_command(&items[1]).cmd, CMD_2);
782        assert_eq!(get_command(&items[1]).description.as_deref(), Some(DESCRIPTION_2));
783        assert_eq!(get_command(&items[2]).cmd, CMD_3);
784        assert_eq!(get_command(&items[2]).description.as_deref(), Some(DESCRIPTION_3));
785    }
786
787    #[tokio::test]
788    async fn test_parse_import_items_sh_style() {
789        let input = format!(
790            r"# {DESCRIPTION_1}
791              {CMD_1}
792
793              # {DESCRIPTION_2}
794              {CMD_2}
795
796              # {DESCRIPTION_3}
797              {CMD_3}"
798        );
799        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
800            .try_collect::<Vec<_>>()
801            .await
802            .unwrap();
803
804        assert_eq!(items.len(), 3);
805        assert_eq!(get_command(&items[0]).cmd, CMD_1);
806        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
807        assert_eq!(get_command(&items[1]).cmd, CMD_2);
808        assert_eq!(get_command(&items[1]).description.as_deref(), Some(DESCRIPTION_2));
809        assert_eq!(get_command(&items[2]).cmd, CMD_3);
810        assert_eq!(get_command(&items[2]).description.as_deref(), Some(DESCRIPTION_3));
811    }
812
813    #[tokio::test]
814    async fn test_parse_import_items_tldr_style() {
815        // https://github.com/tldr-pages/tldr/blob/main/CONTRIBUTING.md#markdown-format
816        let input = format!(
817            r"# command-name
818
819              > Short, snappy description.
820              > Preferably one line; two are acceptable if necessary.
821              > More information: <https://url-to-upstream.tld>.
822
823              - {DESCRIPTION_1}:
824              
825              `{CMD_1}`
826
827              - {DESCRIPTION_2}:
828
829              `{CMD_2}`
830
831              - {DESCRIPTION_3}:
832
833              `{CMD_3}`"
834        );
835        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
836            .try_collect::<Vec<_>>()
837            .await
838            .unwrap();
839
840        assert_eq!(items.len(), 3);
841        assert_eq!(get_command(&items[0]).cmd, CMD_1);
842        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
843        assert_eq!(get_command(&items[1]).cmd, CMD_2);
844        assert_eq!(get_command(&items[1]).description.as_deref(), Some(DESCRIPTION_2));
845        assert_eq!(get_command(&items[2]).cmd, CMD_3);
846        assert_eq!(get_command(&items[2]).description.as_deref(), Some(DESCRIPTION_3));
847    }
848
849    #[tokio::test]
850    async fn test_parse_import_items_discard_orphan_descriptions() {
851        let input = format!(
852            r"# This is a comment without a command
853
854              # {DESCRIPTION_1}
855              {CMD_1}"
856        );
857        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
858            .try_collect::<Vec<_>>()
859            .await
860            .unwrap();
861
862        assert_eq!(items.len(), 1);
863        assert_eq!(get_command(&items[0]).cmd, CMD_1);
864        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
865    }
866
867    #[tokio::test]
868    async fn test_parse_import_items_inline_description_takes_precedence() {
869        let input = format!(
870            r"# {DESCRIPTION_2}
871              {CMD_1} ## {DESCRIPTION_1}"
872        );
873        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
874            .try_collect::<Vec<_>>()
875            .await
876            .unwrap();
877
878        assert_eq!(items.len(), 1);
879        assert_eq!(get_command(&items[0]).cmd, CMD_1);
880        assert_eq!(get_command(&items[0]).description.as_deref(), Some(DESCRIPTION_1));
881    }
882
883    #[tokio::test]
884    async fn test_parse_import_items_multiline_description() {
885        let input = format!(
886            r"# {DESCRIPTION_1}
887              # 
888              # {DESCRIPTION_2}
889              {CMD_1}"
890        );
891        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
892            .try_collect::<Vec<_>>()
893            .await
894            .unwrap();
895
896        assert_eq!(items.len(), 1);
897        let cmd = get_command(&items[0]);
898        assert_eq!(cmd.cmd, CMD_1);
899        assert_eq!(
900            cmd.description.as_ref(),
901            Some(&format!("{DESCRIPTION_1}\n\n{DESCRIPTION_2}"))
902        );
903    }
904
905    #[tokio::test]
906    async fn test_parse_import_items_multiline() {
907        let input = format!(
908            r"# {DESCRIPTION_1}
909              {CMD_MULTI_1} \
910                  # inner comment, not part of the description or command
911                  {CMD_MULTI_2} \ 
912                  {CMD_MULTI_3}"
913        );
914        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
915            .try_collect::<Vec<_>>()
916            .await
917            .unwrap();
918
919        assert_eq!(items.len(), 1);
920        let cmd = get_command(&items[0]);
921        assert_eq!(cmd.cmd, format!("{CMD_MULTI_1} {CMD_MULTI_2} {CMD_MULTI_3}"));
922        assert_eq!(cmd.description.as_deref(), Some(DESCRIPTION_1));
923    }
924
925    #[tokio::test]
926    async fn test_parse_import_items_with_tags_no_description() {
927        let input = CMD_1;
928        let tags = vec!["#test".to_string(), "#tag2".to_string()];
929        let items = parse_import_items(input.as_bytes(), tags, CATEGORY_USER, SOURCE_IMPORT)
930            .try_collect::<Vec<_>>()
931            .await
932            .unwrap();
933
934        assert_eq!(items.len(), 1);
935        let cmd = get_command(&items[0]);
936        assert_eq!(cmd.cmd, CMD_1);
937        assert_eq!(cmd.description.as_deref(), Some("#test #tag2"));
938    }
939
940    #[tokio::test]
941    async fn test_parse_import_items_with_tags_simple_description() {
942        let input = format!(
943            r"# {DESCRIPTION_1}
944              {CMD_1}
945                    
946              {CMD_2} ## {DESCRIPTION_2}"
947        );
948        let tags = vec!["#test".to_string()];
949        let items = parse_import_items(input.as_bytes(), tags, CATEGORY_USER, SOURCE_IMPORT)
950            .try_collect::<Vec<_>>()
951            .await
952            .unwrap();
953
954        assert_eq!(items.len(), 2);
955        let cmd0 = get_command(&items[0]);
956        assert_eq!(cmd0.cmd, CMD_1);
957        assert_eq!(cmd0.description.as_ref(), Some(&format!("{DESCRIPTION_1} #test")));
958        let cmd1 = get_command(&items[1]);
959        assert_eq!(cmd1.cmd, CMD_2);
960        assert_eq!(cmd1.description.as_ref(), Some(&format!("{DESCRIPTION_2} #test")));
961    }
962
963    #[tokio::test]
964    async fn test_parse_import_items_with_tags_and_multiline_description() {
965        let input = format!(
966            r"# {DESCRIPTION_1}
967              # {DESCRIPTION_2}
968              {CMD_1}"
969        );
970        let tags = vec!["#test".to_string()];
971        let items = parse_import_items(input.as_bytes(), tags, CATEGORY_USER, SOURCE_IMPORT)
972            .try_collect::<Vec<_>>()
973            .await
974            .unwrap();
975
976        assert_eq!(items.len(), 1);
977        let cmd = get_command(&items[0]);
978        assert_eq!(cmd.cmd, CMD_1);
979        assert_eq!(
980            cmd.description.as_ref(),
981            Some(&format!("{DESCRIPTION_1}\n{DESCRIPTION_2}\n#test"))
982        );
983    }
984
985    #[tokio::test]
986    async fn test_parse_import_items_skips_existing_tags() {
987        let input = format!(
988            r"# {DESCRIPTION_1} #test
989              {CMD_1}"
990        );
991        let tags = vec!["#test".to_string(), "#new".to_string()];
992        let items = parse_import_items(input.as_bytes(), tags, CATEGORY_USER, SOURCE_IMPORT)
993            .try_collect::<Vec<_>>()
994            .await
995            .unwrap();
996
997        assert_eq!(items.len(), 1);
998        let cmd = get_command(&items[0]);
999        assert_eq!(cmd.cmd, CMD_1);
1000        assert_eq!(cmd.description.as_ref(), Some(&format!("{DESCRIPTION_1} #test #new")));
1001    }
1002
1003    #[tokio::test]
1004    async fn test_parse_import_items_with_aliases() {
1005        let input = format!(
1006            r"# [alias:{ALIAS_1}] {DESCRIPTION_1}
1007              {CMD_1}
1008
1009              # [alias:{ALIAS_2}] 
1010              # {DESCRIPTION_2}
1011              # {DESCRIPTION_2}
1012              {CMD_2}
1013
1014              # [alias:{ALIAS_3}]
1015              {CMD_3}"
1016        );
1017        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
1018            .try_collect::<Vec<_>>()
1019            .await
1020            .unwrap();
1021
1022        assert_eq!(items.len(), 3);
1023        let cmd0 = get_command(&items[0]);
1024        assert_eq!(cmd0.cmd, CMD_1);
1025        assert_eq!(cmd0.description.as_deref(), Some(DESCRIPTION_1));
1026        assert_eq!(cmd0.alias.as_deref(), Some(ALIAS_1));
1027
1028        let cmd1 = get_command(&items[1]);
1029        assert_eq!(cmd1.cmd, CMD_2);
1030        assert_eq!(
1031            cmd1.description.as_ref(),
1032            Some(&format!("{DESCRIPTION_2}\n{DESCRIPTION_2}"))
1033        );
1034        assert_eq!(cmd1.alias.as_deref(), Some(ALIAS_2));
1035
1036        let cmd2 = get_command(&items[2]);
1037        assert_eq!(cmd2.cmd, CMD_3);
1038        assert!(cmd2.description.is_none());
1039        assert_eq!(cmd2.alias.as_deref(), Some(ALIAS_3));
1040    }
1041
1042    #[tokio::test]
1043    async fn test_parse_import_items_completions() {
1044        let input = r#"
1045            # A command to ensure both types are handled
1046            ls -l ## list files
1047
1048            # Completions
1049            $(git) branch: git branch --all
1050            $ file: ls -F
1051            $ (az) group: az group list --output tsv
1052            $ (my_script.sh) arg: printf 'value'
1053            $ (./my_script.sh) arg2: printf 'value2'
1054            $ (C:\tools\my_script.bat) win_arg: printf 'win'
1055            $ (~/bin/my_script.sh) tilde_arg: printf 'tilde'
1056            "#;
1057
1058        let items = parse_import_items(input.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
1059            .try_collect::<Vec<_>>()
1060            .await
1061            .unwrap();
1062
1063        assert_eq!(items.len(), 8);
1064
1065        let cmd = get_command(&items[0]);
1066        assert_eq!(cmd.cmd, "ls -l");
1067        assert_eq!(cmd.description.as_deref(), Some("list files"));
1068
1069        if let ImportExportItem::Completion(c) = &items[1] {
1070            assert_eq!(c.flat_root_cmd, "git");
1071            assert_eq!(c.flat_variable, "branch");
1072            assert_eq!(c.suggestions_provider, "git branch --all");
1073        } else {
1074            panic!("Expected a Completion at index 1");
1075        }
1076
1077        if let ImportExportItem::Completion(c) = &items[2] {
1078            assert_eq!(c.flat_root_cmd, ""); // Global
1079            assert_eq!(c.flat_variable, "file");
1080            assert_eq!(c.suggestions_provider, "ls -F");
1081        } else {
1082            panic!("Expected a Completion at index 2");
1083        }
1084
1085        if let ImportExportItem::Completion(c) = &items[3] {
1086            assert_eq!(c.flat_root_cmd, "az");
1087            assert_eq!(c.flat_variable, "group");
1088            assert_eq!(c.suggestions_provider, "az group list --output tsv");
1089        } else {
1090            panic!("Expected a Completion at index 3");
1091        }
1092
1093        if let ImportExportItem::Completion(c) = &items[4] {
1094            assert_eq!(c.flat_root_cmd, "myscriptsh");
1095            assert_eq!(c.flat_variable, "arg");
1096            assert_eq!(c.suggestions_provider, "printf 'value'");
1097        } else {
1098            panic!("Expected a Completion at index 4");
1099        }
1100
1101        if let ImportExportItem::Completion(c) = &items[5] {
1102            assert_eq!(c.flat_root_cmd, "myscriptsh");
1103            assert_eq!(c.flat_variable, "arg2");
1104            assert_eq!(c.suggestions_provider, "printf 'value2'");
1105        } else {
1106            panic!("Expected a Completion at index 5");
1107        }
1108
1109        if let ImportExportItem::Completion(c) = &items[6] {
1110            assert_eq!(c.flat_root_cmd, "ctoolsmyscriptbat");
1111            assert_eq!(c.flat_variable, "win_arg");
1112            assert_eq!(c.suggestions_provider, "printf 'win'");
1113        } else {
1114            panic!("Expected a Completion at index 6");
1115        }
1116
1117        if let ImportExportItem::Completion(c) = &items[7] {
1118            assert_eq!(c.flat_root_cmd, "binmyscriptsh");
1119            assert_eq!(c.flat_variable, "tilde_arg");
1120            assert_eq!(c.suggestions_provider, "printf 'tilde'");
1121        } else {
1122            panic!("Expected a Completion at index 7");
1123        }
1124    }
1125
1126    #[tokio::test]
1127    async fn test_parse_import_items_invalid_completion_format() {
1128        let line = "$ invalid completion format";
1129        let result = parse_import_items(line.as_bytes(), Vec::new(), CATEGORY_USER, SOURCE_IMPORT)
1130            .try_collect::<Vec<_>>()
1131            .await;
1132
1133        assert!(result.is_err());
1134        if let Err(err) = result {
1135            assert!(
1136                matches!(err, AppError::UserFacing(UserFacingError::ImportCompletionInvalidFormat(s)) if s == line)
1137            );
1138        }
1139    }
1140
1141    /// Test helper to extract a Command from an ImportExportItem, panicking if it's the wrong variant
1142    fn get_command(item: &ImportExportItem) -> &Command {
1143        match item {
1144            ImportExportItem::Command(command) => command,
1145            ImportExportItem::Completion(_) => panic!("Expected ImportExportItem::Command, found completion"),
1146        }
1147    }
1148}
intelli_shell/service/import.rs

intelli_shell/service/
import.rs
