aichat 0.30.0

All-in-one LLM CLI Tool
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

use super::*;

use anyhow::{anyhow, Context, Result};
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;

pub const EXTENSION_METADATA: &str = "__extension__";

pub type DocumentMetadata = IndexMap<String, String>;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoadedDocument {
    pub path: String,
    pub contents: String,
    #[serde(default)]
    pub metadata: DocumentMetadata,
}

impl LoadedDocument {
    pub fn new(path: String, contents: String, metadata: DocumentMetadata) -> Self {
        Self {
            path,
            contents,
            metadata,
        }
    }
}

pub async fn load_recursive_url(
    loaders: &HashMap<String, String>,
    path: &str,
) -> Result<Vec<LoadedDocument>> {
    let extension = RECURSIVE_URL_LOADER;
    let pages: Vec<Page> = match loaders.get(extension) {
        Some(loader_command) => {
            let contents = run_loader_command(path, extension, loader_command)?;
            serde_json::from_str(&contents).context(r#"The crawler response is invalid. It should follow the JSON format: `[{"path":"...", "text":"..."}]`."#)?
        }
        None => {
            let options = CrawlOptions::preset(path);
            crawl_website(path, options).await?
        }
    };
    let output = pages
        .into_iter()
        .map(|v| {
            let Page { path, text } = v;
            let mut metadata: DocumentMetadata = Default::default();
            metadata.insert(EXTENSION_METADATA.into(), "md".into());
            LoadedDocument::new(path, text, metadata)
        })
        .collect();
    Ok(output)
}

pub async fn load_file(loaders: &HashMap<String, String>, path: &str) -> Result<LoadedDocument> {
    let extension = get_patch_extension(path).unwrap_or_else(|| DEFAULT_EXTENSION.into());
    match loaders.get(&extension) {
        Some(loader_command) => load_with_command(path, &extension, loader_command),
        None => load_plain(path, &extension).await,
    }
}

pub async fn load_url(loaders: &HashMap<String, String>, path: &str) -> Result<LoadedDocument> {
    let (contents, extension) = fetch_with_loaders(loaders, path, false).await?;
    let mut metadata: DocumentMetadata = Default::default();
    metadata.insert(EXTENSION_METADATA.into(), extension);
    Ok(LoadedDocument::new(path.into(), contents, metadata))
}

async fn load_plain(path: &str, extension: &str) -> Result<LoadedDocument> {
    let contents = tokio::fs::read_to_string(path).await?;
    let mut metadata: DocumentMetadata = Default::default();
    metadata.insert(EXTENSION_METADATA.into(), extension.to_string());
    Ok(LoadedDocument::new(path.into(), contents, metadata))
}

fn load_with_command(path: &str, extension: &str, loader_command: &str) -> Result<LoadedDocument> {
    let contents = run_loader_command(path, extension, loader_command)?;
    let mut metadata: DocumentMetadata = Default::default();
    metadata.insert(EXTENSION_METADATA.into(), DEFAULT_EXTENSION.to_string());
    Ok(LoadedDocument::new(path.into(), contents, metadata))
}

pub fn is_loader_protocol(loaders: &HashMap<String, String>, path: &str) -> bool {
    match path.split_once(':') {
        Some((protocol, _)) => loaders.contains_key(protocol),
        None => false,
    }
}

pub fn load_protocol_path(
    loaders: &HashMap<String, String>,
    path: &str,
) -> Result<Vec<LoadedDocument>> {
    let (protocol, loader_command, new_path) = path
        .split_once(':')
        .and_then(|(protocol, path)| {
            let loader_command = loaders.get(protocol)?;
            Some((protocol, loader_command, path))
        })
        .ok_or_else(|| anyhow!("No document loader for '{}'", path))?;
    let contents = run_loader_command(new_path, protocol, loader_command)?;
    let output = if let Ok(list) = serde_json::from_str::<Vec<LoadedDocument>>(&contents) {
        list.into_iter()
            .map(|mut v| {
                if v.path.starts_with(path) {
                } else if v.path.starts_with(new_path) {
                    v.path = format!("{}:{}", protocol, v.path);
                } else {
                    v.path = format!("{}/{}", path, v.path);
                }
                v
            })
            .collect()
    } else {
        vec![LoadedDocument::new(
            path.into(),
            contents,
            Default::default(),
        )]
    };
    Ok(output)
}