1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154

//! LLM Service integration for [Flows.network](https://flows.network)
//!
//! # Quick Start
//!
//! To get started, let's write a tiny flow function.
//!
//! ```rust
//! use llmservice_flows::{
//!     chat::ChatOptions,
//!     LLMServiceFlows,
//! };
//! use lambda_flows::{request_received, send_response};
//! use serde_json::Value;
//! use std::collections::HashMap;
//!
//! #[no_mangle]
//! #[tokio::main(flavor = "current_thread")]
//! pub async fn run() {
//!     request_received(handler).await;
//! }
//!
//! async fn handler(_qry: HashMap<String, Value>, body: Vec<u8>) {
//!     let co = ChatOptions {
//!       model: Some("gpt-4"),
//!       token_limit: 8192,
//!       ..Default::default()
//!     };
//!     let mut lf = LLMServiceFlows::new("https://api.openai.com/v1");
//!     lf.set_api_key("your api key");
//!
//!     let r = match lf.chat_completion(
//!         "any_conversation_id",
//!         String::from_utf8_lossy(&body).into_owned().as_str(),
//!         &co,
//!     )
//!     .await
//!     {
//!         Ok(c) => c.choice,
//!         Err(e) => e,
//!     };
//!     
//!     send_response(
//!         200,
//!         vec![(
//!             String::from("content-type"),
//!             String::from("text/plain; charset=UTF-8"),
//!         )],
//!         r.as_bytes().to_vec(),
//!     );
//! }
//! ```
//!
//! When the Lambda request is received, chat
//! using [chat_completion] then send the response.
//!
//! ```

use lazy_static::lazy_static;
use std::thread::sleep;
use std::time::Duration;

pub mod chat;
pub mod embeddings;

lazy_static! {
    static ref LLM_API_PREFIX: String =
        String::from(std::option_env!("LLM_API_PREFIX").unwrap_or("https://llm.flows.network/api"));
}

extern "C" {
    fn get_flows_user(p: *mut u8) -> i32;
    fn get_flow_id(p: *mut u8) -> i32;
}

unsafe fn _get_flows_user() -> String {
    let mut flows_user = Vec::<u8>::with_capacity(100);
    let c = get_flows_user(flows_user.as_mut_ptr());
    flows_user.set_len(c as usize);
    String::from_utf8(flows_user).unwrap()
}

unsafe fn _get_flow_id() -> String {
    let mut flow_id = Vec::<u8>::with_capacity(100);
    let c = get_flow_id(flow_id.as_mut_ptr());
    if c == 0 {
        panic!("Failed to get flow id");
    }
    flow_id.set_len(c as usize);
    String::from_utf8(flow_id).unwrap()
}

const MAX_RETRY_TIMES: u8 = 10;
const RETRY_INTERVAL: u64 = 10; // Wait 10 seconds before retry

/// The main struct for setting the basic configuration
/// for LLM Service interface.
pub struct LLMServiceFlows<'a> {
    /// Exposed url of the LLM Service
    service_endpoint: &'a str,

    /// API Key
    api_key: Option<&'a str>,

    /// Use retry_times to set the number of retries when requesting
    /// LLM Service's api encounters a problem. Default is 0 and max number is 10.
    retry_times: u8,
}

impl<'a> LLMServiceFlows<'a> {
    pub fn new(service_endpoint: &'a str) -> LLMServiceFlows<'a> {
        LLMServiceFlows {
            service_endpoint,
            api_key: None,
            retry_times: 0,
        }
    }

    pub fn set_retry_times(&mut self, retry_times: u8) {
        self.retry_times = retry_times;
    }

    pub fn set_api_key(&mut self, api_key: &'a str) {
        self.api_key = Some(api_key);
    }

    fn keep_trying<T, F>(&self, f: F) -> Result<T, String>
    where
        F: Fn(&str, &str) -> Retry<T>,
    {
        let mut retry_times = match self.retry_times {
            r if r > MAX_RETRY_TIMES => MAX_RETRY_TIMES,
            r => r,
        };

        loop {
            match f(self.service_endpoint, self.api_key.unwrap_or_default()) {
                Retry::Yes(s) => match retry_times > 0 {
                    true => {
                        sleep(Duration::from_secs(crate::RETRY_INTERVAL));
                        retry_times = retry_times - 1;
                        continue;
                    }
                    false => return Err(s),
                },
                Retry::No(r) => return r,
            }
        }
    }
}

enum Retry<T> {
    Yes(String),
    No(Result<T, String>),
}