auto_pilot/
summarize.rs

1use crate::{
2    action::send_message_to_openai,
3    constants::MAX_TOKENS,
4    parsers::format_summary_prompt,
5    screen::capture_screen_with_cursor,
6    types::{
7        ImageMessage, ImageMessageContent, ImageUrl, Message, OpenAIRequest, Role, TextMessage,
8    },
9};
10use base64::{engine::general_purpose, Engine as _};
11use std::{error::Error, fs, path::Path};
12
13pub async fn summarize(
14    messages: &mut Vec<Message>,
15    objective: &str,
16) -> Result<String, Box<dyn Error>> {
17    let screenshots_dir = "screenshots";
18    if !Path::new(screenshots_dir).exists() {
19        fs::create_dir(screenshots_dir)?;
20    }
21
22    let screenshot_filename = format!("{}/summary_screenshot.png", screenshots_dir);
23
24    capture_screen_with_cursor(&screenshot_filename)?;
25
26    let img_file = fs::read(&screenshot_filename)?;
27    let img_base64 = general_purpose::STANDARD.encode(&img_file);
28
29    let summary_prompt = format_summary_prompt(objective);
30
31    let vision_message = Message::ImageMessage(ImageMessage {
32        role: Role::User,
33        content: vec![
34            ImageMessageContent::Text {
35                text: summary_prompt,
36            },
37            ImageMessageContent::ImageUrl {
38                image_url: ImageUrl {
39                    url: format!("data:image/jpeg;base64,{}", img_base64),
40                },
41            },
42        ],
43    });
44
45    let mut messages_clone = messages.clone();
46    messages_clone.push(vision_message);
47
48    let payload = OpenAIRequest {
49        model: "gpt-4-vision-preview".to_string(),
50        messages: messages_clone,
51        max_tokens: MAX_TOKENS,
52    };
53
54    let content = send_message_to_openai(payload)
55        .await
56        .map_err(|e| format!("Error sending message to OpenAI: {}", e))?;
57
58    messages.push(Message::TextMessage(TextMessage {
59        role: Role::User,
60        content: "summary_screenshot.png".to_string(),
61    }));
62
63    messages.push(Message::TextMessage(TextMessage {
64        role: Role::Assistant,
65        content: content.to_string(),
66    }));
67
68    Ok(content)
69}
auto_pilot/summarize.rs

auto_pilot/
summarize.rs
