1pub mod actions;
3
4pub use actions::S3ClientProtocol;
5
6use crate::llm::actions::client_trait::{Client, ClientActionResult};
7use crate::llm::actions::protocol_trait::Protocol;
8use anyhow::{Context, Result};
9use std::net::SocketAddr;
10use std::sync::Arc;
11use tokio::sync::mpsc;
12use tracing::{debug, error, info};
13
14use crate::client::llm_budget::call_llm_for_client;
15use crate::client::s3::actions::S3_CLIENT_RESPONSE_RECEIVED_EVENT;
16use crate::llm::ollama_client::OllamaClient;
17use crate::llm::ClientLlmResult;
18use crate::protocol::{Event, StartupParams};
19use crate::state::app_state::AppState;
20use crate::state::client_handles::{ClientCommand, ClientSendOutcome};
21use crate::state::{AccessLogOwner, ClientId, ClientStatus};
22use crate::utils::truncate::truncate_for_log;
23
24fn normalize_endpoint(endpoint: &str) -> String {
27 let trimmed = endpoint.trim();
28 if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
29 trimmed.to_string()
30 } else {
31 format!("http://{trimmed}")
32 }
33}
34
35pub struct S3Client;
37
38impl S3Client {
39 pub async fn connect_with_llm_actions(
41 remote_addr: String,
42 _llm_client: OllamaClient,
43 app_state: Arc<AppState>,
44 status_tx: mpsc::UnboundedSender<String>,
45 client_id: ClientId,
46 startup_params: Option<StartupParams>,
47 ) -> Result<SocketAddr> {
48 info!("S3 client {} initializing for {}", client_id, remote_addr);
49
50 let access_key_id = startup_params
59 .as_ref()
60 .map(|p| p.get_optional_string("access_key_id"))
61 .transpose()?
62 .flatten()
63 .unwrap_or_default();
64
65 let secret_access_key = startup_params
66 .as_ref()
67 .map(|p| p.get_optional_string("secret_access_key"))
68 .transpose()?
69 .flatten()
70 .unwrap_or_default();
71
72 let region = startup_params
73 .as_ref()
74 .map(|p| p.get_optional_string("region"))
75 .transpose()?
76 .flatten()
77 .unwrap_or_else(|| "us-east-1".to_string());
78
79 let endpoint_url = startup_params
82 .as_ref()
83 .map(|p| p.get_optional_string("endpoint_url"))
84 .transpose()?
85 .flatten()
86 .unwrap_or_else(|| remote_addr.clone());
87
88 use aws_config::BehaviorVersion;
90 use aws_sdk_s3::config::{Credentials, Region};
91
92 let creds = Credentials::new(
93 &access_key_id,
94 &secret_access_key,
95 None, None, "netget-s3-client",
98 );
99
100 let mut config_builder = aws_sdk_s3::config::Builder::new()
101 .behavior_version(BehaviorVersion::latest())
102 .region(Region::new(region.clone()))
103 .credentials_provider(creds);
104
105 if !endpoint_url.is_empty() {
110 config_builder = config_builder.endpoint_url(normalize_endpoint(&endpoint_url));
111 }
112
113 let config = config_builder.build();
116 let _ = aws_sdk_s3::Client::from_conf(config);
117
118 app_state
120 .with_client_mut(client_id, |client| {
121 client.set_protocol_field(
122 "s3_client_initialized".to_string(),
123 serde_json::json!(true),
124 );
125 client.set_protocol_field("endpoint".to_string(), serde_json::json!(endpoint_url));
126 client.set_protocol_field("region".to_string(), serde_json::json!(region));
127 client.set_protocol_field(
128 "access_key_id".to_string(),
129 serde_json::json!(access_key_id),
130 );
131 client.set_protocol_field(
132 "secret_access_key".to_string(),
133 serde_json::json!(secret_access_key),
134 );
135 })
136 .await;
137
138 app_state
140 .update_client_status(client_id, ClientStatus::Connected)
141 .await;
142 let _ = status_tx.send(format!(
143 "[CLIENT] S3 client {} ready for {}",
144 client_id, remote_addr
145 ));
146 let _ = status_tx.send("__UPDATE_UI__".to_string());
147
148 let command_rx =
153 crate::client::command_support::register_command_channel(&app_state, client_id).await;
154 let cmd_task = tokio::spawn(Self::command_loop(
155 command_rx,
156 client_id,
157 app_state.clone(),
158 _llm_client.clone(),
159 status_tx.clone(),
160 ));
161 app_state.register_client_task(client_id, cmd_task).await;
162
163 let remote_addr_clone = remote_addr.clone();
165 let llm_client_clone = _llm_client.clone();
166 let app_state_clone = app_state.clone();
167 let status_tx_clone = status_tx.clone();
168 let region_clone = region.clone();
169
170 let task_registrar = app_state.clone();
173 let task_handle = tokio::spawn(async move {
174 use crate::client::s3::actions::S3_CLIENT_CONNECTED_EVENT;
175
176 let instruction = match app_state_clone.get_instruction_for_client(client_id).await {
178 Some(instr) => instr,
179 None => {
180 error!("S3 client {} has no instruction", client_id);
181 return;
182 }
183 };
184
185 let protocol = Arc::new(crate::client::s3::actions::S3ClientProtocol::new());
186 let event = Event::new(
187 &S3_CLIENT_CONNECTED_EVENT,
188 serde_json::json!({
189 "endpoint": remote_addr_clone,
190 "region": region_clone,
191 }),
192 );
193
194 let memory = app_state_clone
195 .get_memory_for_client(client_id)
196 .await
197 .unwrap_or_default();
198
199 match call_llm_for_client(
200 &llm_client_clone,
201 &app_state_clone,
202 client_id.to_string(),
203 &instruction,
204 &memory,
205 Some(&event),
206 protocol.as_ref(),
207 &status_tx_clone,
208 )
209 .await
210 {
211 Ok(ClientLlmResult {
212 actions,
213 memory_updates,
214 }) => {
215 if let Some(mem) = memory_updates {
217 app_state_clone.set_memory_for_client(client_id, mem).await;
218 }
219
220 for action in actions {
222 let result = match protocol.execute_action(action) {
223 Ok(result) => result,
224 Err(e) => {
225 error!("S3 client {} action error: {}", client_id, e);
226 continue;
227 }
228 };
229 let outcome = Self::apply_action(
230 result,
231 client_id,
232 &app_state_clone,
233 &llm_client_clone,
234 &status_tx_clone,
235 )
236 .await;
237 match outcome {
238 ClientSendOutcome::Disconnected => {
239 info!("S3 client {} disconnecting", client_id);
240 app_state_clone
241 .update_client_status(client_id, ClientStatus::Disconnected)
242 .await;
243 app_state_clone.remove_client_handle(client_id).await;
244 let _ = status_tx_clone.send("__UPDATE_UI__".to_string());
245 return;
246 }
247 other => debug!("S3 client {} connect action: {:?}", client_id, other),
248 }
249 }
250 }
251 Err(e) => {
252 error!("LLM error for S3 client {}: {}", client_id, e);
253 }
254 }
255 });
256 task_registrar
257 .register_client_task(client_id, task_handle)
258 .await;
259
260 Ok("0.0.0.0:0".parse().unwrap())
262 }
263
264 pub async fn run_operation_once(
275 client_id: ClientId,
276 operation_name: &str,
277 operation_data: serde_json::Value,
278 app_state: &Arc<AppState>,
279 ) -> Result<serde_json::Value> {
280 info!(
281 "S3 client {} executing operation: {}",
282 client_id, operation_name
283 );
284
285 let (endpoint_url, region, access_key_id, secret_access_key) = app_state
287 .with_client_mut(client_id, |client| {
288 let endpoint = client
289 .get_protocol_field("endpoint")
290 .and_then(|v| v.as_str())
291 .unwrap_or("")
292 .to_string();
293
294 let region = client
295 .get_protocol_field("region")
296 .and_then(|v| v.as_str())
297 .unwrap_or("us-east-1")
298 .to_string();
299
300 let access_key = client
301 .get_protocol_field("access_key_id")
302 .and_then(|v| v.as_str())
303 .unwrap_or("")
304 .to_string();
305
306 let secret_key = client
307 .get_protocol_field("secret_access_key")
308 .and_then(|v| v.as_str())
309 .unwrap_or("")
310 .to_string();
311
312 (endpoint, region, access_key, secret_key)
313 })
314 .await
315 .unwrap_or_else(|| {
316 (
317 String::new(),
318 "us-east-1".to_string(),
319 String::new(),
320 String::new(),
321 )
322 });
323
324 use aws_config::BehaviorVersion;
326 use aws_sdk_s3::config::{Credentials, Region};
327
328 let creds = Credentials::new(
329 &access_key_id,
330 &secret_access_key,
331 None,
332 None,
333 "netget-s3-client",
334 );
335
336 let mut config_builder = aws_sdk_s3::config::Builder::new()
337 .behavior_version(BehaviorVersion::latest())
338 .region(Region::new(region))
339 .credentials_provider(creds);
340
341 if !endpoint_url.is_empty() {
342 config_builder = config_builder.endpoint_url(&endpoint_url);
343 }
344
345 let config = config_builder.build();
346 let s3_client = aws_sdk_s3::Client::from_conf(config);
347
348 let result = match operation_name {
350 "s3_put_object" => Self::put_object(&s3_client, operation_data).await,
351 "s3_get_object" => Self::get_object(&s3_client, operation_data).await,
352 "s3_list_buckets" => Self::list_buckets(&s3_client).await,
353 "s3_list_objects" => Self::list_objects(&s3_client, operation_data).await,
354 "s3_delete_object" => Self::delete_object(&s3_client, operation_data).await,
355 "s3_head_object" => Self::head_object(&s3_client, operation_data).await,
356 "s3_create_bucket" => Self::create_bucket(&s3_client, operation_data).await,
357 "s3_delete_bucket" => Self::delete_bucket(&s3_client, operation_data).await,
358 _ => Err(anyhow::anyhow!("Unknown S3 operation: {}", operation_name)),
359 };
360
361 result
362 }
363
364 pub async fn execute_operation(
365 client_id: ClientId,
366 operation_name: String,
367 operation_data: serde_json::Value,
368 app_state: Arc<AppState>,
369 llm_client: OllamaClient,
370 status_tx: mpsc::UnboundedSender<String>,
371 ) -> Result<serde_json::Value> {
372 info!(
373 "S3 client {} executing operation: {}",
374 client_id, operation_name
375 );
376
377 let result =
378 Self::run_operation_once(client_id, &operation_name, operation_data, &app_state).await;
379
380 let event_data = match &result {
386 Ok(response_data) => serde_json::json!({
387 "operation": operation_name,
388 "success": true,
389 "result": response_data,
390 }),
391 Err(e) => serde_json::json!({
392 "operation": operation_name,
393 "success": false,
394 "error": e.to_string(),
395 }),
396 };
397 let notify = tokio::spawn(Self::notify_response(
398 client_id,
399 event_data,
400 app_state.clone(),
401 llm_client,
402 status_tx,
403 ));
404 app_state.register_client_task(client_id, notify).await;
405
406 result
407 }
408
409 async fn apply_action(
416 result: ClientActionResult,
417 client_id: ClientId,
418 app_state: &Arc<AppState>,
419 llm_client: &OllamaClient,
420 status_tx: &mpsc::UnboundedSender<String>,
421 ) -> ClientSendOutcome {
422 match result {
423 ClientActionResult::Custom { name, data } => {
424 match Self::execute_operation(
425 client_id,
426 name.clone(),
427 data,
428 app_state.clone(),
429 llm_client.clone(),
430 status_tx.clone(),
431 )
432 .await
433 {
434 Ok(value) => ClientSendOutcome::Executed {
435 detail: format!(
436 "{} completed: {}",
437 name,
438 truncate_for_log(&value.to_string(), 200)
439 ),
440 },
441 Err(e) => {
442 error!("S3 client {} operation {} failed: {}", client_id, name, e);
443 let _ = status_tx.send(format!("[ERROR] S3 operation failed: {}", e));
444 ClientSendOutcome::Rejected {
448 error: format!(
449 "{} failed: {}",
450 name,
451 truncate_for_log(&e.to_string(), 200)
452 ),
453 }
454 }
455 }
456 }
457 ClientActionResult::Disconnect => ClientSendOutcome::Disconnected,
458 ClientActionResult::WaitForMore => ClientSendOutcome::Executed {
459 detail: "wait_for_more".to_string(),
460 },
461 ClientActionResult::NoAction => ClientSendOutcome::Executed {
462 detail: "no_action".to_string(),
463 },
464 ClientActionResult::SendData(_) => ClientSendOutcome::Executed {
465 detail: "send_data has no meaning for the S3 client: it speaks the S3 REST API \
466 through the AWS SDK, not a socket this client owns"
467 .to_string(),
468 },
469 ClientActionResult::Multiple(_) => ClientSendOutcome::Executed {
470 detail: "the S3 client's own verbs never produce Multiple; nothing was applied"
471 .to_string(),
472 },
473 }
474 }
475
476 async fn command_loop(
485 mut command_rx: mpsc::Receiver<ClientCommand>,
486 client_id: ClientId,
487 app_state: Arc<AppState>,
488 llm_client: OllamaClient,
489 status_tx: mpsc::UnboundedSender<String>,
490 ) {
491 let protocol = crate::client::s3::actions::S3ClientProtocol::new();
492
493 while let Some(command) = command_rx.recv().await {
494 let action = command.action.clone();
495 let outcome = match protocol.execute_action(action.clone()) {
496 Err(e) => ClientSendOutcome::Rejected {
497 error: e.to_string(),
498 },
499 Ok(result) => {
500 Self::apply_action(result, client_id, &app_state, &llm_client, &status_tx).await
501 }
502 };
503 let disconnected = matches!(outcome, ClientSendOutcome::Disconnected);
504
505 app_state
506 .record_access_log(
507 AccessLogOwner::Client(client_id.as_u32()),
508 protocol.protocol_name(),
509 None,
510 "injected_action",
511 action,
512 vec![serde_json::to_value(&outcome).unwrap_or(serde_json::Value::Null)],
513 )
514 .await;
515 let _ = status_tx.send("__UPDATE_UI__".to_string());
516 crate::client::command_support::reply(command, Ok(outcome));
517
518 if disconnected {
519 info!("S3 client {} disconnecting on injected action", client_id);
520 app_state
521 .update_client_status(client_id, ClientStatus::Disconnected)
522 .await;
523 break;
524 }
525 }
526
527 app_state.remove_client_handle(client_id).await;
529 let _ = status_tx.send("__UPDATE_UI__".to_string());
530 }
531
532 async fn notify_response(
536 client_id: ClientId,
537 event_data: serde_json::Value,
538 app_state: Arc<AppState>,
539 llm_client: OllamaClient,
540 status_tx: mpsc::UnboundedSender<String>,
541 ) {
542 let Some(instruction) = app_state.get_instruction_for_client(client_id).await else {
543 return;
544 };
545 let protocol = crate::client::s3::actions::S3ClientProtocol::new();
546 let event = Event::new(&S3_CLIENT_RESPONSE_RECEIVED_EVENT, event_data);
547 let memory = app_state
548 .get_memory_for_client(client_id)
549 .await
550 .unwrap_or_default();
551
552 match call_llm_for_client(
553 &llm_client,
554 &app_state,
555 client_id.to_string(),
556 &instruction,
557 &memory,
558 Some(&event),
559 &protocol,
560 &status_tx,
561 )
562 .await
563 {
564 Ok(ClientLlmResult {
565 actions,
566 memory_updates,
567 }) => {
568 if let Some(mem) = memory_updates {
569 app_state.set_memory_for_client(client_id, mem).await;
570 }
571
572 use crate::llm::actions::client_trait::{Client, ClientActionResult};
580 let protocol = crate::client::s3::actions::S3ClientProtocol::new();
581 for action in actions {
582 let (name, data) = match protocol.execute_action(action.clone()) {
587 Ok(ClientActionResult::Custom { name, data }) => (name, data),
588 Ok(ClientActionResult::Disconnect) => {
589 info!(
590 "S3 client {} disconnecting: the model answered the response \
591 event with `disconnect`",
592 client_id
593 );
594 app_state
595 .update_client_status(client_id, ClientStatus::Disconnected)
596 .await;
597 app_state.remove_client_handle(client_id).await;
598 let _ = status_tx
599 .send(format!("[CLIENT] S3 client {client_id} disconnected"));
600 let _ = status_tx.send("__UPDATE_UI__".to_string());
601 return;
602 }
603 Ok(other) => {
604 info!(
605 "S3 client {} follow-up action {:?} needs no S3 operation",
606 client_id, other
607 );
608 continue;
609 }
610 Err(e) => {
611 error!(
612 "S3 client {} could not execute follow-up action {}: {}",
613 client_id, action, e
614 );
615 continue;
616 }
617 };
618 match Self::run_operation_once(client_id, &name, data, &app_state).await {
619 Ok(v) => info!(
620 "S3 client {} follow-up {} -> {}",
621 client_id,
622 name,
623 crate::utils::truncate_for_log(&v.to_string(), 120)
624 ),
625 Err(e) => {
626 error!("S3 client {} follow-up {} failed: {}", client_id, name, e)
627 }
628 }
629 }
630 }
631 Err(e) => {
632 error!("LLM error for S3 client {}: {}", client_id, e);
633 }
634 }
635 }
636
637 async fn put_object(
638 client: &aws_sdk_s3::Client,
639 data: serde_json::Value,
640 ) -> Result<serde_json::Value> {
641 let bucket = data["bucket"].as_str().context("Missing bucket")?;
642 let key = data["key"].as_str().context("Missing key")?;
643 let body = data["body"].as_str().context("Missing body")?;
644 let content_type = data["content_type"].as_str();
645
646 let body_bytes = match data["encoding"].as_str().unwrap_or("utf8") {
652 "utf8" => body.as_bytes().to_vec(),
653 "base64" => {
654 use base64::Engine as _;
655 base64::engine::general_purpose::STANDARD
656 .decode(body)
657 .context(
658 "put_object was given encoding=\"base64\" but `body` is not valid \
659 base64. Send the bytes as base64, or set encoding=\"utf8\" to store \
660 the text as written.",
661 )?
662 }
663 other => anyhow::bail!(
664 "Unknown encoding {:?} for put_object; use \"utf8\" (default) or \"base64\"",
665 other
666 ),
667 };
668
669 let mut request = client
670 .put_object()
671 .bucket(bucket)
672 .key(key)
673 .body(body_bytes.into());
674
675 if let Some(ct) = content_type {
676 request = request.content_type(ct);
677 }
678
679 let response = request.send().await.context("Failed to put object")?;
680
681 Ok(serde_json::json!({
682 "bucket": bucket,
683 "key": key,
684 "etag": response.e_tag().unwrap_or(""),
685 }))
686 }
687
688 async fn get_object(
689 client: &aws_sdk_s3::Client,
690 data: serde_json::Value,
691 ) -> Result<serde_json::Value> {
692 let bucket = data["bucket"].as_str().context("Missing bucket")?;
693 let key = data["key"].as_str().context("Missing key")?;
694
695 let response = client
696 .get_object()
697 .bucket(bucket)
698 .key(key)
699 .send()
700 .await
701 .context("Failed to get object")?;
702
703 let content_type = response
704 .content_type()
705 .map(|s| s.to_string())
706 .unwrap_or_default();
707 let content_length = response.content_length().unwrap_or(0);
708
709 let body_bytes = response
710 .body
711 .collect()
712 .await
713 .context("Failed to read object body")?
714 .into_bytes();
715
716 let (body, body_encoding) = match std::str::from_utf8(&body_bytes) {
721 Ok(text) => (text.to_string(), "utf8"),
722 Err(_) => {
723 use base64::Engine as _;
724 (
725 base64::engine::general_purpose::STANDARD.encode(&body_bytes),
726 "base64",
727 )
728 }
729 };
730
731 Ok(serde_json::json!({
732 "bucket": bucket,
733 "key": key,
734 "content_type": content_type,
735 "content_length": content_length,
736 "body": body,
737 "body_encoding": body_encoding,
739 }))
740 }
741
742 async fn list_buckets(client: &aws_sdk_s3::Client) -> Result<serde_json::Value> {
743 let response = client
744 .list_buckets()
745 .send()
746 .await
747 .context("Failed to list buckets")?;
748
749 let buckets: Vec<serde_json::Value> = response
750 .buckets()
751 .iter()
752 .map(|b| {
753 serde_json::json!({
754 "name": b.name().unwrap_or(""),
755 "creation_date": b.creation_date()
756 .map(|d| d.to_string())
757 .unwrap_or_default(),
758 })
759 })
760 .collect();
761
762 Ok(serde_json::json!({
763 "buckets": buckets,
764 "count": buckets.len(),
765 }))
766 }
767
768 async fn list_objects(
769 client: &aws_sdk_s3::Client,
770 data: serde_json::Value,
771 ) -> Result<serde_json::Value> {
772 let bucket = data["bucket"].as_str().context("Missing bucket")?;
773 let prefix = data["prefix"].as_str();
774 let max_keys = data["max_keys"].as_i64().map(|n| n as i32);
775
776 let mut request = client.list_objects_v2().bucket(bucket);
777
778 if let Some(p) = prefix {
779 request = request.prefix(p);
780 }
781
782 if let Some(mk) = max_keys {
783 request = request.max_keys(mk);
784 }
785
786 let response = request.send().await.context("Failed to list objects")?;
787
788 let objects: Vec<serde_json::Value> = response
789 .contents()
790 .iter()
791 .map(|obj| {
792 serde_json::json!({
793 "key": obj.key().unwrap_or(""),
794 "size": obj.size().unwrap_or(0),
795 "last_modified": obj.last_modified()
796 .map(|d| d.to_string())
797 .unwrap_or_default(),
798 "etag": obj.e_tag().unwrap_or(""),
799 })
800 })
801 .collect();
802
803 Ok(serde_json::json!({
804 "bucket": bucket,
805 "objects": objects,
806 "count": objects.len(),
807 "is_truncated": response.is_truncated(),
808 }))
809 }
810
811 async fn delete_object(
812 client: &aws_sdk_s3::Client,
813 data: serde_json::Value,
814 ) -> Result<serde_json::Value> {
815 let bucket = data["bucket"].as_str().context("Missing bucket")?;
816 let key = data["key"].as_str().context("Missing key")?;
817
818 client
819 .delete_object()
820 .bucket(bucket)
821 .key(key)
822 .send()
823 .await
824 .context("Failed to delete object")?;
825
826 Ok(serde_json::json!({
827 "bucket": bucket,
828 "key": key,
829 "deleted": true,
830 }))
831 }
832
833 async fn head_object(
834 client: &aws_sdk_s3::Client,
835 data: serde_json::Value,
836 ) -> Result<serde_json::Value> {
837 let bucket = data["bucket"].as_str().context("Missing bucket")?;
838 let key = data["key"].as_str().context("Missing key")?;
839
840 let response = client
841 .head_object()
842 .bucket(bucket)
843 .key(key)
844 .send()
845 .await
846 .context("Failed to head object")?;
847
848 Ok(serde_json::json!({
849 "bucket": bucket,
850 "key": key,
851 "content_type": response.content_type().unwrap_or(""),
852 "content_length": response.content_length().unwrap_or(0),
853 "etag": response.e_tag().unwrap_or(""),
854 "last_modified": response.last_modified()
855 .map(|d| d.to_string())
856 .unwrap_or_default(),
857 }))
858 }
859
860 async fn create_bucket(
861 client: &aws_sdk_s3::Client,
862 data: serde_json::Value,
863 ) -> Result<serde_json::Value> {
864 let bucket = data["bucket"].as_str().context("Missing bucket")?;
865
866 client
867 .create_bucket()
868 .bucket(bucket)
869 .send()
870 .await
871 .context("Failed to create bucket")?;
872
873 Ok(serde_json::json!({
874 "bucket": bucket,
875 "created": true,
876 }))
877 }
878
879 async fn delete_bucket(
880 client: &aws_sdk_s3::Client,
881 data: serde_json::Value,
882 ) -> Result<serde_json::Value> {
883 let bucket = data["bucket"].as_str().context("Missing bucket")?;
884
885 client
886 .delete_bucket()
887 .bucket(bucket)
888 .send()
889 .await
890 .context("Failed to delete bucket")?;
891
892 Ok(serde_json::json!({
893 "bucket": bucket,
894 "deleted": true,
895 }))
896 }
897}