1use std::time::Duration;
21
22use super::client::{IntelError, Provider, Transport, resolve};
23use super::health::{BreakerConfig, HealthRecord};
24
25const TOKEN_ENV: &str = "AGENTD_INTELLIGENCE_TOKEN";
28
29const TOKEN_ENV_NEUTRAL: &str = "AGENT_INTELLIGENCE_TOKEN";
33
34pub struct Endpoint {
37 pub(super) transport: Transport,
39 pub(super) http_path: String,
40 pub(super) host_header: String,
41 pub(super) token: Option<String>,
43 pub(super) provider: Provider,
44 pub(super) scheme: &'static str,
47 pub(super) addr: String,
50 pub(super) extra_headers: Vec<(String, String)>,
53 pub(super) signer: Option<std::sync::Arc<dyn ::mcp::http::RequestSigner>>,
57 pub health: HealthRecord,
59}
60
61pub struct EndpointList {
63 eps: Vec<Endpoint>,
64 active: usize,
68 breaker: BreakerConfig,
69}
70
71fn env(name: &str) -> Option<String> {
73 std::env::var(name).ok()
74}
75
76impl EndpointList {
77 pub fn parse(uri: &str, default_token: Option<String>) -> Result<EndpointList, IntelError> {
83 Self::parse_with_env(uri, default_token, &env)
84 }
85
86 pub fn parse_with_env(
88 uri: &str,
89 default_token: Option<String>,
90 env: &dyn Fn(&str) -> Option<String>,
91 ) -> Result<EndpointList, IntelError> {
92 let provider = Provider::OpenAiCompatible;
93 let parts: Vec<&str> = uri
94 .split(',')
95 .map(str::trim)
96 .filter(|s| !s.is_empty())
97 .collect();
98 if parts.is_empty() {
99 return Err(IntelError::Unsupported(
100 "empty intelligence endpoint list".into(),
101 ));
102 }
103 let mut eps = Vec::with_capacity(parts.len());
104 for (i, part) in parts.iter().enumerate() {
105 let (transport, http_path, host_header) = resolve(part, provider)?;
106 let token = resolve_token(i, default_token.as_deref(), env)?;
107 let (scheme, addr) = scheme_and_addr(part);
108 eps.push(Endpoint {
109 transport,
110 http_path,
111 host_header,
112 token,
113 provider,
114 scheme,
115 addr,
116 extra_headers: Vec::new(),
117 signer: None,
118 health: HealthRecord::new(),
119 });
120 }
121 Ok(EndpointList {
122 eps,
123 active: 0,
124 breaker: BreakerConfig::default(),
125 })
126 }
127
128 pub fn set_extra_headers(&mut self, headers: Vec<(String, String)>) {
131 for e in &mut self.eps {
132 e.extra_headers = headers.clone();
133 }
134 }
135
136 pub fn set_signer(&mut self, signer: std::sync::Arc<dyn ::mcp::http::RequestSigner>) {
138 for e in &mut self.eps {
139 e.signer = Some(signer.clone());
140 }
141 }
142
143 pub fn set_provider(&mut self, provider: Provider) {
149 let default = provider.default_path();
150 for e in &mut self.eps {
151 e.provider = provider;
152 if e.http_path == super::openai::DEFAULT_PATH {
153 e.http_path = default.to_string();
154 }
155 }
156 }
157
158 pub fn len(&self) -> usize {
159 self.eps.len()
160 }
161
162 pub fn is_empty(&self) -> bool {
163 self.eps.is_empty()
164 }
165
166 pub fn active(&self) -> usize {
167 self.active
168 }
169
170 pub fn breaker_config(&self) -> &BreakerConfig {
171 &self.breaker
172 }
173
174 pub fn ep(&self, idx: usize) -> &Endpoint {
175 &self.eps[idx]
176 }
177
178 pub fn iter(&self) -> impl Iterator<Item = &Endpoint> {
179 self.eps.iter()
180 }
181
182 pub fn attempt_order(&self) -> Vec<usize> {
187 let mut order = Vec::with_capacity(self.eps.len());
188 if self.eps[self.active].health.available(&self.breaker) {
189 order.push(self.active);
190 }
191 for idx in 0..self.eps.len() {
192 if idx == self.active {
193 continue;
194 }
195 if self.eps[idx].health.available(&self.breaker) {
196 order.push(idx);
197 }
198 }
199 order
200 }
201
202 pub fn prefer_lowest_healthy(&mut self) -> Option<usize> {
207 let target = (0..self.eps.len()).find(|&i| self.eps[i].health.is_up());
208 if let Some(t) = target
209 && t != self.active
210 {
211 self.active = t;
212 return Some(t);
213 }
214 None
215 }
216
217 pub fn set_active(&mut self, idx: usize) -> Option<usize> {
220 if idx != self.active {
221 self.active = idx;
222 Some(idx)
223 } else {
224 None
225 }
226 }
227
228 pub fn all_down(&self) -> bool {
230 self.attempt_order().is_empty()
231 }
232
233 pub fn active_identity(&self) -> (usize, &'static str) {
238 (self.active, self.eps[self.active].scheme)
239 }
240
241 pub fn body(&self, model: Option<&str>) -> serde_json::Value {
247 use serde_json::json;
248 let cfg = &self.breaker;
249 let endpoints: Vec<serde_json::Value> = self
250 .eps
251 .iter()
252 .enumerate()
253 .map(|(i, ep)| {
254 let h = &ep.health;
255 let mut e = json!({
256 "index": i,
257 "transport": ep.scheme,
258 "addr": ep.addr,
259 "state": h.state().as_str(),
260 "active": i == self.active,
261 "ewma_latency_ms": h.ewma_latency_ms(),
262 "error_rate": h.error_rate(),
263 "consec_fail": h.consec_fail(),
264 });
265 if let serde_json::Value::Object(m) = &mut e {
266 if let Some(ms) = h.last_ok_ms_ago() {
267 m.insert("last_ok_ms_ago".into(), json!(ms));
268 }
269 if h.state() == super::health::BreakerState::Open {
270 if let Some(ms) = h.opened_ms_ago() {
271 m.insert("opened_ms_ago".into(), json!(ms));
272 }
273 m.insert(
274 "cooldown_ms".into(),
275 json!(h.cooldown(cfg).as_millis() as u64),
276 );
277 m.insert("last_err".into(), json!(h.last_err_kind().as_str()));
278 }
279 }
280 e
281 })
282 .collect();
283 json!({
284 "active": self.active,
285 "all_down": self.all_down(),
286 "model": model,
287 "endpoints": endpoints,
288 })
289 }
290}
291
292fn resolve_token(
299 idx: usize,
300 default_token: Option<&str>,
301 env: &dyn Fn(&str) -> Option<String>,
302) -> Result<Option<String>, IntelError> {
303 let (inline_var, file_var, inline_var_n, file_var_n) = if idx == 0 {
307 (
308 TOKEN_ENV.to_string(),
309 format!("{TOKEN_ENV}_FILE"),
310 TOKEN_ENV_NEUTRAL.to_string(),
311 format!("{TOKEN_ENV_NEUTRAL}_FILE"),
312 )
313 } else {
314 let n = idx + 1;
315 (
316 format!("{TOKEN_ENV}_{n}"),
317 format!("{TOKEN_ENV}_{n}_FILE"),
318 format!("{TOKEN_ENV_NEUTRAL}_{n}"),
319 format!("{TOKEN_ENV_NEUTRAL}_{n}_FILE"),
320 )
321 };
322 if let Some(v) = env(&inline_var_n).or_else(|| env(&inline_var)) {
326 return Ok(Some(v));
327 }
328 if let Some(path) = env(&file_var_n).or_else(|| env(&file_var)) {
329 let tok = crate::sec::secret::read_token_file(&path).map_err(IntelError::Unsupported)?;
330 return Ok(Some(tok));
331 }
332 if idx == 0 {
333 return Ok(default_token.map(str::to_string));
334 }
335 Ok(None)
336}
337
338fn scheme_and_addr(uri: &str) -> (&'static str, String) {
343 if let Some(rest) = uri.strip_prefix("https://") {
344 ("https", host_only(rest))
345 } else if let Some(rest) = uri.strip_prefix("http://") {
346 ("http", host_only(rest))
347 } else {
348 ("unknown", String::new())
349 }
350}
351
352fn host_only(rest: &str) -> String {
355 rest.split('/').next().unwrap_or(rest).to_string()
356}
357
358fn wire_tool_name(name: &str) -> String {
363 name.chars()
364 .map(|c| {
365 if c.is_ascii_alphanumeric() || c == '_' || c == '-' {
366 c
367 } else {
368 '_'
369 }
370 })
371 .collect()
372}
373
374impl Endpoint {
375 #[cfg(feature = "aauth")]
383 fn aauth_headers(&self, method: &str, path: &str, body: &[u8]) -> Vec<(String, String)> {
384 match crate::aauth::signer() {
385 Some(signer) => signer.sign(method, &self.host_header, path, body),
386 None => Vec::new(),
387 }
388 }
389 #[cfg(not(feature = "aauth"))]
390 fn aauth_headers(&self, _method: &str, _path: &str, _body: &[u8]) -> Vec<(String, String)> {
391 Vec::new()
392 }
393
394 pub(super) fn complete_once(
399 &self,
400 req: &crate::wire::intel::Request,
401 timeout: Duration,
402 trace_id: Option<&str>,
403 ) -> Result<(crate::wire::intel::Response, Duration), IntelError> {
404 use super::{anthropic, openai};
405 use crate::net::http;
406 use std::collections::HashMap;
407 use std::time::Instant;
408
409 use crate::wire::intel::Message;
417 let dirty = |n: &str| wire_tool_name(n) != n;
418 let must_sanitize = req.tools.iter().any(|t| dirty(&t.name))
419 || req.messages.iter().any(|m| {
420 matches!(m, Message::Assistant { tool_calls, .. }
421 if tool_calls.iter().any(|tc| dirty(&tc.name)))
422 });
423 let mut wire_to_orig: HashMap<String, String> = HashMap::new();
424 let owned_req;
425 let req: &crate::wire::intel::Request = if must_sanitize {
426 let mut r = req.clone();
427 for t in &mut r.tools {
428 let w = wire_tool_name(&t.name);
429 if w != t.name {
430 wire_to_orig.insert(w.clone(), t.name.clone());
431 t.name = w;
432 }
433 }
434 for m in &mut r.messages {
435 if let Message::Assistant { tool_calls, .. } = m {
436 for tc in tool_calls {
437 let w = wire_tool_name(&tc.name);
438 if w != tc.name {
439 wire_to_orig.insert(w.clone(), tc.name.clone());
440 tc.name = w;
441 }
442 }
443 }
444 }
445 owned_req = r;
446 &owned_req
447 } else {
448 req
449 };
450
451 use super::bedrock;
452 let (body, mut headers) = match self.provider {
453 Provider::OpenAiCompatible => openai::build_request(req, self.token.as_deref()),
454 Provider::Anthropic => anthropic::build_request(req, self.token.as_deref()),
455 Provider::Bedrock => bedrock::build_request(req, self.token.as_deref()),
456 };
457 let path = self.provider.request_path(&self.http_path, req);
462 if let Some(tid) = trace_id {
463 headers.push((
464 "traceparent".into(),
465 crate::obs::trace::outbound_traceparent(tid),
466 ));
467 }
468 for (k, v) in &self.extra_headers {
472 headers.push((k.clone(), v.clone()));
473 }
474 for (k, v) in self.aauth_headers("POST", &path, &body) {
477 headers.push((k, v));
478 }
479 if let Some(signer) = &self.signer {
483 for (k, v) in signer.sign("POST", &self.host_header, &path, &body) {
484 headers.push((k, v));
485 }
486 }
487 let header_refs: Vec<(&str, &str)> = headers
488 .iter()
489 .map(|(k, v)| (k.as_str(), v.as_str()))
490 .collect();
491
492 const TRANSIENT_RETRIES: u32 = 2;
502 let mut attempt: u32 = 0;
503 let (resp, latency) = loop {
504 let start = Instant::now();
505 let mut stream = self.transport.connect(timeout)?;
506 let resp = http::send(
507 stream.as_mut(),
508 &self.host_header,
509 "POST",
510 &path,
511 &header_refs,
512 &body,
513 )?;
514 let latency = start.elapsed();
515 if resp.is_success() {
516 break (resp, latency);
517 }
518 if super::failover::is_transient_status(resp.status) && attempt < TRANSIENT_RETRIES {
519 attempt += 1;
520 std::thread::sleep(Duration::from_millis(250 * (1u64 << (attempt - 1))));
523 continue;
524 }
525 let snippet: String = resp.body_str().chars().take(512).collect();
526 return Err(IntelError::Http(resp.status, snippet));
527 };
528
529 let mut parsed = match self.provider {
530 Provider::OpenAiCompatible => openai::parse_response(&resp.body),
531 Provider::Anthropic => anthropic::parse_response(&resp.body),
532 Provider::Bedrock => bedrock::parse_response(&resp.body),
533 }
534 .map_err(IntelError::Parse)?;
535 if !wire_to_orig.is_empty() {
537 for tc in &mut parsed.tool_calls {
538 if let Some(orig) = wire_to_orig.get(&tc.name) {
539 tc.name = orig.clone();
540 }
541 }
542 }
543 Ok((parsed, latency))
544 }
545
546 pub(super) fn discover_models(&self, timeout: Duration) -> Vec<String> {
558 use super::openai;
559 use crate::net::http;
560
561 if self.provider != Provider::OpenAiCompatible {
564 return Vec::new();
565 }
566
567 let path = openai::models_path(&self.http_path);
568 let mut headers: Vec<(String, String)> = Vec::new();
570 if let Some(tok) = self.token.as_deref() {
571 headers.push(("authorization".into(), format!("Bearer {tok}")));
572 }
573 for (k, v) in self.aauth_headers("GET", &path, &[]) {
576 headers.push((k, v));
577 }
578 let header_refs: Vec<(&str, &str)> = headers
579 .iter()
580 .map(|(k, v)| (k.as_str(), v.as_str()))
581 .collect();
582
583 let Ok(mut stream) = self.transport.connect(timeout) else {
585 return Vec::new();
586 };
587 let Ok(resp) = http::send(
588 stream.as_mut(),
589 &self.host_header,
590 "GET",
591 &path,
592 &header_refs,
593 &[],
594 ) else {
595 return Vec::new();
596 };
597 if !resp.is_success() {
598 return Vec::new();
600 }
601 openai::parse_models(&resp.body)
602 }
603}
604
605#[cfg(test)]
606mod tests {
607 use super::*;
608
609 #[test]
610 fn extra_headers_apply_to_every_endpoint() {
611 let mut list = EndpointList::parse(
614 "https://a.example/v1,https://b.example/v1",
615 Some("tok".into()),
616 )
617 .unwrap();
618 assert!(list.eps.iter().all(|e| e.extra_headers.is_empty()));
619 list.set_extra_headers(vec![("X-Team".into(), "ops".into())]);
620 for e in &list.eps {
621 assert_eq!(
622 e.extra_headers,
623 vec![("X-Team".to_string(), "ops".to_string())]
624 );
625 }
626 }
627
628 #[test]
629 fn wire_tool_name_maps_only_illegal_chars() {
630 assert_eq!(wire_tool_name("resource.read"), "resource_read");
633 assert_eq!(wire_tool_name("subagent.spawn"), "subagent_spawn");
634 assert_eq!(wire_tool_name("math.factorial"), "math_factorial");
635 assert_eq!(wire_tool_name("get_weather"), "get_weather");
637 assert_eq!(wire_tool_name("list-files"), "list-files");
638 assert_eq!(
639 wire_tool_name("calculate_triangle_area"),
640 "calculate_triangle_area"
641 );
642 assert_eq!(wire_tool_name("a b/c"), "a_b_c");
644 }
645
646 fn env_of<'a>(pairs: &'a [(&'a str, &'a str)]) -> impl Fn(&str) -> Option<String> + 'a {
647 move |k: &str| {
648 pairs
649 .iter()
650 .find(|(n, _)| *n == k)
651 .map(|(_, v)| (*v).to_string())
652 }
653 }
654
655 #[test]
656 fn comma_list_parses_to_n_endpoints_in_order() {
657 let env = env_of(&[]);
658 let list = EndpointList::parse_with_env(
659 "https://gw-a.example:8443,https://gw-b.example:8444,https://intel.example",
660 None,
661 &env,
662 )
663 .unwrap();
664 assert_eq!(list.len(), 3);
665 assert_eq!(list.ep(0).scheme, "https");
666 assert_eq!(list.ep(0).addr, "gw-a.example:8443");
667 assert_eq!(list.ep(1).addr, "gw-b.example:8444");
668 assert_eq!(list.ep(2).scheme, "https");
669 assert_eq!(list.active(), 0);
670 }
671
672 #[test]
673 fn whitespace_around_elements_is_trimmed() {
674 let env = env_of(&[]);
675 let list =
676 EndpointList::parse_with_env(" https://a.example , https://b.example ", None, &env)
677 .unwrap();
678 assert_eq!(list.len(), 2);
679 assert_eq!(list.ep(0).addr, "a.example");
680 assert_eq!(list.ep(1).addr, "b.example");
681 }
682
683 #[test]
684 fn empty_list_is_an_error() {
685 let env = env_of(&[]);
686 assert!(EndpointList::parse_with_env("", None, &env).is_err());
687 assert!(EndpointList::parse_with_env(" , ,", None, &env).is_err());
688 }
689
690 #[test]
691 fn bad_element_scheme_is_an_error() {
692 let env = env_of(&[]);
693 let r = EndpointList::parse_with_env("https://a.example,ftp://nope", None, &env);
694 assert!(matches!(r, Err(IntelError::Unsupported(_))));
695 for uri in ["unix:/a", "vsock:3:8080", "http://not-loopback.example"] {
697 let r = EndpointList::parse_with_env(uri, None, &env);
698 assert!(matches!(r, Err(IntelError::Unsupported(_))), "{uri}");
699 }
700 }
701
702 #[test]
703 fn per_endpoint_token_env_resolves_by_position() {
704 let env = env_of(&[
706 ("AGENTD_INTELLIGENCE_TOKEN", "tok-a"),
707 ("AGENTD_INTELLIGENCE_TOKEN_2", "tok-b"),
708 ]);
709 let list = EndpointList::parse_with_env("https://a.example,https://b.example", None, &env)
710 .unwrap();
711 assert_eq!(list.ep(0).token.as_deref(), Some("tok-a"));
712 assert_eq!(list.ep(1).token.as_deref(), Some("tok-b"));
713 }
714
715 #[test]
716 fn endpoint_0_falls_back_to_default_token_when_env_unset() {
717 let env = env_of(&[]);
718 let list = EndpointList::parse_with_env(
719 "https://a.example,https://b.example",
720 Some("default".into()),
721 &env,
722 )
723 .unwrap();
724 assert_eq!(list.ep(0).token.as_deref(), Some("default"));
726 assert_eq!(list.ep(1).token, None);
727 }
728
729 #[test]
730 fn per_endpoint_env_override_wins_over_default() {
731 let env = env_of(&[("AGENTD_INTELLIGENCE_TOKEN", "from-env")]);
732 let list = EndpointList::parse_with_env("https://a.example", Some("default".into()), &env)
733 .unwrap();
734 assert_eq!(list.ep(0).token.as_deref(), Some("from-env"));
735 }
736
737 #[test]
738 fn neutral_token_env_is_accepted_as_an_alias() {
739 let env = env_of(&[
742 ("AGENT_INTELLIGENCE_TOKEN", "neutral-a"),
743 ("AGENT_INTELLIGENCE_TOKEN_2", "neutral-b"),
744 ]);
745 let list = EndpointList::parse_with_env("https://a.example,https://b.example", None, &env)
746 .unwrap();
747 assert_eq!(list.ep(0).token.as_deref(), Some("neutral-a"));
748 assert_eq!(list.ep(1).token.as_deref(), Some("neutral-b"));
749 }
750
751 #[test]
752 fn branded_token_env_wins_over_neutral_on_conflict() {
753 let env = env_of(&[
756 ("AGENT_INTELLIGENCE_TOKEN", "neutral"),
757 ("AGENTD_INTELLIGENCE_TOKEN", "branded"),
758 ]);
759 let list = EndpointList::parse_with_env("https://a.example", None, &env).unwrap();
760 assert_eq!(list.ep(0).token.as_deref(), Some("neutral"));
761
762 let env = env_of(&[("AGENTD_INTELLIGENCE_TOKEN", "branded")]);
764 let list = EndpointList::parse_with_env("https://a.example", None, &env).unwrap();
765 assert_eq!(list.ep(0).token.as_deref(), Some("branded"));
766 }
767
768 #[test]
769 fn token_file_variant_reads_from_disk() {
770 use std::io::Write;
771 let mut f = tempfile::NamedTempFile::new().unwrap();
772 writeln!(f, "file-secret").unwrap();
773 let path = f.path().to_str().unwrap().to_string();
774 let pairs = [("AGENTD_INTELLIGENCE_TOKEN_2_FILE", path.as_str())];
775 let env = env_of(&pairs);
776 let list = EndpointList::parse_with_env("https://a.example,https://b.example", None, &env)
777 .unwrap();
778 assert_eq!(list.ep(1).token.as_deref(), Some("file-secret"));
779 }
780
781 #[test]
782 fn single_element_list_is_rfc_0006() {
783 let env = env_of(&[]);
784 let list = EndpointList::parse_with_env("https://intel.example", None, &env).unwrap();
785 assert_eq!(list.len(), 1);
786 assert_eq!(list.attempt_order(), vec![0]);
788 assert!(!list.all_down());
789 }
790
791 #[test]
792 fn attempt_order_skips_open_endpoint_and_snaps_back() {
793 use super::super::health::ErrKind;
794 let env = env_of(&[]);
795 let mut list =
796 EndpointList::parse_with_env("https://a.example,https://b.example", None, &env)
797 .unwrap();
798 let cfg = *list.breaker_config();
799 for _ in 0..3 {
801 list.ep(0).health.record_failure(ErrKind::Refused, &cfg);
802 }
803 assert_eq!(list.attempt_order(), vec![1]);
805 assert_eq!(list.prefer_lowest_healthy(), Some(1));
807 assert_eq!(list.active(), 1);
808 list.ep(0).health.record_success(Duration::from_millis(5));
810 assert_eq!(list.prefer_lowest_healthy(), Some(0));
811 assert_eq!(list.active(), 0);
812 }
813
814 #[test]
815 fn resource_body_has_health_and_no_url_or_token() {
816 use super::super::health::ErrKind;
817 let env = env_of(&[("AGENTD_INTELLIGENCE_TOKEN", "super-secret-tok")]);
818 let list = EndpointList::parse_with_env(
819 "https://gw-a.example:8443,https://gw-b.example/v1/secret-path",
820 None,
821 &env,
822 )
823 .unwrap();
824 list.ep(0).health.record_success(Duration::from_millis(41));
826 let cfg = *list.breaker_config();
827 for _ in 0..3 {
828 list.ep(1).health.record_failure(ErrKind::Refused, &cfg);
829 }
830 let body = list.body(Some("claude-opus-4"));
831 let text = body.to_string();
832 assert_eq!(body["active"], 0);
834 assert_eq!(body["model"], "claude-opus-4");
835 assert_eq!(body["endpoints"][0]["transport"], "https");
836 assert_eq!(body["endpoints"][0]["addr"], "gw-a.example:8443");
837 assert_eq!(body["endpoints"][0]["state"], "closed");
838 assert_eq!(body["endpoints"][0]["active"], true);
839 assert_eq!(body["endpoints"][0]["ewma_latency_ms"], 41);
840 assert_eq!(body["endpoints"][1]["state"], "open");
841 assert_eq!(body["endpoints"][1]["last_err"], "refused");
842 assert!(!text.contains("super-secret-tok"), "token leaked: {text}");
844 assert!(!text.contains("https://"), "full URI leaked: {text}");
845 assert!(!text.contains("secret-path"), "URL path leaked: {text}");
846 }
847
848 #[test]
849 fn all_down_when_every_breaker_open() {
850 use super::super::health::ErrKind;
851 let env = env_of(&[]);
852 let list = EndpointList::parse_with_env("https://a.example,https://b.example", None, &env)
853 .unwrap();
854 let cfg = *list.breaker_config();
855 for ep in list.iter() {
856 for _ in 0..3 {
857 ep.health.record_failure(ErrKind::Refused, &cfg);
858 }
859 }
860 assert!(list.all_down());
861 assert!(list.attempt_order().is_empty());
862 }
863}