snm_brightdata_client/tools/
commodity.rs

1// src/tools/commodity.rs - PATCHED: Enhanced with priority-aware filtering and token budget management
2use crate::tool::{Tool, ToolResult, McpContent};
3use crate::error::BrightDataError;
4use crate::extras::logger::JSON_LOGGER;
5use crate::filters::{ResponseFilter, ResponseStrategy, ResponseType};
6use async_trait::async_trait;
7use reqwest::Client;
8use serde_json::{json, Value};
9use std::env;
10use std::time::Duration;
11use std::collections::HashMap;
12use log::info;
13
14pub struct CommodityDataTool;
15
16#[async_trait]
17impl Tool for CommodityDataTool {
18    fn name(&self) -> &str {
19        "get_commodity_data"
20    }
21
22    fn description(&self) -> &str {
23        "Get comprehensive commodity prices and market data with real-time updates and market-specific sources"
24    }
25
26    fn input_schema(&self) -> Value {
27        json!({
28            "type": "object",
29            "properties": {
30                "query": {
31                    "type": "string",
32                    "description": "Commodity name (gold, silver, crude oil), commodity symbol (GC, SI, CL), or commodity market overview"
33                },
34                "commodity_type": {
35                    "type": "string",
36                    "enum": ["precious_metals", "energy", "agricultural", "industrial_metals", "livestock", "all"],
37                    "default": "all",
38                    "description": "Category of commodity for targeted data sources"
39                },
40                "market_region": {
41                    "type": "string",
42                    "enum": ["global", "us", "asia", "europe", "india"],
43                    "default": "global",
44                    "description": "Regional market focus"
45                },
46                "data_source": {
47                    "type": "string",
48                    "enum": ["search", "direct", "auto"],
49                    "default": "auto",
50                    "description": "Data source strategy - search (SERP), direct (commodity exchanges), auto (smart selection)"
51                },
52                "time_range": {
53                    "type": "string",
54                    "enum": ["realtime", "day", "week", "month", "year"],
55                    "default": "realtime",
56                    "description": "Time range for price data"
57                },
58                "include_futures": {
59                    "type": "boolean",
60                    "default": true,
61                    "description": "Include futures contract prices"
62                },
63                "include_analysis": {
64                    "type": "boolean",
65                    "default": false,
66                    "description": "Include market analysis and trends"
67                },
68                "currency": {
69                    "type": "string",
70                    "enum": ["USD", "EUR", "INR", "CNY", "JPY"],
71                    "default": "USD",
72                    "description": "Currency for price display"
73                }
74            },
75            "required": ["query"]
76        })
77    }
78
79    // FIXED: Remove the execute method override to use the default one with metrics logging
80    // async fn execute(&self, parameters: Value) -> Result<ToolResult, BrightDataError> {
81    //     self.execute_internal(parameters).await
82    // }
83
84    async fn execute_internal(&self, parameters: Value) -> Result<ToolResult, BrightDataError> {
85        let query = parameters
86            .get("query")
87            .and_then(|v| v.as_str())
88            .ok_or_else(|| BrightDataError::ToolError("Missing 'query' parameter".into()))?;
89
90        let commodity_type = parameters
91            .get("commodity_type")
92            .and_then(|v| v.as_str())
93            .unwrap_or("all");
94
95        let market_region = parameters
96            .get("market_region")
97            .and_then(|v| v.as_str())
98            .unwrap_or("global");
99
100        let data_source = parameters
101            .get("data_source")
102            .and_then(|v| v.as_str())
103            .unwrap_or("auto");
104
105        let time_range = parameters
106            .get("time_range")
107            .and_then(|v| v.as_str())
108            .unwrap_or("realtime");
109
110        let include_futures = parameters
111            .get("include_futures")
112            .and_then(|v| v.as_bool())
113            .unwrap_or(true);
114
115        let include_analysis = parameters
116            .get("include_analysis")
117            .and_then(|v| v.as_bool())
118            .unwrap_or(false);
119
120        let currency = parameters
121            .get("currency")
122            .and_then(|v| v.as_str())
123            .unwrap_or("USD");
124
125        // ENHANCED: Priority classification and token allocation
126        let query_priority = ResponseStrategy::classify_query_priority(query);
127        let recommended_tokens = ResponseStrategy::get_recommended_token_allocation(query);
128
129        // Early validation using strategy only if TRUNCATE_FILTER is enabled
130        if std::env::var("TRUNCATE_FILTER")
131            .map(|v| v.to_lowercase() == "true")
132            .unwrap_or(false) {
133            
134            // Budget check for commodity queries
135            let (_, remaining_tokens) = ResponseStrategy::get_token_budget_status();
136            if remaining_tokens < 150 && !matches!(query_priority, crate::filters::strategy::QueryPriority::Critical) {
137                return Ok(ResponseStrategy::create_response("", query, "commodity", "budget_limit", json!({}), ResponseType::Skip));
138            }
139        }
140
141        let execution_id = format!("commodity_{}", chrono::Utc::now().format("%Y%m%d_%H%M%S%.3f"));
142        
143        match self.fetch_commodity_data_with_fallbacks_and_priority(
144            query, commodity_type, market_region, data_source, time_range,
145            include_futures, include_analysis, currency, query_priority, recommended_tokens, &execution_id
146        ).await {
147            Ok(result) => {
148                let content = result.get("content").and_then(|c| c.as_str()).unwrap_or("");
149                let source_used = result.get("source_used").and_then(|s| s.as_str()).unwrap_or("Unknown");
150                
151                // Create appropriate response based on whether filtering is enabled
152                let tool_result = if std::env::var("TRUNCATE_FILTER")
153                    .map(|v| v.to_lowercase() == "true")
154                    .unwrap_or(false) {
155                    
156                    ResponseStrategy::create_financial_response(
157                        "commodity", query, market_region, source_used, content, result.clone()
158                    )
159                } else {
160                    // No filtering - create standard response
161                    let mcp_content = vec![McpContent::text(format!(
162                        "🥇 **Commodity Data for: {}**\n\nType: {} | Region: {} | Priority: {:?} | Tokens: {}\nSource: {} | Time Range: {} | Currency: {} | Futures: {} | Analysis: {}\nExecution ID: {}\n\n{}",
163                        query, commodity_type, market_region, query_priority, recommended_tokens, source_used, time_range, currency, include_futures, include_analysis, execution_id, content
164                    ))];
165                    ToolResult::success_with_raw(mcp_content, result)
166                };
167                
168                if std::env::var("TRUNCATE_FILTER")
169                    .map(|v| v.to_lowercase() == "true")
170                    .unwrap_or(false) {
171                    Ok(ResponseStrategy::apply_size_limits(tool_result))
172                } else {
173                    Ok(tool_result)
174                }
175            }
176            Err(e) => {
177                if std::env::var("TRUNCATE_FILTER")
178                    .map(|v| v.to_lowercase() == "true")
179                    .unwrap_or(false) {
180                    Ok(ResponseStrategy::create_error_response(query, &e.to_string()))
181                } else {
182                    Err(e)
183                }
184            }
185        }
186    }
187}
188
189impl CommodityDataTool {
190    // ENHANCED: Priority-aware commodity data fetching with token management
191    async fn fetch_commodity_data_with_fallbacks_and_priority(
192        &self,
193        query: &str,
194        commodity_type: &str,
195        market_region: &str,
196        data_source: &str,
197        time_range: &str,
198        include_futures: bool,
199        include_analysis: bool,
200        currency: &str,
201        query_priority: crate::filters::strategy::QueryPriority,
202        token_budget: usize,
203        execution_id: &str,
204    ) -> Result<Value, BrightDataError> {
205        let sources_to_try = self.build_prioritized_sources_with_priority(query, commodity_type, market_region, data_source, query_priority);
206        let mut last_error = None;
207
208        for (sequence, (source_type, url_or_query, source_name)) in sources_to_try.iter().enumerate() {
209            match source_type.as_str() {
210                "direct" => {
211                    match self.fetch_direct_commodity_data_with_priority(
212                        url_or_query, query, commodity_type, market_region, 
213                        source_name, query_priority, token_budget, execution_id, sequence as u64
214                    ).await {
215                        Ok(mut result) => {
216                            let content = result.get("content").and_then(|c| c.as_str()).unwrap_or("");
217                            
218                            if std::env::var("TRUNCATE_FILTER")
219                                .map(|v| v.to_lowercase() == "true")
220                                .unwrap_or(false) {
221                                
222                                if ResponseStrategy::should_try_next_source(content) {
223                                    last_error = Some(BrightDataError::ToolError(format!(
224                                        "{} returned low-quality content", source_name
225                                    )));
226                                    continue;
227                                }
228                            }
229                            
230                            result["source_used"] = json!(source_name);
231                            result["data_source_type"] = json!("direct");
232                            result["priority"] = json!(format!("{:?}", query_priority));
233                            return Ok(result);
234                        }
235                        Err(e) => last_error = Some(e),
236                    }
237                }
238                "search" => {
239                    match self.fetch_search_commodity_data_with_priority(
240                        url_or_query, commodity_type, market_region, time_range,
241                        include_futures, include_analysis, currency, source_name, 
242                        query_priority, token_budget, execution_id, sequence as u64
243                    ).await {
244                        Ok(mut result) => {
245                            let content = result.get("content").and_then(|c| c.as_str()).unwrap_or("");
246                            
247                            if std::env::var("TRUNCATE_FILTER")
248                                .map(|v| v.to_lowercase() == "true")
249                                .unwrap_or(false) {
250                                
251                                if ResponseStrategy::should_try_next_source(content) {
252                                    last_error = Some(BrightDataError::ToolError(format!(
253                                        "{} returned low-quality content", source_name
254                                    )));
255                                    continue;
256                                }
257                            }
258                            
259                            result["source_used"] = json!(source_name);
260                            result["data_source_type"] = json!("search");
261                            result["priority"] = json!(format!("{:?}", query_priority));
262                            return Ok(result);
263                        }
264                        Err(e) => last_error = Some(e),
265                    }
266                }
267                _ => continue,
268            }
269        }
270
271        Err(last_error.unwrap_or_else(|| BrightDataError::ToolError("All commodity data sources failed".into())))
272    }
273
274    // ENHANCED: Priority-aware source building
275    fn build_prioritized_sources_with_priority(&self, query: &str, commodity_type: &str, market_region: &str, data_source: &str, priority: crate::filters::strategy::QueryPriority) -> Vec<(String, String, String)> {
276        let mut sources = Vec::new();
277        let query_lower = query.to_lowercase();
278
279        match data_source {
280            "direct" => {
281                sources.extend(self.get_direct_sources_with_priority(commodity_type, market_region, priority));
282            }
283            "search" => {
284                sources.extend(self.get_search_sources_with_priority(query, commodity_type, market_region, priority));
285            }
286            "auto" | _ => {
287                // Priority-aware smart selection
288                match priority {
289                    crate::filters::strategy::QueryPriority::Critical => {
290                        // For critical queries, prioritize direct sources for real-time data
291                        sources.extend(self.get_direct_sources_with_priority(commodity_type, market_region, priority));
292                        sources.extend(self.get_search_sources_with_priority(query, commodity_type, market_region, priority));
293                    }
294                    _ => {
295                        // Smart selection based on query content
296                        if query_lower.contains("price") || query_lower.contains("futures") || query_lower.contains("contract") {
297                            sources.extend(self.get_direct_sources_with_priority(commodity_type, market_region, priority));
298                            sources.extend(self.get_search_sources_with_priority(query, commodity_type, market_region, priority));
299                        } else {
300                            sources.extend(self.get_search_sources_with_priority(query, commodity_type, market_region, priority));
301                            sources.extend(self.get_direct_sources_with_priority(commodity_type, market_region, priority));
302                        }
303                    }
304                }
305            }
306        }
307
308        // Limit sources based on priority to save tokens
309        let max_sources = match priority {
310            crate::filters::strategy::QueryPriority::Critical => sources.len(), // No limit for critical
311            crate::filters::strategy::QueryPriority::High => std::cmp::min(sources.len(), 3),
312            crate::filters::strategy::QueryPriority::Medium => std::cmp::min(sources.len(), 2),
313            crate::filters::strategy::QueryPriority::Low => std::cmp::min(sources.len(), 1),
314        };
315
316        sources.truncate(max_sources);
317        sources
318    }
319
320    fn get_direct_sources_with_priority(&self, commodity_type: &str, market_region: &str, priority: crate::filters::strategy::QueryPriority) -> Vec<(String, String, String)> {
321        let mut sources = Vec::new();
322
323        match market_region {
324            "india" => {
325                sources.push(("direct".to_string(), "https://www.mcxindia.com/market-data/live-rates".to_string(), "MCX India".to_string()));
326                if matches!(priority, crate::filters::strategy::QueryPriority::Critical | crate::filters::strategy::QueryPriority::High) {
327                    sources.push(("direct".to_string(), "https://www.ncdex.com/market/live-rates".to_string(), "NCDEX".to_string()));
328                }
329                if commodity_type == "precious_metals" || commodity_type == "all" {
330                    sources.push(("direct".to_string(), "https://www.goldpriceindia.com/".to_string(), "Gold Price India".to_string()));
331                }
332            }
333            "us" => {
334                sources.push(("direct".to_string(), "https://www.cmegroup.com/markets.html".to_string(), "CME Group".to_string()));
335                if matches!(priority, crate::filters::strategy::QueryPriority::Critical | crate::filters::strategy::QueryPriority::High) {
336                    sources.push(("direct".to_string(), "https://www.theice.com/market-data".to_string(), "ICE Markets".to_string()));
337                }
338                if commodity_type == "energy" || commodity_type == "all" {
339                    sources.push(("direct".to_string(), "https://www.eia.gov/petroleum/".to_string(), "EIA Energy".to_string()));
340                }
341            }
342            "europe" => {
343                sources.push(("direct".to_string(), "https://www.theice.com/market-data/dashboard".to_string(), "ICE Europe".to_string()));
344                if !matches!(priority, crate::filters::strategy::QueryPriority::Low) {
345                    sources.push(("direct".to_string(), "https://www.lme.com/Metals".to_string(), "London Metal Exchange".to_string()));
346                }
347            }
348            "asia" => {
349                sources.push(("direct".to_string(), "https://www.tocom.or.jp/market/".to_string(), "TOCOM".to_string()));
350                if !matches!(priority, crate::filters::strategy::QueryPriority::Low) {
351                    sources.push(("direct".to_string(), "https://www.shfe.com.cn/en/".to_string(), "Shanghai Futures".to_string()));
352                }
353            }
354            "global" | _ => {
355                // Global commodity sources with priority filtering
356                sources.push(("direct".to_string(), "https://www.investing.com/commodities/".to_string(), "Investing.com Commodities".to_string()));
357                if matches!(priority, crate::filters::strategy::QueryPriority::Critical | crate::filters::strategy::QueryPriority::High) {
358                    sources.push(("direct".to_string(), "https://www.bloomberg.com/markets/commodities".to_string(), "Bloomberg Commodities".to_string()));
359                    sources.push(("direct".to_string(), "https://www.marketwatch.com/investing/commodities".to_string(), "MarketWatch Commodities".to_string()));
360                }
361                
362                // Commodity-specific sources (only for higher priority)
363                if !matches!(priority, crate::filters::strategy::QueryPriority::Low) {
364                    match commodity_type {
365                        "precious_metals" => {
366                            sources.push(("direct".to_string(), "https://www.kitco.com/market/".to_string(), "Kitco Metals".to_string()));
367                            sources.push(("direct".to_string(), "https://www.lbma.org.uk/prices-and-data".to_string(), "LBMA".to_string()));
368                        }
369                        "energy" => {
370                            sources.push(("direct".to_string(), "https://oilprice.com/".to_string(), "Oil Price".to_string()));
371                            sources.push(("direct".to_string(), "https://www.eia.gov/petroleum/".to_string(), "EIA".to_string()));
372                        }
373                        "agricultural" => {
374                            sources.push(("direct".to_string(), "https://www.cbot.com/".to_string(), "CBOT".to_string()));
375                            sources.push(("direct".to_string(), "https://www.usda.gov/topics/data".to_string(), "USDA".to_string()));
376                        }
377                        "industrial_metals" => {
378                            sources.push(("direct".to_string(), "https://www.lme.com/Metals".to_string(), "LME".to_string()));
379                        }
380                        _ => {}
381                    }
382                }
383            }
384        }
385
386        sources
387    }
388
389    fn get_search_sources_with_priority(&self, query: &str, commodity_type: &str, market_region: &str, priority: crate::filters::strategy::QueryPriority) -> Vec<(String, String, String)> {
390        let mut sources = Vec::new();
391        
392        let region_terms = match market_region {
393            "india" => "india MCX NCDEX commodity exchange rupee INR",
394            "us" => "united states CME CBOT futures contract dollar USD",
395            "europe" => "europe ICE LME futures contract euro EUR",
396            "asia" => "asia TOCOM shanghai futures contract",
397            "global" => "global international commodity futures trading",
398            _ => "commodity futures trading market price"
399        };
400
401        let commodity_terms = match commodity_type {
402            "precious_metals" => "gold silver platinum palladium precious metals spot price",
403            "energy" => "crude oil natural gas gasoline heating oil energy futures",
404            "agricultural" => "wheat corn soybeans rice agricultural commodity farming",
405            "industrial_metals" => "copper aluminum zinc nickel industrial metals LME",
406            "livestock" => "cattle hogs pork livestock futures meat",
407            _ => "commodity futures spot price market trading"
408        };
409
410        // Priority-based search query construction
411        match priority {
412            crate::filters::strategy::QueryPriority::Critical => {
413                sources.push(("search".to_string(), 
414                    format!("{} {} live current real-time price", query, region_terms),
415                    "Critical Commodity Search".to_string()));
416            }
417            crate::filters::strategy::QueryPriority::High => {
418                sources.push(("search".to_string(), 
419                    format!("{} {} {} current price futures today", query, region_terms, commodity_terms),
420                    "High Priority Commodity Search".to_string()));
421                sources.push(("search".to_string(), 
422                    format!("{} {} latest trading data market analysis", query, commodity_terms),
423                    "Commodity Market Analysis".to_string()));
424            }
425            _ => {
426                sources.push(("search".to_string(), 
427                    format!("{} {} price chart trends technical analysis", query, region_terms),
428                    "Commodity Trends Search".to_string()));
429            }
430        }
431
432        sources
433    }
434
435    // ENHANCED: Priority-aware direct commodity data fetching
436    async fn fetch_direct_commodity_data_with_priority(
437        &self,
438        url: &str,
439        query: &str,
440        commodity_type: &str,
441        market_region: &str,
442        source_name: &str,
443        priority: crate::filters::strategy::QueryPriority,
444        token_budget: usize,
445        execution_id: &str,
446        sequence: u64,
447    ) -> Result<Value, BrightDataError> {
448        let api_token = env::var("BRIGHTDATA_API_TOKEN")
449            .or_else(|_| env::var("API_TOKEN"))
450            .map_err(|_| BrightDataError::ToolError("Missing BRIGHTDATA_API_TOKEN".into()))?;
451
452        let base_url = env::var("BRIGHTDATA_BASE_URL")
453            .unwrap_or_else(|_| "https://api.brightdata.com".to_string());
454
455        let zone = env::var("WEB_UNLOCKER_ZONE")
456            .unwrap_or_else(|_| "default".to_string());
457
458        info!("🥇 Priority {} direct commodity data fetch from {} using zone: {} (execution: {})", 
459              format!("{:?}", priority), source_name, zone, execution_id);
460
461        let mut payload = json!({
462            "url": url,
463            "zone": zone,
464            "format": "raw",
465            "data_format": "markdown",
466            "render": true
467        });
468
469        // Add priority processing hints
470        if std::env::var("TRUNCATE_FILTER")
471            .map(|v| v.to_lowercase() == "true")
472            .unwrap_or(false) {
473            
474            payload["processing_priority"] = json!(format!("{:?}", priority));
475            payload["token_budget"] = json!(token_budget);
476        }
477
478        let client = Client::builder()
479            .timeout(Duration::from_secs(120))
480            .build()
481            .map_err(|e| BrightDataError::ToolError(e.to_string()))?;
482
483        let response = client
484            .post(&format!("{}/request", base_url))
485            .header("Authorization", format!("Bearer {}", api_token))
486            .header("Content-Type", "application/json")
487            .json(&payload)
488            .send()
489            .await
490            .map_err(|e| BrightDataError::ToolError(format!("Direct commodity data request failed: {}", e)))?;
491
492        let status = response.status().as_u16();
493        let response_headers: HashMap<String, String> = response
494            .headers()
495            .iter()
496            .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
497            .collect();
498
499        // Log BrightData request
500        if let Err(e) = JSON_LOGGER.log_brightdata_request(
501            &format!("{}_{}", execution_id, sequence),
502            &zone,
503            url,
504            payload.clone(),
505            status,
506            response_headers,
507            "markdown"
508        ).await {
509            log::warn!("Failed to log BrightData request: {}", e);
510        }
511
512        if !response.status().is_success() {
513            let error_text = response.text().await.unwrap_or_default();
514            return Err(BrightDataError::ToolError(format!(
515                "BrightData direct commodity data error {}: {}",
516                status, error_text
517            )));
518        }
519
520        let raw_content = response.text().await
521            .map_err(|e| BrightDataError::ToolError(e.to_string()))?;
522
523        // Apply priority-aware filters
524        let filtered_content = if std::env::var("TRUNCATE_FILTER")
525            .map(|v| v.to_lowercase() == "true")
526            .unwrap_or(false) {
527            
528            if ResponseFilter::is_error_page(&raw_content) {
529                return Err(BrightDataError::ToolError(format!("{} returned error page", source_name)));
530            } else {
531                let max_tokens = token_budget / 2; // Reserve tokens for formatting
532                ResponseFilter::extract_high_value_financial_data(&raw_content, max_tokens)
533            }
534        } else {
535            raw_content.clone()
536        };
537
538        Ok(json!({
539            "content": filtered_content,
540            "query": query,
541            "commodity_type": commodity_type,
542            "market_region": market_region,
543            "priority": format!("{:?}", priority),
544            "token_budget": token_budget,
545            "execution_id": execution_id,
546            "sequence": sequence,
547            "success": true
548        }))
549    }
550
551    // ENHANCED: Priority-aware search commodity data fetching
552    async fn fetch_search_commodity_data_with_priority(
553        &self,
554        search_query: &str,
555        commodity_type: &str,
556        market_region: &str,
557        time_range: &str,
558        include_futures: bool,
559        include_analysis: bool,
560        currency: &str,
561        source_name: &str,
562        priority: crate::filters::strategy::QueryPriority,
563        token_budget: usize,
564        execution_id: &str,
565        sequence: u64,
566    ) -> Result<Value, BrightDataError> {
567        let api_token = env::var("BRIGHTDATA_API_TOKEN")
568            .or_else(|_| env::var("API_TOKEN"))
569            .map_err(|_| BrightDataError::ToolError("Missing BRIGHTDATA_API_TOKEN".into()))?;
570
571        let base_url = env::var("BRIGHTDATA_BASE_URL")
572            .unwrap_or_else(|_| "https://api.brightdata.com".to_string());
573
574        let zone = env::var("BRIGHTDATA_SERP_ZONE")
575            .unwrap_or_else(|_| "serp_api2".to_string());
576
577        // Build enhanced search query with priority awareness
578        let mut enhanced_query = search_query.to_string();
579        
580        // Add terms based on priority
581        match priority {
582            crate::filters::strategy::QueryPriority::Critical => {
583                enhanced_query.push_str(" live current real time");
584            }
585            crate::filters::strategy::QueryPriority::High => {
586                if include_futures {
587                    enhanced_query.push_str(" futures contract trading");
588                }
589                if include_analysis {
590                    enhanced_query.push_str(" market analysis trends forecast");
591                }
592            }
593            _ => {
594                // Basic terms for lower priority
595                enhanced_query.push_str(" overview");
596            }
597        }
598
599        if currency != "USD" && !matches!(priority, crate::filters::strategy::QueryPriority::Low) {
600            enhanced_query.push_str(&format!(" price {}", currency));
601        }
602
603        // Add time range only for higher priority
604        if !matches!(priority, crate::filters::strategy::QueryPriority::Low) {
605            match time_range {
606                "realtime" => enhanced_query.push_str(" live current real time"),
607                "day" => enhanced_query.push_str(" today daily"),
608                "week" => enhanced_query.push_str(" this week weekly"),
609                "month" => enhanced_query.push_str(" this month monthly"),
610                "year" => enhanced_query.push_str(" this year annual"),
611                _ => {}
612            }
613        }
614
615        // Build SERP API query parameters with priority-based limits
616        let mut query_params = HashMap::new();
617        query_params.insert("q".to_string(), enhanced_query.clone());
618        
619        // Adjust results based on priority
620        let num_results = match priority {
621            crate::filters::strategy::QueryPriority::Critical => "20",
622            crate::filters::strategy::QueryPriority::High => "15",
623            crate::filters::strategy::QueryPriority::Medium => "10",
624            crate::filters::strategy::QueryPriority::Low => "5",
625        };
626        query_params.insert("num".to_string(), num_results.to_string());
627        
628        // Set geographic location based on market region
629        let country_code = match market_region {
630            "india" => "in",
631            "us" => "us",
632            "europe" => "de", // Default to Germany for Europe
633            "asia" => "jp", // Default to Japan for Asia
634            _ => "us" // Default to US for global
635        };
636        query_params.insert("gl".to_string(), country_code.to_string());
637        query_params.insert("hl".to_string(), "en".to_string());
638        
639        // Time-based filtering for recent data (skip for low priority)
640        if time_range != "year" && !matches!(priority, crate::filters::strategy::QueryPriority::Low) {
641            let tbs_value = match time_range {
642                "realtime" | "day" => "qdr:d",
643                "week" => "qdr:w",
644                "month" => "qdr:m",
645                _ => ""
646            };
647            if !tbs_value.is_empty() {
648                query_params.insert("tbs".to_string(), tbs_value.to_string());
649            }
650        }
651
652        info!("🔍 Priority {} enhanced commodity search: {} using zone: {} (execution: {})", 
653              format!("{:?}", priority), enhanced_query.clone(), zone.clone(), execution_id.clone());
654
655        let mut payload = json!({
656            "zone": zone,
657            "url": "https://www.google.com/search",
658            "format": "json",
659            "query_params": query_params,
660            "render": true,
661            "data_format": "markdown"
662        });
663
664        // Add priority processing hints
665        if std::env::var("TRUNCATE_FILTER")
666            .map(|v| v.to_lowercase() == "true")
667            .unwrap_or(false) {
668            
669            payload["processing_priority"] = json!(format!("{:?}", priority));
670            payload["token_budget"] = json!(token_budget);
671        }
672
673        let client = Client::builder()
674            .timeout(Duration::from_secs(120))
675            .build()
676            .map_err(|e| BrightDataError::ToolError(e.to_string()))?;
677
678        let response = client
679            .post(&format!("{}/request", base_url))
680            .header("Authorization", format!("Bearer {}", api_token))
681            .header("Content-Type", "application/json")
682            .json(&payload)
683            .send()
684            .await
685            .map_err(|e| BrightDataError::ToolError(format!("Commodity search request failed: {}", e)))?;
686
687        let status = response.status().as_u16();
688        let response_headers: HashMap<String, String> = response
689            .headers()
690            .iter()
691            .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
692            .collect();
693
694        // Log BrightData request
695        if let Err(e) = JSON_LOGGER.log_brightdata_request(
696            &format!("{}_{}", execution_id, sequence),
697            &zone,
698            &format!("Commodity Search: {}", enhanced_query),
699            payload.clone(),
700            status,
701            response_headers,
702            "markdown"
703        ).await {
704            log::warn!("Failed to log BrightData request: {}", e);
705        }
706
707        if !response.status().is_success() {
708            let error_text = response.text().await.unwrap_or_default();
709            return Err(BrightDataError::ToolError(format!(
710                "BrightData commodity search error {}: {}",
711                status, error_text
712            )));
713        }
714
715        let raw_content = response.text().await
716            .map_err(|e| BrightDataError::ToolError(e.to_string()))?;
717
718        // Apply priority-aware filters
719        let filtered_content = if std::env::var("TRUNCATE_FILTER")
720            .map(|v| v.to_lowercase() == "true")
721            .unwrap_or(false) {
722            
723            if ResponseFilter::is_error_page(&raw_content) {
724                return Err(BrightDataError::ToolError(format!("{} search returned error page", source_name)));
725            } else {
726                let max_tokens = token_budget / 3; // Reserve tokens for formatting
727                ResponseFilter::extract_high_value_financial_data(&raw_content, max_tokens)
728            }
729        } else {
730            raw_content.clone()
731        };
732
733        Ok(json!({
734            "content": filtered_content,
735            "search_query": enhanced_query,
736            "commodity_type": commodity_type,
737            "market_region": market_region,
738            "time_range": time_range,
739            "currency": currency,
740            "priority": format!("{:?}", priority),
741            "token_budget": token_budget,
742            "execution_id": execution_id,
743            "sequence": sequence,
744            "success": true
745        }))
746    }
747}
snm_brightdata_client/tools/commodity.rs

snm_brightdata_client/tools/
commodity.rs
