tarzi 0.2.0

Rust-native lite search for AI applications
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192

//! Constants used throughout the Tarzi application
//!
//! This module contains all magic values and constants to avoid duplication
//! and make maintenance easier.

use std::time::Duration;

// ============================================================================
// Network & WebDriver Constants
// ============================================================================

/// Default ChromeDriver URL
pub const CHROMEDRIVER_DEFAULT_URL: &str = "http://localhost:9515";

/// Default ChromeDriver port
pub const CHROMEDRIVER_DEFAULT_PORT: u16 = 9515;

/// Default GeckoDriver port
pub const GECKODRIVER_DEFAULT_PORT: u16 = 4444;

/// Default HTTP client user agent
pub const DEFAULT_USER_AGENT: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";

/// Default ChromeDriver
pub const CHROMEDRIVER: &str = "chromedriver";

/// Default GeckoDriver
pub const GECKODRIVER: &str = "geckodriver";

// ============================================================================
// Timeout Constants
// ============================================================================

/// Default timeout in seconds for various operations
pub const DEFAULT_TIMEOUT_SECS: u64 = 30;

/// Default timeout duration for various operations
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(DEFAULT_TIMEOUT_SECS);

/// WebDriver availability check timeout in seconds
pub const WEBDRIVER_CHECK_TIMEOUT_SECS: u64 = 2;

/// WebDriver availability check timeout duration
pub const WEBDRIVER_CHECK_TIMEOUT: Duration = Duration::from_secs(WEBDRIVER_CHECK_TIMEOUT_SECS);

/// Browser launch timeout in seconds
pub const BROWSER_LAUNCH_TIMEOUT_SECS: u64 = 60;

/// Browser launch timeout duration
pub const BROWSER_LAUNCH_TIMEOUT: Duration = Duration::from_secs(BROWSER_LAUNCH_TIMEOUT_SECS);

/// Page load wait time in seconds
pub const PAGE_LOAD_WAIT_SECS: u64 = 2;

/// Page load wait duration
pub const PAGE_LOAD_WAIT: Duration = Duration::from_secs(PAGE_LOAD_WAIT_SECS);

// ============================================================================
// Test URLs
// ============================================================================

/// HTTPBin HTML test endpoint
pub const HTTPBIN_HTML_URL: &str = "https://httpbin.org/html";

/// HTTPBin JSON test endpoint
pub const HTTPBIN_JSON_URL: &str = "https://httpbin.org/json";

/// HTTPBin XML test endpoint
pub const HTTPBIN_XML_URL: &str = "https://httpbin.org/xml";

/// HTTPBin 404 status test endpoint
pub const HTTPBIN_404_URL: &str = "https://httpbin.org/status/404";

/// HTTPBin 500 status test endpoint
pub const HTTPBIN_500_URL: &str = "https://httpbin.org/status/500";

/// HTTPBin large response test endpoint (10KB)
pub const HTTPBIN_LARGE_URL: &str = "https://httpbin.org/bytes/10000";

/// Example domain for testing
pub const EXAMPLE_URL: &str = "https://example.com";

/// Example proxy URL for testing
pub const EXAMPLE_PROXY_URL: &str = "http://example.com:8080";

// ============================================================================
// Browser Arguments
// ============================================================================

/// Chrome/Chromium browser arguments for headless mode
pub const CHROME_HEADLESS_ARGS: &[&str] = &[
    "--headless",
    "--disable-gpu",
    "--disable-dev-shm-usage",
    "--no-sandbox",
];

/// Chrome/Chromium browser arguments for driver
pub const CHROME_DRIVER_ARGS: &[&str] =
    &["--disable-gpu", "--no-sandbox", "--disable-dev-shm-usage"];

/// Firefox browser arguments
pub const FIREFOX_DRIVER_ARGS: &[&str] = &["--log=warn"];

// ============================================================================
// Default Configuration Values
// ============================================================================

/// Default log level
pub const DEFAULT_LOG_LEVEL: &str = "info";

/// Default search limit
pub const DEFAULT_SEARCH_LIMIT: usize = 5;

/// Default fetcher mode string
pub const DEFAULT_FETCH_MODE: &str = "browser_headless";

/// Default converter format string
pub const DEFAULT_FORMAT: &str = "markdown";

/// Default search engine
pub const DEFAULT_SEARCH_ENGINE: &str = "bing";

// Default log levels
pub const LOG_LEVEL_INFO: &str = "info";
pub const LOG_LEVEL_DEBUG: &str = "debug";
pub const LOG_LEVEL_WARN: &str = "warn";

// Default formats
pub const FORMAT_MARKDOWN: &str = "markdown";
pub const FORMAT_JSON: &str = "json";
pub const FORMAT_YAML: &str = "yaml";
pub const FORMAT_HTML: &str = "html";

// Default fetcher modes
pub const FETCHER_MODE_BROWSER_HEADLESS: &str = "browser_headless";
pub const FETCHER_MODE_BROWSER_HEAD: &str = "browser_head";
pub const FETCHER_MODE_PLAIN_REQUEST: &str = "plain_request";
pub const FETCHER_MODE_HEAD: &str = "head";
pub const FETCHER_MODE_HEADLESS: &str = "headless";
pub const FETCHER_MODE_PLAIN: &str = "plain";

// Default search engines
pub const SEARCH_ENGINE_DUCKDUCKGO: &str = "duckduckgo";
pub const SEARCH_ENGINE_BING: &str = "bing";
pub const SEARCH_ENGINE_GOOGLE: &str = "google";
pub const SEARCH_ENGINE_GOOGLE_SERPER: &str = "google_serper";
pub const SEARCH_ENGINE_SERPER_ALIAS: &str = "serper";
pub const SEARCH_ENGINE_BRAVE: &str = "brave";
pub const SEARCH_ENGINE_BAIDU: &str = "baidu";
pub const SEARCH_ENGINE_SOUGOU_WEIXIN: &str = "sogou_weixin";

// Search modes
pub const SEARCH_MODE_AUTO: &str = "auto";
pub const SEARCH_MODE_APIQUERY: &str = "apiquery";
pub const SEARCH_MODE_WEBQUERY: &str = "webquery";

/// Default search mode
pub const DEFAULT_SEARCH_MODE: &str = SEARCH_MODE_AUTO;

// API key environment variable names
pub const ENV_BRAVE_API_KEY: &str = "BRAVE_API_KEY";
pub const ENV_SERPER_API_KEY: &str = "SERPER_API_KEY";

// ============================================================================
// Search Engine Query Patterns
// ============================================================================

/// Browser / JS SERP query patterns
pub const DUCKDUCKGO_QUERY_PATTERN: &str = "https://duckduckgo.com/?q={query}";
/// Plain HTML DuckDuckGo endpoint (no JS required)
pub const DUCKDUCKGO_PLAIN_QUERY_PATTERN: &str = "https://html.duckduckgo.com/html/?q={query}";
pub const BING_QUERY_PATTERN: &str = "https://www.bing.com/search?q={query}";
pub const GOOGLE_QUERY_PATTERN: &str = "https://www.google.com/search?q={query}";
pub const BRAVE_QUERY_PATTERN: &str = "https://search.brave.com/search?q={query}&source=web";
pub const BAIDU_QUERY_PATTERN: &str = "https://www.baidu.com/s?wd={query}";
pub const SOUGOU_WEIXIN_QUERY_PATTERN: &str =
    "https://weixin.sogou.com/weixin?type=2&s_from=input&&ie=utf8&query={query}";

/// Brave Search API endpoint pattern
pub const BRAVE_API_QUERY_PATTERN: &str =
    "https://api.search.brave.com/res/v1/web/search?q={query}&count={limit}";

/// Serper Google search API endpoint
pub const SERPER_API_URL: &str = "https://google.serper.dev/search";

// ============================================================================
// Default Values
// ============================================================================

/// Default query pattern (Bing)
pub const DEFAULT_QUERY_PATTERN: &str = BING_QUERY_PATTERN;