Struct Spider 

pub struct Spider {
    pub api_key: String,
    pub client: Client,
    pub rate_limit: RateLimitInfo,
}

Represents a Spider with API key and HTTP client.

Fields

api_key: String

The Spider API key.

client: Client

The Spider Client to re-use.

rate_limit: RateLimitInfo

The latest rate limit info from API responses.

Implementations

impl Spider

pub fn browser(&self, options: Option<BrowserOptions>) -> SpiderBrowser

Create a new SpiderBrowser instance using this client’s API key. The returned browser must be initialized with init() before use.

use spider_client::{BrowserOptions, Spider};

#[tokio::main]
async fn main() {
    let spider = Spider::new(Some("myspiderapikey".into())).expect("API key must be provided");

    let mut browser = spider.browser(Some(BrowserOptions::new().with_browser("chrome")));
    browser.init().await.expect("Failed to connect to the browser fleet");

    browser.page().goto("https://example.com").await.expect("Failed to navigate");
    let html = browser.page().content(1_000, 0).await.expect("Failed to get the page content");
    println!("{}", html);

    browser.close();
}

impl Spider

pub fn new(api_key: Option<String>) -> Result<Self, &'static str>

Creates a new instance of Spider.

Arguments

• api_key - An optional API key. Defaults to using the ‘SPIDER_API_KEY’ env variable.

Returns

A new instance of Spider or an error string if no API key is provided.

pub fn new_with_client( api_key: Option<String>, client: Client, ) -> Result<Self, &'static str>

Creates a new instance of Spider.

Arguments

• api_key - An optional API key. Defaults to using the ‘SPIDER_API_KEY’ env variable.
• client - A custom client to pass in.

Returns

A new instance of Spider or an error string if no API key is provided.

pub async fn api_post( &self, endpoint: &str, data: impl Serialize + Debug + Clone + Send + Sync, content_type: &str, ) -> Result<Response, Error>

Sends a POST request to the API.

Arguments

• endpoint - The API endpoint.
• data - The request data as a HashMap.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API.

pub async fn api_get<T: Serialize>( &self, endpoint: &str, query_params: Option<&T>, ) -> Result<Value, Error>

Sends a GET request to the API.

Arguments

• endpoint - The API endpoint.

Returns

The response from the API as a JSON value.

pub async fn api_delete( &self, endpoint: &str, params: Option<HashMap<String, Value>>, ) -> Result<Response, Error>

Sends a DELETE request to the API.

Arguments

• endpoint - The API endpoint.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API.

pub async fn scrape_url( &self, url: &str, params: Option<RequestParams>, content_type: &str, ) -> Result<Value, Error>

Scrapes a URL.

Arguments

• url - The URL to scrape.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn multi_scrape_url( &self, params: Option<Vec<RequestParams>>, content_type: &str, ) -> Result<Value, Error>

Scrapes multi URLs.

Arguments

• url - The URL to scrape.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn crawl_url( &self, url: &str, params: Option<RequestParams>, stream: bool, content_type: &str, callback: Option<impl Fn(Value) + Send>, ) -> Result<Value, Error>

Crawls a URL.

Arguments

• url - The URL to crawl.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.
• callback - Optional callback function to handle each streamed chunk.

Returns

The response from the API as a JSON value.

pub async fn multi_crawl_url( &self, params: Option<Vec<RequestParams>>, stream: bool, content_type: &str, callback: Option<impl Fn(Value) + Send>, ) -> Result<Value, Error>

Crawls multiple URLs.

Arguments

• url - The URL to crawl.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.
• callback - Optional callback function to handle each streamed chunk.

Returns

The response from the API as a JSON value.

pub async fn links( &self, url: &str, params: Option<RequestParams>, _stream: bool, content_type: &str, ) -> Result<Value, Error>

Fetches links from a URL.

Arguments

• url - The URL to fetch links from.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn multi_links( &self, params: Option<Vec<RequestParams>>, _stream: bool, content_type: &str, ) -> Result<Value, Error>

Fetches links from a URLs.

Arguments

• url - The URL to fetch links from.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn screenshot( &self, url: &str, params: Option<RequestParams>, _stream: bool, content_type: &str, ) -> Result<Value, Error>

Takes a screenshot of a URL.

Arguments

• url - The URL to take a screenshot of.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn multi_screenshot( &self, params: Option<Vec<RequestParams>>, _stream: bool, content_type: &str, ) -> Result<Value, Error>

Takes a screenshot of multiple URLs.

Arguments

• url - The URL to take a screenshot of.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn search( &self, q: &str, params: Option<SearchRequestParams>, _stream: bool, content_type: &str, ) -> Result<Value, Error>

Searches for a query.

Arguments

• q - The query to search for.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn multi_search( &self, params: Option<Vec<SearchRequestParams>>, content_type: &str, ) -> Result<Value, Error>

Searches for multiple querys.

Arguments

• q - The query to search for.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn ai_crawl( &self, url: &str, prompt: &str, params: Option<RequestParams>, content_type: &str, ) -> Result<Value, Error>

AI-guided crawling of a website using a natural language prompt.

Requires an active AI Studio subscription (billed separately from credits): https://spider.cloud/ai/pricing

Arguments

• url - The URL to start crawling.
• prompt - Natural language instruction for what to crawl and extract.
• params - Optional request parameters.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn ai_scrape( &self, url: &str, prompt: &str, params: Option<RequestParams>, content_type: &str, ) -> Result<Value, Error>

AI-guided scraping of a URL using a natural language prompt.

Requires an active AI Studio subscription (billed separately from credits): https://spider.cloud/ai/pricing

Arguments

• url - The URL to scrape.
• prompt - Natural language description of the data to extract.
• params - Optional request parameters.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn ai_search( &self, prompt: &str, params: Option<RequestParams>, content_type: &str, ) -> Result<Value, Error>

AI-enhanced web search using a natural language query.

Requires an active AI Studio subscription (billed separately from credits): https://spider.cloud/ai/pricing

Arguments

• prompt - Natural language search query.
• params - Optional request parameters.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn ai_browser( &self, url: &str, prompt: &str, params: Option<RequestParams>, content_type: &str, ) -> Result<Value, Error>

AI-guided browser automation using natural language commands.

Requires an active AI Studio subscription (billed separately from credits): https://spider.cloud/ai/pricing

Arguments

• url - The URL to automate.
• prompt - Natural language description of the browser actions.
• params - Optional request parameters.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn ai_links( &self, url: &str, prompt: &str, params: Option<RequestParams>, content_type: &str, ) -> Result<Value, Error>

AI-guided link extraction and filtering using a natural language prompt.

Requires an active AI Studio subscription (billed separately from credits): https://spider.cloud/ai/pricing

Arguments

• url - The URL to extract links from.
• prompt - Natural language description of the links to find.
• params - Optional request parameters.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn unlimited_scrape( &self, url: &str, params: Option<RequestParams>, content_type: &str, ) -> Result<Value, Error>

Scrapes a URL on the Unlimited plan. Takes the same parameters and returns the same responses as Spider::scrape_url.

Requires an active Unlimited subscription — a flat monthly rate billed by purchased concurrency seats (the number of requests in flight at once) instead of per-request credits. Without one the API returns 403 with {"error":"unlimited_plan_required"} or {"error":"unlimited_plan_inactive"}.

There is no queueing: when all purchased seats are in flight the API returns an immediate 429 with a Retry-After header — callers should retry with backoff (this client already retries with exponential backoff). Every response carries X-Concurrency-Limit and X-Concurrency-Active headers. AI/LLM extraction params (prompt, extraction_schema, model/vision params) are not allowed and are rejected with 400.

Docs: https://spider.cloud/docs/api/unlimited — Pricing: https://spider.cloud/pricing?plan=unlimited

Arguments

• url - The URL to scrape.
• params - Optional request parameters.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn unlimited_crawl( &self, url: &str, params: Option<RequestParams>, stream: bool, content_type: &str, callback: Option<impl Fn(Value) + Send>, ) -> Result<Value, Error>

Crawls a URL on the Unlimited plan. Takes the same parameters and returns the same responses as Spider::crawl_url.

Requires an active Unlimited subscription — a flat monthly rate billed by purchased concurrency seats (the number of requests in flight at once) instead of per-request credits. Without one the API returns 403 with {"error":"unlimited_plan_required"} or {"error":"unlimited_plan_inactive"}.

There is no queueing: when all purchased seats are in flight the API returns an immediate 429 with a Retry-After header — callers should retry with backoff (this client already retries with exponential backoff). Every response carries X-Concurrency-Limit and X-Concurrency-Active headers. AI/LLM extraction params (prompt, extraction_schema, model/vision params) are not allowed and are rejected with 400.

Docs: https://spider.cloud/docs/api/unlimited — Pricing: https://spider.cloud/pricing?plan=unlimited

Arguments

• url - The URL to crawl.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.
• callback - Optional callback function to handle each streamed chunk.

Returns

The response from the API as a JSON value.

pub async fn unlimited_links( &self, url: &str, params: Option<RequestParams>, _stream: bool, content_type: &str, ) -> Result<Value, Error>

Fetches links from a URL on the Unlimited plan. Takes the same parameters and returns the same responses as Spider::links.

Requires an active Unlimited subscription — a flat monthly rate billed by purchased concurrency seats (the number of requests in flight at once) instead of per-request credits. Without one the API returns 403 with {"error":"unlimited_plan_required"} or {"error":"unlimited_plan_inactive"}.

There is no queueing: when all purchased seats are in flight the API returns an immediate 429 with a Retry-After header — callers should retry with backoff (this client already retries with exponential backoff). Every response carries X-Concurrency-Limit and X-Concurrency-Active headers. AI/LLM extraction params (prompt, extraction_schema, model/vision params) are not allowed and are rejected with 400.

Docs: https://spider.cloud/docs/api/unlimited — Pricing: https://spider.cloud/pricing?plan=unlimited

Arguments

• url - The URL to fetch links from.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn unblock_url( &self, url: &str, params: Option<RequestParams>, content_type: &str, ) -> Result<Value, Error>

Unblock a URL.

Arguments

• url - The URL to scrape.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn multi_unblock_url( &self, params: Option<Vec<RequestParams>>, content_type: &str, ) -> Result<Value, Error>

Unblock multi URLs.

Arguments

• url - The URL to scrape.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn transform( &self, data: Vec<HashMap<&str, &str>>, params: Option<TransformParams>, _stream: bool, content_type: &str, ) -> Result<Value, Error>

Transforms data.

Arguments

• data - The data to transform.
• params - Optional request parameters.
• stream - Whether streaming is enabled.
• content_type - The content type of the request.

Returns

The response from the API as a JSON value.

pub async fn get_credits(&self) -> Result<Value, Error>

Get the account credits left.

pub async fn data_post( &self, table: &str, data: Option<RequestParams>, ) -> Result<Value, Error>

Send a request for a data record.

pub async fn data_get( &self, table: &str, params: Option<RequestParams>, ) -> Result<Value, Error>

Get a table record.

Trait Implementations

impl Debug for Spider

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for Spider

fn default() -> Spider

Returns the “default value” for a type.