spider_agent 2.51.205

//! # Spider Agent
//!
//! A concurrent-safe multimodal agent for web automation and research.
//!
//! ## Features
//!
//! - **Concurrent-safe**: Designed to be wrapped in `Arc` for multi-task access
//! - **Feature-gated**: Only include dependencies you need
//! - **Multiple LLM providers**: OpenAI, OpenAI-compatible APIs
//! - **Multiple search providers**: Serper, Brave, Bing, Tavily
//! - **Browser automation**: Chrome support via chromiumoxide
//!
//! ## Quick Start
//!
//! ```rust,ignore
//! use spider_agent::{Agent, AgentConfig};
//! use std::sync::Arc;
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
//!     let agent = Arc::new(Agent::builder()
//!         .with_openai("sk-...", "gpt-4o-mini")
//!         .with_search_serper("serper-key")
//!         .build()?);
//!
//!     // Search
//!     let results = agent.search("rust web frameworks").await?;
//!     println!("Found {} results", results.len());
//!
//!     // Extract from first result
//!     let html = agent.fetch(&results.results[0].url).await?.html;
//!     let data = agent.extract(&html, "Extract framework name and features").await?;
//!     println!("{}", serde_json::to_string_pretty(&data)?);
//!
//!     Ok(())
//! }
//! ```
//!
//! ## Concurrent Execution
//!
//! ```rust,ignore
//! use spider_agent::Agent;
//! use std::sync::Arc;
//!
//! let agent = Arc::new(Agent::builder()
//!     .with_openai("sk-...", "gpt-4o")
//!     .with_search_serper("serper-key")
//!     .with_max_concurrent_llm_calls(10)
//!     .build()?);
//!
//! // Execute multiple searches concurrently
//! let queries = vec!["rust async", "rust web frameworks", "rust databases"];
//! let mut handles = Vec::new();
//!
//! for query in queries {
//!     let agent = agent.clone();
//!     let query = query.to_string();
//!     handles.push(tokio::spawn(async move {
//!         agent.search(&query).await
//!     }));
//! }
//!
//! // Collect results
//! for handle in handles {
//!     let result = handle.await??;
//!     println!("Found {} results", result.results.len());
//! }
//! ```
//!
//! ## Feature Flags
//!
//! - `openai` - OpenAI/OpenAI-compatible LLM provider
//! - `chrome` - Browser automation via CDP (local or remote via Spider Browser Cloud)
//! - `search` - Base search functionality
//! - `search_serper` - Serper.dev search provider
//! - `search_brave` - Brave Search provider
//! - `search_bing` - Bing Search provider
//! - `search_tavily` - Tavily AI Search provider
//! - `webdriver` - Browser automation via thirtyfour
//! - `fs` - Tempfile-backed disk storage helpers
//! - `skills` - Dynamic skill loading for web challenge solving
//! - `memvid` - Long-term experience memory via semantic search
//! - `scripting` - Embedded pure-Rust Python + JavaScript interpreters
//!   ([`scripting`]) for LLM-callable `RunPython` / `RunJavaScript` actions
//! - `full` - All features
//!
//! ## Scripting (pure-Rust embedded Python + JS)
//!
//! With `features = ["scripting"]`, the agent can evaluate arbitrary Python
//! (via `rustpython-vm` with the frozen pure-Python stdlib) and JavaScript
//! (via `boa_engine`) entirely inside the spider_agent process — no CPython,
//! no V8, no subprocesses. Workers run on dedicated `std::thread`s
//! (not tokio's blocking pool), with `flume::bounded` + `tokio::oneshot`
//! bridging the async caller; the design has no `Mutex`/`RwLock` in the
//! scripting module and no `unwrap`/`expect` that can panic the worker.
//!
//! Scripts run with an injected `agent` object exposing:
//!
//! - `agent.url`, `agent.title`, `agent.html`, `agent.memory`, `agent.tmpdir` — read-only
//! - `agent.log(...)` / `print(...)` / `console.log(...)` — captured to `stdout`
//! - `agent.fetch(url, opts?)` — reuses spider_agent's reqwest client (gated on `allow_network`)
//! - `agent.read_file(rel)` / `agent.write_file(rel, content)` — sandboxed to a
//!   per-call tmpdir via `cap-std`; path escapes are structurally impossible
//! - `agent.check_interrupted()` — cooperative cancel poll honoring per-call timeout
//!
//! ```rust,ignore
//! use spider_agent::scripting::{ScriptConfig, ScriptContext, ScriptEngine};
//! use std::time::Duration;
//!
//! # async fn run() -> Result<(), Box<dyn std::error::Error>> {
//! let engine = ScriptEngine::new(ScriptConfig {
//!     enabled: true,
//!     allow_network: true,
//!     default_timeout: Duration::from_secs(5),
//!     ..ScriptConfig::default()
//! });
//!
//! let result = engine.run_python(
//!     r#"
//! import json, re
//! numbers = [int(n) for n in re.findall(r"\d+", agent.html)]
//! print(json.dumps({"url": agent.url, "found": numbers}))
//! "#.to_string(),
//!     ScriptContext {
//!         url: Some("https://shop.example.com".into()),
//!         html: Some("<p>price 1999, stock 42</p>".into()),
//!         ..ScriptContext::default()
//!     },
//!     None,
//! ).await;
//!
//! println!("{}", result.stdout);
//! # Ok(()) }
//! ```
//!
//! When wired into the chrome action dispatcher
//! (`RemoteMultimodalEngine::with_script_engine(...)`), the LLM can emit
//! `{"RunPython": {"code": "..."}}` or `{"RunJavaScript": {"code": "..."}}`
//! actions and the engine routes them through the worker pool, surfacing
//! stdout back to the next LLM round.
//!
//! See `examples/scripting.rs` for the full end-to-end demo:
//! `cargo run --example scripting --features scripting`.

#![warn(missing_docs)]

mod agent;
pub mod automation;
mod config;
mod error;
mod llm;
mod memory;
pub mod tools;

#[cfg(feature = "search")]
pub mod search;

#[cfg(feature = "chrome")]
pub mod browser;

#[cfg(feature = "webdriver")]
pub mod webdriver;

#[cfg(feature = "fs")]
pub mod temp;

#[cfg(feature = "scripting")]
pub mod scripting;

// Re-exports
pub use agent::{Agent, AgentBuilder, FetchResult, PageExtraction};
pub use config::{
    AgentConfig, HtmlCleaningMode, LimitType, ResearchOptions, RetryConfig, SearchOptions,
    TimeRange, UsageLimits, UsageSnapshot, UsageStats,
};
pub use error::{AgentError, AgentResult, SearchError};
pub use llm::{
    CompletionOptions, CompletionResponse, LLMProvider, Message, MessageContent, TokenUsage,
};
pub use memory::AgentMemory;
pub use tools::{
    AuthConfig, CustomTool, CustomToolRegistry, CustomToolResult, HttpMethod,
    SpiderBrowserToolConfig, SpiderCloudToolConfig,
};

// Automation re-exports - core types
pub use automation::{
    ActResult, ActionRecord, ActionResult, ActionType, AutomationConfig, AutomationResult,
    AutomationUsage, CaptureProfile, ChainBuilder, ChainCondition, ChainContext, ChainResult,
    ChainStep, ChainStepResult, CleaningIntent, ClipViewport, ContentAnalysis, CostTier,
    ExtractionSchema, FormField, FormInfo, HtmlCleaningProfile, InteractiveElement,
    ModelCapabilities, ModelEndpoint, ModelInfoEntry, ModelPolicy, ModelPricing, ModelProfile,
    ModelRanks, NavigationOption, PageObservation, PromptUrlGate, ReasoningEffort,
    RecoveryStrategy, RetryPolicy, SelectorCache, SelectorCacheEntry, StructuredOutputConfig,
    VisionRouteMode, MODEL_INFO,
};

// Automation re-exports - engine and configuration
pub use automation::{RemoteMultimodalConfig, RemoteMultimodalConfigs, RemoteMultimodalEngine};

// Automation re-exports - engine error types
pub use automation::{EngineError, EngineResult};

// Automation re-exports - helper functions
pub use automation::{
    best_effort_parse_json_object, extract_assistant_content, extract_last_code_block,
    extract_last_json_array, extract_last_json_boundaries, extract_last_json_object, extract_usage,
    fnv1a64, truncate_utf8_tail,
};

// Automation re-exports - HTML cleaning
pub use automation::{
    clean_html, clean_html_base, clean_html_full, clean_html_raw, clean_html_slim,
    clean_html_with_profile, clean_html_with_profile_and_intent, smart_clean_html,
};

// Automation re-exports - map result types
pub use automation::{categories, DiscoveredUrl, MapResult};

// Automation re-exports - memory operations
pub use automation::{AutomationMemory, MemoryOperation};

// Automation re-exports - system prompts
pub use automation::{
    ACT_SYSTEM_PROMPT, CONFIGURATION_SYSTEM_PROMPT, DEFAULT_SYSTEM_PROMPT, EXTRACT_SYSTEM_PROMPT,
    MAP_SYSTEM_PROMPT, OBSERVE_SYSTEM_PROMPT,
};

// Automation re-exports - concurrent chain types
pub use automation::{
    ConcurrentChainConfig, ConcurrentChainResult, DependencyGraph, DependentStep, StepResult,
};

// Automation re-exports - confidence types
pub use automation::{
    Alternative, ConfidenceRetryStrategy, ConfidenceSummary, ConfidenceTracker, ConfidentStep,
    Verification, VerificationType,
};

// Automation re-exports - tool calling types
pub use automation::{
    ActionToolSchemas, FunctionCall, FunctionDefinition, ToolCall, ToolCallingMode, ToolDefinition,
};

// Automation re-exports - HTML diff types
pub use automation::{
    ChangeType, DiffStats, ElementChange, HtmlDiffMode, HtmlDiffResult, PageStateDiff,
};

// Automation re-exports - planning types
pub use automation::{
    Checkpoint, CheckpointResult, CheckpointType, ExecutionPlan, PageState, PlanExecutionState,
    PlannedStep, PlanningModeConfig, ReplanContext,
};

// Automation re-exports - self-healing types
pub use automation::{
    HealedSelectorCache, HealingDiagnosis, HealingRequest, HealingResult, HealingStats,
    SelectorIssueType, SelfHealingConfig,
};

// Automation re-exports - synthesis types
pub use automation::{
    MultiPageContext, PageContext, PageContribution, SynthesisConfig, SynthesisResult,
};

// Automation re-exports - schema generation types
pub use automation::{
    build_schema_generation_prompt, generate_schema, infer_schema, infer_schema_from_examples,
    refine_schema, GeneratedSchema, SchemaCache, SchemaGenerationRequest,
};

// Automation re-exports - self-healing helper functions
pub use automation::extract_html_context;

// Automation re-exports - tool calling helper functions
pub use automation::{parse_tool_calls, tool_calls_to_steps};

// Automation re-exports - concurrent chain execution
pub use automation::execute_graph;

// Performance re-exports
pub use automation::cache::{CacheStats, CacheValue, SmartCache};
pub use automation::executor::{BatchExecutor, ChainExecutor, PrefetchManager};
pub use automation::router::{
    auto_policy, estimate_tokens, ModelRequirements, ModelRouter, ModelSelector, RoutingDecision,
    ScoredModel, SelectionStrategy, TaskAnalysis, TaskCategory,
};

#[cfg(feature = "search")]
pub use agent::ResearchResult;

#[cfg(feature = "search")]
pub use search::{SearchProvider, SearchResult, SearchResults};

#[cfg(feature = "openai")]
pub use llm::OpenAIProvider;

#[cfg(feature = "search_serper")]
pub use search::SerperProvider;

#[cfg(feature = "search_brave")]
pub use search::BraveProvider;

#[cfg(feature = "search_bing")]
pub use search::BingProvider;

#[cfg(feature = "search_tavily")]
pub use search::TavilyProvider;

#[cfg(feature = "memvid")]
pub use automation::{
    ExperienceMemory, ExperienceMemoryConfig, ExperienceOutcome, ExperienceRecord,
    MemoryStats as ExperienceMemoryStats, RecalledExperience,
};

#[cfg(feature = "chrome")]
pub use browser::BrowserContext;

#[cfg(feature = "chrome")]
pub use automation::{
    run_remote_multimodal_with_page, run_spawn_pages_concurrent, run_spawn_pages_with_factory,
    run_spawn_pages_with_options, PageFactory, PageSetupFn, SpawnPageOptions, SpawnedPageResult,
};

#[cfg(feature = "webdriver")]
pub use webdriver::WebDriverContext;

#[cfg(feature = "fs")]
pub use temp::{TempFile, TempStorage};