essence-engine 0.2.0

A fast web retrieval engine with HTTP-to-browser fallback, producing LLM-ready Markdown
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

use axum::{
    http::StatusCode,
    response::{IntoResponse, Response},
    Json,
};
use serde_json::json;
use thiserror::Error;

#[derive(Debug, Error)]
pub enum ScrapeError {
    #[error("HTTP request failed: {0}")]
    RequestFailed(#[from] reqwest::Error),

    #[error("Invalid URL: {0}")]
    InvalidUrl(String),

    #[error("Timeout occurred")]
    Timeout,

    #[error("Failed to parse HTML: {0}")]
    ParseError(String),

    #[error("Robots.txt disallows scraping")]
    RobotsDisallowed,

    #[error("Unsupported format: {0}")]
    UnsupportedFormat(String),

    #[error("Internal error: {0}")]
    Internal(String),

    #[error("Browser error: {0}")]
    BrowserError(String),

    #[error("Browser launch failed: {0}")]
    BrowserLaunchFailed(String),

    #[error("Navigation failed: {0}")]
    NavigationFailed(String),

    #[error("Element not found: {0}")]
    ElementNotFound(String),

    #[error("Validation failed")]
    ValidationFailed(Vec<String>),

    #[error("Browser not found: {0}")]
    BrowserNotFound(String),

    #[error("Invalid request: {0}")]
    InvalidRequest(String),

    #[error("Resource limit exceeded: {0}")]
    ResourceLimit(String),

    #[error("Unauthorized")]
    Unauthorized,

    #[error("SSRF attempt detected: {0}")]
    SsrfAttempt(String),

    #[error("Empty content: {0}")]
    EmptyContent(String),

    #[error("Low quality content: {0}")]
    LowQuality(String),

    #[error("Error page detected: {0}")]
    ErrorPage(String),

    #[error("Configuration error: {0}")]
    Configuration(String),
}

impl IntoResponse for ScrapeError {
    fn into_response(self) -> Response {
        let (status, error_message) = match self {
            ScrapeError::RequestFailed(ref e) => {
                if e.is_timeout() {
                    (StatusCode::REQUEST_TIMEOUT, "Request timeout".to_string())
                } else if e.is_connect() {
                    (
                        StatusCode::BAD_GATEWAY,
                        "Failed to connect to target".to_string(),
                    )
                } else {
                    (StatusCode::BAD_GATEWAY, format!("Request failed: {}", e))
                }
            }
            ScrapeError::InvalidUrl(_) => (StatusCode::BAD_REQUEST, self.to_string()),
            ScrapeError::Timeout => (StatusCode::REQUEST_TIMEOUT, self.to_string()),
            ScrapeError::ParseError(_) => (StatusCode::UNPROCESSABLE_ENTITY, self.to_string()),
            ScrapeError::RobotsDisallowed => (StatusCode::FORBIDDEN, self.to_string()),
            ScrapeError::UnsupportedFormat(_) => (StatusCode::BAD_REQUEST, self.to_string()),
            ScrapeError::Internal(_) => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()),
            ScrapeError::BrowserError(_) => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()),
            ScrapeError::BrowserLaunchFailed(_) => {
                (StatusCode::SERVICE_UNAVAILABLE, self.to_string())
            }
            ScrapeError::NavigationFailed(_) => (StatusCode::BAD_GATEWAY, self.to_string()),
            ScrapeError::ElementNotFound(_) => (StatusCode::UNPROCESSABLE_ENTITY, self.to_string()),
            ScrapeError::ValidationFailed(ref errors) => (
                StatusCode::BAD_REQUEST,
                format!("Validation failed: {}", errors.join(", ")),
            ),
            ScrapeError::BrowserNotFound(_) => {
                (StatusCode::SERVICE_UNAVAILABLE, self.to_string())
            }
            ScrapeError::InvalidRequest(_) => (StatusCode::BAD_REQUEST, self.to_string()),
            ScrapeError::ResourceLimit(_) => (StatusCode::PAYLOAD_TOO_LARGE, self.to_string()),
            ScrapeError::Unauthorized => (StatusCode::UNAUTHORIZED, "Unauthorized".to_string()),
            ScrapeError::SsrfAttempt(_) => (StatusCode::FORBIDDEN, self.to_string()),
            ScrapeError::EmptyContent(_) => (StatusCode::UNPROCESSABLE_ENTITY, self.to_string()),
            ScrapeError::LowQuality(_) => (StatusCode::UNPROCESSABLE_ENTITY, self.to_string()),
            ScrapeError::ErrorPage(_) => (StatusCode::BAD_GATEWAY, self.to_string()),
            ScrapeError::Configuration(_) => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()),
        };

        let body = Json(json!({
            "success": false,
            "error": error_message,
        }));

        (status, body).into_response()
    }
}

pub type Result<T> = std::result::Result<T, ScrapeError>;