pagesift_api/jina/
reader_route.rs

1//! Jina Reader protocol handler (`GET /r/{*url}` and `POST /r`).
2//! Provides direct Markdown extraction and JSON responses for AI Agents.
3
4use axum::Json;
5use axum::extract::{Path, State};
6use axum::http::{HeaderMap, StatusCode, header};
7use axum::response::{IntoResponse, Response};
8use serde::Deserialize;
9use std::sync::Arc;
10
11use super::stream_sse::stream_markdown_sse;
12use crate::engine::Engine;
13use crate::types::ScrapeOptions;
14
15#[derive(Debug, Deserialize)]
16pub struct JinaPostRequest {
17    pub url: String,
18}
19
20pub struct JinaState {
21    pub engine: Arc<dyn Engine>,
22}
23
24/// Handler for `GET /r/{*target_url}`.
25pub async fn handle_jina_get(
26    State(engine): State<Arc<dyn Engine>>,
27    headers: HeaderMap,
28    Path(raw_url): Path<String>,
29) -> Response {
30    let clean_url = parse_target_url(&raw_url);
31    process_jina_scrape(engine, headers, &clean_url).await
32}
33
34/// Handler for `POST /r`.
35pub async fn handle_jina_post(
36    State(engine): State<Arc<dyn Engine>>,
37    headers: HeaderMap,
38    Json(payload): Json<JinaPostRequest>,
39) -> Response {
40    let clean_url = parse_target_url(&payload.url);
41    process_jina_scrape(engine, headers, &clean_url).await
42}
43
44fn parse_target_url(raw: &str) -> String {
45    let trimmed = raw.trim();
46    if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
47        trimmed.to_string()
48    } else if let Some(idx) = trimmed.find("http:/") {
49        if trimmed[idx..].starts_with("http://") {
50            trimmed[idx..].to_string()
51        } else {
52            format!("http://{}", &trimmed[idx + 6..].trim_start_matches('/'))
53        }
54    } else if let Some(idx) = trimmed.find("https:/") {
55        if trimmed[idx..].starts_with("https://") {
56            trimmed[idx..].to_string()
57        } else {
58            format!("https://{}", &trimmed[idx + 7..].trim_start_matches('/'))
59        }
60    } else {
61        format!("https://{}", trimmed.trim_start_matches('/'))
62    }
63}
64
65async fn process_jina_scrape(
66    engine: Arc<dyn Engine>,
67    headers: HeaderMap,
68    target_url: &str,
69) -> Response {
70    if let Err(e) = crate::validate::check_url(target_url) {
71        return e.into_response();
72    }
73
74    let opts = ScrapeOptions {
75        only_main_content: true,
76        ..Default::default()
77    };
78
79    match engine.scrape(target_url, &opts).await {
80        Ok(doc) => {
81            let markdown = doc.markdown.unwrap_or_default();
82            let title = doc.metadata.title.as_deref();
83
84            let accept = headers
85                .get(header::ACCEPT)
86                .and_then(|v| v.to_str().ok())
87                .unwrap_or("");
88
89            if accept.contains("text/event-stream") {
90                stream_markdown_sse(target_url, title, &markdown)
91            } else if accept.contains("application/json") {
92                let json_data = serde_json::json!({
93                    "code": 200,
94                    "status": 20000,
95                    "data": {
96                        "title": title.unwrap_or(""),
97                        "description": doc.metadata.description.unwrap_or_default(),
98                        "url": target_url,
99                        "content": markdown,
100                    }
101                });
102                Json(json_data).into_response()
103            } else {
104                // Default: Direct Raw Markdown for LLM Agents
105                Response::builder()
106                    .header(header::CONTENT_TYPE, "text/markdown; charset=utf-8")
107                    .body(axum::body::Body::from(markdown))
108                    .unwrap_or_else(|_| {
109                        (StatusCode::INTERNAL_SERVER_ERROR, "response builder error")
110                            .into_response()
111                    })
112            }
113        }
114        Err(e) => e.into_response(),
115    }
116}
pagesift_api/jina/reader_route.rs

pagesift_api/jina/
reader_route.rs
