data_gov_catalog/
client.rs

1//! HTTP client and error types for the data.gov Catalog API.
2
3use crate::models;
4use serde::de::DeserializeOwned;
5use serde_json::Value;
6use std::sync::Arc;
7
8/// Configuration for the Catalog API client.
9///
10/// The defaults target the public data.gov endpoint. Override `base_path`
11/// to point at a staging instance or at `https://api.data.gov/catalog`
12/// once the announced migration lands.
13#[derive(Debug, Clone)]
14pub struct Configuration {
15    /// Base URL for the Catalog API (e.g. `https://catalog.data.gov`).
16    pub base_path: String,
17    /// User-Agent header sent with every request.
18    pub user_agent: Option<String>,
19    /// Shared reqwest client. Cheap to clone; reuse across requests.
20    pub client: reqwest::Client,
21}
22
23impl Configuration {
24    /// Build a [`Configuration`] with default values.
25    pub fn new() -> Self {
26        Self::default()
27    }
28}
29
30impl Default for Configuration {
31    fn default() -> Self {
32        Self {
33            base_path: "https://catalog.data.gov".to_owned(),
34            user_agent: Some(concat!("data-gov-rs/", env!("CARGO_PKG_VERSION")).to_owned()),
35            client: reqwest::Client::new(),
36        }
37    }
38}
39
40/// Async client for the Catalog API.
41///
42/// Holds an [`Arc<Configuration>`] so it's cheap to clone and share across
43/// tasks. Every method is `async` and returns [`Result<_, CatalogError>`].
44pub struct CatalogClient {
45    configuration: Arc<Configuration>,
46}
47
48impl std::fmt::Debug for CatalogClient {
49    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
50        f.debug_struct("CatalogClient")
51            .field("base_path", &self.configuration.base_path)
52            .finish()
53    }
54}
55
56/// Errors returned by the Catalog API client.
57#[derive(Debug)]
58pub enum CatalogError {
59    /// Network, TLS, or HTTP-protocol failure.
60    RequestError(Box<dyn std::error::Error + Send + Sync>),
61    /// JSON could not be deserialized into the expected shape.
62    ParseError(serde_json::Error),
63    /// The server returned a non-2xx status code.
64    ApiError {
65        /// HTTP status code.
66        status: u16,
67        /// Server-provided response body (often a JSON error document).
68        message: String,
69    },
70}
71
72impl std::fmt::Display for CatalogError {
73    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
74        match self {
75            CatalogError::RequestError(e) => write!(f, "Request error: {e}"),
76            CatalogError::ParseError(e) => write!(f, "Parse error: {e}"),
77            CatalogError::ApiError { status, message } => {
78                write!(f, "Catalog API error ({status}): {message}")
79            }
80        }
81    }
82}
83
84impl std::error::Error for CatalogError {}
85
86/// Parameters for [`CatalogClient::search`].
87///
88/// Constructed with a builder: start from [`SearchParams::new`] and chain
89/// setters. All fields are optional; the server defaults apply when a field
90/// is left unset.
91#[derive(Debug, Default, Clone)]
92pub struct SearchParams {
93    /// Full-text query.
94    pub q: Option<String>,
95    /// Sort order (`relevance`, `popularity`, `distance`, `last_harvested_date`).
96    pub sort: Option<String>,
97    /// Results per page.
98    pub per_page: Option<i32>,
99    /// Filter by organization slug (e.g. `nasa`).
100    pub org_slug: Option<String>,
101    /// Filter by organization type (e.g. `Federal Government`).
102    pub org_type: Option<String>,
103    /// Exact-match keyword filters. Repeated on the wire.
104    pub keyword: Vec<String>,
105    /// `geospatial` or `non-geospatial`.
106    pub spatial_filter: Option<String>,
107    /// GeoJSON geometry used for bounding-box / shape queries.
108    pub spatial_geometry: Option<Value>,
109    /// Whether to require containment (true) vs. intersection (false).
110    pub spatial_within: Option<bool>,
111    /// Opaque cursor from a previous [`SearchResponse::after`](models::SearchResponse::after).
112    pub after: Option<String>,
113    /// Exact-match slug filter (single dataset lookup).
114    pub slug: Option<String>,
115}
116
117impl SearchParams {
118    /// Construct empty [`SearchParams`].
119    pub fn new() -> Self {
120        Self::default()
121    }
122
123    /// Set the full-text query.
124    pub fn q(mut self, q: impl Into<String>) -> Self {
125        self.q = Some(q.into());
126        self
127    }
128
129    /// Set the sort order.
130    pub fn sort(mut self, sort: impl Into<String>) -> Self {
131        self.sort = Some(sort.into());
132        self
133    }
134
135    /// Set page size.
136    pub fn per_page(mut self, per_page: i32) -> Self {
137        self.per_page = Some(per_page);
138        self
139    }
140
141    /// Filter by organization slug.
142    pub fn org_slug(mut self, slug: impl Into<String>) -> Self {
143        self.org_slug = Some(slug.into());
144        self
145    }
146
147    /// Filter by organization type.
148    pub fn org_type(mut self, org_type: impl Into<String>) -> Self {
149        self.org_type = Some(org_type.into());
150        self
151    }
152
153    /// Append a keyword filter (exact match).
154    pub fn keyword(mut self, keyword: impl Into<String>) -> Self {
155        self.keyword.push(keyword.into());
156        self
157    }
158
159    /// Replace the keyword list.
160    pub fn keywords<I, S>(mut self, keywords: I) -> Self
161    where
162        I: IntoIterator<Item = S>,
163        S: Into<String>,
164    {
165        self.keyword = keywords.into_iter().map(Into::into).collect();
166        self
167    }
168
169    /// Set the spatial-filter mode.
170    pub fn spatial_filter(mut self, mode: impl Into<String>) -> Self {
171        self.spatial_filter = Some(mode.into());
172        self
173    }
174
175    /// Set the GeoJSON geometry for spatial queries.
176    pub fn spatial_geometry(mut self, geometry: Value) -> Self {
177        self.spatial_geometry = Some(geometry);
178        self
179    }
180
181    /// Require containment vs. intersection for spatial matches.
182    pub fn spatial_within(mut self, within: bool) -> Self {
183        self.spatial_within = Some(within);
184        self
185    }
186
187    /// Set the pagination cursor.
188    pub fn after(mut self, after: impl Into<String>) -> Self {
189        self.after = Some(after.into());
190        self
191    }
192
193    /// Filter to an exact slug match (single-dataset lookup).
194    pub fn slug(mut self, slug: impl Into<String>) -> Self {
195        self.slug = Some(slug.into());
196        self
197    }
198
199    /// Serialize to the repeated `(key, value)` form reqwest expects.
200    fn to_query(&self) -> Vec<(&'static str, String)> {
201        let mut q: Vec<(&'static str, String)> = Vec::new();
202        if let Some(v) = &self.q {
203            q.push(("q", v.clone()));
204        }
205        if let Some(v) = &self.sort {
206            q.push(("sort", v.clone()));
207        }
208        if let Some(v) = self.per_page {
209            q.push(("per_page", v.to_string()));
210        }
211        if let Some(v) = &self.org_slug {
212            q.push(("org_slug", v.clone()));
213        }
214        if let Some(v) = &self.org_type {
215            q.push(("org_type", v.clone()));
216        }
217        for kw in &self.keyword {
218            q.push(("keyword", kw.clone()));
219        }
220        if let Some(v) = &self.spatial_filter {
221            q.push(("spatial_filter", v.clone()));
222        }
223        if let Some(v) = &self.spatial_geometry {
224            q.push(("spatial_geometry", v.to_string()));
225        }
226        if let Some(v) = self.spatial_within {
227            q.push(("spatial_within", v.to_string()));
228        }
229        if let Some(v) = &self.after {
230            q.push(("after", v.clone()));
231        }
232        if let Some(v) = &self.slug {
233            q.push(("slug", v.clone()));
234        }
235        q
236    }
237}
238
239impl CatalogClient {
240    /// Construct a new client from a shared [`Configuration`].
241    pub fn new(configuration: Arc<Configuration>) -> Self {
242        Self { configuration }
243    }
244
245    /// Build a URL by joining `path` onto the configured base.
246    fn url(&self, path: &str) -> String {
247        let base = self.configuration.base_path.trim_end_matches('/');
248        format!("{base}{path}")
249    }
250
251    /// Issue a GET with optional query parameters and deserialize the JSON body.
252    async fn get_json<T: DeserializeOwned, Q: serde::Serialize + ?Sized>(
253        &self,
254        path: &str,
255        params: &Q,
256    ) -> Result<T, CatalogError> {
257        let mut req = self.configuration.client.get(self.url(path)).query(params);
258        if let Some(ua) = &self.configuration.user_agent {
259            req = req.header(reqwest::header::USER_AGENT, ua);
260        }
261        let response = req
262            .send()
263            .await
264            .map_err(|e| CatalogError::RequestError(Box::new(e)))?;
265
266        if !response.status().is_success() {
267            let status = response.status().as_u16();
268            let message = response
269                .text()
270                .await
271                .unwrap_or_else(|_| "<no body>".to_string());
272            return Err(CatalogError::ApiError { status, message });
273        }
274
275        let bytes = response
276            .bytes()
277            .await
278            .map_err(|e| CatalogError::RequestError(Box::new(e)))?;
279        serde_json::from_slice(&bytes).map_err(CatalogError::ParseError)
280    }
281
282    /// Search datasets. See the module docs for parameters.
283    ///
284    /// # Errors
285    ///
286    /// Returns [`CatalogError::ApiError`] if the server returns non-2xx,
287    /// [`CatalogError::RequestError`] for network/TLS failure, and
288    /// [`CatalogError::ParseError`] if the response isn't a valid
289    /// [`SearchResponse`](models::SearchResponse).
290    pub async fn search(
291        &self,
292        params: SearchParams,
293    ) -> Result<models::SearchResponse, CatalogError> {
294        let query = params.to_query();
295        self.get_json("/search", &query).await
296    }
297
298    /// Fetch a single dataset by its data.gov slug.
299    ///
300    /// Returns `Ok(None)` if no dataset with that slug exists. The returned
301    /// [`SearchHit`](models::SearchHit) carries the denormalized fields and a
302    /// nested `dcat` record with the full DCAT-US 3 metadata.
303    ///
304    /// # Notes
305    ///
306    /// The Catalog API does not actually honor a `slug=` query parameter
307    /// today — it returns the top relevance hit regardless of the value.
308    /// We work around this by using a full-text query (`q=<slug>`) and then
309    /// scanning the first page for a hit whose `slug` exactly matches.
310    /// Slug-shaped queries reliably rank the exact match first when it
311    /// exists; we look at the top 20 results to leave headroom for ties.
312    pub async fn dataset_by_slug(
313        &self,
314        slug: &str,
315    ) -> Result<Option<models::SearchHit>, CatalogError> {
316        let params = SearchParams::new().q(slug).per_page(20);
317        let response: models::SearchResponse = self.search(params).await?;
318        Ok(response
319            .results
320            .into_iter()
321            .find(|hit| hit.slug.as_deref() == Some(slug)))
322    }
323
324    /// List all organizations known to the catalog.
325    ///
326    /// The endpoint returns the full list in one response; there is no
327    /// pagination today.
328    pub async fn organizations(&self) -> Result<models::OrganizationsResponse, CatalogError> {
329        self.get_json("/api/organizations", &[(); 0]).await
330    }
331
332    /// Return the top keywords ranked by document frequency.
333    ///
334    /// `size` caps the number of rows (server default 100, max 1000).
335    /// `min_count` drops keywords with fewer than that many datasets.
336    pub async fn keywords(
337        &self,
338        size: Option<i32>,
339        min_count: Option<i32>,
340    ) -> Result<models::KeywordsResponse, CatalogError> {
341        let mut params: Vec<(&str, String)> = Vec::new();
342        if let Some(s) = size {
343            params.push(("size", s.to_string()));
344        }
345        if let Some(m) = min_count {
346            params.push(("min_count", m.to_string()));
347        }
348        self.get_json("/api/keywords", &params).await
349    }
350
351    /// Autocomplete against known locations.
352    pub async fn locations_search(
353        &self,
354        q: &str,
355        size: Option<i32>,
356    ) -> Result<models::LocationsResponse, CatalogError> {
357        let mut params: Vec<(&str, String)> = vec![("q", q.to_string())];
358        if let Some(s) = size {
359            params.push(("size", s.to_string()));
360        }
361        self.get_json("/api/locations/search", &params).await
362    }
363
364    /// Fetch the GeoJSON geometry for a given location id.
365    ///
366    /// The response is returned as a raw [`serde_json::Value`] because the
367    /// shape is unconstrained GeoJSON and callers typically hand it straight
368    /// to a mapping library.
369    pub async fn location_geometry(&self, id: &str) -> Result<Value, CatalogError> {
370        let path = format!("/api/location/{id}");
371        self.get_json(&path, &[(); 0]).await
372    }
373
374    /// Retrieve a harvest record's metadata envelope.
375    pub async fn harvest_record(&self, id: &str) -> Result<models::HarvestRecord, CatalogError> {
376        let path = format!("/harvest_record/{id}");
377        self.get_json(&path, &[(); 0]).await
378    }
379
380    /// Retrieve the raw (pre-transform) payload a harvester ingested.
381    ///
382    /// The payload is not constrained to a single shape — agencies post JSON,
383    /// XML fragments, and DCAT records through the same surface — so the
384    /// result is returned as [`serde_json::Value`].
385    pub async fn harvest_record_raw(&self, id: &str) -> Result<Value, CatalogError> {
386        let path = format!("/harvest_record/{id}/raw");
387        self.get_json(&path, &[(); 0]).await
388    }
389
390    /// Retrieve the DCAT-US 3 transform of a harvest record.
391    pub async fn harvest_record_transformed(
392        &self,
393        id: &str,
394    ) -> Result<models::Dataset, CatalogError> {
395        let path = format!("/harvest_record/{id}/transformed");
396        self.get_json(&path, &[(); 0]).await
397    }
398}
data_gov_catalog/client.rs

data_gov_catalog/
client.rs
