apify_client/clients/
dataset.rs

1//! Client for a single dataset (`/v2/datasets/{datasetId}` and run-nested variants).
2
3use serde::de::DeserializeOwned;
4use serde::Serialize;
5use serde_json::Value;
6
7use crate::clients::base::{delete_resource, get_resource, update_resource, ResourceContext};
8use crate::common::{parse_data_envelope, sign_storage_content, PaginationList, QueryParams};
9use crate::error::ApifyClientResult;
10use crate::http_client::{HttpClient, HttpMethod, HttpRequest};
11use crate::models::Dataset;
12
13/// Options for listing or downloading dataset items.
14///
15/// Covers the filtering, projection and transformation parameters of
16/// `GET /v2/datasets/{datasetId}/items`.
17#[derive(Debug, Default, Clone)]
18pub struct DatasetListItemsOptions {
19    /// Number of items to skip.
20    pub offset: Option<i64>,
21    /// Maximum number of items to return.
22    pub limit: Option<i64>,
23    /// Return items newest-first.
24    pub desc: Option<bool>,
25    /// Only include these fields.
26    pub fields: Option<Vec<String>>,
27    /// Positionally renames the fields selected by `fields` in the output (requires `fields`
28    /// to be set). The i-th name here becomes the output name of the i-th `fields` entry.
29    pub output_fields: Option<Vec<String>>,
30    /// Exclude these fields.
31    pub omit: Option<Vec<String>>,
32    /// Skip empty items.
33    pub skip_empty: Option<bool>,
34    /// Skip hidden fields (those starting with `#`).
35    pub skip_hidden: Option<bool>,
36    /// Only return clean (non-empty, non-hidden) items.
37    pub clean: Option<bool>,
38    /// Unwind these fields (each array element becomes a separate item).
39    pub unwind: Option<Vec<String>>,
40    /// Flatten these nested fields into dot-notation keys.
41    pub flatten: Option<Vec<String>>,
42    /// Use a predefined dataset view for field selection.
43    pub view: Option<String>,
44    /// Return simplified (flattened, cleaned) items.
45    pub simplified: Option<bool>,
46    /// Skip items that come from failed pages.
47    pub skip_failed_pages: Option<bool>,
48    /// Pre-shared URL signature granting access to a private dataset without an API token.
49    pub signature: Option<String>,
50}
51
52impl DatasetListItemsOptions {
53    fn apply(&self, params: &mut QueryParams) {
54        params
55            .add_int("offset", self.offset)
56            .add_int("limit", self.limit)
57            .add_bool("desc", self.desc)
58            .add_csv("fields", self.fields.as_deref())
59            .add_csv("outputFields", self.output_fields.as_deref())
60            .add_csv("omit", self.omit.as_deref())
61            .add_bool("skipEmpty", self.skip_empty)
62            .add_bool("skipHidden", self.skip_hidden)
63            .add_bool("clean", self.clean)
64            .add_csv("unwind", self.unwind.as_deref())
65            .add_csv("flatten", self.flatten.as_deref())
66            .add_str("view", self.view.clone())
67            .add_bool("simplified", self.simplified)
68            .add_bool("skipFailedPages", self.skip_failed_pages)
69            .add_str("signature", self.signature.clone());
70    }
71}
72
73/// Output formats supported by [`DatasetClient::download_items`].
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
75pub enum DownloadItemsFormat {
76    /// JSON array.
77    Json,
78    /// Newline-delimited JSON.
79    Jsonl,
80    /// Comma-separated values.
81    Csv,
82    /// Microsoft Excel (XLSX).
83    Xlsx,
84    /// XML.
85    Xml,
86    /// RSS feed.
87    Rss,
88    /// HTML table.
89    Html,
90}
91
92impl DownloadItemsFormat {
93    fn as_str(&self) -> &'static str {
94        match self {
95            DownloadItemsFormat::Json => "json",
96            DownloadItemsFormat::Jsonl => "jsonl",
97            DownloadItemsFormat::Csv => "csv",
98            DownloadItemsFormat::Xlsx => "xlsx",
99            DownloadItemsFormat::Xml => "xml",
100            DownloadItemsFormat::Rss => "rss",
101            DownloadItemsFormat::Html => "html",
102        }
103    }
104}
105
106/// Format-specific options for [`DatasetClient::download_items`], on top of the shared
107/// filtering/projection options.
108#[derive(Debug, Default, Clone)]
109pub struct DatasetDownloadOptions {
110    /// Shared item filtering/projection options.
111    pub items: DatasetListItemsOptions,
112    /// Set `Content-Disposition: attachment` on the response.
113    pub attachment: Option<bool>,
114    /// Prepend a UTF-8 BOM (useful for Excel-compatible CSV).
115    pub bom: Option<bool>,
116    /// CSV field delimiter (default `,`).
117    pub delimiter: Option<String>,
118    /// Omit the CSV header row.
119    pub skip_header_row: Option<bool>,
120    /// Name of the root XML element (default `items`).
121    pub xml_root: Option<String>,
122    /// Name of the per-item XML element (default `item`).
123    pub xml_row: Option<String>,
124    /// Title to use for RSS/Atom feed exports.
125    pub feed_title: Option<String>,
126    /// Description to use for RSS/Atom feed exports.
127    pub feed_description: Option<String>,
128}
129
130impl DatasetDownloadOptions {
131    fn apply(&self, params: &mut QueryParams) {
132        self.items.apply(params);
133        params
134            .add_bool("attachment", self.attachment)
135            .add_bool("bom", self.bom)
136            .add_str("delimiter", self.delimiter.clone())
137            .add_bool("skipHeaderRow", self.skip_header_row)
138            .add_str("xmlRoot", self.xml_root.clone())
139            .add_str("xmlRow", self.xml_row.clone())
140            .add_str("feedTitle", self.feed_title.clone())
141            .add_str("feedDescription", self.feed_description.clone());
142    }
143}
144
145/// Client for a specific dataset.
146#[derive(Debug, Clone)]
147pub struct DatasetClient {
148    ctx: ResourceContext,
149}
150
151impl DatasetClient {
152    pub(crate) fn new(http: HttpClient, base_url: &str, resource_path: &str, id: &str) -> Self {
153        Self {
154            ctx: ResourceContext::single(http, base_url, resource_path, id),
155        }
156    }
157
158    /// Creates a dataset client for a run's default dataset (no ID; nested path only).
159    pub(crate) fn nested(http: HttpClient, base_url: &str, sub_path: &str) -> Self {
160        Self {
161            ctx: ResourceContext::collection(http, base_url, sub_path),
162        }
163    }
164
165    /// Sets the public origin used when building shareable URLs.
166    pub(crate) fn with_public_base(mut self, public_base_url: &str) -> Self {
167        self.ctx = self.ctx.with_public_origin(public_base_url);
168        self
169    }
170
171    /// Fetches the dataset metadata, or `None` if it does not exist.
172    pub async fn get(&self) -> ApifyClientResult<Option<Dataset>> {
173        get_resource(&self.ctx, None, &QueryParams::new()).await
174    }
175
176    /// Updates the dataset metadata (e.g. `name`, `title`).
177    pub async fn update<T: Serialize>(&self, new_fields: &T) -> ApifyClientResult<Dataset> {
178        update_resource(&self.ctx, None, new_fields).await
179    }
180
181    /// Deletes the dataset.
182    pub async fn delete(&self) -> ApifyClientResult<()> {
183        delete_resource(&self.ctx, None).await
184    }
185
186    /// Lists items from the dataset.
187    ///
188    /// The dataset items endpoint returns a bare JSON array (not a `data` envelope) and
189    /// reports pagination via `X-Apify-Pagination-*` headers, which are surfaced in the
190    /// returned [`PaginationList`].
191    pub async fn list_items<T: DeserializeOwned>(
192        &self,
193        options: DatasetListItemsOptions,
194    ) -> ApifyClientResult<PaginationList<T>> {
195        let mut params = QueryParams::new();
196        options.apply(&mut params);
197        let url = params.apply_to_url(&self.ctx.url(Some("items")));
198        let response = self
199            .ctx
200            .http
201            .call(HttpRequest {
202                method: HttpMethod::Get,
203                url,
204                headers: Default::default(),
205                body: None,
206                timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT,
207            })
208            .await?;
209
210        let items: Vec<T> = serde_json::from_slice(&response.body)?;
211        let count = items.len() as i64;
212        let total = response
213            .header("x-apify-pagination-total")
214            .and_then(|v| v.parse().ok())
215            .unwrap_or(count);
216        let offset = response
217            .header("x-apify-pagination-offset")
218            .and_then(|v| v.parse().ok())
219            .unwrap_or(0);
220        let limit = response
221            .header("x-apify-pagination-limit")
222            .and_then(|v| v.parse().ok())
223            .unwrap_or(count);
224
225        Ok(PaginationList {
226            total,
227            offset,
228            limit,
229            count,
230            desc: options.desc.unwrap_or(false),
231            items,
232        })
233    }
234
235    /// Downloads dataset items serialized in the given `format`, returning the raw bytes.
236    ///
237    /// Unlike [`list_items`](Self::list_items), which returns parsed items, this returns the
238    /// items already serialized to JSON, CSV, XLSX, XML, RSS or HTML — useful for exporting.
239    /// Use [`DatasetDownloadOptions`] to control export-specific behaviour (BOM, CSV
240    /// delimiter/header, XML element names, attachment disposition).
241    pub async fn download_items(
242        &self,
243        format: DownloadItemsFormat,
244        options: DatasetDownloadOptions,
245    ) -> ApifyClientResult<Vec<u8>> {
246        let mut params = QueryParams::new();
247        params.add_str("format", Some(format.as_str()));
248        options.apply(&mut params);
249        let url = params.apply_to_url(&self.ctx.url(Some("items")));
250        let response = self
251            .ctx
252            .http
253            .call(HttpRequest {
254                method: HttpMethod::Get,
255                url,
256                headers: Default::default(),
257                body: None,
258                timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT,
259            })
260            .await?;
261        Ok(response.body)
262    }
263
264    /// Pushes one or more items to the dataset.
265    ///
266    /// `items` must serialize to a JSON object or an array of objects.
267    pub async fn push_items<T: Serialize>(&self, items: &T) -> ApifyClientResult<()> {
268        let body = serde_json::to_vec(items)?;
269        let url = self.ctx.url(Some("items"));
270        let mut headers = std::collections::HashMap::new();
271        headers.insert(
272            "Content-Type".to_string(),
273            "application/json; charset=utf-8".to_string(),
274        );
275        self.ctx
276            .http
277            .call(HttpRequest {
278                method: HttpMethod::Post,
279                url,
280                headers,
281                body: Some(body),
282                timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT,
283            })
284            .await?;
285        Ok(())
286    }
287
288    /// Builds a public URL for downloading this dataset's items.
289    ///
290    /// Mirrors the reference client's `createItemsPublicUrl`: it fetches the dataset, and if
291    /// the dataset exposes a URL-signing secret key (i.e. it is private), appends an
292    /// HMAC-SHA256 `signature` so the URL grants access without an API token. `expires_in_secs`
293    /// optionally bounds the validity of a signed URL. The URL is built from the configured
294    /// public base URL.
295    pub async fn create_items_public_url(
296        &self,
297        options: DatasetListItemsOptions,
298        expires_in_secs: Option<i64>,
299    ) -> ApifyClientResult<String> {
300        let mut params = QueryParams::new();
301        options.apply(&mut params);
302
303        if let Some(dataset) = self.get().await? {
304            if let Some(secret) = dataset
305                .extra
306                .get("urlSigningSecretKey")
307                .and_then(|v| v.as_str())
308            {
309                let signature = sign_storage_content(secret, &dataset.id, expires_in_secs);
310                params.add_str("signature", Some(signature));
311            }
312        }
313        Ok(params.apply_to_url(&self.ctx.public_url(Some("items"))))
314    }
315
316    /// Returns statistical information about the dataset, or `None` if unavailable.
317    pub async fn get_statistics(&self) -> ApifyClientResult<Option<Value>> {
318        let result: ApifyClientResult<Value> = async {
319            let response = self
320                .ctx
321                .http
322                .call(HttpRequest {
323                    method: HttpMethod::Get,
324                    url: self.ctx.url(Some("statistics")),
325                    headers: Default::default(),
326                    body: None,
327                    timeout: crate::clients::base::DEFAULT_REQUEST_TIMEOUT,
328                })
329                .await?;
330            parse_data_envelope(&response.body)
331        }
332        .await;
333        crate::common::catch_not_found(result)
334    }
335}
apify_client/clients/dataset.rs

apify_client/clients/
dataset.rs
