pub struct FsDataset { /* private fields */ }Expand description
A file-system-backed, append-only JSON dataset.
Trait Implementations§
Source§impl Dataset for FsDataset
impl Dataset for FsDataset
Source§fn info<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = StorageResult<DatasetInfo>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn info<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = StorageResult<DatasetInfo>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Returns metadata derived from directory and item-file timestamps.
File systems do not expose Crawlee’s logical creation metadata, so the directory creation time (falling back to its modification time) and the newest directory or item modification time are approximations.
Source§fn push_json<'life0, 'async_trait>(
&'life0 self,
item: Value,
) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn push_json<'life0, 'async_trait>(
&'life0 self,
item: Value,
) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Appends one raw JSON value.
Source§fn push_json_batch<'life0, 'async_trait>(
&'life0 self,
items: Vec<Value>,
) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn push_json_batch<'life0, 'async_trait>(
&'life0 self,
items: Vec<Value>,
) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Appends a batch of raw JSON values.
Source§fn list_raw<'life0, 'async_trait>(
&'life0 self,
opts: ListOptions,
) -> Pin<Box<dyn Future<Output = StorageResult<Page<Value>>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn list_raw<'life0, 'async_trait>(
&'life0 self,
opts: ListOptions,
) -> Pin<Box<dyn Future<Output = StorageResult<Page<Value>>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Lists raw JSON values according to the supplied options.
Source§fn stream_raw(&self, opts: ListOptions) -> BoxStream<'_, StorageResult<Value>>
fn stream_raw(&self, opts: ListOptions) -> BoxStream<'_, StorageResult<Value>>
Streams raw JSON values according to the supplied options.
Source§fn export_json<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn export_json<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Exports the complete dataset as JSON.
Source§fn export_csv<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn export_csv<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = StorageResult<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Exports the complete dataset as CSV.
Auto Trait Implementations§
impl !Freeze for FsDataset
impl !RefUnwindSafe for FsDataset
impl !UnwindSafe for FsDataset
impl Send for FsDataset
impl Sync for FsDataset
impl Unpin for FsDataset
impl UnsafeUnpin for FsDataset
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<D> DatasetExt for D
impl<D> DatasetExt for D
Source§fn push<'life0, 'life1, 'async_trait, T>(
&'life0 self,
item: &'life1 T,
) -> Pin<Box<dyn Future<Output = Result<(), StorageError>> + Send + 'async_trait>>
fn push<'life0, 'life1, 'async_trait, T>( &'life0 self, item: &'life1 T, ) -> Pin<Box<dyn Future<Output = Result<(), StorageError>> + Send + 'async_trait>>
Serializes and appends one item.
Source§fn push_batch<'life0, 'life1, 'async_trait, T>(
&'life0 self,
items: &'life1 [T],
) -> Pin<Box<dyn Future<Output = Result<(), StorageError>> + Send + 'async_trait>>
fn push_batch<'life0, 'life1, 'async_trait, T>( &'life0 self, items: &'life1 [T], ) -> Pin<Box<dyn Future<Output = Result<(), StorageError>> + Send + 'async_trait>>
Serializes and appends a batch of items.
Source§fn list<'life0, 'async_trait, T>(
&'life0 self,
opts: ListOptions,
) -> Pin<Box<dyn Future<Output = Result<Page<T>, StorageError>> + Send + 'async_trait>>where
'life0: 'async_trait,
T: 'async_trait + DeserializeOwned,
Self: Sync + 'async_trait,
fn list<'life0, 'async_trait, T>(
&'life0 self,
opts: ListOptions,
) -> Pin<Box<dyn Future<Output = Result<Page<T>, StorageError>> + Send + 'async_trait>>where
'life0: 'async_trait,
T: 'async_trait + DeserializeOwned,
Self: Sync + 'async_trait,
Lists and deserializes typed items.
Source§fn stream<T>(
&self,
opts: ListOptions,
) -> Pin<Box<dyn Stream<Item = Result<T, StorageError>> + Send + '_>>where
T: DeserializeOwned + Send + 'static,
fn stream<T>(
&self,
opts: ListOptions,
) -> Pin<Box<dyn Stream<Item = Result<T, StorageError>> + Send + '_>>where
T: DeserializeOwned + Send + 'static,
Streams and deserializes typed items.