Struct RestStreamConfig 

pub struct RestStreamConfig {

    pub base_url: String,
    pub path: String,
    pub method: Method,
    pub auth: AuthSpec<Auth>,
    pub headers: HeaderMap,
    pub query_params: HashMap<String, String>,
    pub body: Option<Value>,
    pub pagination: PaginationStyle,
    pub records_path: Option<String>,
    pub max_pages: Option<usize>,
    pub request_delay: Option<Duration>,
    pub timeout: Option<Duration>,
    pub max_retries: u32,
    pub retry_backoff: Duration,
    pub tolerated_http_errors: Vec<u16>,
    pub replication_method: ReplicationMethod,
    pub replication_key: Option<String>,
    pub start_replication_value: Option<Value>,
    pub state_key: Option<String>,
    pub name: Option<String>,
    pub primary_keys: Vec<String>,
    pub schema: Option<Value>,
    pub schema_sample_size: usize,
    pub partitions: Vec<HashMap<String, Value>>,
    pub partition_concurrency: Option<usize>,
}

Available on crate feature source-rest only.

Configuration for a RestStream.

Fields

base_url: String

path: String

URL path, relative to base_url. May contain {key} placeholders that are substituted per-partition (e.g. "/orgs/{org_id}/users").

method: Method

auth: AuthSpec<Auth>

Authentication: either inline ({ type, config }) or a { ref: <name> } pointer to a shared provider in the CLI’s top-level auth: catalog.

headers: HeaderMap

query_params: HashMap<String, String>

body: Option<Value>

pagination: PaginationStyle

records_path: Option<String>

max_pages: Option<usize>

request_delay: Option<Duration>

timeout: Option<Duration>

max_retries: u32

retry_backoff: Duration

tolerated_http_errors: Vec<u16>

HTTP status codes that should not cause an error. Responses with these codes are treated as empty pages (no records, no further pages).

replication_method: ReplicationMethod

replication_key: Option<String>

Field name (not a JSONPath) used for incremental replication bookmarking.

start_replication_value: Option<Value>

Bookmark value: records where record[replication_key] <= start_replication_value are filtered out when replication_method is Incremental.

state_key: Option<String>

Opt-in identifier used by Pipeline::with_state_store to persist this stream’s bookmark across runs. When set, the pipeline will load any previously-stored bookmark before fetching and write the new bookmark only after the sink confirms the batch.

Keys must satisfy faucet_core::state::validate_state_key.

name: Option<String>

Human-readable stream name (used in logging and Singer SCHEMA messages).

primary_keys: Vec<String>

Field names that uniquely identify a record (Singer key_properties).

schema: Option<Value>

JSON Schema describing the structure of each record.

schema_sample_size: usize

Maximum number of records to sample when inferring the schema via crate::stream::RestStream::infer_schema. 0 means sample all available records (up to max_pages). Defaults to 100.

partitions: Vec<HashMap<String, Value>>

Each entry is a context map whose values are substituted into path placeholders. The stream is executed once per partition and results are concatenated. Empty means run once with no substitution.

partition_concurrency: Option<usize>

Maximum number of partitions to fetch concurrently. None means sequential processing (backward compatible default).

Implementations

impl RestStreamConfig

pub fn new(base_url: &str, path: &str) -> RestStreamConfig

pub fn method(self, m: Method) -> RestStreamConfig

pub fn auth(self, a: Auth) -> RestStreamConfig

pub fn header(self, k: &str, v: &str) -> RestStreamConfig

pub fn query(self, k: &str, v: &str) -> RestStreamConfig

pub fn body(self, b: Value) -> RestStreamConfig

pub fn pagination(self, p: PaginationStyle) -> RestStreamConfig

pub fn records_path(self, p: &str) -> RestStreamConfig

pub fn max_pages(self, n: usize) -> RestStreamConfig

pub fn request_delay(self, d: Duration) -> RestStreamConfig

pub fn timeout(self, d: Duration) -> RestStreamConfig

pub fn max_retries(self, n: u32) -> RestStreamConfig

pub fn retry_backoff(self, d: Duration) -> RestStreamConfig

pub fn tolerate_http_error(self, status: u16) -> RestStreamConfig

HTTP status codes that should be silently ignored (treated as empty pages).

pub fn replication_method(self, m: ReplicationMethod) -> RestStreamConfig

pub fn replication_key(self, key: &str) -> RestStreamConfig

Field name (not JSONPath) used as the incremental replication bookmark.

pub fn start_replication_value(self, v: Value) -> RestStreamConfig

Bookmark start value: records at or before this value are filtered out when using ReplicationMethod::Incremental.

pub fn state_key(self, key: &str) -> RestStreamConfig

Opt the stream into resumable runs by giving it a stable state key. When this is set and the Pipeline is configured with a state store, the previously persisted bookmark is applied to the stream before fetching.

pub fn name(self, n: &str) -> RestStreamConfig

Human-readable stream name.

pub fn primary_keys(self, keys: Vec<String>) -> RestStreamConfig

Field names that uniquely identify a record (Singer key_properties).

pub fn schema(self, s: Value) -> RestStreamConfig

JSON Schema for the stream’s records.

pub fn schema_sample_size(self, n: usize) -> RestStreamConfig

Maximum records to sample for schema inference (0 = unlimited).

pub fn add_partition(self, ctx: HashMap<String, Value>) -> RestStreamConfig

Add a partition context. The stream will execute once for each partition, substituting {key} placeholders in path with values from the context.

pub fn partition_concurrency( self, concurrency: Option<usize>, ) -> RestStreamConfig

Set the maximum number of partitions to fetch concurrently. None (default) means sequential processing.

Trait Implementations

impl Clone for RestStreamConfig

fn clone(&self) -> RestStreamConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for RestStreamConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Default for RestStreamConfig

fn default() -> RestStreamConfig

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for RestStreamConfig

fn deserialize<__D>( __deserializer: __D, ) -> Result<RestStreamConfig, <__D as Deserializer<'de>>::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl JsonSchema for RestStreamConfig

fn schema_name() -> Cow<'static, str>

The name of the generated JSON Schema.

fn schema_id() -> Cow<'static, str>

Returns a string that uniquely identifies the schema produced by this type.

fn json_schema(generator: &mut SchemaGenerator) -> Schema

Generates a JSON Schema for this type.

fn inline_schema() -> bool

Whether JSON Schemas generated for this type should be included directly in parent schemas, rather than being re-used where possible using the $ref keyword.

impl Serialize for RestStreamConfig

fn serialize<__S>( &self, __serializer: __S, ) -> Result<<__S as Serializer>::Ok, <__S as Serializer>::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.