{
"openapi": "3.0.3",
"info": {
"title": "Feldera API",
"description": "\nWith Feldera, users create data pipelines out of SQL programs.\nA SQL program comprises tables and views, and includes as well the definition of\ninput and output connectors for each respectively. A connector defines a data\nsource or data sink to feed input data into tables or receive output data\ncomputed by the views respectively.\n\n## Pipeline\n\nThe API is centered around the **pipeline**, which most importantly consists\nout of the SQL program, but also has accompanying metadata and configuration parameters\n(e.g., compilation profile, number of workers, etc.).\n\n* A pipeline is identified and referred to by a user-provided unique name.\n* The pipeline program is asynchronously compiled when the pipeline is first created or\n its program code or configuration is updated.\n* Running the pipeline (*deployment*) is only possible once the program is compiled\n* A pipeline cannot be updated while it is running\n\n## Concurrency\n\nBoth the pipeline and its program have an associated *version*.\nA version is a monotonically increasing number.\nAnytime the core fields (name, description, runtime_config, program_code, program_config) are modified,\nthe pipeline version is incremented.\nAnytime the program core fields (program_code, program_config) are modified,\nthe program version is incremented.\nThe program version is used internally by the compiler to know when to recompile.",
"license": {
"name": "MIT OR Apache-2.0"
},
"version": "0.27.0"
},
"paths": {
"/config": {
"get": {
"tags": [
"Configuration"
],
"summary": "Retrieve general configuration.",
"operationId": "get_config",
"responses": {
"200": {
"description": "The response body contains basic configuration information about this host.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Configuration"
}
}
}
},
"500": {
"description": "Request failed.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
}
}
},
"/config/authentication": {
"get": {
"tags": [
"Configuration"
],
"summary": "Retrieve authentication provider configuration.",
"operationId": "get_config_authentication",
"responses": {
"200": {
"description": "The response body contains Authentication Provider configuration, or is empty if no auth is configured.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AuthProvider"
}
}
}
},
"500": {
"description": "Request failed.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
}
}
},
"/v0/api_keys": {
"get": {
"tags": [
"API keys"
],
"summary": "Retrieve the list of API keys.",
"operationId": "list_api_keys",
"responses": {
"200": {
"description": "API keys retrieved successfully",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiKeyDescr"
}
}
}
}
},
"500": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"post": {
"tags": [
"API keys"
],
"summary": "Create a new API key.",
"operationId": "post_api_key",
"requestBody": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/NewApiKeyRequest"
}
}
},
"required": true
},
"responses": {
"201": {
"description": "API key created successfully",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/NewApiKeyResponse"
}
}
}
},
"409": {
"description": "API key with that name already exists",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": null,
"error_code": "DuplicateName",
"message": "An entity with this name already exists"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/api_keys/{api_key_name}": {
"get": {
"tags": [
"API keys"
],
"summary": "Retrieve an API key.",
"operationId": "get_api_key",
"parameters": [
{
"name": "api_key_name",
"in": "path",
"description": "Unique API key name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "API key retrieved successfully",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ApiKeyDescr"
}
}
}
},
"404": {
"description": "API key with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"name": "unknown_api_key"
},
"error_code": "UnknownApiKey",
"message": "Unknown API key 'unknown_api_key'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"delete": {
"tags": [
"API keys"
],
"summary": "Delete an API key.",
"operationId": "delete_api_key",
"parameters": [
{
"name": "api_key_name",
"in": "path",
"description": "Unique API key name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "API key deleted successfully"
},
"404": {
"description": "API key with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"name": "unknown_api_key"
},
"error_code": "UnknownApiKey",
"message": "Unknown API key 'unknown_api_key'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/config/demos": {
"get": {
"tags": [
"Configuration"
],
"summary": "Retrieve the list of demos.",
"operationId": "get_config_demos",
"responses": {
"200": {
"description": "List of demos.",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Demo"
}
}
}
}
},
"500": {
"description": "Failed to read demos from the demos directory.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/metrics": {
"get": {
"tags": [
"Metrics"
],
"summary": "Retrieve the metrics of all running pipelines belonging to this tenant.",
"description": "The metrics are collected by making individual HTTP requests to `/metrics`\nendpoint of each pipeline, of which only successful responses are included\nin the returned list.",
"operationId": "get_metrics",
"responses": {
"200": {
"description": "Metrics of all running pipelines belonging to this tenant in Prometheus format",
"content": {
"text/plain": {
"schema": {
"type": "string",
"format": "binary"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve the list of pipelines.",
"description": "Inclusion of program code is configured with by the `code` boolean query parameter.",
"operationId": "list_pipelines",
"parameters": [
{
"name": "code",
"in": "query",
"description": "Whether to include program code in the response (default: `true`).\nPassing `false` reduces the response size, which is particularly handy\nwhen frequently monitoring the endpoint over low bandwidth connections.",
"required": false,
"schema": {
"type": "boolean"
}
}
],
"responses": {
"200": {
"description": "List of pipelines retrieved successfully",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ExtendedPipelineDescrOptionalCode"
}
},
"example": [
{
"created_at": "1970-01-01T00:00:00Z",
"deployment_config": null,
"deployment_desired_status": "Shutdown",
"deployment_error": null,
"deployment_location": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"description": "Description of the pipeline example1",
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"program_binary_url": null,
"program_code": null,
"program_config": {
"profile": "optimized"
},
"program_info": null,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_version": 2,
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 0,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": null,
"storage_class": null,
"storage_mb_max": null
},
"storage": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 16
},
"version": 4
},
{
"created_at": "1970-01-01T00:00:00Z",
"deployment_config": null,
"deployment_desired_status": "Shutdown",
"deployment_error": null,
"deployment_location": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"description": "Description of the pipeline example2",
"id": "67e55044-10b1-426f-9247-bb680e5fe0c9",
"name": "example2",
"program_binary_url": null,
"program_code": null,
"program_config": {
"profile": "unoptimized"
},
"program_info": null,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_version": 1,
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 100000,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": 1000,
"storage_class": null,
"storage_mb_max": 10000
},
"storage": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 10
},
"version": 1
}
]
}
}
},
"500": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"post": {
"tags": [
"Pipelines"
],
"summary": "Create a new pipeline.",
"operationId": "post_pipeline",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PipelineDescr"
},
"example": {
"description": "Description of the pipeline example1",
"name": "example1",
"program_code": "CREATE TABLE table1 ( col1 INT );",
"program_config": {
"profile": "optimized"
},
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 0,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": null,
"storage_class": null,
"storage_mb_max": null
},
"storage": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 16
}
}
}
},
"required": true
},
"responses": {
"201": {
"description": "Pipeline successfully created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ExtendedPipelineDescr"
},
"example": {
"created_at": "1970-01-01T00:00:00Z",
"deployment_config": null,
"deployment_desired_status": "Shutdown",
"deployment_error": null,
"deployment_location": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"description": "Description of the pipeline example1",
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"program_binary_url": null,
"program_code": "CREATE TABLE table1 ( col1 INT );",
"program_config": {
"profile": "optimized"
},
"program_info": null,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_version": 2,
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 0,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": null,
"storage_class": null,
"storage_mb_max": null
},
"storage": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 16
},
"version": 4
}
}
}
},
"409": {
"description": "Cannot create pipeline as the name already exists",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": null,
"error_code": "DuplicateName",
"message": "An entity with this name already exists"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve a pipeline.",
"operationId": "get_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline retrieved successfully",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ExtendedPipelineDescr"
},
"example": {
"created_at": "1970-01-01T00:00:00Z",
"deployment_config": null,
"deployment_desired_status": "Shutdown",
"deployment_error": null,
"deployment_location": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"description": "Description of the pipeline example1",
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"program_binary_url": null,
"program_code": "CREATE TABLE table1 ( col1 INT );",
"program_config": {
"profile": "optimized"
},
"program_info": null,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_version": 2,
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 0,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": null,
"storage_class": null,
"storage_mb_max": null
},
"storage": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 16
},
"version": 4
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"put": {
"tags": [
"Pipelines"
],
"summary": "Fully update a pipeline if it already exists, otherwise create a new pipeline.",
"operationId": "put_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PipelineDescr"
},
"example": {
"description": "Description of the pipeline example1",
"name": "example1",
"program_code": "CREATE TABLE table1 ( col1 INT );",
"program_config": {
"profile": "optimized"
},
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 0,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": null,
"storage_class": null,
"storage_mb_max": null
},
"storage": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 16
}
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Pipeline successfully updated",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ExtendedPipelineDescr"
},
"example": {
"created_at": "1970-01-01T00:00:00Z",
"deployment_config": null,
"deployment_desired_status": "Shutdown",
"deployment_error": null,
"deployment_location": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"description": "Description of the pipeline example1",
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"program_binary_url": null,
"program_code": "CREATE TABLE table1 ( col1 INT );",
"program_config": {
"profile": "optimized"
},
"program_info": null,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_version": 2,
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 0,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": null,
"storage_class": null,
"storage_mb_max": null
},
"storage": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 16
},
"version": 4
}
}
}
},
"201": {
"description": "Pipeline successfully created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ExtendedPipelineDescr"
},
"example": {
"created_at": "1970-01-01T00:00:00Z",
"deployment_config": null,
"deployment_desired_status": "Shutdown",
"deployment_error": null,
"deployment_location": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"description": "Description of the pipeline example1",
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"program_binary_url": null,
"program_code": "CREATE TABLE table1 ( col1 INT );",
"program_config": {
"profile": "optimized"
},
"program_info": null,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_version": 2,
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 0,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": null,
"storage_class": null,
"storage_mb_max": null
},
"storage": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 16
},
"version": 4
}
}
}
},
"400": {
"description": "Pipeline needs to be shutdown to be modified",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": null,
"error_code": "CannotUpdateNonShutdownPipeline",
"message": "Cannot update a pipeline which is not fully shutdown. Shutdown the pipeline first by invoking the '/shutdown' endpoint."
}
}
}
},
"409": {
"description": "Cannot rename pipeline as the name already exists",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": null,
"error_code": "DuplicateName",
"message": "An entity with this name already exists"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"delete": {
"tags": [
"Pipelines"
],
"summary": "Delete a pipeline.",
"operationId": "delete_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline successfully deleted"
},
"400": {
"description": "Pipeline needs to be shutdown to be deleted",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": null,
"error_code": "CannotDeleteNonShutdownPipeline",
"message": "Cannot delete a pipeline which is not fully shutdown. Shutdown the pipeline first by invoking the '/shutdown' endpoint."
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"patch": {
"tags": [
"Pipelines"
],
"summary": "Partially update a pipeline.",
"operationId": "patch_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PatchPipeline"
},
"example": {
"description": "This is a new description",
"name": null,
"program_code": "CREATE TABLE table3 ( col3 INT );",
"program_config": null,
"runtime_config": null
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Pipeline successfully updated",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ExtendedPipelineDescr"
},
"example": {
"created_at": "1970-01-01T00:00:00Z",
"deployment_config": null,
"deployment_desired_status": "Shutdown",
"deployment_error": null,
"deployment_location": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"description": "Description of the pipeline example1",
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"program_binary_url": null,
"program_code": "CREATE TABLE table1 ( col1 INT );",
"program_config": {
"profile": "optimized"
},
"program_info": null,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_version": 2,
"runtime_config": {
"clock_resolution_usecs": 100000,
"cpu_profiler": false,
"max_buffering_delay_usecs": 0,
"min_batch_size_records": 0,
"min_storage_bytes": null,
"resources": {
"cpu_cores_max": null,
"cpu_cores_min": null,
"memory_mb_max": null,
"memory_mb_min": null,
"storage_class": null,
"storage_mb_max": null
},
"storage": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"workers": 16
},
"version": 4
}
}
}
},
"400": {
"description": "Pipeline needs to be shutdown to be modified",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": null,
"error_code": "CannotUpdateNonShutdownPipeline",
"message": "Cannot update a pipeline which is not fully shutdown. Shutdown the pipeline first by invoking the '/shutdown' endpoint."
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
},
"409": {
"description": "Cannot rename pipeline as the name already exists",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": null,
"error_code": "DuplicateName",
"message": "An entity with this name already exists"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/circuit_profile": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve the circuit performance profile of a running or paused pipeline.",
"operationId": "get_pipeline_circuit_profile",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Obtains a circuit performance profile.",
"content": {
"application/zip": {
"schema": {
"type": "object"
}
}
}
},
"400": {
"description": "Pipeline is not running or paused",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
},
"error_code": "PipelineNotRunningOrPaused",
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused."
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/egress/{table_name}": {
"post": {
"tags": [
"HTTP input/output"
],
"summary": "Subscribe to a stream of updates from a SQL view or table.",
"description": "The pipeline responds with a continuous stream of changes to the specified\ntable or view, encoded using the format specified in the `?format=`\nparameter. Updates are split into `Chunk`s.\n\nThe pipeline continues sending updates until the client closes the\nconnection or the pipeline is shut down.",
"operationId": "http_output",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "table_name",
"in": "path",
"description": "SQL table name. Unquoted SQL names have to be capitalized. Quoted SQL names have to exactly match the case from the SQL program.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "format",
"in": "query",
"description": "Output data format, e.g., 'csv' or 'json'.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "array",
"in": "query",
"description": "Set to `true` to group updates in this stream into JSON arrays (used in conjunction with `format=json`). The default value is `false`",
"required": false,
"schema": {
"type": "boolean",
"nullable": true
}
},
{
"name": "backpressure",
"in": "query",
"description": "Apply backpressure on the pipeline when the HTTP client cannot receive data fast enough.\n When this flag is set to false (the default), the HTTP connector drops data chunks if the client is not keeping up with its output. This prevents a slow HTTP client from slowing down the entire pipeline.\n When the flag is set to true, the connector waits for the client to receive each chunk and blocks the pipeline if the client cannot keep up.",
"required": false,
"schema": {
"type": "boolean",
"nullable": true
}
}
],
"responses": {
"200": {
"description": "Connection to the endpoint successfully established. The body of the response contains a stream of data chunks.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Chunk"
}
}
}
},
"400": {
"description": "Unknown data format specified in the '?format=' argument.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"404": {
"description": "Specified table or view does not exist.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"410": {
"description": "Pipeline is not currently running because it has been shutdown or not yet started.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
},
"error_code": "PipelineNotRunningOrPaused",
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused."
}
}
}
},
"500": {
"description": "Request failed.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/heap_profile": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve the heap profile of a running or paused pipeline.",
"operationId": "get_pipeline_heap_profile",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline's heap usage profile as a gzipped protobuf that can be inspected by the pprof tool",
"content": {
"application/protobuf": {
"schema": {
"type": "string",
"format": "binary"
}
}
}
},
"400": {
"description": "Pipeline is not running or paused, or getting a heap profile is not supported on this platform",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
},
"error_code": "PipelineNotRunningOrPaused",
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused."
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/ingress/{table_name}": {
"post": {
"tags": [
"HTTP input/output"
],
"summary": "Push data to a SQL table.",
"description": "The client sends data encoded using the format specified in the `?format=`\nparameter as a body of the request. The contents of the data must match\nthe SQL table schema specified in `table_name`\n\nThe pipeline ingests data as it arrives without waiting for the end of\nthe request. Successful HTTP response indicates that all data has been\ningested successfully.",
"operationId": "http_input",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "table_name",
"in": "path",
"description": "SQL table name. Unquoted SQL names have to be capitalized. Quoted SQL names have to exactly match the case from the SQL program.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "force",
"in": "query",
"description": "When `true`, push data to the pipeline even if the pipeline is paused. The default value is `false`",
"required": true,
"schema": {
"type": "boolean"
}
},
{
"name": "format",
"in": "query",
"description": "Input data format, e.g., 'csv' or 'json'.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "array",
"in": "query",
"description": "Set to `true` if updates in this stream are packaged into JSON arrays (used in conjunction with `format=json`). The default values is `false`.",
"required": false,
"schema": {
"type": "boolean",
"nullable": true
}
},
{
"name": "update_format",
"in": "query",
"description": "JSON data change event format (used in conjunction with `format=json`). The default value is 'insert_delete'.",
"required": false,
"schema": {
"allOf": [
{
"$ref": "#/components/schemas/JsonUpdateFormat"
}
],
"nullable": true
}
}
],
"requestBody": {
"description": "Contains the new input data in CSV.",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Data successfully delivered to the pipeline."
},
"400": {
"description": "Error parsing input data.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"404": {
"description": "Pipeline is not currently running because it has been shutdown or not yet started.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
},
"error_code": "PipelineNotRunningOrPaused",
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused."
}
}
}
},
"500": {
"description": "Request failed.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/input_endpoints/{endpoint_name}/{action}": {
"post": {
"tags": [
"Pipelines"
],
"summary": "Change the desired state of an input endpoint.",
"description": "The following values of the `action` argument are accepted by this endpoint:\n\n- 'start': Start processing data.\n- 'pause': Pause the pipeline.",
"operationId": "input_endpoint_action",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "endpoint_name",
"in": "path",
"description": "Input endpoint name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "action",
"in": "path",
"description": "Endpoint action [start, pause]",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"202": {
"description": "Request accepted."
},
"404": {
"description": "Specified endpoint does not exist.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/logs": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve pipeline logs as a stream.",
"description": "The logs stream catches up to the extent of the internally configured per-pipeline\ncircular logs buffer (limited to a certain byte size and number of lines, whichever\nis reached first). After the catch-up, new lines are pushed whenever they become\navailable.\n\nThe logs stream will end when the pipeline is shut down. It is also possible for the\nlogs stream to end prematurely due to the runner back-end (temporarily) losing\nconnectivity to the pipeline instance (e.g., process). In this case, it is needed\nto issue again a new request to this endpoint.",
"operationId": "get_pipeline_logs",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline logs retrieved successfully",
"content": {
"text/plain": {
"schema": {
"type": "string",
"format": "binary"
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/query": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Execute an ad-hoc query in a running or paused pipeline.",
"operationId": "pipeline_adhoc_sql",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "sql",
"in": "query",
"description": "The SQL query to execute.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "format",
"in": "query",
"description": "Input data format, e.g., 'text', 'json' or 'parquet'.",
"required": true,
"schema": {
"$ref": "#/components/schemas/AdHocResultFormat"
}
}
],
"responses": {
"200": {
"description": "Executes an ad-hoc SQL query in a running or paused pipeline. The evaluation is not incremental.",
"content": {
"text/plain": {
"schema": {
"type": "string",
"format": "binary"
}
}
}
},
"400": {
"description": "Pipeline is shutdown or an invalid SQL query was supplied",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
},
"error_code": "PipelineNotRunningOrPaused",
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused."
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
},
"500": {
"description": "A fatal error occurred during query processing (after streaming response was already initiated)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"error": "Failed to connect to host: Internal error: connector has been disconnected",
"pipeline_id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"pipeline_name": "example",
"url": "https://localhost:1234/query"
},
"error_code": "PipelineEndpointSendError",
"message": "Sending request to URL https://localhost:1234/query of pipeline example (67e55044-10b1-426f-9247-bb680e5fe0c8) failed: Failed to connect to host: Internal error: connector has been disconnected"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/stats": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve pipeline statistics (e.g., metrics, performance counters).",
"operationId": "get_pipeline_stats",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline metrics retrieved successfully",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
},
"400": {
"description": "Pipeline is not running or paused",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
},
"error_code": "PipelineNotRunningOrPaused",
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused."
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/{action}": {
"post": {
"tags": [
"Pipelines"
],
"summary": "Start, pause or shutdown a pipeline.",
"description": "The endpoint returns immediately after performing initial request validation\n(e.g., upon start checking the program is compiled) and initiating the relevant\nprocedure (e.g., informing the runner or the already running pipeline).\nThe state changes completely asynchronously. On error, the pipeline\ntransitions to the `Failed` state. The user can monitor the current status\nof the pipeline by polling the `GET /pipelines` and\n`GET /pipelines/{pipeline_name}` endpoint.\n\nThe following values of the `action` argument are accepted:\n- `start`: Start the pipeline\n- `pause`: Pause the pipeline\n- `shutdown`: Terminate the pipeline",
"operationId": "post_pipeline_action",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "action",
"in": "path",
"description": "Pipeline action (one of: start, pause, shutdown)",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"202": {
"description": "Action accepted and is being performed"
},
"400": {
"description": "Unable to accept action",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"examples": {
"Illegal action": {
"description": "Action is not applicable in the current state",
"value": {
"details": {
"desired_status": "Shutdown",
"hint": "Cannot restart the pipeline while it is shutting down. Wait for the shutdown to complete before starting a new instance of the pipeline.",
"requested_desired_status": "Running",
"status": "ShuttingDown"
},
"error_code": "IllegalPipelineStateTransition",
"message": "Deployment status (current: 'ShuttingDown', desired: 'Shutdown') cannot have desired changed to 'Running'. Cannot restart the pipeline while it is shutting down. Wait for the shutdown to complete before starting a new instance of the pipeline."
}
},
"Invalid action": {
"description": "Invalid action specified",
"value": {
"details": {
"action": "dance"
},
"error_code": "InvalidPipelineAction",
"message": "Invalid pipeline action 'dance'; valid actions are: 'deploy', 'start', 'pause', or 'shutdown'"
}
},
"Program failed compilation": {
"description": "Program failed compilation",
"value": {
"details": null,
"error_code": "ProgramFailedToCompile",
"message": "The program did not compile successfully"
}
},
"Program not (yet) compiled": {
"description": "Program has not (yet) been compiled",
"value": {
"details": null,
"error_code": "ProgramNotYetCompiled",
"message": "The program hasn't been compiled yet"
}
}
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
},
"error_code": "UnknownPipeline",
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
}
},
"components": {
"schemas": {
"AdHocResultFormat": {
"type": "string",
"description": "URL-encoded `format` argument to the `/query` endpoint.",
"enum": [
"text",
"json",
"parquet"
]
},
"AdhocQueryArgs": {
"type": "object",
"description": "URL-encoded arguments to the `/query` endpoint.",
"required": [
"sql"
],
"properties": {
"format": {
"$ref": "#/components/schemas/AdHocResultFormat"
},
"sql": {
"type": "string",
"description": "The SQL query to run."
}
}
},
"ApiKeyDescr": {
"type": "object",
"description": "API key descriptor.",
"required": [
"id",
"name",
"scopes"
],
"properties": {
"id": {
"$ref": "#/components/schemas/ApiKeyId"
},
"name": {
"type": "string"
},
"scopes": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiPermission"
}
}
}
},
"ApiKeyId": {
"type": "string",
"format": "uuid",
"description": "API key identifier."
},
"ApiPermission": {
"type": "string",
"description": "Permission types for invoking API endpoints.",
"enum": [
"Read",
"Write"
]
},
"AuthProvider": {
"oneOf": [
{
"type": "object",
"required": [
"AwsCognito"
],
"properties": {
"AwsCognito": {
"$ref": "#/components/schemas/ProviderAwsCognito"
}
}
},
{
"type": "object",
"required": [
"GoogleIdentity"
],
"properties": {
"GoogleIdentity": {
"$ref": "#/components/schemas/ProviderGoogleIdentity"
}
}
}
]
},
"Chunk": {
"type": "object",
"description": "A set of updates to a SQL table or view.\n\nThe `sequence_number` field stores the offset of the chunk relative to the\nstart of the stream and can be used to implement reliable delivery.\nThe payload is stored in the `bin_data`, `text_data`, or `json_data` field\ndepending on the data format used.",
"required": [
"sequence_number"
],
"properties": {
"bin_data": {
"type": "string",
"format": "binary",
"description": "Base64 encoded binary payload, e.g., bincode.",
"nullable": true
},
"json_data": {
"type": "object",
"description": "JSON payload.",
"nullable": true
},
"sequence_number": {
"type": "integer",
"format": "int64",
"minimum": 0
},
"text_data": {
"type": "string",
"description": "Text payload, e.g., CSV.",
"nullable": true
}
}
},
"ColumnType": {
"type": "object",
"description": "A SQL column type description.\n\nMatches the Calcite JSON format.",
"required": [
"nullable"
],
"properties": {
"component": {
"allOf": [
{
"$ref": "#/components/schemas/ColumnType"
}
],
"nullable": true
},
"fields": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Field"
},
"description": "The fields of the type (if available).\n\nFor example this would specify the fields of a `CREATE TYPE` construct.\n\n```sql\nCREATE TYPE person_typ AS (\nfirstname VARCHAR(30),\nlastname VARCHAR(30),\naddress ADDRESS_TYP\n);\n```\n\nWould lead to the following `fields` value:\n\n```sql\n[\nColumnType { name: \"firstname, ... },\nColumnType { name: \"lastname\", ... },\nColumnType { name: \"address\", fields: [ ... ] }\n]\n```",
"nullable": true
},
"key": {
"allOf": [
{
"$ref": "#/components/schemas/ColumnType"
}
],
"nullable": true
},
"nullable": {
"type": "boolean",
"description": "Does the type accept NULL values?"
},
"precision": {
"type": "integer",
"format": "int64",
"description": "Precision of the type.\n\n# Examples\n- `VARCHAR` sets precision to `-1`.\n- `VARCHAR(255)` sets precision to `255`.\n- `BIGINT`, `DATE`, `FLOAT`, `DOUBLE`, `GEOMETRY`, etc. sets precision\nto None\n- `TIME`, `TIMESTAMP` set precision to `0`.",
"nullable": true
},
"scale": {
"type": "integer",
"format": "int64",
"description": "The scale of the type.\n\n# Example\n- `DECIMAL(1,2)` sets scale to `2`.",
"nullable": true
},
"type": {
"$ref": "#/components/schemas/SqlType"
},
"value": {
"allOf": [
{
"$ref": "#/components/schemas/ColumnType"
}
],
"nullable": true
}
}
},
"CompilationProfile": {
"type": "string",
"description": "Enumeration of possible compilation profiles that can be passed to the Rust compiler\nas an argument via `cargo build --profile <>`. A compilation profile affects among\nother things the compilation speed (how long till the program is ready to be run)\nand runtime speed (the performance while running).",
"enum": [
"dev",
"unoptimized",
"optimized"
]
},
"Configuration": {
"type": "object",
"required": [
"telemetry",
"version"
],
"properties": {
"telemetry": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"ConnectorConfig": {
"allOf": [
{
"$ref": "#/components/schemas/OutputBufferConfig"
},
{
"type": "object",
"required": [
"transport"
],
"properties": {
"format": {
"allOf": [
{
"$ref": "#/components/schemas/FormatConfig"
}
],
"nullable": true
},
"max_batch_size": {
"type": "integer",
"format": "int64",
"description": "Maximum batch size, in records.\n\nThis is the maximum number of records to process in one batch through\nthe circuit. The time and space cost of processing a batch is\nasymptotically superlinear in the size of the batch, but very small\nbatches are less efficient due to constant factors.\n\nThis should usually be less than `max_queued_records`, to give the\nconnector a round-trip time to restart and refill the buffer while\nbatches are being processed.\n\nSome input adapters might not honor this setting.\n\nThe default is 10,000.",
"minimum": 0
},
"max_queued_records": {
"type": "integer",
"format": "int64",
"description": "Backpressure threshold.\n\nMaximal number of records queued by the endpoint before the endpoint\nis paused by the backpressure mechanism.\n\nFor input endpoints, this setting bounds the number of records that have\nbeen received from the input transport but haven't yet been consumed by\nthe circuit since the circuit, since the circuit is still busy processing\nprevious inputs.\n\nFor output endpoints, this setting bounds the number of records that have\nbeen produced by the circuit but not yet sent via the output transport endpoint\nnor stored in the output buffer (see `enable_output_buffer`).\n\nNote that this is not a hard bound: there can be a small delay between\nthe backpressure mechanism is triggered and the endpoint is paused, during\nwhich more data may be queued.\n\nThe default is 1 million.",
"minimum": 0
},
"paused": {
"type": "boolean",
"description": "Create connector in paused state.\n\nThe default is `false`."
},
"transport": {
"$ref": "#/components/schemas/TransportConfig"
}
}
}
],
"description": "A data connector's configuration"
},
"DatagenInputConfig": {
"type": "object",
"description": "Configuration for generating random data for a table.",
"properties": {
"plan": {
"type": "array",
"items": {
"$ref": "#/components/schemas/GenerationPlan"
},
"description": "The sequence of generations to perform.\n\nIf not set, the generator will produce a single sequence with default settings.\nIf set, the generator will produce the specified sequences in sequential order.\n\nNote that if one of the sequences before the last one generates an unlimited number of rows\nthe following sequences will not be executed."
},
"seed": {
"type": "integer",
"format": "int64",
"description": "Optional seed for the random generator.\n\nSetting this to a fixed value will make the generator produce the same sequence of records\nevery time the pipeline is run.\n\n# Notes\n- To ensure the set of generated input records is deterministic across multiple runs,\napart from setting a seed, `workers` also needs to remain unchanged.\n- The input will arrive in non-deterministic order if `workers > 1`.",
"nullable": true,
"minimum": 0
},
"workers": {
"type": "integer",
"description": "Number of workers to use for generating data.",
"minimum": 0
}
},
"additionalProperties": false
},
"DatagenStrategy": {
"type": "string",
"description": "Strategy used to generate values.",
"enum": [
"increment",
"uniform",
"zipf",
"word",
"words",
"sentence",
"sentences",
"paragraph",
"paragraphs",
"first_name",
"last_name",
"title",
"suffix",
"name",
"name_with_title",
"domain_suffix",
"email",
"username",
"password",
"field",
"position",
"seniority",
"job_title",
"ipv4",
"ipv6",
"ip",
"mac_address",
"user_agent",
"rfc_status_code",
"valid_status_code",
"company_suffix",
"company_name",
"buzzword",
"buzzword_middle",
"buzzword_tail",
"catch_phrase",
"bs_verb",
"bs_adj",
"bs_noun",
"bs",
"profession",
"industry",
"currency_code",
"currency_name",
"currency_symbol",
"credit_card_number",
"city_prefix",
"city_suffix",
"city_name",
"country_name",
"country_code",
"street_suffix",
"street_name",
"time_zone",
"state_name",
"state_abbr",
"secondary_address_type",
"secondary_address",
"zip_code",
"post_code",
"building_number",
"latitude",
"longitude",
"isbn",
"isbn13",
"isbn10",
"phone_number",
"cell_number",
"file_path",
"file_name",
"file_extension",
"dir_path"
]
},
"DeltaTableIngestMode": {
"type": "string",
"description": "Delta table read mode.\n\nThree options are available:\n\n* `snapshot` - read a snapshot of the table and stop.\n\n* `follow` - continuously ingest changes to the table, starting from a specified version\nor timestamp.\n\n* `snapshot_and_follow` - read a snapshot of the table before switching to continuous ingestion\nmode.",
"enum": [
"snapshot",
"follow",
"snapshot_and_follow"
]
},
"DeltaTableReaderConfig": {
"type": "object",
"description": "Delta table output connector configuration.",
"required": [
"uri",
"mode"
],
"properties": {
"datetime": {
"type": "string",
"description": "Optional timestamp for the snapshot in the ISO-8601/RFC-3339 format, e.g.,\n\"2024-12-09T16:09:53+00:00.\n\nWhen this option is set, the connector finds and opens the version of the table as of the\nspecified point in time. In `snapshot` and `snapshot_and_follow` modes, it retrieves the\nsnapshot of this version of the table (based on the server time recorded in the transaction\nlog, not the event time encoded in the data). In `follow` and `snapshot_and_follow` modes, it\nfollows transaction log records **after** this version.\n\nNote: at most one of `version` and `datetime` options can be specified.\nWhen neither of the two options is specified, the latest committed version of the table\nis used.",
"nullable": true
},
"mode": {
"$ref": "#/components/schemas/DeltaTableIngestMode"
},
"snapshot_filter": {
"type": "string",
"description": "Optional row filter.\n\nThis option is only valid when `mode` is set to `snapshot` or `snapshot_and_follow`.\n\nWhen specified, only rows that satisfy the filter condition are included in the\nsnapshot. The condition must be a valid SQL Boolean expression that can be used in\nthe `where` clause of the `select * from snapshot where ...` query.\n\nThis option can be used to specify the range of event times to include in the snapshot,\ne.g.: `ts BETWEEN '2005-01-01 00:00:00' AND '2010-12-31 23:59:59'`.",
"nullable": true
},
"timestamp_column": {
"type": "string",
"description": "Table column that serves as an event timestamp.\n\n\nWhen this option is specified, and `mode` is one of `snapshot` or `snapshot_and_follow`,\nthe snapshot of the table is ingested in the timestamp order. This setting is required\nfor tables declared with the\n[`LATENESS`](https://docs.feldera.com/sql/streaming#lateness-expressions) attribute\nin Feldera SQL. It impacts the performance of the connector, since data must be sorted\nbefore pushing it to the pipeline; therefore it is not recommended to use this\nsettings for tables without `LATENESS`.",
"nullable": true
},
"uri": {
"type": "string",
"description": "Table URI.\n\nExample: \"s3://feldera-fraud-detection-data/demographics_train\""
},
"version": {
"type": "integer",
"format": "int64",
"description": "Optional table version.\n\nWhen this option is set, the connector finds and opens the specified version of the table.\nIn `snapshot` and `snapshot_and_follow` modes, it retrieves the snapshot of this version of\nthe table. In `follow` and `snapshot_and_follow` modes, it follows transaction log records\n**after** this version.\n\nNote: at most one of `version` and `datetime` options can be specified.\nWhen neither of the two options is specified, the latest committed version of the table\nis used.",
"nullable": true
}
},
"additionalProperties": {
"type": "string",
"description": "Storage options for configuring backend object store.\n\nFor specific options available for different storage backends, see:\n* [Azure options](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)\n* [Amazon S3 options](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)\n* [Google Cloud Storage options](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)"
}
},
"DeltaTableWriteMode": {
"type": "string",
"description": "Delta table write mode.\n\nDetermines how the Delta table connector handles an existing table at the target location.",
"enum": [
"append",
"truncate",
"error_if_exists"
]
},
"DeltaTableWriterConfig": {
"type": "object",
"description": "Delta table output connector configuration.",
"required": [
"uri"
],
"properties": {
"mode": {
"$ref": "#/components/schemas/DeltaTableWriteMode"
},
"uri": {
"type": "string",
"description": "Table URI."
}
},
"additionalProperties": {
"type": "string",
"description": "Storage options for configuring backend object store.\n\nFor specific options available for different storage backends, see:\n* [Azure options](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)\n* [Amazon S3 options](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)\n* [Google Cloud Storage options](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)"
}
},
"Demo": {
"type": "object",
"required": [
"title",
"pipeline"
],
"properties": {
"pipeline": {
"$ref": "#/components/schemas/PipelineDescr"
},
"title": {
"type": "string",
"description": "Demo title."
}
}
},
"ErrorResponse": {
"type": "object",
"description": "Information returned by REST API endpoints on error.",
"required": [
"message",
"error_code",
"details"
],
"properties": {
"details": {
"type": "object",
"description": "Detailed error metadata.\nThe contents of this field is determined by `error_code`."
},
"error_code": {
"type": "string",
"description": "Error code is a string that specifies this error type.",
"example": "UnknownInputFormat"
},
"message": {
"type": "string",
"description": "Human-readable error message.",
"example": "Unknown input format 'xml'."
}
}
},
"ExtendedPipelineDescr": {
"type": "object",
"description": "Pipeline descriptor which besides the basic fields in direct regular control of the user\nalso has all additional fields generated and maintained by the back-end.",
"required": [
"id",
"name",
"description",
"version",
"created_at",
"runtime_config",
"program_code",
"program_config",
"program_version",
"program_status",
"program_status_since",
"deployment_status",
"deployment_status_since",
"deployment_desired_status"
],
"properties": {
"created_at": {
"type": "string",
"format": "date-time",
"description": "Timestamp when the pipeline was originally created."
},
"deployment_config": {
"allOf": [
{
"$ref": "#/components/schemas/PipelineConfig"
}
],
"nullable": true
},
"deployment_desired_status": {
"$ref": "#/components/schemas/PipelineDesiredStatus"
},
"deployment_error": {
"allOf": [
{
"$ref": "#/components/schemas/ErrorResponse"
}
],
"nullable": true
},
"deployment_location": {
"type": "string",
"description": "Location where the pipeline can be reached at runtime\n(e.g., a TCP port number or a URI).",
"nullable": true
},
"deployment_status": {
"$ref": "#/components/schemas/PipelineStatus"
},
"deployment_status_since": {
"type": "string",
"format": "date-time",
"description": "Time when the pipeline was assigned its current status\nof the pipeline."
},
"description": {
"type": "string",
"description": "Pipeline description."
},
"id": {
"$ref": "#/components/schemas/PipelineId"
},
"name": {
"type": "string",
"description": "Pipeline name."
},
"program_binary_url": {
"type": "string",
"description": "URL where to download the program binary from.",
"nullable": true
},
"program_code": {
"type": "string",
"description": "Program SQL code."
},
"program_config": {
"$ref": "#/components/schemas/ProgramConfig"
},
"program_info": {
"allOf": [
{
"$ref": "#/components/schemas/ProgramInfo"
}
],
"nullable": true
},
"program_status": {
"$ref": "#/components/schemas/ProgramStatus"
},
"program_status_since": {
"type": "string",
"format": "date-time",
"description": "Timestamp when the current program status was set."
},
"program_version": {
"$ref": "#/components/schemas/Version"
},
"runtime_config": {
"$ref": "#/components/schemas/RuntimeConfig"
},
"version": {
"$ref": "#/components/schemas/Version"
}
}
},
"ExtendedPipelineDescrOptionalCode": {
"type": "object",
"description": "Extended pipeline descriptor with code being optionally included.",
"required": [
"id",
"name",
"description",
"version",
"created_at",
"runtime_config",
"program_config",
"program_version",
"program_status",
"program_status_since",
"deployment_status",
"deployment_status_since",
"deployment_desired_status"
],
"properties": {
"created_at": {
"type": "string",
"format": "date-time"
},
"deployment_config": {
"allOf": [
{
"$ref": "#/components/schemas/PipelineConfig"
}
],
"nullable": true
},
"deployment_desired_status": {
"$ref": "#/components/schemas/PipelineDesiredStatus"
},
"deployment_error": {
"allOf": [
{
"$ref": "#/components/schemas/ErrorResponse"
}
],
"nullable": true
},
"deployment_location": {
"type": "string",
"nullable": true
},
"deployment_status": {
"$ref": "#/components/schemas/PipelineStatus"
},
"deployment_status_since": {
"type": "string",
"format": "date-time"
},
"description": {
"type": "string"
},
"id": {
"$ref": "#/components/schemas/PipelineId"
},
"name": {
"type": "string"
},
"program_binary_url": {
"type": "string",
"nullable": true
},
"program_code": {
"type": "string",
"nullable": true
},
"program_config": {
"$ref": "#/components/schemas/ProgramConfig"
},
"program_info": {
"allOf": [
{
"$ref": "#/components/schemas/ProgramInfo"
}
],
"nullable": true
},
"program_status": {
"$ref": "#/components/schemas/ProgramStatus"
},
"program_status_since": {
"type": "string",
"format": "date-time"
},
"program_version": {
"$ref": "#/components/schemas/Version"
},
"runtime_config": {
"$ref": "#/components/schemas/RuntimeConfig"
},
"version": {
"$ref": "#/components/schemas/Version"
}
}
},
"Field": {
"allOf": [
{
"$ref": "#/components/schemas/SqlIdentifier"
},
{
"type": "object",
"required": [
"columntype"
],
"properties": {
"columntype": {
"$ref": "#/components/schemas/ColumnType"
}
}
}
],
"description": "A SQL field.\n\nMatches the SQL compiler JSON format."
},
"FileInputConfig": {
"type": "object",
"description": "Configuration for reading data from a file with `FileInputTransport`",
"required": [
"path"
],
"properties": {
"buffer_size_bytes": {
"type": "integer",
"description": "Read buffer size.\n\nDefault: when this parameter is not specified, a platform-specific\ndefault is used.",
"nullable": true,
"minimum": 0
},
"follow": {
"type": "boolean",
"description": "Enable file following.\n\nWhen `false`, the endpoint outputs an `InputConsumer::eoi`\nmessage and stops upon reaching the end of file. When `true`, the\nendpoint will keep watching the file and outputting any new content\nappended to it."
},
"path": {
"type": "string",
"description": "File path."
}
}
},
"FileOutputConfig": {
"type": "object",
"description": "Configuration for writing data to a file with `FileOutputTransport`.",
"required": [
"path"
],
"properties": {
"path": {
"type": "string",
"description": "File path."
}
}
},
"FormatConfig": {
"type": "object",
"description": "Data format specification used to parse raw data received from the\nendpoint or to encode data sent to the endpoint.",
"required": [
"name"
],
"properties": {
"config": {
"type": "object",
"description": "Format-specific parser or encoder configuration."
},
"name": {
"type": "string",
"description": "Format name, e.g., \"csv\", \"json\", \"bincode\", etc."
}
}
},
"GenerationPlan": {
"type": "object",
"description": "A random generation plan for a table that generates either a limited amount of rows or runs continuously.",
"properties": {
"fields": {
"type": "object",
"description": "Specifies the values that the generator should produce.",
"additionalProperties": {
"$ref": "#/components/schemas/RngFieldSettings"
}
},
"limit": {
"type": "integer",
"description": "Total number of new rows to generate.\n\nIf not set, the generator will produce new/unique records as long as the pipeline is running.\nIf set to 0, the table will always remain empty.\nIf set, the generator will produce new records until the specified limit is reached.\n\nNote that if the table has one or more primary keys that don't use the `increment` strategy to\ngenerate the key there is a potential that an update is generated instead of an insert. In\nthis case it's possible the total number of records is less than the specified limit.",
"nullable": true,
"minimum": 0
},
"rate": {
"type": "integer",
"format": "int32",
"description": "Non-zero number of rows to generate per second.\n\nIf not set, the generator will produce rows as fast as possible.",
"nullable": true,
"minimum": 0
},
"worker_chunk_size": {
"type": "integer",
"description": "When multiple workers are used, each worker will pick a consecutive \"chunk\" of\nrecords to generate.\n\nBy default, if not specified, the generator will use the formula `min(rate, 10_000)`\nto determine it. This works well in most situations. However, if you're\nrunning tests with lateness and many workers you can e.g., reduce the\nchunk size to make sure a smaller range of records is being ingested in parallel.\n\n# Example\nAssume you generate a total of 125 records with 4 workers and a chunk size of 25.\nIn this case, worker A will generate records 0..25, worker B will generate records 25..50,\netc. A, B, C, and D will generate records in parallel. The first worker to finish its chunk\nwill pick up the last chunk of records (100..125) to generate.",
"nullable": true,
"minimum": 0
}
},
"additionalProperties": false
},
"InputEndpointConfig": {
"allOf": [
{
"$ref": "#/components/schemas/ConnectorConfig"
},
{
"type": "object",
"required": [
"stream"
],
"properties": {
"stream": {
"type": "string",
"description": "The name of the input stream of the circuit that this endpoint is\nconnected to."
}
}
}
],
"description": "Describes an input connector configuration"
},
"IntervalUnit": {
"type": "string",
"description": "The specified units for SQL Interval types.\n\n`INTERVAL 1 DAY`, `INTERVAL 1 DAY TO HOUR`, `INTERVAL 1 DAY TO MINUTE`,\nwould yield `Day`, `DayToHour`, `DayToMinute`, as the `IntervalUnit` respectively.",
"enum": [
"DAY",
"DAYTOHOUR",
"DAYTOMINUTE",
"DAYTOSECOND",
"HOUR",
"HOURTOMINUTE",
"HOURTOSECOND",
"MINUTE",
"MINUTETOSECOND",
"MONTH",
"SECOND",
"YEAR",
"YEARTOMONTH"
]
},
"JsonUpdateFormat": {
"type": "string",
"description": "Supported JSON data change event formats.\n\nEach element in a JSON-formatted input stream specifies\nan update to one or more records in an input table. We support\nseveral different ways to represent such updates.",
"enum": [
"insert_delete",
"weighted",
"debezium",
"snowflake",
"raw"
]
},
"KafkaHeader": {
"type": "object",
"description": "Kafka message header.",
"required": [
"key"
],
"properties": {
"key": {
"type": "string"
},
"value": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaHeaderValue"
}
],
"nullable": true
}
}
},
"KafkaHeaderValue": {
"type": "string",
"format": "binary",
"description": "Kafka header value encoded as a UTF-8 string or a byte array."
},
"KafkaInputConfig": {
"type": "object",
"description": "Configuration for reading data from Kafka topics with `InputTransport`.",
"required": [
"topics"
],
"properties": {
"fault_tolerance": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaInputFtConfig"
}
],
"nullable": true
},
"group_join_timeout_secs": {
"type": "integer",
"format": "int32",
"description": "Maximum timeout in seconds to wait for the endpoint to join the Kafka\nconsumer group during initialization.",
"minimum": 0
},
"kafka_service": {
"type": "string",
"description": "If specified, this service is used to provide defaults for the Kafka options.",
"nullable": true
},
"log_level": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaLogLevel"
}
],
"nullable": true
},
"poller_threads": {
"type": "integer",
"description": "Set to 1 or more to fix the number of threads used to poll\n`rdkafka`. Multiple threads can increase performance with small Kafka\nmessages; for large messages, one thread is enough. In either case, too\nmany threads can harm performance. If unset, the default is 3, which\nhelps with small messages but will not harm performance with large\nmessagee",
"nullable": true,
"minimum": 0
},
"topics": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of topics to subscribe to."
}
},
"additionalProperties": {
"type": "string",
"description": "Options passed directly to `rdkafka`.\n\n[`librdkafka` options](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)\nused to configure the Kafka consumer. Not all options are valid with\nthis Kafka adapter:\n\n* \"enable.auto.commit\", if present, must be set to \"false\",\n* \"enable.auto.offset.store\", if present, must be set to \"false\""
}
},
"KafkaInputFtConfig": {
"type": "object",
"description": "Fault tolerance configuration for Kafka input connector.",
"properties": {
"consumer_options": {
"type": "object",
"description": "Options passed to `rdkafka` for consumers only, as documented at\n[`librdkafka`\noptions](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).\n\nThese options override `kafka_options` for consumers, and may be empty.",
"additionalProperties": {
"type": "string"
}
},
"create_missing_index": {
"type": "boolean",
"description": "If this is true or unset, then the connector will create missing index\ntopics as needed. If this is false, then a missing index topic is a\nfatal error.",
"nullable": true
},
"index_suffix": {
"type": "string",
"description": "Suffix to append to each data topic name, to give the name of a topic\nthat the connector uses for recording the division of the corresponding\ndata topic into steps. Defaults to `_input-index`.\n\nAn index topic must have the same number of partitions as its\ncorresponding data topic.\n\nIf two or more fault-tolerant Kafka endpoints read from overlapping sets\nof topics, they must specify different `index_suffix` values.",
"nullable": true
},
"max_step_bytes": {
"type": "integer",
"format": "int64",
"description": "Maximum number of bytes in a step. Any individual message bigger than\nthis will be given a step of its own.",
"nullable": true,
"minimum": 0
},
"max_step_messages": {
"type": "integer",
"format": "int64",
"description": "Maximum number of messages in a step.",
"nullable": true,
"minimum": 0
},
"producer_options": {
"type": "object",
"description": "Options passed to `rdkafka` for producers only, as documented at\n[`librdkafka`\noptions](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).\n\nThese options override `kafka_options` for producers, and may be empty.",
"additionalProperties": {
"type": "string"
}
}
}
},
"KafkaLogLevel": {
"type": "string",
"description": "Kafka logging levels.",
"enum": [
"emerg",
"alert",
"critical",
"error",
"warning",
"notice",
"info",
"debug"
]
},
"KafkaOutputConfig": {
"type": "object",
"description": "Configuration for writing data to a Kafka topic with `OutputTransport`.",
"required": [
"topic"
],
"properties": {
"fault_tolerance": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaOutputFtConfig"
}
],
"nullable": true
},
"headers": {
"type": "array",
"items": {
"$ref": "#/components/schemas/KafkaHeader"
},
"description": "Kafka headers to be added to each message produced by this connector."
},
"initialization_timeout_secs": {
"type": "integer",
"format": "int32",
"description": "Maximum timeout in seconds to wait for the endpoint to connect to\na Kafka broker.\n\nDefaults to 60.",
"minimum": 0
},
"kafka_service": {
"type": "string",
"description": "If specified, this service is used to provide defaults for the Kafka options.",
"nullable": true
},
"log_level": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaLogLevel"
}
],
"nullable": true
},
"topic": {
"type": "string",
"description": "Topic to write to."
}
},
"additionalProperties": {
"type": "string",
"description": "Options passed directly to `rdkafka`.\n\nSee [`librdkafka` options](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)\nused to configure the Kafka producer."
}
},
"KafkaOutputFtConfig": {
"type": "object",
"description": "Fault tolerance configuration for Kafka output connector.",
"properties": {
"consumer_options": {
"type": "object",
"description": "Options passed to `rdkafka` for consumers only, as documented at\n[`librdkafka`\noptions](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).\n\nThese options override `kafka_options` for consumers, and may be empty.",
"additionalProperties": {
"type": "string"
}
},
"producer_options": {
"type": "object",
"description": "Options passed to `rdkafka` for producers only, as documented at\n[`librdkafka`\noptions](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).\n\nThese options override `kafka_options` for producers, and may be empty.",
"additionalProperties": {
"type": "string"
}
}
}
},
"ListPipelinesQueryParameters": {
"type": "object",
"description": "Query parameters for GET the list of pipelines.",
"properties": {
"code": {
"type": "boolean",
"description": "Whether to include program code in the response (default: `true`).\nPassing `false` reduces the response size, which is particularly handy\nwhen frequently monitoring the endpoint over low bandwidth connections."
}
}
},
"NewApiKeyRequest": {
"type": "object",
"description": "Request to create a new API key.",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Key name.",
"example": "my-api-key"
}
}
},
"NewApiKeyResponse": {
"type": "object",
"description": "Response to a successful API key creation.",
"required": [
"id",
"name",
"api_key"
],
"properties": {
"api_key": {
"type": "string",
"description": "Generated secret API key. There is no way to retrieve this\nkey again through the API, so store it securely.",
"example": "apikey:v5y5QNtlPNVMwkmNjKwFU8bbIu5lMge3yHbyddxAOdXlEo84SEoNn32DUhQaf1KLeI9aOOfnJjhQ1pYzMrU4wQXON6pm6BS7Zgzj46U2b8pwz1280vYBEtx41hiDBRP"
},
"id": {
"$ref": "#/components/schemas/ApiKeyId"
},
"name": {
"type": "string",
"description": "API key name provided by the user.",
"example": "my-api-key"
}
}
},
"NexmarkInputConfig": {
"type": "object",
"description": "Configuration for generating Nexmark input data.\n\nThis connector must be used exactly three times in a pipeline if it is used\nat all, once for each [`NexmarkTable`].",
"required": [
"table"
],
"properties": {
"options": {
"allOf": [
{
"$ref": "#/components/schemas/NexmarkInputOptions"
}
],
"nullable": true
},
"table": {
"$ref": "#/components/schemas/NexmarkTable"
}
}
},
"NexmarkInputOptions": {
"type": "object",
"description": "Configuration for generating Nexmark input data.",
"properties": {
"batch_size_per_thread": {
"type": "integer",
"format": "int64",
"description": "Number of events to generate and submit together, per thread.\n\nEach thread generates this many records, which are then combined with\nthe records generated by the other threads, to form combined input\nbatches of size `threads × batch_size_per_thread`.",
"default": 1000,
"minimum": 0
},
"events": {
"type": "integer",
"format": "int64",
"description": "Number of events to generate.",
"default": 100000000,
"minimum": 0
},
"max_step_size": {
"type": "integer",
"format": "int64",
"description": "Maximum number of events to submit in a single step. This should be a\nmultiple of `batch_size`.\n\nThis stands in for `max_batch_size` from the connector configuration\nbecause it must be a constant across all three of the nexmark tables.",
"default": 4000000,
"minimum": 0
},
"threads": {
"type": "integer",
"description": "Number of event generator threads.\n\nIt's reasonable to choose the same number of generator threads as worker\nthreads.",
"default": 4,
"minimum": 0
}
}
},
"NexmarkTable": {
"type": "string",
"description": "Table in Nexmark.",
"enum": [
"bid",
"auction",
"person"
]
},
"OutputBufferConfig": {
"type": "object",
"properties": {
"enable_output_buffer": {
"type": "boolean",
"description": "Enable output buffering.\n\nThe output buffering mechanism allows decoupling the rate at which the pipeline\npushes changes to the output transport from the rate of input changes.\n\nBy default, output updates produced by the pipeline are pushed directly to\nthe output transport. Some destinations may prefer to receive updates in fewer\nbigger batches. For instance, when writing Parquet files, producing\none bigger file every few minutes is usually better than creating\nsmall files every few milliseconds.\n\nTo achieve such input/output decoupling, users can enable output buffering by\nsetting the `enable_output_buffer` flag to `true`. When buffering is enabled, output\nupdates produced by the pipeline are consolidated in an internal buffer and are\npushed to the output transport when one of several conditions is satisfied:\n\n* data has been accumulated in the buffer for more than `max_output_buffer_time_millis`\nmilliseconds.\n* buffer size exceeds `max_output_buffer_size_records` records.\n\nThis flag is `false` by default."
},
"max_output_buffer_size_records": {
"type": "integer",
"description": "Maximum number of updates to be kept in the output buffer.\n\nThis parameter bounds the maximal size of the buffer.\nNote that the size of the buffer is not always equal to the\ntotal number of updates output by the pipeline. Updates to the\nsame record can overwrite or cancel previous updates.\n\nBy default, the buffer can grow indefinitely until one of\nthe other output conditions is satisfied.\n\nNOTE: this configuration option requires the `enable_output_buffer` flag\nto be set.",
"minimum": 0
},
"max_output_buffer_time_millis": {
"type": "integer",
"description": "Maximum time in milliseconds data is kept in the output buffer.\n\nBy default, data is kept in the buffer indefinitely until one of\nthe other output conditions is satisfied. When this option is\nset the buffer will be flushed at most every\n`max_output_buffer_time_millis` milliseconds.\n\nNOTE: this configuration option requires the `enable_output_buffer` flag\nto be set.",
"minimum": 0
}
}
},
"OutputEndpointConfig": {
"allOf": [
{
"$ref": "#/components/schemas/ConnectorConfig"
},
{
"type": "object",
"required": [
"stream"
],
"properties": {
"stream": {
"type": "string",
"description": "The name of the output stream of the circuit that this endpoint is\nconnected to."
}
}
}
],
"description": "Describes an output connector configuration"
},
"PatchPipeline": {
"type": "object",
"description": "Patch (partially) update the pipeline.\n\nNote that the patching only applies to the main fields, not subfields.\nFor instance, it is not possible to update only the number of workers;\nit is required to again pass the whole runtime configuration with the\nchange.",
"properties": {
"description": {
"type": "string",
"nullable": true
},
"name": {
"type": "string",
"nullable": true
},
"program_code": {
"type": "string",
"nullable": true
},
"program_config": {
"allOf": [
{
"$ref": "#/components/schemas/ProgramConfig"
}
],
"nullable": true
},
"runtime_config": {
"allOf": [
{
"$ref": "#/components/schemas/RuntimeConfig"
}
],
"nullable": true
}
}
},
"PipelineConfig": {
"allOf": [
{
"type": "object",
"description": "Global pipeline configuration settings. This is the publicly\nexposed type for users to configure pipelines.",
"properties": {
"clock_resolution_usecs": {
"type": "integer",
"format": "int64",
"description": "Real-time clock resolution in microseconds.\n\nThis parameter controls the execution of queries that use the `NOW()` function. The output of such\nqueries depends on the real-time clock and can change over time without any external\ninputs. The pipeline will update the clock value and trigger incremental recomputation\nat most each `clock_resolution_usecs` microseconds.\n\nIt is set to 100 milliseconds (100,000 microseconds) by default.\n\nSet to `null` to disable periodic clock updates.",
"nullable": true,
"minimum": 0
},
"cpu_profiler": {
"type": "boolean",
"description": "Enable CPU profiler.\n\nThe default value is `true`."
},
"max_buffering_delay_usecs": {
"type": "integer",
"format": "int64",
"description": "Maximal delay in microseconds to wait for `min_batch_size_records` to\nget buffered by the controller, defaults to 0.",
"minimum": 0
},
"min_batch_size_records": {
"type": "integer",
"format": "int64",
"description": "Minimal input batch size.\n\nThe controller delays pushing input records to the circuit until at\nleast `min_batch_size_records` records have been received (total\nacross all endpoints) or `max_buffering_delay_usecs` microseconds\nhave passed since at least one input records has been buffered.\nDefaults to 0.",
"minimum": 0
},
"min_storage_bytes": {
"type": "integer",
"description": "The minimum estimated number of bytes in a batch of data to write it to\nstorage. This is provided for debugging and fine-tuning and should\nordinarily be left unset. It only has an effect when `storage` is set to\ntrue.\n\nA value of 0 will write even empty batches to storage, and nonzero\nvalues provide a threshold. `usize::MAX` would effectively disable\nstorage.",
"nullable": true,
"minimum": 0
},
"resources": {
"$ref": "#/components/schemas/ResourceConfig"
},
"storage": {
"type": "boolean",
"description": "Should persistent storage be enabled for this pipeline?\n\n- If `false` (default), the pipeline's state is kept in in-memory data-structures.\nThis is useful if the pipeline is ephemeral and does not need to be recovered\nafter a restart. The pipeline will most likely run faster since it does not\nneed to read from, or write to disk\n\n- If `true`, the pipeline state is stored in the specified location,\nis persisted across restarts, and can be checkpointed and recovered.\nThis feature is currently experimental."
},
"tracing": {
"type": "boolean",
"description": "Enable pipeline tracing."
},
"tracing_endpoint_jaeger": {
"type": "string",
"description": "Jaeger tracing endpoint to send tracing information to."
},
"workers": {
"type": "integer",
"format": "int32",
"description": "Number of DBSP worker threads.",
"minimum": 0
}
}
},
{
"type": "object",
"required": [
"inputs"
],
"properties": {
"inputs": {
"type": "object",
"description": "Input endpoint configuration.",
"additionalProperties": {
"$ref": "#/components/schemas/InputEndpointConfig"
}
},
"name": {
"type": "string",
"description": "Pipeline name.",
"nullable": true
},
"outputs": {
"type": "object",
"description": "Output endpoint configuration.",
"additionalProperties": {
"$ref": "#/components/schemas/OutputEndpointConfig"
}
},
"storage_config": {
"allOf": [
{
"$ref": "#/components/schemas/StorageConfig"
}
],
"nullable": true
}
}
}
],
"description": "Pipeline deployment configuration.\nIt represents configuration entries directly provided by the user\n(e.g., runtime configuration) and entries derived from the schema\nof the compiled program (e.g., connectors). Storage configuration,\nif applicable, is set by the runner."
},
"PipelineDescr": {
"type": "object",
"description": "Pipeline descriptor.",
"required": [
"name",
"description",
"runtime_config",
"program_code",
"program_config"
],
"properties": {
"description": {
"type": "string",
"description": "Pipeline description."
},
"name": {
"type": "string",
"description": "Pipeline name."
},
"program_code": {
"type": "string",
"description": "Program SQL code."
},
"program_config": {
"$ref": "#/components/schemas/ProgramConfig"
},
"runtime_config": {
"$ref": "#/components/schemas/RuntimeConfig"
}
}
},
"PipelineDesiredStatus": {
"type": "string",
"enum": [
"Shutdown",
"Paused",
"Running"
]
},
"PipelineId": {
"type": "string",
"format": "uuid",
"description": "Pipeline identifier."
},
"PipelineStatus": {
"type": "string",
"description": "Pipeline status.\n\nThis type represents the state of the pipeline tracked by the pipeline\nrunner and observed by the API client via the `GET /v0/pipelines/{name}` endpoint.\n\n### The lifecycle of a pipeline\n\nThe following automaton captures the lifecycle of the pipeline. Individual\nstates and transitions of the automaton are described below.\n\n* In addition to the transitions shown in the diagram, all states have an\nimplicit \"forced shutdown\" transition to the `Shutdown` state. This\ntransition is triggered when the pipeline runner is unable to communicate\nwith the pipeline and thereby forces a shutdown.\n\n* States labeled with the hourglass symbol (⌛) are **timed** states. The\nautomaton stays in timed state until the corresponding operation completes\nor until the runner performs a forced shutdown of the pipeline after a\npre-defined timeout period.\n\n* State transitions labeled with API endpoint names (`/start`, `/pause`,\n`/shutdown`) are triggered by invoking corresponding endpoint,\ne.g., `POST /v0/pipelines/{name}/start`.\n\n```text\nShutdown◄────┐\n│ │\n/deploy│ │\n│ ⌛ShuttingDown\n▼ ▲\n⌛Provisioning │\n│ │\n│ │\n▼ │/shutdown\n⌛Initializing │\n│ │\n┌────────┴─────────┴─┐\n│ ▼ │\n│ Paused │\n│ │ ▲ │\n│/start│ │/pause │\n│ ▼ │ │\n│ Running │\n└──────────┬─────────┘\n│\n▼\nFailed\n```\n\n### Desired and actual status\n\nWe use the desired state model to manage the lifecycle of a pipeline.\nIn this model, the pipeline has two status attributes associated with\nit at runtime: the **desired** status, which represents what the user\nwould like the pipeline to do, and the **current** status, which\nrepresents the actual state of the pipeline. The pipeline runner\nservice continuously monitors both fields and steers the pipeline\ntowards the desired state specified by the user.\nOnly three of the states in the pipeline automaton above can be\nused as desired statuses: `Paused`, `Running`, and `Shutdown`.\nThese statuses are selected by invoking REST endpoints shown\nin the diagram.\n\nThe user can monitor the current state of the pipeline via the\n`GET /v0/pipelines/{name}` endpoint, which returns an object of type `ExtendedPipelineDescr`.\nIn a typical scenario, the user first sets\nthe desired state, e.g., by invoking the `/deploy` endpoint, and\nthen polls the `GET /v0/pipelines/{name}` endpoint to monitor the actual status\nof the pipeline until its `deployment_status` attribute changes\nto \"paused\" indicating that the pipeline has been successfully\ninitialized, or \"failed\", indicating an error.",
"enum": [
"Shutdown",
"Provisioning",
"Initializing",
"Paused",
"Running",
"ShuttingDown",
"Failed"
]
},
"ProgramConfig": {
"type": "object",
"description": "Program configuration.",
"properties": {
"profile": {
"allOf": [
{
"$ref": "#/components/schemas/CompilationProfile"
}
],
"nullable": true
}
}
},
"ProgramInfo": {
"type": "object",
"description": "Program information which includes schema, input connectors and output connectors.",
"required": [
"schema",
"input_connectors",
"output_connectors"
],
"properties": {
"input_connectors": {
"type": "object",
"description": "Input connectors derived from the schema.",
"additionalProperties": {
"$ref": "#/components/schemas/InputEndpointConfig"
}
},
"output_connectors": {
"type": "object",
"description": "Output connectors derived from the schema.",
"additionalProperties": {
"$ref": "#/components/schemas/OutputEndpointConfig"
}
},
"schema": {
"$ref": "#/components/schemas/ProgramSchema"
}
}
},
"ProgramSchema": {
"type": "object",
"description": "A struct containing the tables (inputs) and views for a program.\n\nParse from the JSON data-type of the DDL generated by the SQL compiler.",
"required": [
"inputs",
"outputs"
],
"properties": {
"inputs": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Relation"
}
},
"outputs": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Relation"
}
}
}
},
"ProgramStatus": {
"oneOf": [
{
"type": "string",
"description": "Compilation request received from the user; program has been placed\nin the queue.",
"enum": [
"Pending"
]
},
{
"type": "string",
"description": "Compilation of SQL -> Rust in progress.",
"enum": [
"CompilingSql"
]
},
{
"type": "string",
"description": "Compiling Rust -> executable in progress.",
"enum": [
"CompilingRust"
]
},
{
"type": "string",
"description": "Compilation succeeded.",
"enum": [
"Success"
]
},
{
"type": "object",
"required": [
"SqlError"
],
"properties": {
"SqlError": {
"type": "array",
"items": {
"$ref": "#/components/schemas/SqlCompilerMessage"
},
"description": "SQL compiler returned an error."
}
}
},
{
"type": "object",
"required": [
"RustError"
],
"properties": {
"RustError": {
"type": "string",
"description": "Rust compiler returned an error."
}
}
},
{
"type": "object",
"required": [
"SystemError"
],
"properties": {
"SystemError": {
"type": "string",
"description": "System/OS returned an error when trying to invoke commands."
}
}
}
],
"description": "Program compilation status."
},
"PropertyValue": {
"type": "object",
"required": [
"value",
"key_position",
"value_position"
],
"properties": {
"key_position": {
"$ref": "#/components/schemas/SourcePosition"
},
"value": {
"type": "string"
},
"value_position": {
"$ref": "#/components/schemas/SourcePosition"
}
}
},
"ProviderAwsCognito": {
"type": "object",
"required": [
"jwk_uri",
"login_url",
"logout_url"
],
"properties": {
"jwk_uri": {
"type": "string"
},
"login_url": {
"type": "string"
},
"logout_url": {
"type": "string"
}
}
},
"ProviderGoogleIdentity": {
"type": "object",
"required": [
"jwk_uri",
"client_id"
],
"properties": {
"client_id": {
"type": "string"
},
"jwk_uri": {
"type": "string"
}
}
},
"PubSubInputConfig": {
"type": "object",
"description": "Google Pub/Sub input connector configuration.",
"required": [
"subscription"
],
"properties": {
"connect_timeout_seconds": {
"type": "integer",
"format": "int32",
"description": "gRPC connection timeout.",
"nullable": true,
"minimum": 0
},
"credentials": {
"type": "string",
"description": "The content of a Google Cloud credentials JSON file.\n\nWhen this option is specified, the connector will use the provided credentials for\nauthentication. Otherwise, it will use Application Default Credentials (ADC) configured\nin the environment where the Feldera service is running. See\n[Google Cloud documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc)\nfor information on configuring application default credentials.\n\nWhen running Feldera in an environment where ADC are not configured,\ne.g., a Docker container, use this option to ship Google Cloud credentials from another environment.\nFor example, if you use the\n[`gcloud auth application-default login`](https://cloud.google.com/pubsub/docs/authentication#client-libs)\ncommand for authentication in your local development environment, ADC are stored in the\n`.config/gcloud/application_default_credentials.json` file in your home directory.",
"nullable": true
},
"emulator": {
"type": "string",
"description": "Set in order to use a Pub/Sub [emulator](https://cloud.google.com/pubsub/docs/emulator)\ninstead of the production service, e.g., 'localhost:8681'.",
"nullable": true
},
"endpoint": {
"type": "string",
"description": "Override the default service endpoint 'pubsub.googleapis.com'",
"nullable": true
},
"pool_size": {
"type": "integer",
"format": "int32",
"description": "gRPC channel pool size.",
"nullable": true,
"minimum": 0
},
"project_id": {
"type": "string",
"description": "Google Cloud project_id.\n\nWhen not specified, the connector will use the project id associated\nwith the authenticated account.",
"nullable": true
},
"snapshot": {
"type": "string",
"description": "Reset subscription's backlog to a given snapshot on startup,\nusing the Pub/Sub `Seek` API.\n\nThis option is mutually exclusive with the `timestamp` option.",
"nullable": true
},
"subscription": {
"type": "string",
"description": "Subscription name."
},
"timeout_seconds": {
"type": "integer",
"format": "int32",
"description": "gRPC request timeout.",
"nullable": true,
"minimum": 0
},
"timestamp": {
"type": "string",
"description": "Reset subscription's backlog to a given timestamp on startup,\nusing the Pub/Sub `Seek` API.\n\nThe value of this option is an ISO 8601-encoded UTC time, e.g., \"2024-08-17T16:39:57-08:00\".\n\nThis option is mutually exclusive with the `snapshot` option.",
"nullable": true
}
}
},
"Relation": {
"allOf": [
{
"$ref": "#/components/schemas/SqlIdentifier"
},
{
"type": "object",
"required": [
"fields"
],
"properties": {
"fields": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Field"
}
},
"materialized": {
"type": "boolean"
},
"properties": {
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/PropertyValue"
}
}
}
}
],
"description": "A SQL table or view. It has a name and a list of fields.\n\nMatches the Calcite JSON format."
},
"ResourceConfig": {
"type": "object",
"properties": {
"cpu_cores_max": {
"type": "integer",
"format": "int64",
"description": "The maximum number of CPU cores to reserve\nfor an instance of this pipeline",
"nullable": true,
"minimum": 0
},
"cpu_cores_min": {
"type": "integer",
"format": "int64",
"description": "The minimum number of CPU cores to reserve\nfor an instance of this pipeline",
"nullable": true,
"minimum": 0
},
"memory_mb_max": {
"type": "integer",
"format": "int64",
"description": "The maximum memory in Megabytes to reserve\nfor an instance of this pipeline",
"nullable": true,
"minimum": 0
},
"memory_mb_min": {
"type": "integer",
"format": "int64",
"description": "The minimum memory in Megabytes to reserve\nfor an instance of this pipeline",
"nullable": true,
"minimum": 0
},
"storage_class": {
"type": "string",
"description": "Storage class to use for an instance of this pipeline.\nThe class determines storage performance such as IOPS and throughput.",
"nullable": true
},
"storage_mb_max": {
"type": "integer",
"format": "int64",
"description": "The total storage in Megabytes to reserve\nfor an instance of this pipeline",
"nullable": true,
"minimum": 0
}
}
},
"RngFieldSettings": {
"type": "object",
"description": "Configuration for generating random data for a field of a table.",
"required": [
"range"
],
"properties": {
"e": {
"type": "integer",
"format": "int64",
"description": "The frequency rank exponent for the Zipf distribution.\n\n- This value is only used if the strategy is set to `Zipf`.\n- The default value is 1.0."
},
"fields": {
"type": "object",
"description": "Specifies the values that the generator should produce in case the field is a struct type.",
"additionalProperties": {
"$ref": "#/components/schemas/RngFieldSettings"
},
"nullable": true
},
"key": {
"allOf": [
{
"$ref": "#/components/schemas/RngFieldSettings"
}
],
"nullable": true
},
"null_percentage": {
"type": "integer",
"description": "Percentage of records where this field should be set to NULL.\n\nIf not set, the generator will produce only records with non-NULL values.\nIf set to `1..=100`, the generator will produce records with NULL values with the specified percentage.",
"nullable": true,
"minimum": 0
},
"range": {
"type": "object",
"description": "An optional, exclusive range [a, b) to limit the range of values the generator should produce.\n\n- For integer/floating point types specifies min/max values as an integer.\nIf not set, the generator will produce values for the entire range of the type for number types.\n- For string/binary types specifies min/max length as an integer, values are required to be >=0.\nIf not set, a range of [0, 25) is used by default.\n- For timestamp types specifies the min/max as two strings in the RFC 3339 format\n(e.g., [\"2021-01-01T00:00:00Z\", \"2022-01-02T00:00:00Z\"]).\nAlternatively, the range values can be specified as a number of non-leap\nmilliseconds since January 1, 1970 0:00:00.000 UTC (aka “UNIX timestamp”).\nIf not set, a range of [\"1970-01-01T00:00:00Z\", \"2100-01-01T00:00:00Z\") or [0, 4102444800000)\nis used by default.\n- For time types specifies the min/max as two strings in the \"HH:MM:SS\" format.\nAlternatively, the range values can be specified in milliseconds as two positive integers.\nIf not set, the range is 24h.\n- For date types, the min/max range is specified as two strings in the \"YYYY-MM-DD\" format.\nAlternatively, two integers that represent number of days since January 1, 1970 can be used.\nIf not set, a range of [\"1970-01-01\", \"2100-01-01\") or [0, 54787) is used by default.\n- For array types specifies the min/max number of elements as an integer.\nIf not set, a range of [0, 5) is used by default. Range values are required to be >=0.\n- For map types specifies the min/max number of key-value pairs as an integer.\nIf not set, a range of [0, 5) is used by default.\n- For struct/boolean/null types `range` is ignored."
},
"scale": {
"type": "integer",
"format": "int64",
"description": "A scale factor to apply a multiplier to the generated value.\n\n- For integer/floating point types, the value is multiplied by the scale factor.\n- For timestamp types, the generated value (milliseconds) is multiplied by the scale factor.\n- For time types, the generated value (milliseconds) is multiplied by the scale factor.\n- For date types, the generated value (days) is multiplied by the scale factor.\n- For string/binary/array/map/struct/boolean/null types, the scale factor is ignored.\n\n- If `values` is specified, the scale factor is ignored.\n- If `range` is specified and the range is required to be positive (struct, map, array etc.)\nthe scale factor is required to be positive too.\n\nThe default scale factor is 1."
},
"strategy": {
"$ref": "#/components/schemas/DatagenStrategy"
},
"value": {
"allOf": [
{
"$ref": "#/components/schemas/RngFieldSettings"
}
],
"nullable": true
},
"values": {
"type": "array",
"items": {
"type": "object"
},
"description": "An optional set of values the generator will pick from.\n\nIf set, the generator will pick values from the specified set.\nIf not set, the generator will produce values according to the specified range.\nIf set to an empty set, the generator will produce NULL values.\nIf set to a single value, the generator will produce only that value.\n\nNote that `range` is ignored if `values` is set.",
"nullable": true
}
},
"additionalProperties": false
},
"RuntimeConfig": {
"type": "object",
"description": "Global pipeline configuration settings. This is the publicly\nexposed type for users to configure pipelines.",
"properties": {
"clock_resolution_usecs": {
"type": "integer",
"format": "int64",
"description": "Real-time clock resolution in microseconds.\n\nThis parameter controls the execution of queries that use the `NOW()` function. The output of such\nqueries depends on the real-time clock and can change over time without any external\ninputs. The pipeline will update the clock value and trigger incremental recomputation\nat most each `clock_resolution_usecs` microseconds.\n\nIt is set to 100 milliseconds (100,000 microseconds) by default.\n\nSet to `null` to disable periodic clock updates.",
"nullable": true,
"minimum": 0
},
"cpu_profiler": {
"type": "boolean",
"description": "Enable CPU profiler.\n\nThe default value is `true`."
},
"max_buffering_delay_usecs": {
"type": "integer",
"format": "int64",
"description": "Maximal delay in microseconds to wait for `min_batch_size_records` to\nget buffered by the controller, defaults to 0.",
"minimum": 0
},
"min_batch_size_records": {
"type": "integer",
"format": "int64",
"description": "Minimal input batch size.\n\nThe controller delays pushing input records to the circuit until at\nleast `min_batch_size_records` records have been received (total\nacross all endpoints) or `max_buffering_delay_usecs` microseconds\nhave passed since at least one input records has been buffered.\nDefaults to 0.",
"minimum": 0
},
"min_storage_bytes": {
"type": "integer",
"description": "The minimum estimated number of bytes in a batch of data to write it to\nstorage. This is provided for debugging and fine-tuning and should\nordinarily be left unset. It only has an effect when `storage` is set to\ntrue.\n\nA value of 0 will write even empty batches to storage, and nonzero\nvalues provide a threshold. `usize::MAX` would effectively disable\nstorage.",
"nullable": true,
"minimum": 0
},
"resources": {
"$ref": "#/components/schemas/ResourceConfig"
},
"storage": {
"type": "boolean",
"description": "Should persistent storage be enabled for this pipeline?\n\n- If `false` (default), the pipeline's state is kept in in-memory data-structures.\nThis is useful if the pipeline is ephemeral and does not need to be recovered\nafter a restart. The pipeline will most likely run faster since it does not\nneed to read from, or write to disk\n\n- If `true`, the pipeline state is stored in the specified location,\nis persisted across restarts, and can be checkpointed and recovered.\nThis feature is currently experimental."
},
"tracing": {
"type": "boolean",
"description": "Enable pipeline tracing."
},
"tracing_endpoint_jaeger": {
"type": "string",
"description": "Jaeger tracing endpoint to send tracing information to."
},
"workers": {
"type": "integer",
"format": "int32",
"description": "Number of DBSP worker threads.",
"minimum": 0
}
}
},
"S3InputConfig": {
"type": "object",
"description": "Configuration for reading data from AWS S3.",
"required": [
"region",
"bucket_name"
],
"properties": {
"aws_access_key_id": {
"type": "string",
"description": "AWS Access Key id. This property must be specified unless `no_sign_request` is set to `true`.",
"nullable": true
},
"aws_secret_access_key": {
"type": "string",
"description": "Secret Access Key. This property must be specified unless `no_sign_request` is set to `true`.",
"nullable": true
},
"bucket_name": {
"type": "string",
"description": "S3 bucket name to access."
},
"key": {
"type": "string",
"description": "Read a single object specified by a key.",
"nullable": true
},
"no_sign_request": {
"type": "boolean",
"description": "Do not sign requests. This is equivalent to the `--no-sign-request` flag in the AWS CLI."
},
"prefix": {
"type": "string",
"description": "Read all objects whose keys match a prefix. Set to an empty string to read all objects in the bucket.",
"nullable": true
},
"region": {
"type": "string",
"description": "AWS region."
},
"streaming": {
"type": "boolean",
"description": "Determines how the connector ingests an individual S3 object. When `true`,\nthe connector pushes the object to the pipeline chunk-by-chunk, so that the\npipeline can parse and process initial chunks of the object before the entire\nobject has been retrieved. This mode is suitable for streaming formats such as\nnewline-delimited JSON. When `false`, the connector buffers the entire object\nin memory and pushes it to the pipeline as a single chunk. Appropriate for\nformats like Parquet that cannot be streamed.\n\nThe default value is `false`."
}
}
},
"SourcePosition": {
"type": "object",
"required": [
"start_line_number",
"start_column",
"end_line_number",
"end_column"
],
"properties": {
"end_column": {
"type": "integer",
"minimum": 0
},
"end_line_number": {
"type": "integer",
"minimum": 0
},
"start_column": {
"type": "integer",
"minimum": 0
},
"start_line_number": {
"type": "integer",
"minimum": 0
}
}
},
"SqlCompilerMessage": {
"type": "object",
"description": "A SQL compiler error.\n\nThe SQL compiler returns a list of errors in the following JSON format if\nit's invoked with the `-je` option.\n\n```ignore\n[ {\n\"start_line_number\" : 2,\n\"start_column\" : 4,\n\"end_line_number\" : 2,\n\"end_column\" : 8,\n\"warning\" : false,\n\"error_type\" : \"PRIMARY KEY cannot be nullable\",\n\"message\" : \"PRIMARY KEY column 'C' has type INTEGER, which is nullable\",\n\"snippet\" : \" 2| c INT PRIMARY KEY\\n ^^^^^\\n 3|);\\n\"\n} ]\n```",
"required": [
"start_line_number",
"start_column",
"end_line_number",
"end_column",
"warning",
"error_type",
"message"
],
"properties": {
"end_column": {
"type": "integer",
"minimum": 0
},
"end_line_number": {
"type": "integer",
"minimum": 0
},
"error_type": {
"type": "string"
},
"message": {
"type": "string"
},
"snippet": {
"type": "string",
"nullable": true
},
"start_column": {
"type": "integer",
"minimum": 0
},
"start_line_number": {
"type": "integer",
"minimum": 0
},
"warning": {
"type": "boolean"
}
}
},
"SqlIdentifier": {
"type": "object",
"description": "An SQL identifier.\n\nThis struct is used to represent SQL identifiers in a canonical form.\nWe store table names or field names as identifiers in the schema.",
"required": [
"name",
"case_sensitive"
],
"properties": {
"case_sensitive": {
"type": "boolean"
},
"name": {
"type": "string"
}
}
},
"SqlType": {
"oneOf": [
{
"type": "string",
"description": "SQL `BOOLEAN` type.",
"enum": [
"BOOLEAN"
]
},
{
"type": "string",
"description": "SQL `TINYINT` type.",
"enum": [
"TINYINT"
]
},
{
"type": "string",
"description": "SQL `SMALLINT` or `INT2` type.",
"enum": [
"SMALLINT"
]
},
{
"type": "string",
"description": "SQL `INTEGER`, `INT`, `SIGNED`, `INT4` type.",
"enum": [
"INTEGER"
]
},
{
"type": "string",
"description": "SQL `BIGINT` or `INT64` type.",
"enum": [
"BIGINT"
]
},
{
"type": "string",
"description": "SQL `REAL` or `FLOAT4` or `FLOAT32` type.",
"enum": [
"REAL"
]
},
{
"type": "string",
"description": "SQL `DOUBLE` or `FLOAT8` or `FLOAT64` type.",
"enum": [
"DOUBLE"
]
},
{
"type": "string",
"description": "SQL `DECIMAL` or `DEC` or `NUMERIC` type.",
"enum": [
"DECIMAL"
]
},
{
"type": "string",
"description": "SQL `CHAR(n)` or `CHARACTER(n)` type.",
"enum": [
"CHAR"
]
},
{
"type": "string",
"description": "SQL `VARCHAR`, `CHARACTER VARYING`, `TEXT`, or `STRING` type.",
"enum": [
"VARCHAR"
]
},
{
"type": "string",
"description": "SQL `BINARY(n)` type.",
"enum": [
"BINARY"
]
},
{
"type": "string",
"description": "SQL `VARBINARY` or `BYTEA` type.",
"enum": [
"VARBINARY"
]
},
{
"type": "string",
"description": "SQL `TIME` type.",
"enum": [
"TIME"
]
},
{
"type": "string",
"description": "SQL `DATE` type.",
"enum": [
"DATE"
]
},
{
"type": "string",
"description": "SQL `TIMESTAMP` type.",
"enum": [
"TIMESTAMP"
]
},
{
"type": "object",
"required": [
"Interval"
],
"properties": {
"Interval": {
"$ref": "#/components/schemas/IntervalUnit"
}
}
},
{
"type": "string",
"description": "SQL `ARRAY` type.",
"enum": [
"ARRAY"
]
},
{
"type": "string",
"description": "A complex SQL struct type (`CREATE TYPE x ...`).",
"enum": [
"STRUCT"
]
},
{
"type": "string",
"description": "SQL `MAP` type.",
"enum": [
"MAP"
]
},
{
"type": "string",
"description": "SQL `NULL` type.",
"enum": [
"NULL"
]
},
{
"type": "string",
"description": "SQL `VARIANT` type.",
"enum": [
"VARIANT"
]
}
],
"description": "The available SQL types as specified in `CREATE` statements."
},
"StorageCacheConfig": {
"type": "string",
"description": "How to cache access to storage within a Feldera pipeline.",
"enum": [
"page_cache",
"feldera_cache"
]
},
"StorageConfig": {
"type": "object",
"description": "Configuration for persistent storage in a [`PipelineConfig`].",
"required": [
"path",
"cache"
],
"properties": {
"cache": {
"$ref": "#/components/schemas/StorageCacheConfig"
},
"path": {
"type": "string",
"description": "The location where the pipeline state is stored or will be stored.\n\nIt should point to a path on the file-system of the machine/container\nwhere the pipeline will run. If that path doesn't exist yet, or if it\ndoes not contain any checkpoints, then the pipeline creates it and\nstarts from an initial state in which no data has yet been received. If\nit does exist, then the pipeline starts from the most recent checkpoint\nthat already exists there. In either case, (further) checkpoints will be\nwritten there."
}
}
},
"TransportConfig": {
"oneOf": [
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/FileInputConfig"
},
"name": {
"type": "string",
"enum": [
"file_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/FileOutputConfig"
},
"name": {
"type": "string",
"enum": [
"file_output"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/KafkaInputConfig"
},
"name": {
"type": "string",
"enum": [
"kafka_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/KafkaOutputConfig"
},
"name": {
"type": "string",
"enum": [
"kafka_output"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/PubSubInputConfig"
},
"name": {
"type": "string",
"enum": [
"pub_sub_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/UrlInputConfig"
},
"name": {
"type": "string",
"enum": [
"url_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/S3InputConfig"
},
"name": {
"type": "string",
"enum": [
"s3_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/DeltaTableReaderConfig"
},
"name": {
"type": "string",
"enum": [
"delta_table_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/DeltaTableWriterConfig"
},
"name": {
"type": "string",
"enum": [
"delta_table_output"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/DatagenInputConfig"
},
"name": {
"type": "string",
"enum": [
"datagen"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/NexmarkInputConfig"
},
"name": {
"type": "string",
"enum": [
"nexmark"
]
}
}
},
{
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"enum": [
"http_input"
]
}
}
},
{
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"enum": [
"http_output"
]
}
}
}
],
"description": "Transport-specific endpoint configuration passed to\n`crate::OutputTransport::new_endpoint`\nand `crate::InputTransport::new_endpoint`.",
"discriminator": {
"propertyName": "name"
}
},
"UrlInputConfig": {
"type": "object",
"description": "Configuration for reading data from an HTTP or HTTPS URL with\n`UrlInputTransport`.",
"required": [
"path"
],
"properties": {
"path": {
"type": "string",
"description": "URL."
},
"pause_timeout": {
"type": "integer",
"format": "int32",
"description": "Timeout before disconnection when paused.\n\nIf the pipeline is paused, or if the input adapter reads data faster\nthan the pipeline can process it, then the controller will pause the\ninput adapter. If the input adapter stays paused longer than this\ntimeout, it will drop the network connection to the server. It will\nautomatically reconnect when the input adapter starts running again.",
"minimum": 0
}
}
},
"Version": {
"type": "integer",
"format": "int64",
"description": "Version number."
}
},
"securitySchemes": {
"JSON web token (JWT) or API key": {
"type": "http",
"scheme": "bearer",
"bearerFormat": "JWT",
"description": "Use a JWT token obtained via an OAuth2/OIDC\n login workflow or an API key obtained via\n the `/v0/api-keys` endpoint."
}
}
},
"tags": [
{
"name": "Pipelines",
"description": "Manage pipelines and their deployment."
},
{
"name": "HTTP input/output",
"description": "Interact with running pipelines using HTTP."
},
{
"name": "Authentication",
"description": "Retrieve authentication configuration."
},
{
"name": "Configuration",
"description": "Retrieve general configuration."
},
{
"name": "API keys",
"description": "Manage API keys."
},
{
"name": "Metrics",
"description": "Retrieve pipeline metrics."
}
]
}