{
"openapi": "3.0.3",
"info": {
"title": "Feldera API",
"description": "\nWith Feldera, users create data pipelines out of SQL programs.\nA SQL program comprises tables and views, and includes as well the definition of\ninput and output connectors for each respectively. A connector defines a data\nsource or data sink to feed input data into tables or receive output data\ncomputed by the views respectively.\n\n## Pipeline\n\nThe API is centered around the **pipeline**, which most importantly consists\nout of the SQL program, but also has accompanying metadata and configuration parameters\n(e.g., compilation profile, number of workers, etc.).\n\n* A pipeline is identified and referred to by a user-provided unique name.\n* The pipeline program is asynchronously compiled when the pipeline is first created or\n its program code or configuration is updated.\n* Running the pipeline (*deployment*) is only possible once the program is compiled\n* A pipeline cannot be updated while it is running\n\n## Concurrency\n\nBoth the pipeline and its program have an associated *version*.\nA version is a monotonically increasing number.\nAnytime the core fields (name, description, runtime_config, program_code, program_config) are modified,\nthe pipeline version is incremented.\nAnytime the program core fields (program_code, program_config) are modified,\nthe program version is incremented.\nThe program version is used internally by the compiler to know when to recompile.",
"license": {
"name": "MIT OR Apache-2.0"
},
"version": "0.35.0"
},
"paths": {
"/config": {
"get": {
"tags": [
"Configuration"
],
"summary": "Retrieve general configuration.",
"operationId": "get_config",
"responses": {
"200": {
"description": "The response body contains basic configuration information about this host.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Configuration"
}
}
}
},
"500": {
"description": "Request failed.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
}
}
},
"/config/authentication": {
"get": {
"tags": [
"Configuration"
],
"summary": "Retrieve authentication provider configuration.",
"operationId": "get_config_authentication",
"responses": {
"200": {
"description": "The response body contains Authentication Provider configuration, or is empty if no auth is configured.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AuthProvider"
}
}
}
},
"500": {
"description": "Request failed.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
}
}
},
"/v0/api_keys": {
"get": {
"tags": [
"API keys"
],
"summary": "Retrieve the list of API keys.",
"operationId": "list_api_keys",
"responses": {
"200": {
"description": "API keys retrieved successfully",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiKeyDescr"
}
}
}
}
},
"500": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"post": {
"tags": [
"API keys"
],
"summary": "Create a new API key.",
"operationId": "post_api_key",
"requestBody": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/NewApiKeyRequest"
}
}
},
"required": true
},
"responses": {
"201": {
"description": "API key created successfully",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/NewApiKeyResponse"
}
}
}
},
"409": {
"description": "API key with that name already exists",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "An entity with this name already exists",
"error_code": "DuplicateName",
"details": null
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/api_keys/{api_key_name}": {
"get": {
"tags": [
"API keys"
],
"summary": "Retrieve an API key.",
"operationId": "get_api_key",
"parameters": [
{
"name": "api_key_name",
"in": "path",
"description": "Unique API key name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "API key retrieved successfully",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ApiKeyDescr"
}
}
}
},
"404": {
"description": "API key with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown API key 'unknown_api_key'",
"error_code": "UnknownApiKey",
"details": {
"name": "unknown_api_key"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"delete": {
"tags": [
"API keys"
],
"summary": "Delete an API key.",
"operationId": "delete_api_key",
"parameters": [
{
"name": "api_key_name",
"in": "path",
"description": "Unique API key name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "API key deleted successfully"
},
"404": {
"description": "API key with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown API key 'unknown_api_key'",
"error_code": "UnknownApiKey",
"details": {
"name": "unknown_api_key"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/config/demos": {
"get": {
"tags": [
"Configuration"
],
"summary": "Retrieve the list of demos.",
"operationId": "get_config_demos",
"responses": {
"200": {
"description": "List of demos",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Demo"
}
}
}
}
},
"500": {
"description": "Failed to read demos from the demos directories",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/metrics": {
"get": {
"tags": [
"Metrics"
],
"summary": "Retrieve the metrics of all running pipelines belonging to this tenant.",
"description": "The metrics are collected by making individual HTTP requests to `/metrics`\nendpoint of each pipeline, of which only successful responses are included\nin the returned list.",
"operationId": "get_metrics",
"responses": {
"200": {
"description": "Metrics of all running pipelines belonging to this tenant in Prometheus format",
"content": {
"text/plain": {
"schema": {
"type": "string",
"format": "binary"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve the list of pipelines.",
"description": "Configure which fields are included using the `selector` query parameter.",
"operationId": "list_pipelines",
"parameters": [
{
"name": "selector",
"in": "query",
"description": "The `selector` parameter limits which fields are returned for a pipeline.\nLimiting which fields is particularly handy for instance when frequently\nmonitoring over low bandwidth connections while being only interested\nin pipeline status.",
"required": false,
"schema": {
"$ref": "#/components/schemas/PipelineFieldSelector"
}
}
],
"responses": {
"200": {
"description": "List of pipelines retrieved successfully",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/PipelineSelectedInfo"
}
},
"example": [
{
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"description": "Description of the pipeline example1",
"created_at": "1970-01-01T00:00:00Z",
"version": 4,
"platform_version": "v0",
"runtime_config": {
"workers": 16,
"storage": false,
"fault_tolerance": null,
"cpu_profiler": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 0,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table1 ( col1 INT );",
"udf_rust": "",
"udf_toml": "",
"program_config": {
"profile": "optimized",
"cache": true
},
"program_version": 2,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_info": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"deployment_desired_status": "Shutdown",
"deployment_error": null
},
{
"id": "67e55044-10b1-426f-9247-bb680e5fe0c9",
"name": "example2",
"description": "Description of the pipeline example2",
"created_at": "1970-01-01T00:00:00Z",
"version": 1,
"platform_version": "v0",
"runtime_config": {
"workers": 10,
"storage": true,
"fault_tolerance": null,
"cpu_profiler": false,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 100000,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": 1000,
"memory_mb_max": null,
"storage_mb_max": 10000,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table2 ( col2 VARCHAR );",
"udf_rust": "",
"udf_toml": "",
"program_config": {
"profile": "unoptimized",
"cache": true
},
"program_version": 1,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_info": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"deployment_desired_status": "Shutdown",
"deployment_error": null
}
]
}
}
},
"500": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"post": {
"tags": [
"Pipelines"
],
"summary": "Create a new pipeline.",
"operationId": "post_pipeline",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PostPutPipeline"
},
"example": {
"name": "example1",
"description": "Description of the pipeline example1",
"runtime_config": {
"workers": 16,
"storage": false,
"fault_tolerance": null,
"cpu_profiler": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 0,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table1 ( col1 INT );",
"udf_rust": null,
"udf_toml": null,
"program_config": {
"profile": "optimized",
"cache": true
}
}
}
},
"required": true
},
"responses": {
"201": {
"description": "Pipeline successfully created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PipelineInfo"
},
"example": {
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"description": "Description of the pipeline example1",
"created_at": "1970-01-01T00:00:00Z",
"version": 4,
"platform_version": "v0",
"runtime_config": {
"workers": 16,
"storage": false,
"fault_tolerance": null,
"cpu_profiler": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 0,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table1 ( col1 INT );",
"udf_rust": "",
"udf_toml": "",
"program_config": {
"profile": "optimized",
"cache": true
},
"program_version": 2,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_info": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"deployment_desired_status": "Shutdown",
"deployment_error": null
}
}
}
},
"400": {
"description": "Invalid name specified",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Name 'invalid-name.db' contains characters which are not lowercase (a-z), uppercase (A-Z), numbers (0-9), underscores (_) or hyphens (-)",
"error_code": "NameDoesNotMatchPattern",
"details": {
"name": "invalid-name.db"
}
}
}
}
},
"409": {
"description": "Cannot create pipeline as the name already exists",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "An entity with this name already exists",
"error_code": "DuplicateName",
"details": null
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve a pipeline.",
"description": "Configure which fields are included using the `selector` query parameter.",
"operationId": "get_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "selector",
"in": "query",
"description": "The `selector` parameter limits which fields are returned for a pipeline.\nLimiting which fields is particularly handy for instance when frequently\nmonitoring over low bandwidth connections while being only interested\nin pipeline status.",
"required": false,
"schema": {
"$ref": "#/components/schemas/PipelineFieldSelector"
}
}
],
"responses": {
"200": {
"description": "Pipeline retrieved successfully",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PipelineSelectedInfo"
},
"example": {
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"description": "Description of the pipeline example1",
"created_at": "1970-01-01T00:00:00Z",
"version": 4,
"platform_version": "v0",
"runtime_config": {
"workers": 16,
"storage": false,
"fault_tolerance": null,
"cpu_profiler": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 0,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table1 ( col1 INT );",
"udf_rust": "",
"udf_toml": "",
"program_config": {
"profile": "optimized",
"cache": true
},
"program_version": 2,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_info": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"deployment_desired_status": "Shutdown",
"deployment_error": null
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"put": {
"tags": [
"Pipelines"
],
"summary": "Fully update a pipeline if it already exists, otherwise create a new pipeline.",
"operationId": "put_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PostPutPipeline"
},
"example": {
"name": "example1",
"description": "Description of the pipeline example1",
"runtime_config": {
"workers": 16,
"storage": false,
"fault_tolerance": null,
"cpu_profiler": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 0,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table1 ( col1 INT );",
"udf_rust": null,
"udf_toml": null,
"program_config": {
"profile": "optimized",
"cache": true
}
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Pipeline successfully updated",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PipelineInfo"
},
"example": {
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"description": "Description of the pipeline example1",
"created_at": "1970-01-01T00:00:00Z",
"version": 4,
"platform_version": "v0",
"runtime_config": {
"workers": 16,
"storage": false,
"fault_tolerance": null,
"cpu_profiler": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 0,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table1 ( col1 INT );",
"udf_rust": "",
"udf_toml": "",
"program_config": {
"profile": "optimized",
"cache": true
},
"program_version": 2,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_info": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"deployment_desired_status": "Shutdown",
"deployment_error": null
}
}
}
},
"201": {
"description": "Pipeline successfully created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PipelineInfo"
},
"example": {
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"description": "Description of the pipeline example1",
"created_at": "1970-01-01T00:00:00Z",
"version": 4,
"platform_version": "v0",
"runtime_config": {
"workers": 16,
"storage": false,
"fault_tolerance": null,
"cpu_profiler": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 0,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table1 ( col1 INT );",
"udf_rust": "",
"udf_toml": "",
"program_config": {
"profile": "optimized",
"cache": true
},
"program_version": 2,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_info": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"deployment_desired_status": "Shutdown",
"deployment_error": null
}
}
}
},
"400": {
"description": "Pipeline needs to be shutdown to be modified",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Cannot update a pipeline which is not fully shutdown. Shutdown the pipeline first by invoking the '/shutdown' endpoint.",
"error_code": "CannotUpdateNonShutdownPipeline",
"details": null
}
}
}
},
"409": {
"description": "Cannot rename pipeline as the name already exists",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "An entity with this name already exists",
"error_code": "DuplicateName",
"details": null
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"delete": {
"tags": [
"Pipelines"
],
"summary": "Delete a pipeline.",
"operationId": "delete_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline successfully deleted"
},
"400": {
"description": "Pipeline needs to be shutdown to be deleted",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Cannot delete a pipeline which is not fully shutdown. Shutdown the pipeline first by invoking the '/shutdown' endpoint.",
"error_code": "CannotDeleteNonShutdownPipeline",
"details": null
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
},
"patch": {
"tags": [
"Pipelines"
],
"summary": "Partially update a pipeline.",
"operationId": "patch_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PatchPipeline"
},
"example": {
"name": null,
"description": "This is a new description",
"runtime_config": null,
"program_code": "CREATE TABLE table3 ( col3 INT );",
"udf_rust": null,
"udf_toml": null,
"program_config": null
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Pipeline successfully updated",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/PipelineInfo"
},
"example": {
"id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"name": "example1",
"description": "Description of the pipeline example1",
"created_at": "1970-01-01T00:00:00Z",
"version": 4,
"platform_version": "v0",
"runtime_config": {
"workers": 16,
"storage": false,
"fault_tolerance": null,
"cpu_profiler": true,
"tracing": false,
"tracing_endpoint_jaeger": "",
"min_batch_size_records": 0,
"max_buffering_delay_usecs": 0,
"resources": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
},
"min_storage_bytes": null,
"clock_resolution_usecs": 100000
},
"program_code": "CREATE TABLE table1 ( col1 INT );",
"udf_rust": "",
"udf_toml": "",
"program_config": {
"profile": "optimized",
"cache": true
},
"program_version": 2,
"program_status": "Pending",
"program_status_since": "1970-01-01T00:00:00Z",
"program_info": null,
"deployment_status": "Shutdown",
"deployment_status_since": "1970-01-01T00:00:00Z",
"deployment_desired_status": "Shutdown",
"deployment_error": null
}
}
}
},
"400": {
"description": "Pipeline needs to be shutdown to be modified",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Cannot update a pipeline which is not fully shutdown. Shutdown the pipeline first by invoking the '/shutdown' endpoint.",
"error_code": "CannotUpdateNonShutdownPipeline",
"details": null
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
},
"409": {
"description": "Cannot rename pipeline as the name already exists",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "An entity with this name already exists",
"error_code": "DuplicateName",
"details": null
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/checkpoint": {
"post": {
"tags": [
"Pipelines"
],
"summary": "Checkpoint a running or paused pipeline.",
"operationId": "checkpoint_pipeline",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Checkpoint completed."
},
"400": {
"description": "Pipeline is not running or paused, or fault tolerance is not enabled for this pipeline",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused.",
"error_code": "PipelineNotRunningOrPaused",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
}
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/circuit_profile": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve the circuit performance profile of a running or paused pipeline.",
"operationId": "get_pipeline_circuit_profile",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Obtains a circuit performance profile.",
"content": {
"application/zip": {
"schema": {
"type": "object"
}
}
}
},
"400": {
"description": "Pipeline is not running or paused",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused.",
"error_code": "PipelineNotRunningOrPaused",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
}
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/egress/{table_name}": {
"post": {
"tags": [
"HTTP input/output"
],
"summary": "Subscribe to a stream of updates from a SQL view or table.",
"description": "The pipeline responds with a continuous stream of changes to the specified\ntable or view, encoded using the format specified in the `?format=`\nparameter. Updates are split into `Chunk`s.\n\nThe pipeline continues sending updates until the client closes the\nconnection or the pipeline is shut down.",
"operationId": "http_output",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "table_name",
"in": "path",
"description": "SQL table name. Unquoted SQL names have to be capitalized. Quoted SQL names have to exactly match the case from the SQL program.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "format",
"in": "query",
"description": "Output data format, e.g., 'csv' or 'json'.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "array",
"in": "query",
"description": "Set to `true` to group updates in this stream into JSON arrays (used in conjunction with `format=json`). The default value is `false`",
"required": false,
"schema": {
"type": "boolean",
"nullable": true
}
},
{
"name": "backpressure",
"in": "query",
"description": "Apply backpressure on the pipeline when the HTTP client cannot receive data fast enough.\n When this flag is set to false (the default), the HTTP connector drops data chunks if the client is not keeping up with its output. This prevents a slow HTTP client from slowing down the entire pipeline.\n When the flag is set to true, the connector waits for the client to receive each chunk and blocks the pipeline if the client cannot keep up.",
"required": false,
"schema": {
"type": "boolean",
"nullable": true
}
}
],
"responses": {
"200": {
"description": "Connection to the endpoint successfully established. The body of the response contains a stream of data chunks.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Chunk"
}
}
}
},
"400": {
"description": "Unknown data format specified in the '?format=' argument.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"404": {
"description": "Specified table or view does not exist.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"410": {
"description": "Pipeline is not currently running because it has been shutdown or not yet started.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused.",
"error_code": "PipelineNotRunningOrPaused",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
}
}
}
}
},
"500": {
"description": "Request failed.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/heap_profile": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve the heap profile of a running or paused pipeline.",
"operationId": "get_pipeline_heap_profile",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline's heap usage profile as a gzipped protobuf that can be inspected by the pprof tool",
"content": {
"application/protobuf": {
"schema": {
"type": "string",
"format": "binary"
}
}
}
},
"400": {
"description": "Pipeline is not running or paused, or getting a heap profile is not supported on this platform",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused.",
"error_code": "PipelineNotRunningOrPaused",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
}
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/ingress/{table_name}": {
"post": {
"tags": [
"HTTP input/output"
],
"summary": "Push data to a SQL table.",
"description": "The client sends data encoded using the format specified in the `?format=`\nparameter as a body of the request. The contents of the data must match\nthe SQL table schema specified in `table_name`\n\nThe pipeline ingests data as it arrives without waiting for the end of\nthe request. Successful HTTP response indicates that all data has been\ningested successfully.",
"operationId": "http_input",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "table_name",
"in": "path",
"description": "SQL table name. Unquoted SQL names have to be capitalized. Quoted SQL names have to exactly match the case from the SQL program.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "force",
"in": "query",
"description": "When `true`, push data to the pipeline even if the pipeline is paused. The default value is `false`",
"required": true,
"schema": {
"type": "boolean"
}
},
{
"name": "format",
"in": "query",
"description": "Input data format, e.g., 'csv' or 'json'.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "array",
"in": "query",
"description": "Set to `true` if updates in this stream are packaged into JSON arrays (used in conjunction with `format=json`). The default values is `false`.",
"required": false,
"schema": {
"type": "boolean",
"nullable": true
}
},
{
"name": "update_format",
"in": "query",
"description": "JSON data change event format (used in conjunction with `format=json`). The default value is 'insert_delete'.",
"required": false,
"schema": {
"allOf": [
{
"$ref": "#/components/schemas/JsonUpdateFormat"
}
],
"nullable": true
}
}
],
"requestBody": {
"description": "Contains the new input data in CSV.",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Data successfully delivered to the pipeline."
},
"400": {
"description": "Error parsing input data.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"404": {
"description": "Pipeline is not currently running because it has been shutdown or not yet started.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused.",
"error_code": "PipelineNotRunningOrPaused",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
}
}
}
}
},
"500": {
"description": "Request failed.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/logs": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve pipeline logs as a stream.",
"description": "The logs stream catches up to the extent of the internally configured per-pipeline\ncircular logs buffer (limited to a certain byte size and number of lines, whichever\nis reached first). After the catch-up, new lines are pushed whenever they become\navailable.\n\nThe logs stream will end when the pipeline is shut down. It is also possible for the\nlogs stream to end prematurely due to the runner back-end (temporarily) losing\nconnectivity to the pipeline instance (e.g., process). In this case, it is needed\nto issue again a new request to this endpoint.",
"operationId": "get_pipeline_logs",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline logs retrieved successfully",
"content": {
"text/plain": {
"schema": {
"type": "string",
"format": "binary"
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/query": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Execute an ad-hoc query in a running or paused pipeline.",
"operationId": "pipeline_adhoc_sql",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "sql",
"in": "query",
"description": "The SQL query to execute.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "format",
"in": "query",
"description": "Input data format, e.g., 'text', 'json' or 'parquet'.",
"required": true,
"schema": {
"$ref": "#/components/schemas/AdHocResultFormat"
}
}
],
"responses": {
"200": {
"description": "Executes an ad-hoc SQL query in a running or paused pipeline. The evaluation is not incremental.",
"content": {
"text/plain": {
"schema": {
"type": "string",
"format": "binary"
}
}
}
},
"400": {
"description": "Pipeline is shutdown or an invalid SQL query was supplied",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused.",
"error_code": "PipelineNotRunningOrPaused",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
}
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
},
"500": {
"description": "A fatal error occurred during query processing (after streaming response was already initiated)",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Sending request to URL https://localhost:1234/query of pipeline example (67e55044-10b1-426f-9247-bb680e5fe0c8) failed: Failed to connect to host: Internal error: connector has been disconnected",
"error_code": "PipelineEndpointSendError",
"details": {
"pipeline_id": "67e55044-10b1-426f-9247-bb680e5fe0c8",
"pipeline_name": "example",
"url": "https://localhost:1234/query",
"error": "Failed to connect to host: Internal error: connector has been disconnected"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/stats": {
"get": {
"tags": [
"Pipelines"
],
"summary": "Retrieve pipeline statistics (e.g., metrics, performance counters).",
"operationId": "get_pipeline_stats",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Pipeline metrics retrieved successfully",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
},
"400": {
"description": "Pipeline is not running or paused",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Pipeline example (2e79afe1-ff4d-44d3-af5f-9397de7746c0) is not currently running or paused.",
"error_code": "PipelineNotRunningOrPaused",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0",
"pipeline_name": "example"
}
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/tables/{table_name}/connectors/{connector_name}/{action}": {
"post": {
"tags": [
"Pipelines"
],
"summary": "Start (resume) or pause the input connector.",
"description": "The following values of the `action` argument are accepted: `start` and `pause`.\n\nInput connectors can be in either the `Running` or `Paused` state. By default,\nconnectors are initialized in the `Running` state when a pipeline is deployed.\nIn this state, the connector actively fetches data from its configured data\nsource and forwards it to the pipeline. If needed, a connector can be created\nin the `Paused` state by setting its\n[`paused`](https://docs.feldera.com/connectors/#generic-attributes) property\nto `true`. When paused, the connector remains idle until reactivated using the\n`start` command. Conversely, a connector in the `Running` state can be paused\nat any time by issuing the `pause` command.\n\nThe current connector state can be retrieved via the\n`GET /v0/pipelines/{pipeline_name}/stats` endpoint.\n\nNote that only if both the pipeline *and* the connector state is `Running`,\nis the input connector active.\n```text\nPipeline state Connector state Connector is active?\n-------------- --------------- --------------------\nPaused Paused No\nPaused Running No\nRunning Paused No\nRunning Running Yes\n```",
"operationId": "post_pipeline_input_connector_action",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "table_name",
"in": "path",
"description": "Unique table name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "connector_name",
"in": "path",
"description": "Unique input connector name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "action",
"in": "path",
"description": "Input connector action (one of: start, pause)",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Action has been processed"
},
"404": {
"description": "Input connector with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
},
"/v0/pipelines/{pipeline_name}/{action}": {
"post": {
"tags": [
"Pipelines"
],
"summary": "Start, pause or shutdown a pipeline.",
"description": "The endpoint returns immediately after performing initial request validation\n(e.g., upon start checking the program is compiled) and initiating the relevant\nprocedure (e.g., informing the runner or the already running pipeline).\nThe state changes completely asynchronously. On error, the pipeline\ntransitions to the `Failed` state. The user can monitor the current status\nof the pipeline by polling the `GET /pipelines` and\n`GET /pipelines/{pipeline_name}` endpoint.\n\nThe following values of the `action` argument are accepted:\n- `start`: Start the pipeline\n- `pause`: Pause the pipeline\n- `shutdown`: Terminate the pipeline",
"operationId": "post_pipeline_action",
"parameters": [
{
"name": "pipeline_name",
"in": "path",
"description": "Unique pipeline name",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "action",
"in": "path",
"description": "Pipeline action (one of: start, pause, shutdown)",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"202": {
"description": "Action accepted and is being performed"
},
"400": {
"description": "Unable to accept action",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"examples": {
"Illegal action": {
"description": "Action is not applicable in the current state",
"value": {
"message": "Deployment status (current: 'ShuttingDown', desired: 'Shutdown') cannot have desired changed to 'Running'. Cannot restart the pipeline while it is shutting down. Wait for the shutdown to complete before starting a new instance of the pipeline.",
"error_code": "IllegalPipelineStateTransition",
"details": {
"hint": "Cannot restart the pipeline while it is shutting down. Wait for the shutdown to complete before starting a new instance of the pipeline.",
"status": "ShuttingDown",
"desired_status": "Shutdown",
"requested_desired_status": "Running"
}
}
},
"Invalid action": {
"description": "Invalid action specified",
"value": {
"message": "Invalid pipeline action 'dance'; valid actions are: 'start', 'pause', or 'shutdown'",
"error_code": "InvalidPipelineAction",
"details": {
"action": "dance"
}
}
},
"Program failed compilation": {
"description": "Program failed compilation",
"value": {
"message": "Not possible to start the pipeline because the program failed to compile",
"error_code": "StartFailedDueToFailedCompilation",
"details": {
"compiler_error": "error: failed to compile: error: linking with `cc` failed: exit code: 1"
}
}
}
}
}
}
},
"404": {
"description": "Pipeline with that name does not exist",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
},
"example": {
"message": "Unknown pipeline id '2e79afe1-ff4d-44d3-af5f-9397de7746c0'",
"error_code": "UnknownPipeline",
"details": {
"pipeline_id": "2e79afe1-ff4d-44d3-af5f-9397de7746c0"
}
}
}
}
}
},
"security": [
{
"JSON web token (JWT) or API key": []
}
]
}
}
},
"components": {
"schemas": {
"AdHocInputConfig": {
"type": "object",
"description": "Configuration for inserting data with ad-hoc queries\n\nAn ad-hoc input adapters cannot be usefully configured as part of pipeline\nconfiguration. Instead, use ad-hoc queries through the UI, the REST API, or\nthe `fda` command-line tool.",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Autogenerated name."
}
}
},
"AdHocResultFormat": {
"type": "string",
"description": "URL-encoded `format` argument to the `/query` endpoint.",
"enum": [
"text",
"json",
"parquet"
]
},
"AdhocQueryArgs": {
"type": "object",
"description": "URL-encoded arguments to the `/query` endpoint.",
"required": [
"sql"
],
"properties": {
"format": {
"$ref": "#/components/schemas/AdHocResultFormat"
},
"sql": {
"type": "string",
"description": "The SQL query to run."
}
}
},
"ApiKeyDescr": {
"type": "object",
"description": "API key descriptor.",
"required": [
"id",
"name",
"scopes"
],
"properties": {
"id": {
"$ref": "#/components/schemas/ApiKeyId"
},
"name": {
"type": "string"
},
"scopes": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ApiPermission"
}
}
}
},
"ApiKeyId": {
"type": "string",
"format": "uuid",
"description": "API key identifier."
},
"ApiPermission": {
"type": "string",
"description": "Permission types for invoking API endpoints.",
"enum": [
"Read",
"Write"
]
},
"AuthProvider": {
"oneOf": [
{
"type": "object",
"required": [
"AwsCognito"
],
"properties": {
"AwsCognito": {
"$ref": "#/components/schemas/ProviderAwsCognito"
}
}
},
{
"type": "object",
"required": [
"GoogleIdentity"
],
"properties": {
"GoogleIdentity": {
"$ref": "#/components/schemas/ProviderGoogleIdentity"
}
}
}
]
},
"Chunk": {
"type": "object",
"description": "A set of updates to a SQL table or view.\n\nThe `sequence_number` field stores the offset of the chunk relative to the\nstart of the stream and can be used to implement reliable delivery.\nThe payload is stored in the `bin_data`, `text_data`, or `json_data` field\ndepending on the data format used.",
"required": [
"sequence_number"
],
"properties": {
"bin_data": {
"type": "string",
"format": "binary",
"description": "Base64 encoded binary payload, e.g., bincode.",
"nullable": true
},
"json_data": {
"type": "object",
"description": "JSON payload.",
"nullable": true
},
"sequence_number": {
"type": "integer",
"format": "int64",
"minimum": 0
},
"text_data": {
"type": "string",
"description": "Text payload, e.g., CSV.",
"nullable": true
}
}
},
"ColumnType": {
"type": "object",
"description": "A SQL column type description.\n\nMatches the Calcite JSON format.",
"required": [
"nullable"
],
"properties": {
"component": {
"allOf": [
{
"$ref": "#/components/schemas/ColumnType"
}
],
"nullable": true
},
"fields": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Field"
},
"description": "The fields of the type (if available).\n\nFor example this would specify the fields of a `CREATE TYPE` construct.\n\n```sql\nCREATE TYPE person_typ AS (\nfirstname VARCHAR(30),\nlastname VARCHAR(30),\naddress ADDRESS_TYP\n);\n```\n\nWould lead to the following `fields` value:\n\n```sql\n[\nColumnType { name: \"firstname, ... },\nColumnType { name: \"lastname\", ... },\nColumnType { name: \"address\", fields: [ ... ] }\n]\n```",
"nullable": true
},
"key": {
"allOf": [
{
"$ref": "#/components/schemas/ColumnType"
}
],
"nullable": true
},
"nullable": {
"type": "boolean",
"description": "Does the type accept NULL values?"
},
"precision": {
"type": "integer",
"format": "int64",
"description": "Precision of the type.\n\n# Examples\n- `VARCHAR` sets precision to `-1`.\n- `VARCHAR(255)` sets precision to `255`.\n- `BIGINT`, `DATE`, `FLOAT`, `DOUBLE`, `GEOMETRY`, etc. sets precision\nto None\n- `TIME`, `TIMESTAMP` set precision to `0`.",
"nullable": true
},
"scale": {
"type": "integer",
"format": "int64",
"description": "The scale of the type.\n\n# Example\n- `DECIMAL(1,2)` sets scale to `2`.",
"nullable": true
},
"type": {
"$ref": "#/components/schemas/SqlType"
},
"value": {
"allOf": [
{
"$ref": "#/components/schemas/ColumnType"
}
],
"nullable": true
}
}
},
"CompilationProfile": {
"type": "string",
"description": "Enumeration of possible compilation profiles that can be passed to the Rust compiler\nas an argument via `cargo build --profile <>`. A compilation profile affects among\nother things the compilation speed (how long till the program is ready to be run)\nand runtime speed (the performance while running).",
"enum": [
"dev",
"unoptimized",
"optimized"
]
},
"Configuration": {
"type": "object",
"required": [
"telemetry",
"version"
],
"properties": {
"telemetry": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"ConnectorConfig": {
"allOf": [
{
"$ref": "#/components/schemas/OutputBufferConfig"
},
{
"type": "object",
"required": [
"transport"
],
"properties": {
"format": {
"allOf": [
{
"$ref": "#/components/schemas/FormatConfig"
}
],
"nullable": true
},
"max_batch_size": {
"type": "integer",
"format": "int64",
"description": "Maximum batch size, in records.\n\nThis is the maximum number of records to process in one batch through\nthe circuit. The time and space cost of processing a batch is\nasymptotically superlinear in the size of the batch, but very small\nbatches are less efficient due to constant factors.\n\nThis should usually be less than `max_queued_records`, to give the\nconnector a round-trip time to restart and refill the buffer while\nbatches are being processed.\n\nSome input adapters might not honor this setting.\n\nThe default is 10,000.",
"minimum": 0
},
"max_queued_records": {
"type": "integer",
"format": "int64",
"description": "Backpressure threshold.\n\nMaximal number of records queued by the endpoint before the endpoint\nis paused by the backpressure mechanism.\n\nFor input endpoints, this setting bounds the number of records that have\nbeen received from the input transport but haven't yet been consumed by\nthe circuit since the circuit, since the circuit is still busy processing\nprevious inputs.\n\nFor output endpoints, this setting bounds the number of records that have\nbeen produced by the circuit but not yet sent via the output transport endpoint\nnor stored in the output buffer (see `enable_output_buffer`).\n\nNote that this is not a hard bound: there can be a small delay between\nthe backpressure mechanism is triggered and the endpoint is paused, during\nwhich more data may be queued.\n\nThe default is 1 million.",
"minimum": 0
},
"paused": {
"type": "boolean",
"description": "Create connector in paused state.\n\nThe default is `false`."
},
"transport": {
"$ref": "#/components/schemas/TransportConfig"
}
}
}
],
"description": "A data connector's configuration"
},
"DatagenInputConfig": {
"type": "object",
"description": "Configuration for generating random data for a table.",
"properties": {
"plan": {
"type": "array",
"items": {
"$ref": "#/components/schemas/GenerationPlan"
},
"description": "The sequence of generations to perform.\n\nIf not set, the generator will produce a single sequence with default settings.\nIf set, the generator will produce the specified sequences in sequential order.\n\nNote that if one of the sequences before the last one generates an unlimited number of rows\nthe following sequences will not be executed.",
"default": [
{
"rate": null,
"limit": null,
"worker_chunk_size": null,
"fields": {}
}
]
},
"seed": {
"type": "integer",
"format": "int64",
"description": "Optional seed for the random generator.\n\nSetting this to a fixed value will make the generator produce the same sequence of records\nevery time the pipeline is run.\n\n# Notes\n- To ensure the set of generated input records is deterministic across multiple runs,\napart from setting a seed, `workers` also needs to remain unchanged.\n- The input will arrive in non-deterministic order if `workers > 1`.",
"default": null,
"nullable": true,
"minimum": 0
},
"workers": {
"type": "integer",
"description": "Number of workers to use for generating data.",
"default": 1,
"minimum": 0
}
},
"additionalProperties": false
},
"DatagenStrategy": {
"type": "string",
"description": "Strategy used to generate values.",
"enum": [
"increment",
"uniform",
"zipf",
"word",
"words",
"sentence",
"sentences",
"paragraph",
"paragraphs",
"first_name",
"last_name",
"title",
"suffix",
"name",
"name_with_title",
"domain_suffix",
"email",
"username",
"password",
"field",
"position",
"seniority",
"job_title",
"ipv4",
"ipv6",
"ip",
"mac_address",
"user_agent",
"rfc_status_code",
"valid_status_code",
"company_suffix",
"company_name",
"buzzword",
"buzzword_middle",
"buzzword_tail",
"catch_phrase",
"bs_verb",
"bs_adj",
"bs_noun",
"bs",
"profession",
"industry",
"currency_code",
"currency_name",
"currency_symbol",
"credit_card_number",
"city_prefix",
"city_suffix",
"city_name",
"country_name",
"country_code",
"street_suffix",
"street_name",
"time_zone",
"state_name",
"state_abbr",
"secondary_address_type",
"secondary_address",
"zip_code",
"post_code",
"building_number",
"latitude",
"longitude",
"isbn",
"isbn13",
"isbn10",
"phone_number",
"cell_number",
"file_path",
"file_name",
"file_extension",
"dir_path"
]
},
"DeltaTableIngestMode": {
"type": "string",
"description": "Delta table read mode.\n\nThree options are available:\n\n* `snapshot` - read a snapshot of the table and stop.\n\n* `follow` - continuously ingest changes to the table, starting from a specified version\nor timestamp.\n\n* `snapshot_and_follow` - read a snapshot of the table before switching to continuous ingestion\nmode.",
"enum": [
"snapshot",
"follow",
"snapshot_and_follow"
]
},
"DeltaTableReaderConfig": {
"type": "object",
"description": "Delta table input connector configuration.",
"required": [
"uri",
"mode"
],
"properties": {
"datetime": {
"type": "string",
"description": "Optional timestamp for the snapshot in the ISO-8601/RFC-3339 format, e.g.,\n\"2024-12-09T16:09:53+00:00\".\n\nWhen this option is set, the connector finds and opens the version of the table as of the\nspecified point in time (based on the server time recorded in the transaction log, not the\nevent time encoded in the data). In `snapshot` and `snapshot_and_follow` modes, it\nretrieves the snapshot of this version of the table. In `follow` and `snapshot_and_follow`\nmodes, it follows transaction log records **after** this version.\n\nNote: at most one of `version` and `datetime` options can be specified.\nWhen neither of the two options is specified, the latest committed version of the table\nis used.",
"nullable": true
},
"mode": {
"$ref": "#/components/schemas/DeltaTableIngestMode"
},
"snapshot_filter": {
"type": "string",
"description": "Optional row filter.\n\nThis option is only valid when `mode` is set to `snapshot` or `snapshot_and_follow`.\n\nWhen specified, only rows that satisfy the filter condition are included in the\nsnapshot. The condition must be a valid SQL Boolean expression that can be used in\nthe `where` clause of the `select * from snapshot where ...` query.\n\nThis option can be used to specify the range of event times to include in the snapshot,\ne.g.: `ts BETWEEN TIMESTAMP '2005-01-01 00:00:00' AND TIMESTAMP '2010-12-31 23:59:59'`.",
"nullable": true
},
"timestamp_column": {
"type": "string",
"description": "Table column that serves as an event timestamp.\n\nWhen this option is specified, and `mode` is one of `snapshot` or `snapshot_and_follow`,\ntable rows are ingested in the timestamp order, respecting the\n[`LATENESS`](https://docs.feldera.com/sql/streaming#lateness-expressions)\nproperty of the column: each ingested row has a timestamp no more than `LATENESS`\ntime units earlier than the most recent timestamp of any previously ingested row.\nThe ingestion is performed by partitioning the table into timestamp ranges of width\n`LATENESS`. Each range is processed sequentially, in increasing timestamp order.\n\n# Example\n\nConsider a table with timestamp column of type `TIMESTAMP` and lateness attribute\n`INTERVAL 1 DAY`. Assuming that the oldest timestamp in the table is\n`2024-01-01T00:00:00``, the connector will fetch all records with timestamps\nfrom `2024-01-01`, then all records for `2024-01-02`, `2024-01-03`, etc., until all records\nin the table have been ingested.\n\n# Requirements\n\n* The timestamp column must be of a supported type: integer, `DATE`, or `TIMESTAMP`.\n* The timestamp column must be declared with non-zero `LATENESS`.\n* For efficient ingest, the table must be optimized for timestamp-based\nqueries using partitioning, Z-ordering, or liquid clustering.",
"nullable": true
},
"uri": {
"type": "string",
"description": "Table URI.\n\nExample: \"s3://feldera-fraud-detection-data/demographics_train\""
},
"version": {
"type": "integer",
"format": "int64",
"description": "Optional table version.\n\nWhen this option is set, the connector finds and opens the specified version of the table.\nIn `snapshot` and `snapshot_and_follow` modes, it retrieves the snapshot of this version of\nthe table. In `follow` and `snapshot_and_follow` modes, it follows transaction log records\n**after** this version.\n\nNote: at most one of `version` and `datetime` options can be specified.\nWhen neither of the two options is specified, the latest committed version of the table\nis used.",
"nullable": true
}
},
"additionalProperties": {
"type": "string",
"description": "Storage options for configuring backend object store.\n\nFor specific options available for different storage backends, see:\n* [Azure options](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)\n* [Amazon S3 options](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)\n* [Google Cloud Storage options](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)"
}
},
"DeltaTableWriteMode": {
"type": "string",
"description": "Delta table write mode.\n\nDetermines how the Delta table connector handles an existing table at the target location.",
"enum": [
"append",
"truncate",
"error_if_exists"
]
},
"DeltaTableWriterConfig": {
"type": "object",
"description": "Delta table output connector configuration.",
"required": [
"uri"
],
"properties": {
"mode": {
"$ref": "#/components/schemas/DeltaTableWriteMode"
},
"uri": {
"type": "string",
"description": "Table URI."
}
},
"additionalProperties": {
"type": "string",
"description": "Storage options for configuring backend object store.\n\nFor specific options available for different storage backends, see:\n* [Azure options](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)\n* [Amazon S3 options](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)\n* [Google Cloud Storage options](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)"
}
},
"Demo": {
"type": "object",
"required": [
"name",
"title",
"description",
"program_code",
"udf_rust",
"udf_toml"
],
"properties": {
"description": {
"type": "string",
"description": "Description of the demo (parsed from SQL preamble)."
},
"name": {
"type": "string",
"description": "Name of the demo (parsed from SQL preamble)."
},
"program_code": {
"type": "string",
"description": "Program SQL code."
},
"title": {
"type": "string",
"description": "Title of the demo (parsed from SQL preamble)."
},
"udf_rust": {
"type": "string",
"description": "User defined function (UDF) Rust code."
},
"udf_toml": {
"type": "string",
"description": "User defined function (UDF) TOML dependencies."
}
}
},
"ErrorResponse": {
"type": "object",
"description": "Information returned by REST API endpoints on error.",
"required": [
"message",
"error_code",
"details"
],
"properties": {
"details": {
"type": "object",
"description": "Detailed error metadata.\nThe contents of this field is determined by `error_code`."
},
"error_code": {
"type": "string",
"description": "Error code is a string that specifies this error type.",
"example": "UnknownInputFormat"
},
"message": {
"type": "string",
"description": "Human-readable error message.",
"example": "Unknown input format 'xml'."
}
}
},
"Field": {
"allOf": [
{
"$ref": "#/components/schemas/SqlIdentifier"
},
{
"type": "object",
"required": [
"columntype"
],
"properties": {
"columntype": {
"$ref": "#/components/schemas/ColumnType"
},
"default": {
"type": "string",
"nullable": true
},
"lateness": {
"type": "string",
"nullable": true
},
"watermark": {
"type": "string",
"nullable": true
}
}
}
],
"description": "A SQL field.\n\nMatches the SQL compiler JSON format."
},
"FileInputConfig": {
"type": "object",
"description": "Configuration for reading data from a file with `FileInputTransport`",
"required": [
"path"
],
"properties": {
"buffer_size_bytes": {
"type": "integer",
"description": "Read buffer size.\n\nDefault: when this parameter is not specified, a platform-specific\ndefault is used.",
"nullable": true,
"minimum": 0
},
"follow": {
"type": "boolean",
"description": "Enable file following.\n\nWhen `false`, the endpoint outputs an `InputConsumer::eoi`\nmessage and stops upon reaching the end of file. When `true`, the\nendpoint will keep watching the file and outputting any new content\nappended to it."
},
"path": {
"type": "string",
"description": "File path."
}
}
},
"FileOutputConfig": {
"type": "object",
"description": "Configuration for writing data to a file with `FileOutputTransport`.",
"required": [
"path"
],
"properties": {
"path": {
"type": "string",
"description": "File path."
}
}
},
"FormatConfig": {
"type": "object",
"description": "Data format specification used to parse raw data received from the\nendpoint or to encode data sent to the endpoint.",
"required": [
"name"
],
"properties": {
"config": {
"type": "object",
"description": "Format-specific parser or encoder configuration."
},
"name": {
"type": "string",
"description": "Format name, e.g., \"csv\", \"json\", \"bincode\", etc."
}
}
},
"FtConfig": {
"type": "object",
"description": "Fault-tolerance configuration for runtime startup.",
"properties": {
"checkpoint_interval_secs": {
"type": "integer",
"format": "int64",
"description": "Interval between automatic checkpoints, in seconds.\n\nThe default is 60 seconds. A value of 0 disables automatic\ncheckpointing.",
"default": 60,
"minimum": 0
}
}
},
"GenerationPlan": {
"type": "object",
"description": "A random generation plan for a table that generates either a limited amount of rows or runs continuously.",
"properties": {
"fields": {
"type": "object",
"description": "Specifies the values that the generator should produce.",
"default": {},
"additionalProperties": {
"$ref": "#/components/schemas/RngFieldSettings"
}
},
"limit": {
"type": "integer",
"description": "Total number of new rows to generate.\n\nIf not set, the generator will produce new/unique records as long as the pipeline is running.\nIf set to 0, the table will always remain empty.\nIf set, the generator will produce new records until the specified limit is reached.\n\nNote that if the table has one or more primary keys that don't use the `increment` strategy to\ngenerate the key there is a potential that an update is generated instead of an insert. In\nthis case it's possible the total number of records is less than the specified limit.",
"default": null,
"nullable": true,
"minimum": 0
},
"rate": {
"type": "integer",
"format": "int32",
"description": "Non-zero number of rows to generate per second.\n\nIf not set, the generator will produce rows as fast as possible.",
"default": null,
"nullable": true,
"minimum": 0
},
"worker_chunk_size": {
"type": "integer",
"description": "When multiple workers are used, each worker will pick a consecutive \"chunk\" of\nrecords to generate.\n\nBy default, if not specified, the generator will use the formula `min(rate, 10_000)`\nto determine it. This works well in most situations. However, if you're\nrunning tests with lateness and many workers you can e.g., reduce the\nchunk size to make sure a smaller range of records is being ingested in parallel.\n\n# Example\nAssume you generate a total of 125 records with 4 workers and a chunk size of 25.\nIn this case, worker A will generate records 0..25, worker B will generate records 25..50,\netc. A, B, C, and D will generate records in parallel. The first worker to finish its chunk\nwill pick up the last chunk of records (100..125) to generate.",
"default": null,
"nullable": true,
"minimum": 0
}
},
"additionalProperties": false
},
"GetPipelineParameters": {
"type": "object",
"description": "Query parameters to GET a pipeline or a list of pipelines.",
"properties": {
"selector": {
"$ref": "#/components/schemas/PipelineFieldSelector"
}
}
},
"GlueCatalogConfig": {
"type": "object",
"description": "AWS Glue catalog config.",
"properties": {
"glue.access-key-id": {
"type": "string",
"description": "Access key id used to access the Glue catalog.",
"nullable": true
},
"glue.endpoint": {
"type": "string",
"description": "Configure an alternative endpoint of the Glue service for Glue catalog to access.\n\nExample: `\"https://glue.us-east-1.amazonaws.com\"`",
"nullable": true
},
"glue.id": {
"type": "string",
"description": "The 12-digit ID of the Glue catalog.",
"nullable": true
},
"glue.profile-name": {
"type": "string",
"description": "Profile used to access the Glue catalog.",
"nullable": true
},
"glue.region": {
"type": "string",
"description": "Region of the Glue catalog.",
"nullable": true
},
"glue.secret-access-key": {
"type": "string",
"description": "Secret access key used to access the Glue catalog.",
"nullable": true
},
"glue.session-token": {
"type": "string",
"nullable": true
},
"glue.warehouse": {
"type": "string",
"description": "Location for table metadata.\n\nExample: `\"s3://my-data-warehouse/tables/\"`",
"nullable": true
}
}
},
"HttpInputConfig": {
"type": "object",
"description": "Configuration for reading data via HTTP.\n\nHTTP input adapters cannot be usefully configured as part of pipeline\nconfiguration. Instead, instantiate them through the REST API as\n`/pipelines/{pipeline_name}/ingress/{table_name}`.",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Autogenerated name."
}
}
},
"IcebergCatalogType": {
"type": "string",
"enum": [
"rest",
"glue"
]
},
"IcebergIngestMode": {
"type": "string",
"description": "Iceberg table read mode.\n\nThree options are available:\n\n* `snapshot` - read a snapshot of the table and stop.\n\n* `follow` - continuously ingest changes to the table, starting from a specified snapshot\nor timestamp.\n\n* `snapshot_and_follow` - read a snapshot of the table before switching to continuous ingestion\nmode.",
"enum": [
"snapshot",
"follow",
"snapshot_and_follow"
]
},
"IcebergReaderConfig": {
"allOf": [
{
"$ref": "#/components/schemas/GlueCatalogConfig"
},
{
"$ref": "#/components/schemas/RestCatalogConfig"
},
{
"type": "object",
"required": [
"mode"
],
"properties": {
"catalog_type": {
"allOf": [
{
"$ref": "#/components/schemas/IcebergCatalogType"
}
],
"nullable": true
},
"datetime": {
"type": "string",
"description": "Optional timestamp for the snapshot in the ISO-8601/RFC-3339 format, e.g.,\n\"2024-12-09T16:09:53+00:00\".\n\nWhen this option is set, the connector finds and opens the snapshot of the table as of the\nspecified point in time (based on the server time recorded in the transaction\nlog, not the event time encoded in the data). In `snapshot` and `snapshot_and_follow`\nmodes, it retrieves this snapshot. In `follow` and `snapshot_and_follow` modes, it\nfollows transaction log records **after** this snapshot.\n\nNote: at most one of `snapshot_id` and `datetime` options can be specified.\nWhen neither of the two options is specified, the latest committed version of the table\nis used.",
"nullable": true
},
"metadata_location": {
"type": "string",
"description": "Location of the table metadata JSON file.\n\nThis propery is used to access an Iceberg table without a catalog. It is mutually\nexclusive with the `catalog_type` property.",
"nullable": true
},
"mode": {
"$ref": "#/components/schemas/IcebergIngestMode"
},
"snapshot_filter": {
"type": "string",
"description": "Optional row filter.\n\nThis option is only valid when `mode` is set to `snapshot` or `snapshot_and_follow`.\n\nWhen specified, only rows that satisfy the filter condition are included in the\nsnapshot. The condition must be a valid SQL Boolean expression that can be used in\nthe `where` clause of the `select * from snapshot where ...` query.\n\nThis option can be used to specify the range of event times to include in the snapshot,\ne.g.: `ts BETWEEN '2005-01-01 00:00:00' AND '2010-12-31 23:59:59'`.",
"nullable": true
},
"snapshot_id": {
"type": "integer",
"format": "int64",
"description": "Optional snapshot id.\n\nWhen this option is set, the connector finds the specified snapshot of the table.\nIn `snapshot` and `snapshot_and_follow` modes, it loads this snapshot.\nIn `follow` and `snapshot_and_follow` modes, it follows table updates\n**after** this snapshot.\n\nNote: at most one of `snapshot_id` and `datetime` options can be specified.\nWhen neither of the two options is specified, the latest committed version of the table\nis used.",
"nullable": true
},
"table_name": {
"type": "string",
"description": "Specifies the Iceberg table name in the \"namespace.table\" format.\n\nThis option is applicable when an Iceberg catalog is configured using the `catalog_type` property.",
"nullable": true
},
"timestamp_column": {
"type": "string",
"description": "Table column that serves as an event timestamp.\n\nWhen this option is specified, and `mode` is one of `snapshot` or `snapshot_and_follow`,\ntable rows are ingested in the timestamp order, respecting the\n[`LATENESS`](https://docs.feldera.com/sql/streaming#lateness-expressions)\nproperty of the column: each ingested row has a timestamp no more than `LATENESS`\ntime units earlier than the most recent timestamp of any previously ingested row.\nThe ingestion is performed by partitioning the table into timestamp ranges of width\n`LATENESS`. Each range is processed sequentially, in increasing timestamp order.\n\n# Example\n\nConsider a table with timestamp column of type `TIMESTAMP` and lateness attribute\n`INTERVAL 1 DAY`. Assuming that the oldest timestamp in the table is\n`2024-01-01T00:00:00``, the connector will fetch all records with timestamps\nfrom `2024-01-01`, then all records for `2024-01-02`, `2024-01-03`, etc., until all records\nin the table have been ingested.\n\n# Requirements\n\n* The timestamp column must be of a supported type: integer, `DATE`, or `TIMESTAMP`.\n* The timestamp column must be declared with non-zero `LATENESS`.\n* For efficient ingest, the table must be optimized for timestamp-based\nqueries using partitioning, Z-ordering, or liquid clustering.",
"nullable": true
}
},
"additionalProperties": {
"type": "string",
"description": "Storage options for configuring backend object store.\n\nSee the [list of available options in PyIceberg documentation](https://py.iceberg.apache.org/configuration/#fileio)."
}
}
],
"description": "Iceberg input connector configuration."
},
"InputEndpointConfig": {
"allOf": [
{
"$ref": "#/components/schemas/ConnectorConfig"
},
{
"type": "object",
"required": [
"stream"
],
"properties": {
"stream": {
"type": "string",
"description": "The name of the input stream of the circuit that this endpoint is\nconnected to."
}
}
}
],
"description": "Describes an input connector configuration"
},
"IntervalUnit": {
"type": "string",
"description": "The specified units for SQL Interval types.\n\n`INTERVAL 1 DAY`, `INTERVAL 1 DAY TO HOUR`, `INTERVAL 1 DAY TO MINUTE`,\nwould yield `Day`, `DayToHour`, `DayToMinute`, as the `IntervalUnit` respectively.",
"enum": [
"Day",
"DayToHour",
"DayToMinute",
"DayToSecond",
"Hour",
"HourToMinute",
"HourToSecond",
"Minute",
"MinuteToSecond",
"Month",
"Second",
"Year",
"YearToMonth"
]
},
"JsonLines": {
"type": "string",
"description": "Whether JSON values can span multiple lines.",
"enum": [
"multiple",
"single"
]
},
"JsonUpdateFormat": {
"type": "string",
"description": "Supported JSON data change event formats.\n\nEach element in a JSON-formatted input stream specifies\nan update to one or more records in an input table. We support\nseveral different ways to represent such updates.\n\n### `InsertDelete`\n\nEach element in the input stream consists of an \"insert\" or \"delete\"\ncommand and a record to be inserted to or deleted from the input table.\n\n```json\n{\"insert\": {\"column1\": \"hello, world!\", \"column2\": 100}}\n```\n\n### `Weighted`\n\nEach element in the input stream consists of a record and a weight\nwhich indicates how many times the row appears.\n\n```json\n{\"weight\": 2, \"data\": {\"column1\": \"hello, world!\", \"column2\": 100}}\n```\n\nNote that the line above would be equivalent to the following input in the `InsertDelete` format:\n\n```json\n{\"insert\": {\"column1\": \"hello, world!\", \"column2\": 100}}\n{\"insert\": {\"column1\": \"hello, world!\", \"column2\": 100}}\n```\n\nSimilarly, negative weights are equivalent to deletions:\n\n```json\n{\"weight\": -1, \"data\": {\"column1\": \"hello, world!\", \"column2\": 100}}\n```\n\nis equivalent to in the `InsertDelete` format:\n\n```json\n{\"delete\": {\"column1\": \"hello, world!\", \"column2\": 100}}\n```\n\n### `Debezium`\n\nDebezium CDC format. Refer to [Debezium input connector documentation](https://docs.feldera.com/connectors/sources/debezium) for details.\n\n### `Snowflake`\n\nUses flat structure so that fields can get parsed directly into SQL\ncolumns. Defines three metadata fields:\n\n* `__action` - \"insert\" or \"delete\"\n* `__stream_id` - unique 64-bit ID of the output stream (records within\na stream are totally ordered)\n* `__seq_number` - monotonically increasing sequence number relative to\nthe start of the stream.\n\n```json\n{\"PART\":1,\"VENDOR\":2,\"EFFECTIVE_SINCE\":\"2019-05-21\",\"PRICE\":\"10000\",\"__action\":\"insert\",\"__stream_id\":4523666124030717756,\"__seq_number\":1}\n```\n\n### `Raw`\n\nThis format is suitable for insert-only streams (no deletions).\nEach element in the input stream contains a record without any\nadditional envelope that gets inserted in the input table.",
"enum": [
"insert_delete",
"weighted",
"debezium",
"snowflake",
"raw"
]
},
"KafkaHeader": {
"type": "object",
"description": "Kafka message header.",
"required": [
"key"
],
"properties": {
"key": {
"type": "string"
},
"value": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaHeaderValue"
}
],
"nullable": true
}
}
},
"KafkaHeaderValue": {
"type": "string",
"format": "binary",
"description": "Kafka header value encoded as a UTF-8 string or a byte array."
},
"KafkaInputConfig": {
"type": "object",
"description": "Configuration for reading data from Kafka topics with `InputTransport`.",
"required": [
"topics"
],
"properties": {
"fault_tolerance": {
"type": "string",
"description": "Deprecated.",
"nullable": true
},
"group_join_timeout_secs": {
"type": "integer",
"format": "int32",
"description": "Maximum timeout in seconds to wait for the endpoint to join the Kafka\nconsumer group during initialization.",
"minimum": 0
},
"kafka_service": {
"type": "string",
"description": "Deprecated.",
"nullable": true
},
"log_level": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaLogLevel"
}
],
"nullable": true
},
"poller_threads": {
"type": "integer",
"description": "Set to 1 or more to fix the number of threads used to poll\n`rdkafka`. Multiple threads can increase performance with small Kafka\nmessages; for large messages, one thread is enough. In either case, too\nmany threads can harm performance. If unset, the default is 3, which\nhelps with small messages but will not harm performance with large\nmessagee",
"nullable": true,
"minimum": 0
},
"topics": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of topics to subscribe to."
}
},
"additionalProperties": {
"type": "string",
"description": "Options passed directly to `rdkafka`.\n\n[`librdkafka` options](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)\nused to configure the Kafka consumer. Not all options are valid with\nthis Kafka adapter:\n\n* \"enable.auto.commit\", if present, must be set to \"false\",\n* \"enable.auto.offset.store\", if present, must be set to \"false\""
}
},
"KafkaLogLevel": {
"type": "string",
"description": "Kafka logging levels.",
"enum": [
"emerg",
"alert",
"critical",
"error",
"warning",
"notice",
"info",
"debug"
]
},
"KafkaOutputConfig": {
"type": "object",
"description": "Configuration for writing data to a Kafka topic with `OutputTransport`.",
"required": [
"topic"
],
"properties": {
"fault_tolerance": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaOutputFtConfig"
}
],
"nullable": true
},
"headers": {
"type": "array",
"items": {
"$ref": "#/components/schemas/KafkaHeader"
},
"description": "Kafka headers to be added to each message produced by this connector."
},
"initialization_timeout_secs": {
"type": "integer",
"format": "int32",
"description": "Maximum timeout in seconds to wait for the endpoint to connect to\na Kafka broker.\n\nDefaults to 60.",
"minimum": 0
},
"kafka_service": {
"type": "string",
"description": "If specified, this service is used to provide defaults for the Kafka options.",
"nullable": true
},
"log_level": {
"allOf": [
{
"$ref": "#/components/schemas/KafkaLogLevel"
}
],
"nullable": true
},
"topic": {
"type": "string",
"description": "Topic to write to."
}
},
"additionalProperties": {
"type": "string",
"description": "Options passed directly to `rdkafka`.\n\nSee [`librdkafka` options](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)\nused to configure the Kafka producer."
}
},
"KafkaOutputFtConfig": {
"type": "object",
"description": "Fault tolerance configuration for Kafka output connector.",
"properties": {
"consumer_options": {
"type": "object",
"description": "Options passed to `rdkafka` for consumers only, as documented at\n[`librdkafka`\noptions](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).\n\nThese options override `kafka_options` for consumers, and may be empty.",
"default": {},
"additionalProperties": {
"type": "string"
}
},
"producer_options": {
"type": "object",
"description": "Options passed to `rdkafka` for producers only, as documented at\n[`librdkafka`\noptions](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).\n\nThese options override `kafka_options` for producers, and may be empty.",
"default": {},
"additionalProperties": {
"type": "string"
}
}
}
},
"NewApiKeyRequest": {
"type": "object",
"description": "Request to create a new API key.",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Key name.",
"example": "my-api-key"
}
}
},
"NewApiKeyResponse": {
"type": "object",
"description": "Response to a successful API key creation.",
"required": [
"id",
"name",
"api_key"
],
"properties": {
"api_key": {
"type": "string",
"description": "Generated secret API key. There is no way to retrieve this\nkey again through the API, so store it securely.",
"example": "apikey:v5y5QNtlPNVMwkmNjKwFU8bbIu5lMge3yHbyddxAOdXlEo84SEoNn32DUhQaf1KLeI9aOOfnJjhQ1pYzMrU4wQXON6pm6BS7Zgzj46U2b8pwz1280vYBEtx41hiDBRP"
},
"id": {
"$ref": "#/components/schemas/ApiKeyId"
},
"name": {
"type": "string",
"description": "API key name provided by the user.",
"example": "my-api-key"
}
}
},
"NexmarkInputConfig": {
"type": "object",
"description": "Configuration for generating Nexmark input data.\n\nThis connector must be used exactly three times in a pipeline if it is used\nat all, once for each [`NexmarkTable`].",
"required": [
"table"
],
"properties": {
"options": {
"allOf": [
{
"$ref": "#/components/schemas/NexmarkInputOptions"
}
],
"nullable": true
},
"table": {
"$ref": "#/components/schemas/NexmarkTable"
}
}
},
"NexmarkInputOptions": {
"type": "object",
"description": "Configuration for generating Nexmark input data.",
"properties": {
"batch_size_per_thread": {
"type": "integer",
"format": "int64",
"description": "Number of events to generate and submit together, per thread.\n\nEach thread generates this many records, which are then combined with\nthe records generated by the other threads, to form combined input\nbatches of size `threads × batch_size_per_thread`.",
"default": 1000,
"minimum": 0
},
"events": {
"type": "integer",
"format": "int64",
"description": "Number of events to generate.",
"default": 100000000,
"minimum": 0
},
"max_step_size_per_thread": {
"type": "integer",
"format": "int64",
"description": "Maximum number of events to submit in a single step, per thread.\n\nThis should really be per worker thread, not per generator thread, but\nthe connector does not know how many worker threads there are.\n\nThis stands in for `max_batch_size` from the connector configuration\nbecause it must be a constant across all three of the nexmark tables.",
"default": 10000,
"minimum": 0
},
"threads": {
"type": "integer",
"description": "Number of event generator threads.\n\nIt's reasonable to choose the same number of generator threads as worker\nthreads.",
"default": 4,
"minimum": 0
}
}
},
"NexmarkTable": {
"type": "string",
"description": "Table in Nexmark.",
"enum": [
"bid",
"auction",
"person"
]
},
"OutputBufferConfig": {
"type": "object",
"properties": {
"enable_output_buffer": {
"type": "boolean",
"description": "Enable output buffering.\n\nThe output buffering mechanism allows decoupling the rate at which the pipeline\npushes changes to the output transport from the rate of input changes.\n\nBy default, output updates produced by the pipeline are pushed directly to\nthe output transport. Some destinations may prefer to receive updates in fewer\nbigger batches. For instance, when writing Parquet files, producing\none bigger file every few minutes is usually better than creating\nsmall files every few milliseconds.\n\nTo achieve such input/output decoupling, users can enable output buffering by\nsetting the `enable_output_buffer` flag to `true`. When buffering is enabled, output\nupdates produced by the pipeline are consolidated in an internal buffer and are\npushed to the output transport when one of several conditions is satisfied:\n\n* data has been accumulated in the buffer for more than `max_output_buffer_time_millis`\nmilliseconds.\n* buffer size exceeds `max_output_buffer_size_records` records.\n\nThis flag is `false` by default.",
"default": false
},
"max_output_buffer_size_records": {
"type": "integer",
"description": "Maximum number of updates to be kept in the output buffer.\n\nThis parameter bounds the maximal size of the buffer.\nNote that the size of the buffer is not always equal to the\ntotal number of updates output by the pipeline. Updates to the\nsame record can overwrite or cancel previous updates.\n\nBy default, the buffer can grow indefinitely until one of\nthe other output conditions is satisfied.\n\nNOTE: this configuration option requires the `enable_output_buffer` flag\nto be set.",
"default": 18446744073709551615,
"minimum": 0
},
"max_output_buffer_time_millis": {
"type": "integer",
"description": "Maximum time in milliseconds data is kept in the output buffer.\n\nBy default, data is kept in the buffer indefinitely until one of\nthe other output conditions is satisfied. When this option is\nset the buffer will be flushed at most every\n`max_output_buffer_time_millis` milliseconds.\n\nNOTE: this configuration option requires the `enable_output_buffer` flag\nto be set.",
"default": 18446744073709551615,
"minimum": 0
}
}
},
"OutputEndpointConfig": {
"allOf": [
{
"$ref": "#/components/schemas/ConnectorConfig"
},
{
"type": "object",
"required": [
"stream"
],
"properties": {
"stream": {
"type": "string",
"description": "The name of the output stream of the circuit that this endpoint is\nconnected to."
}
}
}
],
"description": "Describes an output connector configuration"
},
"PatchPipeline": {
"type": "object",
"description": "Partially update the pipeline (PATCH).\n\nNote that the patching only applies to the main fields, not subfields.\nFor instance, it is not possible to update only the number of workers;\nit is required to again pass the whole runtime configuration with the\nchange.",
"properties": {
"description": {
"type": "string",
"nullable": true
},
"name": {
"type": "string",
"nullable": true
},
"program_code": {
"type": "string",
"nullable": true
},
"program_config": {
"allOf": [
{
"$ref": "#/components/schemas/ProgramConfig"
}
],
"nullable": true
},
"runtime_config": {
"allOf": [
{
"$ref": "#/components/schemas/RuntimeConfig"
}
],
"nullable": true
},
"udf_rust": {
"type": "string",
"nullable": true
},
"udf_toml": {
"type": "string",
"nullable": true
}
}
},
"PipelineConfig": {
"allOf": [
{
"type": "object",
"description": "Global pipeline configuration settings. This is the publicly\nexposed type for users to configure pipelines.",
"properties": {
"clock_resolution_usecs": {
"type": "integer",
"format": "int64",
"description": "Real-time clock resolution in microseconds.\n\nThis parameter controls the execution of queries that use the `NOW()` function. The output of such\nqueries depends on the real-time clock and can change over time without any external\ninputs. The pipeline will update the clock value and trigger incremental recomputation\nat most each `clock_resolution_usecs` microseconds.\n\nIt is set to 100 milliseconds (100,000 microseconds) by default.\n\nSet to `null` to disable periodic clock updates.",
"default": 100000,
"nullable": true,
"minimum": 0
},
"cpu_profiler": {
"type": "boolean",
"description": "Enable CPU profiler.\n\nThe default value is `true`.",
"default": true
},
"fault_tolerance": {
"allOf": [
{
"$ref": "#/components/schemas/FtConfig"
}
],
"default": null,
"nullable": true
},
"max_buffering_delay_usecs": {
"type": "integer",
"format": "int64",
"description": "Maximal delay in microseconds to wait for `min_batch_size_records` to\nget buffered by the controller, defaults to 0.",
"default": 0,
"minimum": 0
},
"min_batch_size_records": {
"type": "integer",
"format": "int64",
"description": "Minimal input batch size.\n\nThe controller delays pushing input records to the circuit until at\nleast `min_batch_size_records` records have been received (total\nacross all endpoints) or `max_buffering_delay_usecs` microseconds\nhave passed since at least one input records has been buffered.\nDefaults to 0.",
"default": 0,
"minimum": 0
},
"min_storage_bytes": {
"type": "integer",
"description": "The minimum estimated number of bytes in a batch of data to write it to\nstorage. This is provided for debugging and fine-tuning and should\nordinarily be left unset. It only has an effect when `storage` is set to\ntrue.\n\nA value of 0 will write even empty batches to storage, and nonzero\nvalues provide a threshold. `usize::MAX` would effectively disable\nstorage.",
"default": null,
"nullable": true,
"minimum": 0
},
"resources": {
"allOf": [
{
"$ref": "#/components/schemas/ResourceConfig"
}
],
"default": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
}
},
"storage": {
"type": "boolean",
"description": "Should storage be enabled for this pipeline?\n\n- If `false` (default), the pipeline's state is kept in in-memory\ndata-structures. This is useful if the pipeline's state will fit in\nmemory and if the pipeline is ephemeral and does not need to be\nrecovered after a restart. The pipeline will most likely run faster\nsince it does not need to access storage.\n\n- If `true`, the pipeline's state is kept on storage. This allows the\npipeline to work with state that will not fit into memory. It also\nallows the state to be checkpointed and recovered across restarts.\nThis feature is currently experimental.",
"default": false
},
"tracing": {
"type": "boolean",
"description": "Enable pipeline tracing.",
"default": false
},
"tracing_endpoint_jaeger": {
"type": "string",
"description": "Jaeger tracing endpoint to send tracing information to.",
"default": "127.0.0.1:6831"
},
"workers": {
"type": "integer",
"format": "int32",
"description": "Number of DBSP worker threads.",
"default": 8,
"minimum": 0
}
}
},
{
"type": "object",
"required": [
"inputs"
],
"properties": {
"inputs": {
"type": "object",
"description": "Input endpoint configuration.",
"additionalProperties": {
"$ref": "#/components/schemas/InputEndpointConfig"
}
},
"name": {
"type": "string",
"description": "Pipeline name.",
"nullable": true
},
"outputs": {
"type": "object",
"description": "Output endpoint configuration.",
"additionalProperties": {
"$ref": "#/components/schemas/OutputEndpointConfig"
}
},
"storage_config": {
"allOf": [
{
"$ref": "#/components/schemas/StorageConfig"
}
],
"nullable": true
}
}
}
],
"description": "Pipeline deployment configuration.\nIt represents configuration entries directly provided by the user\n(e.g., runtime configuration) and entries derived from the schema\nof the compiled program (e.g., connectors). Storage configuration,\nif applicable, is set by the runner."
},
"PipelineDesiredStatus": {
"type": "string",
"enum": [
"Shutdown",
"Paused",
"Running"
]
},
"PipelineFieldSelector": {
"type": "string",
"enum": [
"all",
"status"
]
},
"PipelineId": {
"type": "string",
"format": "uuid",
"description": "Pipeline identifier."
},
"PipelineInfo": {
"type": "object",
"description": "Pipeline information.\nIt both includes fields which are user-provided and system-generated.",
"required": [
"id",
"name",
"description",
"created_at",
"version",
"platform_version",
"runtime_config",
"program_code",
"udf_rust",
"udf_toml",
"program_config",
"program_version",
"program_status",
"program_status_since",
"deployment_status",
"deployment_status_since",
"deployment_desired_status"
],
"properties": {
"created_at": {
"type": "string",
"format": "date-time"
},
"deployment_desired_status": {
"$ref": "#/components/schemas/PipelineDesiredStatus"
},
"deployment_error": {
"allOf": [
{
"$ref": "#/components/schemas/ErrorResponse"
}
],
"nullable": true
},
"deployment_status": {
"$ref": "#/components/schemas/PipelineStatus"
},
"deployment_status_since": {
"type": "string",
"format": "date-time"
},
"description": {
"type": "string"
},
"id": {
"$ref": "#/components/schemas/PipelineId"
},
"name": {
"type": "string"
},
"platform_version": {
"type": "string"
},
"program_code": {
"type": "string"
},
"program_config": {
"$ref": "#/components/schemas/ProgramConfig"
},
"program_info": {
"allOf": [
{
"$ref": "#/components/schemas/ProgramInfo"
}
],
"nullable": true
},
"program_status": {
"$ref": "#/components/schemas/ProgramStatus"
},
"program_status_since": {
"type": "string",
"format": "date-time"
},
"program_version": {
"$ref": "#/components/schemas/Version"
},
"runtime_config": {
"$ref": "#/components/schemas/RuntimeConfig"
},
"udf_rust": {
"type": "string"
},
"udf_toml": {
"type": "string"
},
"version": {
"$ref": "#/components/schemas/Version"
}
}
},
"PipelineSelectedInfo": {
"type": "object",
"description": "Pipeline information which has a selected subset of optional fields.\nIt both includes fields which are user-provided and system-generated.\nIf an optional field is not selected (i.e., is `None`), it will not be serialized.",
"required": [
"id",
"name",
"description",
"created_at",
"version",
"platform_version",
"program_version",
"program_status",
"program_status_since",
"deployment_status",
"deployment_status_since",
"deployment_desired_status"
],
"properties": {
"created_at": {
"type": "string",
"format": "date-time"
},
"deployment_desired_status": {
"$ref": "#/components/schemas/PipelineDesiredStatus"
},
"deployment_error": {
"allOf": [
{
"$ref": "#/components/schemas/ErrorResponse"
}
],
"nullable": true
},
"deployment_status": {
"$ref": "#/components/schemas/PipelineStatus"
},
"deployment_status_since": {
"type": "string",
"format": "date-time"
},
"description": {
"type": "string"
},
"id": {
"$ref": "#/components/schemas/PipelineId"
},
"name": {
"type": "string"
},
"platform_version": {
"type": "string"
},
"program_code": {
"type": "string",
"nullable": true
},
"program_config": {
"allOf": [
{
"$ref": "#/components/schemas/ProgramConfig"
}
],
"nullable": true
},
"program_info": {
"allOf": [
{
"$ref": "#/components/schemas/ProgramInfo"
}
],
"nullable": true
},
"program_status": {
"$ref": "#/components/schemas/ProgramStatus"
},
"program_status_since": {
"type": "string",
"format": "date-time"
},
"program_version": {
"$ref": "#/components/schemas/Version"
},
"runtime_config": {
"allOf": [
{
"$ref": "#/components/schemas/RuntimeConfig"
}
],
"nullable": true
},
"udf_rust": {
"type": "string",
"nullable": true
},
"udf_toml": {
"type": "string",
"nullable": true
},
"version": {
"$ref": "#/components/schemas/Version"
}
}
},
"PipelineStatus": {
"type": "string",
"description": "Pipeline status.\n\nThis type represents the state of the pipeline tracked by the pipeline\nrunner and observed by the API client via the `GET /v0/pipelines/{name}` endpoint.\n\n### The lifecycle of a pipeline\n\nThe following automaton captures the lifecycle of the pipeline.\nIndividual states and transitions of the automaton are described below.\n\n* States labeled with the hourglass symbol (⌛) are **timed** states. The\nautomaton stays in timed state until the corresponding operation completes\nor until it transitions to become failed after the pre-defined timeout\nperiod expires.\n\n* State transitions labeled with API endpoint names (`/start`, `/pause`,\n`/shutdown`) are triggered by invoking corresponding endpoint,\ne.g., `POST /v0/pipelines/{name}/start`. Note that these only express\ndesired state, and are applied asynchronously by the automata.\n\n```text\nShutdown◄────────────────────┐\n│ │\n/start or /pause│ ShuttingDown ◄────── Failed\n│ ▲ ▲\n▼ /shutdown │ │\n⌛Provisioning ──────────────────┤ Shutdown, Provisioning,\n│ │ Initializing, Paused,\n│ │ Running, Unavailable\n▼ │ (all states except ShuttingDown\n⌛Initializing ──────────────────┤ can transition to Failed)\n│ │\n┌─────────┼────────────────────────┴─┐\n│ ▼ │\n│ Paused ◄──────► Unavailable │\n│ │ ▲ ▲ │\n│ /start│ │/pause │ │\n│ ▼ │ │ │\n│ Running ◄──────────────┘ │\n└────────────────────────────────────┘\n```\n\n### Desired and actual status\n\nWe use the desired state model to manage the lifecycle of a pipeline.\nIn this model, the pipeline has two status attributes associated with\nit at runtime: the **desired** status, which represents what the user\nwould like the pipeline to do, and the **current** status, which\nrepresents the actual state of the pipeline. The pipeline runner\nservice continuously monitors both fields and steers the pipeline\ntowards the desired state specified by the user.\nOnly three of the states in the pipeline automaton above can be\nused as desired statuses: `Paused`, `Running`, and `Shutdown`.\nThese statuses are selected by invoking REST endpoints shown\nin the diagram.\n\nThe user can monitor the current state of the pipeline via the\n`GET /v0/pipelines/{name}` endpoint. In a typical scenario,\nthe user first sets the desired state, e.g., by invoking the\n`/start` endpoint, and then polls the `GET /v0/pipelines/{name}`\nendpoint to monitor the actual status of the pipeline until its\n`deployment_status` attribute changes to `Running` indicating\nthat the pipeline has been successfully initialized and is\nprocessing data, or `Failed`, indicating an error.",
"enum": [
"Shutdown",
"Provisioning",
"Initializing",
"Paused",
"Running",
"Unavailable",
"Failed",
"ShuttingDown"
]
},
"PostPutPipeline": {
"type": "object",
"description": "Create a new pipeline (POST), or fully update an existing pipeline (PUT).\nFields which are optional and not provided will be set to their empty type value\n(for strings: an empty string `\"\"`, for objects: an empty dictionary `{}`).",
"required": [
"name",
"program_code"
],
"properties": {
"description": {
"type": "string",
"nullable": true
},
"name": {
"type": "string"
},
"program_code": {
"type": "string"
},
"program_config": {
"allOf": [
{
"$ref": "#/components/schemas/ProgramConfig"
}
],
"nullable": true
},
"runtime_config": {
"allOf": [
{
"$ref": "#/components/schemas/RuntimeConfig"
}
],
"nullable": true
},
"udf_rust": {
"type": "string",
"nullable": true
},
"udf_toml": {
"type": "string",
"nullable": true
}
}
},
"ProgramConfig": {
"type": "object",
"description": "Program configuration.",
"properties": {
"cache": {
"type": "boolean",
"description": "If `true` (default), when a prior compilation with the same checksum\nalready exists, the output of that (i.e., binary) is used.\nSet `false` to always trigger a new compilation, which might take longer\nand as well can result in overriding an existing binary.",
"default": true
},
"profile": {
"allOf": [
{
"$ref": "#/components/schemas/CompilationProfile"
}
],
"default": null,
"nullable": true
}
}
},
"ProgramInfo": {
"type": "object",
"description": "Program information is the output of the SQL compiler.\n\nIt includes information needed for Rust compilation (e.g., generated Rust code)\nas well as only for runtime (e.g., schema, input/output connectors).",
"required": [
"schema",
"input_connectors",
"output_connectors"
],
"properties": {
"input_connectors": {
"type": "object",
"description": "Input connectors derived from the schema.",
"additionalProperties": {
"$ref": "#/components/schemas/InputEndpointConfig"
}
},
"main_rust": {
"type": "string",
"description": "Generated main program Rust code: main.rs"
},
"output_connectors": {
"type": "object",
"description": "Output connectors derived from the schema.",
"additionalProperties": {
"$ref": "#/components/schemas/OutputEndpointConfig"
}
},
"schema": {
"$ref": "#/components/schemas/ProgramSchema"
},
"udf_stubs": {
"type": "string",
"description": "Generated user defined function (UDF) stubs Rust code: stubs.rs"
}
}
},
"ProgramSchema": {
"type": "object",
"description": "A struct containing the tables (inputs) and views for a program.\n\nParse from the JSON data-type of the DDL generated by the SQL compiler.",
"required": [
"inputs",
"outputs"
],
"properties": {
"inputs": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Relation"
}
},
"outputs": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Relation"
}
}
}
},
"ProgramStatus": {
"oneOf": [
{
"type": "string",
"description": "Awaiting to be picked up for SQL compilation.",
"enum": [
"Pending"
]
},
{
"type": "string",
"description": "Compilation of SQL is ongoing.",
"enum": [
"CompilingSql"
]
},
{
"type": "string",
"description": "Compilation of SQL has been completed; awaiting to be picked up for Rust compilation.",
"enum": [
"SqlCompiled"
]
},
{
"type": "string",
"description": "Compilation of Rust is ongoing.",
"enum": [
"CompilingRust"
]
},
{
"type": "string",
"description": "Compilation (both SQL and Rust) succeeded.",
"enum": [
"Success"
]
},
{
"type": "object",
"required": [
"SqlError"
],
"properties": {
"SqlError": {
"type": "array",
"items": {
"$ref": "#/components/schemas/SqlCompilerMessage"
},
"description": "SQL compiler returned one or more errors."
}
}
},
{
"type": "object",
"required": [
"RustError"
],
"properties": {
"RustError": {
"type": "string",
"description": "Rust compiler returned an error."
}
}
},
{
"type": "object",
"required": [
"SystemError"
],
"properties": {
"SystemError": {
"type": "string",
"description": "System/OS returned an error when trying to invoke commands."
}
}
}
],
"description": "Program compilation status."
},
"PropertyValue": {
"type": "object",
"required": [
"value",
"key_position",
"value_position"
],
"properties": {
"key_position": {
"$ref": "#/components/schemas/SourcePosition"
},
"value": {
"type": "string"
},
"value_position": {
"$ref": "#/components/schemas/SourcePosition"
}
}
},
"ProviderAwsCognito": {
"type": "object",
"required": [
"jwk_uri",
"login_url",
"logout_url"
],
"properties": {
"jwk_uri": {
"type": "string"
},
"login_url": {
"type": "string"
},
"logout_url": {
"type": "string"
}
}
},
"ProviderGoogleIdentity": {
"type": "object",
"required": [
"jwk_uri",
"client_id"
],
"properties": {
"client_id": {
"type": "string"
},
"jwk_uri": {
"type": "string"
}
}
},
"PubSubInputConfig": {
"type": "object",
"description": "Google Pub/Sub input connector configuration.",
"required": [
"subscription"
],
"properties": {
"connect_timeout_seconds": {
"type": "integer",
"format": "int32",
"description": "gRPC connection timeout.",
"nullable": true,
"minimum": 0
},
"credentials": {
"type": "string",
"description": "The content of a Google Cloud credentials JSON file.\n\nWhen this option is specified, the connector will use the provided credentials for\nauthentication. Otherwise, it will use Application Default Credentials (ADC) configured\nin the environment where the Feldera service is running. See\n[Google Cloud documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc)\nfor information on configuring application default credentials.\n\nWhen running Feldera in an environment where ADC are not configured,\ne.g., a Docker container, use this option to ship Google Cloud credentials from another environment.\nFor example, if you use the\n[`gcloud auth application-default login`](https://cloud.google.com/pubsub/docs/authentication#client-libs)\ncommand for authentication in your local development environment, ADC are stored in the\n`.config/gcloud/application_default_credentials.json` file in your home directory.",
"nullable": true
},
"emulator": {
"type": "string",
"description": "Set in order to use a Pub/Sub [emulator](https://cloud.google.com/pubsub/docs/emulator)\ninstead of the production service, e.g., 'localhost:8681'.",
"nullable": true
},
"endpoint": {
"type": "string",
"description": "Override the default service endpoint 'pubsub.googleapis.com'",
"nullable": true
},
"pool_size": {
"type": "integer",
"format": "int32",
"description": "gRPC channel pool size.",
"nullable": true,
"minimum": 0
},
"project_id": {
"type": "string",
"description": "Google Cloud project_id.\n\nWhen not specified, the connector will use the project id associated\nwith the authenticated account.",
"nullable": true
},
"snapshot": {
"type": "string",
"description": "Reset subscription's backlog to a given snapshot on startup,\nusing the Pub/Sub `Seek` API.\n\nThis option is mutually exclusive with the `timestamp` option.",
"nullable": true
},
"subscription": {
"type": "string",
"description": "Subscription name."
},
"timeout_seconds": {
"type": "integer",
"format": "int32",
"description": "gRPC request timeout.",
"nullable": true,
"minimum": 0
},
"timestamp": {
"type": "string",
"description": "Reset subscription's backlog to a given timestamp on startup,\nusing the Pub/Sub `Seek` API.\n\nThe value of this option is an ISO 8601-encoded UTC time, e.g., \"2024-08-17T16:39:57-08:00\".\n\nThis option is mutually exclusive with the `snapshot` option.",
"nullable": true
}
}
},
"Relation": {
"allOf": [
{
"$ref": "#/components/schemas/SqlIdentifier"
},
{
"type": "object",
"required": [
"fields"
],
"properties": {
"fields": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Field"
}
},
"materialized": {
"type": "boolean"
},
"properties": {
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/PropertyValue"
}
}
}
}
],
"description": "A SQL table or view. It has a name and a list of fields.\n\nMatches the Calcite JSON format."
},
"ResourceConfig": {
"type": "object",
"properties": {
"cpu_cores_max": {
"type": "integer",
"format": "int64",
"description": "The maximum number of CPU cores to reserve\nfor an instance of this pipeline",
"default": null,
"nullable": true,
"minimum": 0
},
"cpu_cores_min": {
"type": "integer",
"format": "int64",
"description": "The minimum number of CPU cores to reserve\nfor an instance of this pipeline",
"default": null,
"nullable": true,
"minimum": 0
},
"memory_mb_max": {
"type": "integer",
"format": "int64",
"description": "The maximum memory in Megabytes to reserve\nfor an instance of this pipeline",
"default": null,
"nullable": true,
"minimum": 0
},
"memory_mb_min": {
"type": "integer",
"format": "int64",
"description": "The minimum memory in Megabytes to reserve\nfor an instance of this pipeline",
"default": null,
"nullable": true,
"minimum": 0
},
"storage_class": {
"type": "string",
"description": "Storage class to use for an instance of this pipeline.\nThe class determines storage performance such as IOPS and throughput.",
"default": null,
"nullable": true
},
"storage_mb_max": {
"type": "integer",
"format": "int64",
"description": "The total storage in Megabytes to reserve\nfor an instance of this pipeline",
"default": null,
"nullable": true,
"minimum": 0
}
}
},
"RestCatalogConfig": {
"type": "object",
"description": "Iceberg REST catalog config.",
"properties": {
"rest.audience": {
"type": "string",
"description": "Logical name of target resource or service.",
"nullable": true
},
"rest.credential": {
"type": "string",
"description": "Credential to use for OAuth2 credential flow when initializing the catalog.\n\nA key and secret pair separated by \":\" (key is optional).",
"nullable": true
},
"rest.headers": {
"type": "array",
"items": {
"type": "array",
"items": {
"allOf": [
{
"type": "string"
},
{
"type": "string"
}
]
}
},
"description": "Additional HTTP request headers added to each catalog REST API call.",
"nullable": true
},
"rest.oauth2-server-uri": {
"type": "string",
"description": "Authentication URL to use for client credentials authentication (default: uri + 'v1/oauth/tokens')",
"nullable": true
},
"rest.prefix": {
"type": "string",
"description": "Customize table storage paths.\n\nWhen combined with the `warehouse` property, the prefix determines\nhow table data is organized within the storage.",
"nullable": true
},
"rest.resource": {
"type": "string",
"description": "URI for the target resource or service.",
"nullable": true
},
"rest.scope": {
"type": "string",
"nullable": true
},
"rest.token": {
"type": "string",
"description": "Bearer token value to use for `Authorization` header.",
"nullable": true
},
"rest.uri": {
"type": "string",
"description": "URI identifying the REST catalog server.",
"nullable": true
},
"rest.warehouse": {
"type": "string",
"description": "The default location for managed tables created by the catalog.",
"nullable": true
}
}
},
"RngFieldSettings": {
"type": "object",
"description": "Configuration for generating random data for a field of a table.",
"properties": {
"e": {
"type": "integer",
"format": "int64",
"description": "The frequency rank exponent for the Zipf distribution.\n\n- This value is only used if the strategy is set to `Zipf`.\n- The default value is 1.0.",
"default": 1
},
"fields": {
"type": "object",
"description": "Specifies the values that the generator should produce in case the field is a struct type.",
"default": null,
"additionalProperties": {
"$ref": "#/components/schemas/RngFieldSettings"
},
"nullable": true
},
"key": {
"allOf": [
{
"$ref": "#/components/schemas/RngFieldSettings"
}
],
"default": null,
"nullable": true
},
"null_percentage": {
"type": "integer",
"description": "Percentage of records where this field should be set to NULL.\n\nIf not set, the generator will produce only records with non-NULL values.\nIf set to `1..=100`, the generator will produce records with NULL values with the specified percentage.",
"default": null,
"nullable": true,
"minimum": 0
},
"range": {
"type": "object",
"description": "An optional, exclusive range [a, b) to limit the range of values the generator should produce.\n\n- For integer/floating point types specifies min/max values as an integer.\nIf not set, the generator will produce values for the entire range of the type for number types.\n- For string/binary types specifies min/max length as an integer, values are required to be >=0.\nIf not set, a range of [0, 25) is used by default.\n- For timestamp types specifies the min/max as two strings in the RFC 3339 format\n(e.g., [\"2021-01-01T00:00:00Z\", \"2022-01-02T00:00:00Z\"]).\nAlternatively, the range values can be specified as a number of non-leap\nmilliseconds since January 1, 1970 0:00:00.000 UTC (aka “UNIX timestamp”).\nIf not set, a range of [\"1970-01-01T00:00:00Z\", \"2100-01-01T00:00:00Z\") or [0, 4102444800000)\nis used by default.\n- For time types specifies the min/max as two strings in the \"HH:MM:SS\" format.\nAlternatively, the range values can be specified in milliseconds as two positive integers.\nIf not set, the range is 24h.\n- For date types, the min/max range is specified as two strings in the \"YYYY-MM-DD\" format.\nAlternatively, two integers that represent number of days since January 1, 1970 can be used.\nIf not set, a range of [\"1970-01-01\", \"2100-01-01\") or [0, 54787) is used by default.\n- For array types specifies the min/max number of elements as an integer.\nIf not set, a range of [0, 5) is used by default. Range values are required to be >=0.\n- For map types specifies the min/max number of key-value pairs as an integer.\nIf not set, a range of [0, 5) is used by default.\n- For struct/boolean/null types `range` is ignored."
},
"scale": {
"type": "integer",
"format": "int64",
"description": "A scale factor to apply a multiplier to the generated value.\n\n- For integer/floating point types, the value is multiplied by the scale factor.\n- For timestamp types, the generated value (milliseconds) is multiplied by the scale factor.\n- For time types, the generated value (milliseconds) is multiplied by the scale factor.\n- For date types, the generated value (days) is multiplied by the scale factor.\n- For string/binary/array/map/struct/boolean/null types, the scale factor is ignored.\n\n- If `values` is specified, the scale factor is ignored.\n- If `range` is specified and the range is required to be positive (struct, map, array etc.)\nthe scale factor is required to be positive too.\n\nThe default scale factor is 1.",
"default": 1
},
"strategy": {
"allOf": [
{
"$ref": "#/components/schemas/DatagenStrategy"
}
],
"default": "increment"
},
"value": {
"allOf": [
{
"$ref": "#/components/schemas/RngFieldSettings"
}
],
"default": null,
"nullable": true
},
"values": {
"type": "array",
"items": {
"type": "object"
},
"description": "An optional set of values the generator will pick from.\n\nIf set, the generator will pick values from the specified set.\nIf not set, the generator will produce values according to the specified range.\nIf set to an empty set, the generator will produce NULL values.\nIf set to a single value, the generator will produce only that value.\n\nNote that `range` is ignored if `values` is set.",
"default": null,
"nullable": true
}
},
"additionalProperties": false
},
"RuntimeConfig": {
"type": "object",
"description": "Global pipeline configuration settings. This is the publicly\nexposed type for users to configure pipelines.",
"properties": {
"clock_resolution_usecs": {
"type": "integer",
"format": "int64",
"description": "Real-time clock resolution in microseconds.\n\nThis parameter controls the execution of queries that use the `NOW()` function. The output of such\nqueries depends on the real-time clock and can change over time without any external\ninputs. The pipeline will update the clock value and trigger incremental recomputation\nat most each `clock_resolution_usecs` microseconds.\n\nIt is set to 100 milliseconds (100,000 microseconds) by default.\n\nSet to `null` to disable periodic clock updates.",
"default": 100000,
"nullable": true,
"minimum": 0
},
"cpu_profiler": {
"type": "boolean",
"description": "Enable CPU profiler.\n\nThe default value is `true`.",
"default": true
},
"fault_tolerance": {
"allOf": [
{
"$ref": "#/components/schemas/FtConfig"
}
],
"default": null,
"nullable": true
},
"max_buffering_delay_usecs": {
"type": "integer",
"format": "int64",
"description": "Maximal delay in microseconds to wait for `min_batch_size_records` to\nget buffered by the controller, defaults to 0.",
"default": 0,
"minimum": 0
},
"min_batch_size_records": {
"type": "integer",
"format": "int64",
"description": "Minimal input batch size.\n\nThe controller delays pushing input records to the circuit until at\nleast `min_batch_size_records` records have been received (total\nacross all endpoints) or `max_buffering_delay_usecs` microseconds\nhave passed since at least one input records has been buffered.\nDefaults to 0.",
"default": 0,
"minimum": 0
},
"min_storage_bytes": {
"type": "integer",
"description": "The minimum estimated number of bytes in a batch of data to write it to\nstorage. This is provided for debugging and fine-tuning and should\nordinarily be left unset. It only has an effect when `storage` is set to\ntrue.\n\nA value of 0 will write even empty batches to storage, and nonzero\nvalues provide a threshold. `usize::MAX` would effectively disable\nstorage.",
"default": null,
"nullable": true,
"minimum": 0
},
"resources": {
"allOf": [
{
"$ref": "#/components/schemas/ResourceConfig"
}
],
"default": {
"cpu_cores_min": null,
"cpu_cores_max": null,
"memory_mb_min": null,
"memory_mb_max": null,
"storage_mb_max": null,
"storage_class": null
}
},
"storage": {
"type": "boolean",
"description": "Should storage be enabled for this pipeline?\n\n- If `false` (default), the pipeline's state is kept in in-memory\ndata-structures. This is useful if the pipeline's state will fit in\nmemory and if the pipeline is ephemeral and does not need to be\nrecovered after a restart. The pipeline will most likely run faster\nsince it does not need to access storage.\n\n- If `true`, the pipeline's state is kept on storage. This allows the\npipeline to work with state that will not fit into memory. It also\nallows the state to be checkpointed and recovered across restarts.\nThis feature is currently experimental.",
"default": false
},
"tracing": {
"type": "boolean",
"description": "Enable pipeline tracing.",
"default": false
},
"tracing_endpoint_jaeger": {
"type": "string",
"description": "Jaeger tracing endpoint to send tracing information to.",
"default": "127.0.0.1:6831"
},
"workers": {
"type": "integer",
"format": "int32",
"description": "Number of DBSP worker threads.",
"default": 8,
"minimum": 0
}
}
},
"S3InputConfig": {
"type": "object",
"description": "Configuration for reading data from AWS S3.",
"required": [
"region",
"bucket_name"
],
"properties": {
"aws_access_key_id": {
"type": "string",
"description": "AWS Access Key id. This property must be specified unless `no_sign_request` is set to `true`.",
"nullable": true
},
"aws_secret_access_key": {
"type": "string",
"description": "Secret Access Key. This property must be specified unless `no_sign_request` is set to `true`.",
"nullable": true
},
"bucket_name": {
"type": "string",
"description": "S3 bucket name to access."
},
"endpoint_url": {
"type": "string",
"description": "The endpoint URL used to communicate with this service. Can be used to make this connector\ntalk to non-AWS services with an S3 API.",
"nullable": true
},
"key": {
"type": "string",
"description": "Read a single object specified by a key.",
"nullable": true
},
"no_sign_request": {
"type": "boolean",
"description": "Do not sign requests. This is equivalent to the `--no-sign-request` flag in the AWS CLI."
},
"prefix": {
"type": "string",
"description": "Read all objects whose keys match a prefix. Set to an empty string to read all objects in the bucket.",
"nullable": true
},
"region": {
"type": "string",
"description": "AWS region."
}
}
},
"SourcePosition": {
"type": "object",
"required": [
"start_line_number",
"start_column",
"end_line_number",
"end_column"
],
"properties": {
"end_column": {
"type": "integer",
"minimum": 0
},
"end_line_number": {
"type": "integer",
"minimum": 0
},
"start_column": {
"type": "integer",
"minimum": 0
},
"start_line_number": {
"type": "integer",
"minimum": 0
}
}
},
"SqlCompilerMessage": {
"type": "object",
"description": "A SQL compiler error.\n\nThe SQL compiler returns a list of errors in the following JSON format if\nit's invoked with the `-je` option.\n\n```ignore\n[ {\n\"start_line_number\" : 2,\n\"start_column\" : 4,\n\"end_line_number\" : 2,\n\"end_column\" : 8,\n\"warning\" : false,\n\"error_type\" : \"PRIMARY KEY cannot be nullable\",\n\"message\" : \"PRIMARY KEY column 'C' has type INTEGER, which is nullable\",\n\"snippet\" : \" 2| c INT PRIMARY KEY\\n ^^^^^\\n 3|);\\n\"\n} ]\n```",
"required": [
"start_line_number",
"start_column",
"end_line_number",
"end_column",
"warning",
"error_type",
"message"
],
"properties": {
"end_column": {
"type": "integer",
"minimum": 0
},
"end_line_number": {
"type": "integer",
"minimum": 0
},
"error_type": {
"type": "string"
},
"message": {
"type": "string"
},
"snippet": {
"type": "string",
"nullable": true
},
"start_column": {
"type": "integer",
"minimum": 0
},
"start_line_number": {
"type": "integer",
"minimum": 0
},
"warning": {
"type": "boolean"
}
}
},
"SqlIdentifier": {
"type": "object",
"description": "An SQL identifier.\n\nThis struct is used to represent SQL identifiers in a canonical form.\nWe store table names or field names as identifiers in the schema.",
"required": [
"name",
"case_sensitive"
],
"properties": {
"case_sensitive": {
"type": "boolean"
},
"name": {
"type": "string"
}
}
},
"SqlType": {
"oneOf": [
{
"type": "string",
"description": "SQL `BOOLEAN` type.",
"enum": [
"Boolean"
]
},
{
"type": "string",
"description": "SQL `TINYINT` type.",
"enum": [
"TinyInt"
]
},
{
"type": "string",
"description": "SQL `SMALLINT` or `INT2` type.",
"enum": [
"SmallInt"
]
},
{
"type": "string",
"description": "SQL `INTEGER`, `INT`, `SIGNED`, `INT4` type.",
"enum": [
"Int"
]
},
{
"type": "string",
"description": "SQL `BIGINT` or `INT64` type.",
"enum": [
"BigInt"
]
},
{
"type": "string",
"description": "SQL `REAL` or `FLOAT4` or `FLOAT32` type.",
"enum": [
"Real"
]
},
{
"type": "string",
"description": "SQL `DOUBLE` or `FLOAT8` or `FLOAT64` type.",
"enum": [
"Double"
]
},
{
"type": "string",
"description": "SQL `DECIMAL` or `DEC` or `NUMERIC` type.",
"enum": [
"Decimal"
]
},
{
"type": "string",
"description": "SQL `CHAR(n)` or `CHARACTER(n)` type.",
"enum": [
"Char"
]
},
{
"type": "string",
"description": "SQL `VARCHAR`, `CHARACTER VARYING`, `TEXT`, or `STRING` type.",
"enum": [
"Varchar"
]
},
{
"type": "string",
"description": "SQL `BINARY(n)` type.",
"enum": [
"Binary"
]
},
{
"type": "string",
"description": "SQL `VARBINARY` or `BYTEA` type.",
"enum": [
"Varbinary"
]
},
{
"type": "string",
"description": "SQL `TIME` type.",
"enum": [
"Time"
]
},
{
"type": "string",
"description": "SQL `DATE` type.",
"enum": [
"Date"
]
},
{
"type": "string",
"description": "SQL `TIMESTAMP` type.",
"enum": [
"Timestamp"
]
},
{
"type": "object",
"required": [
"Interval"
],
"properties": {
"Interval": {
"$ref": "#/components/schemas/IntervalUnit"
}
}
},
{
"type": "string",
"description": "SQL `ARRAY` type.",
"enum": [
"Array"
]
},
{
"type": "string",
"description": "A complex SQL struct type (`CREATE TYPE x ...`).",
"enum": [
"Struct"
]
},
{
"type": "string",
"description": "SQL `MAP` type.",
"enum": [
"Map"
]
},
{
"type": "string",
"description": "SQL `NULL` type.",
"enum": [
"Null"
]
},
{
"type": "string",
"description": "SQL `UUID` type.",
"enum": [
"Uuid"
]
},
{
"type": "string",
"description": "SQL `VARIANT` type.",
"enum": [
"Variant"
]
}
],
"description": "The available SQL types as specified in `CREATE` statements."
},
"StorageCacheConfig": {
"type": "string",
"description": "How to cache access to storage within a Feldera pipeline.",
"enum": [
"page_cache",
"feldera_cache"
]
},
"StorageConfig": {
"type": "object",
"description": "Configuration for persistent storage in a [`PipelineConfig`].",
"required": [
"path"
],
"properties": {
"cache": {
"$ref": "#/components/schemas/StorageCacheConfig"
},
"path": {
"type": "string",
"description": "The location where the pipeline state is stored or will be stored.\n\nIt should point to a path on the file-system of the machine/container\nwhere the pipeline will run. If that path doesn't exist yet, or if it\ndoes not contain any checkpoints, then the pipeline creates it and\nstarts from an initial state in which no data has yet been received. If\nit does exist, then the pipeline starts from the most recent checkpoint\nthat already exists there. In either case, (further) checkpoints will be\nwritten there."
}
}
},
"TransportConfig": {
"oneOf": [
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/FileInputConfig"
},
"name": {
"type": "string",
"enum": [
"file_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/FileOutputConfig"
},
"name": {
"type": "string",
"enum": [
"file_output"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/KafkaInputConfig"
},
"name": {
"type": "string",
"enum": [
"kafka_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/KafkaOutputConfig"
},
"name": {
"type": "string",
"enum": [
"kafka_output"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/PubSubInputConfig"
},
"name": {
"type": "string",
"enum": [
"pub_sub_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/UrlInputConfig"
},
"name": {
"type": "string",
"enum": [
"url_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/S3InputConfig"
},
"name": {
"type": "string",
"enum": [
"s3_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/DeltaTableReaderConfig"
},
"name": {
"type": "string",
"enum": [
"delta_table_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/DeltaTableWriterConfig"
},
"name": {
"type": "string",
"enum": [
"delta_table_output"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/IcebergReaderConfig"
},
"name": {
"type": "string",
"enum": [
"iceberg_input"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/DatagenInputConfig"
},
"name": {
"type": "string",
"enum": [
"datagen"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/NexmarkInputConfig"
},
"name": {
"type": "string",
"enum": [
"nexmark"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/HttpInputConfig"
},
"name": {
"type": "string",
"enum": [
"http_input"
]
}
}
},
{
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"enum": [
"http_output"
]
}
}
},
{
"type": "object",
"required": [
"name",
"config"
],
"properties": {
"config": {
"$ref": "#/components/schemas/AdHocInputConfig"
},
"name": {
"type": "string",
"enum": [
"ad_hoc_input"
]
}
}
}
],
"description": "Transport-specific endpoint configuration passed to\n`crate::OutputTransport::new_endpoint`\nand `crate::InputTransport::new_endpoint`.",
"discriminator": {
"propertyName": "name"
}
},
"UrlInputConfig": {
"type": "object",
"description": "Configuration for reading data from an HTTP or HTTPS URL with\n`UrlInputTransport`.",
"required": [
"path"
],
"properties": {
"path": {
"type": "string",
"description": "URL."
},
"pause_timeout": {
"type": "integer",
"format": "int32",
"description": "Timeout before disconnection when paused, in seconds.\n\nIf the pipeline is paused, or if the input adapter reads data faster\nthan the pipeline can process it, then the controller will pause the\ninput adapter. If the input adapter stays paused longer than this\ntimeout, it will drop the network connection to the server. It will\nautomatically reconnect when the input adapter starts running again.",
"minimum": 0
}
}
},
"Version": {
"type": "integer",
"format": "int64",
"description": "Version number."
}
},
"securitySchemes": {
"JSON web token (JWT) or API key": {
"type": "http",
"scheme": "bearer",
"bearerFormat": "JWT",
"description": "Use a JWT token obtained via an OAuth2/OIDC\n login workflow or an API key obtained via\n the `/v0/api-keys` endpoint."
}
}
},
"tags": [
{
"name": "Pipelines",
"description": "Manage pipelines and their deployment."
},
{
"name": "HTTP input/output",
"description": "Interact with running pipelines using HTTP."
},
{
"name": "Authentication",
"description": "Retrieve authentication configuration."
},
{
"name": "Configuration",
"description": "Retrieve general configuration."
},
{
"name": "API keys",
"description": "Manage API keys."
},
{
"name": "Metrics",
"description": "Retrieve pipeline metrics."
}
]
}