ballista_executor/lib.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18#![doc = include_str!("../README.md")]
19#![warn(missing_docs)]
20
21/// Connection pool for `BallistaClient` instances.
22mod client_pool;
23/// Execution plan for collecting distributed query results into a single partition.
24pub mod collect;
25/// Command-line configuration for the executor binary.
26#[cfg(feature = "build-binary")]
27pub mod config;
28/// Extension point for custom query stage execution engines.
29pub mod execution_engine;
30/// Pull-based task execution loop that polls the scheduler for work.
31pub mod execution_loop;
32/// Core executor implementation for running distributed query tasks.
33pub mod executor;
34/// Executor process lifecycle management and configuration.
35pub mod executor_process;
36/// gRPC server for receiving pushed tasks from the scheduler.
37pub mod executor_server;
38/// Arrow Flight service for streaming shuffle data between executors.
39pub mod flight_service;
40/// Metrics collection for executor runtime statistics.
41pub mod metrics;
42/// Session-scoped cache of shared executor runtime environments.
43pub mod runtime_cache;
44/// Graceful shutdown coordination for executor components.
45pub mod shutdown;
46/// Signal handling for process termination.
47pub mod terminate;
48
49mod cpu_bound_executor;
50mod standalone;
51
52use ballista_core::error::BallistaError;
53use log::debug;
54use std::net::SocketAddr;
55
56pub use standalone::new_standalone_executor;
57pub use standalone::new_standalone_executor_from_builder;
58pub use standalone::new_standalone_executor_from_state;
59
60use log::info;
61
62use crate::shutdown::Shutdown;
63use ballista_core::serde::protobuf::{
64 FailedTask, OperatorMetricsSet, ShuffleWritePartition, SuccessfulTask, TaskStatus,
65 task_status,
66};
67use ballista_core::serde::scheduler::PartitionId;
68use ballista_core::utils::GrpcServerConfig;
69
70/// [ArrowFlightServerProvider] provides a function which creates a new Arrow Flight server.
71///
72/// The function should take four arguments:
73/// [String] - executor work directory
74/// [SocketAddr] - the address to bind the server to
75/// [Shutdown] - a shutdown signal to gracefully shutdown the server
76/// [GrpcServerConfig] - the gRPC server configuration for timeout settings
77/// Returns a [tokio::task::JoinHandle] which will be registered as service handler
78///
79pub type ArrowFlightServerProvider = dyn Fn(
80 String,
81 SocketAddr,
82 Shutdown,
83 GrpcServerConfig,
84 ) -> tokio::task::JoinHandle<Result<(), BallistaError>>
85 + Send
86 + Sync;
87
88/// Timestamps capturing the lifecycle of a task execution.
89#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
90pub struct TaskExecutionTimes {
91 /// Timestamp when the task was launched by the scheduler (milliseconds since epoch).
92 launch_time: u64,
93 /// Timestamp when task execution started on the executor (milliseconds since epoch).
94 start_exec_time: u64,
95 /// Timestamp when task execution completed (milliseconds since epoch).
96 end_exec_time: u64,
97}
98
99/// Converts a task execution result into a [`TaskStatus`] protobuf message.
100///
101/// This function wraps the outcome of task execution (success or failure)
102/// along with timing and metrics information into a status message that
103/// can be sent back to the scheduler.
104pub fn as_task_status(
105 execution_result: ballista_core::error::Result<Vec<ShuffleWritePartition>>,
106 executor_id: String,
107 task_id: usize,
108 stage_attempt_num: usize,
109 partition_id: PartitionId,
110 operator_metrics: Option<Vec<OperatorMetricsSet>>,
111 execution_times: TaskExecutionTimes,
112) -> TaskStatus {
113 let metrics = operator_metrics.unwrap_or_default();
114 match execution_result {
115 Ok(partitions) => {
116 debug!(
117 "Task {:?} finished with operator_metrics array size {}",
118 task_id,
119 metrics.len()
120 );
121 TaskStatus {
122 task_id: task_id as u32,
123 job_id: partition_id.job_id.into(),
124 stage_id: partition_id.stage_id as u32,
125 stage_attempt_num: stage_attempt_num as u32,
126 partition_id: partition_id.partition_id as u32,
127 launch_time: execution_times.launch_time,
128 start_exec_time: execution_times.start_exec_time,
129 end_exec_time: execution_times.end_exec_time,
130 metrics,
131 status: Some(task_status::Status::Successful(SuccessfulTask {
132 executor_id,
133 partitions,
134 })),
135 }
136 }
137 Err(e) => {
138 let error_msg = e.to_string();
139 info!("Task {task_id:?} failed: {error_msg}");
140
141 TaskStatus {
142 task_id: task_id as u32,
143 job_id: partition_id.job_id.into(),
144 stage_id: partition_id.stage_id as u32,
145 stage_attempt_num: stage_attempt_num as u32,
146 partition_id: partition_id.partition_id as u32,
147 launch_time: execution_times.launch_time,
148 start_exec_time: execution_times.start_exec_time,
149 end_exec_time: execution_times.end_exec_time,
150 metrics,
151 status: Some(task_status::Status::Failed(FailedTask::from(e))),
152 }
153 }
154 }
155}