ballista_executor/lib.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18#![doc = include_str!("../README.md")]
19#![warn(missing_docs)]
20
21/// Connection pool for `BallistaClient` instances.
22mod client_pool;
23/// Execution plan for collecting distributed query results into a single partition.
24pub mod collect;
25/// Command-line configuration for the executor binary.
26#[cfg(feature = "build-binary")]
27pub mod config;
28/// Extension point for custom query stage execution engines.
29pub mod execution_engine;
30/// Pull-based task execution loop that polls the scheduler for work.
31pub mod execution_loop;
32/// Core executor implementation for running distributed query tasks.
33pub mod executor;
34/// Executor process lifecycle management and configuration.
35pub mod executor_process;
36/// gRPC server for receiving pushed tasks from the scheduler.
37pub mod executor_server;
38/// Arrow Flight service for streaming shuffle data between executors.
39pub mod flight_service;
40/// Metrics collection for executor runtime statistics.
41pub mod metrics;
42/// Graceful shutdown coordination for executor components.
43pub mod shutdown;
44/// Signal handling for process termination.
45pub mod terminate;
46
47mod cpu_bound_executor;
48mod standalone;
49
50use ballista_core::error::BallistaError;
51use std::net::SocketAddr;
52
53pub use standalone::new_standalone_executor;
54pub use standalone::new_standalone_executor_from_builder;
55pub use standalone::new_standalone_executor_from_state;
56
57use log::info;
58
59use crate::shutdown::Shutdown;
60use ballista_core::serde::protobuf::{
61 FailedTask, OperatorMetricsSet, ShuffleWritePartition, SuccessfulTask, TaskStatus,
62 task_status,
63};
64use ballista_core::serde::scheduler::PartitionId;
65use ballista_core::utils::GrpcServerConfig;
66
67/// [ArrowFlightServerProvider] provides a function which creates a new Arrow Flight server.
68///
69/// The function should take four arguments:
70/// [String] - executor work directory
71/// [SocketAddr] - the address to bind the server to
72/// [Shutdown] - a shutdown signal to gracefully shutdown the server
73/// [GrpcServerConfig] - the gRPC server configuration for timeout settings
74/// Returns a [tokio::task::JoinHandle] which will be registered as service handler
75///
76pub type ArrowFlightServerProvider = dyn Fn(
77 String,
78 SocketAddr,
79 Shutdown,
80 GrpcServerConfig,
81 ) -> tokio::task::JoinHandle<Result<(), BallistaError>>
82 + Send
83 + Sync;
84
85/// Timestamps capturing the lifecycle of a task execution.
86#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
87pub struct TaskExecutionTimes {
88 /// Timestamp when the task was launched by the scheduler (milliseconds since epoch).
89 launch_time: u64,
90 /// Timestamp when task execution started on the executor (milliseconds since epoch).
91 start_exec_time: u64,
92 /// Timestamp when task execution completed (milliseconds since epoch).
93 end_exec_time: u64,
94}
95
96/// Converts a task execution result into a [`TaskStatus`] protobuf message.
97///
98/// This function wraps the outcome of task execution (success or failure)
99/// along with timing and metrics information into a status message that
100/// can be sent back to the scheduler.
101pub fn as_task_status(
102 execution_result: ballista_core::error::Result<Vec<ShuffleWritePartition>>,
103 executor_id: String,
104 task_id: usize,
105 stage_attempt_num: usize,
106 partition_id: PartitionId,
107 operator_metrics: Option<Vec<OperatorMetricsSet>>,
108 execution_times: TaskExecutionTimes,
109) -> TaskStatus {
110 let metrics = operator_metrics.unwrap_or_default();
111 match execution_result {
112 Ok(partitions) => {
113 info!(
114 "Task {:?} finished with operator_metrics array size {}",
115 task_id,
116 metrics.len()
117 );
118 TaskStatus {
119 task_id: task_id as u32,
120 job_id: partition_id.job_id,
121 stage_id: partition_id.stage_id as u32,
122 stage_attempt_num: stage_attempt_num as u32,
123 partition_id: partition_id.partition_id as u32,
124 launch_time: execution_times.launch_time,
125 start_exec_time: execution_times.start_exec_time,
126 end_exec_time: execution_times.end_exec_time,
127 metrics,
128 status: Some(task_status::Status::Successful(SuccessfulTask {
129 executor_id,
130 partitions,
131 })),
132 }
133 }
134 Err(e) => {
135 let error_msg = e.to_string();
136 info!("Task {task_id:?} failed: {error_msg}");
137
138 TaskStatus {
139 task_id: task_id as u32,
140 job_id: partition_id.job_id,
141 stage_id: partition_id.stage_id as u32,
142 stage_attempt_num: stage_attempt_num as u32,
143 partition_id: partition_id.partition_id as u32,
144 launch_time: execution_times.launch_time,
145 start_exec_time: execution_times.start_exec_time,
146 end_exec_time: execution_times.end_exec_time,
147 metrics,
148 status: Some(task_status::Status::Failed(FailedTask::from(e))),
149 }
150 }
151 }
152}