datafusion-distributed 2.0.0

Framework for enhancing Apache DataFusion with distributed capabilities
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

syntax = "proto3";
package observability;

service ObservabilityService {
  rpc Ping (PingRequest) returns (PingResponse);
  rpc GetTaskProgress (GetTaskProgressRequest) returns (GetTaskProgressResponse);
  rpc GetClusterWorkers (GetClusterWorkersRequest) returns (GetClusterWorkersResponse);
}

message PingRequest {}

message PingResponse {
  uint32 value = 1;
}

message GetTaskProgressRequest {}

message TaskKey {
  bytes query_id = 1;
  uint64 stage_id = 2;
  uint64 task_number = 3;
}

// Progress information for a single task
message TaskProgress {
  TaskKey task_key = 1;
  uint64 total_partitions = 2;
  uint64 completed_partitions = 3;
  TaskStatus status = 4;
  uint64 output_rows = 5;
}

enum TaskStatus {
  TASK_STATUS_UNSPECIFIED = 0;
  TASK_STATUS_RUNNING = 1;
}

// Worker-level system metrics
message WorkerMetrics {
  uint64 rss_bytes = 1;
  double cpu_usage_percent = 2;
}

message GetTaskProgressResponse {
  repeated TaskProgress tasks = 1;
  WorkerMetrics worker_metrics = 2;
}

message GetClusterWorkersRequest {}

message GetClusterWorkersResponse {
  repeated string worker_urls = 1;
}