/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* These .proto interfaces are private and stable.
* Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
* for what changes are allowed for a *stable* .proto interface.
*/
syntax="proto2";
// This file contains protocol buffers that are used to transfer data
// to and from the datanode, as well as between datanodes.
option java_package = "org.apache.hadoop.hdfs.protocol.proto";
option java_outer_classname = "DataTransferProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdfs;
import "Security.proto";
import "hdfs.proto";
message DataTransferEncryptorMessageProto {
enum DataTransferEncryptorStatus {
SUCCESS = 0;
ERROR_UNKNOWN_KEY = 1;
ERROR = 2;
}
required DataTransferEncryptorStatus status = 1;
optional bytes payload = 2;
optional string message = 3;
repeated CipherOptionProto cipherOption = 4;
optional HandshakeSecretProto handshakeSecret = 5;
optional bool accessTokenError = 6;
}
message HandshakeSecretProto {
required bytes secret = 1;
required string bpid = 2;
}
message BaseHeaderProto {
required ExtendedBlockProto block = 1;
optional hadoop.common.TokenProto token = 2;
optional DataTransferTraceInfoProto traceInfo = 3;
}
message DataTransferTraceInfoProto {
optional uint64 traceId = 1;
optional uint64 parentId = 2;
optional bytes spanContext = 3;
}
message ClientOperationHeaderProto {
required BaseHeaderProto baseHeader = 1;
required string clientName = 2;
}
message CachingStrategyProto {
optional bool dropBehind = 1;
optional int64 readahead = 2;
}
message OpReadBlockProto {
required ClientOperationHeaderProto header = 1;
required uint64 offset = 2;
required uint64 len = 3;
optional bool sendChecksums = 4 [default = true];
optional CachingStrategyProto cachingStrategy = 5;
}
message ChecksumProto {
required ChecksumTypeProto type = 1;
required uint32 bytesPerChecksum = 2;
}
message OpWriteBlockProto {
required ClientOperationHeaderProto header = 1;
repeated DatanodeInfoProto targets = 2;
optional DatanodeInfoProto source = 3;
enum BlockConstructionStage {
PIPELINE_SETUP_APPEND = 0;
// pipeline set up for failed PIPELINE_SETUP_APPEND recovery
PIPELINE_SETUP_APPEND_RECOVERY = 1;
// data streaming
DATA_STREAMING = 2;
// pipeline setup for failed data streaming recovery
PIPELINE_SETUP_STREAMING_RECOVERY = 3;
// close the block and pipeline
PIPELINE_CLOSE = 4;
// Recover a failed PIPELINE_CLOSE
PIPELINE_CLOSE_RECOVERY = 5;
// pipeline set up for block creation
PIPELINE_SETUP_CREATE = 6;
// transfer RBW for adding datanodes
TRANSFER_RBW = 7;
// transfer Finalized for adding datanodes
TRANSFER_FINALIZED = 8;
}
required BlockConstructionStage stage = 4;
required uint32 pipelineSize = 5;
required uint64 minBytesRcvd = 6;
required uint64 maxBytesRcvd = 7;
required uint64 latestGenerationStamp = 8;
/**
* The requested checksum mechanism for this block write.
*/
required ChecksumProto requestedChecksum = 9;
optional CachingStrategyProto cachingStrategy = 10;
optional StorageTypeProto storageType = 11 [default = DISK];
repeated StorageTypeProto targetStorageTypes = 12;
/**
* Hint to the DataNode that the block can be allocated on transient
* storage i.e. memory and written to disk lazily. The DataNode is free
* to ignore this hint.
*/
optional bool allowLazyPersist = 13 [default = false];
//whether to pin the block, so Balancer won't move it.
optional bool pinning = 14 [default = false];
repeated bool targetPinnings = 15;
optional string storageId = 16;
repeated string targetStorageIds = 17;
}
message OpTransferBlockProto {
required ClientOperationHeaderProto header = 1;
repeated DatanodeInfoProto targets = 2;
repeated StorageTypeProto targetStorageTypes = 3;
repeated string targetStorageIds = 4;
}
message OpReplaceBlockProto {
required BaseHeaderProto header = 1;
required string delHint = 2;
required DatanodeInfoProto source = 3;
optional StorageTypeProto storageType = 4 [default = DISK];
optional string storageId = 5;
}
message OpCopyBlockProto {
required BaseHeaderProto header = 1;
}
message OpBlockChecksumProto {
required BaseHeaderProto header = 1;
optional BlockChecksumOptionsProto blockChecksumOptions = 2;
}
message OpBlockGroupChecksumProto {
required BaseHeaderProto header = 1;
required DatanodeInfosProto datanodes = 2;
// each internal block has a block token
repeated hadoop.common.TokenProto blockTokens = 3;
required ErasureCodingPolicyProto ecPolicy = 4;
repeated uint32 blockIndices = 5;
required uint64 requestedNumBytes = 6;
optional BlockChecksumOptionsProto blockChecksumOptions = 7;
}
/**
* An ID uniquely identifying a shared memory segment.
*/
message ShortCircuitShmIdProto {
required int64 hi = 1;
required int64 lo = 2;
}
/**
* An ID uniquely identifying a slot within a shared memory segment.
*/
message ShortCircuitShmSlotProto {
required ShortCircuitShmIdProto shmId = 1;
required int32 slotIdx = 2;
}
message OpRequestShortCircuitAccessProto {
required BaseHeaderProto header = 1;
/** In order to get short-circuit access to block data, clients must set this
* to the highest version of the block data that they can understand.
* Currently 1 is the only version, but more versions may exist in the future
* if the on-disk format changes.
*/
required uint32 maxVersion = 2;
/**
* The shared memory slot to use, if we are using one.
*/
optional ShortCircuitShmSlotProto slotId = 3;
/**
* True if the client supports verifying that the file descriptor has been
* sent successfully.
*/
optional bool supportsReceiptVerification = 4 [default = false];
}
message ReleaseShortCircuitAccessRequestProto {
required ShortCircuitShmSlotProto slotId = 1;
optional DataTransferTraceInfoProto traceInfo = 2;
}
message ReleaseShortCircuitAccessResponseProto {
required Status status = 1;
optional string error = 2;
}
message ShortCircuitShmRequestProto {
// The name of the client requesting the shared memory segment. This is
// purely for logging / debugging purposes.
required string clientName = 1;
optional DataTransferTraceInfoProto traceInfo = 2;
}
message ShortCircuitShmResponseProto {
required Status status = 1;
optional string error = 2;
optional ShortCircuitShmIdProto id = 3;
}
message PacketHeaderProto {
// All fields must be fixed-length!
required sfixed64 offsetInBlock = 1;
required sfixed64 seqno = 2;
required bool lastPacketInBlock = 3;
required sfixed32 dataLen = 4;
optional bool syncBlock = 5 [default = false];
}
// Status is a 4-bit enum
enum Status {
SUCCESS = 0;
ERROR = 1;
ERROR_CHECKSUM = 2;
ERROR_INVALID = 3;
ERROR_EXISTS = 4;
ERROR_ACCESS_TOKEN = 5;
CHECKSUM_OK = 6;
ERROR_UNSUPPORTED = 7;
OOB_RESTART = 8; // Quick restart
OOB_RESERVED1 = 9; // Reserved
OOB_RESERVED2 = 10; // Reserved
OOB_RESERVED3 = 11; // Reserved
IN_PROGRESS = 12;
ERROR_BLOCK_PINNED = 13;
}
enum ShortCircuitFdResponse {
DO_NOT_USE_RECEIPT_VERIFICATION = 0;
USE_RECEIPT_VERIFICATION = 1;
}
message PipelineAckProto {
required sint64 seqno = 1;
repeated Status reply = 2;
optional uint64 downstreamAckTimeNanos = 3 [default = 0];
repeated uint32 flag = 4 [packed=true];
}
/**
* Sent as part of the BlockOpResponseProto
* for READ_BLOCK and COPY_BLOCK operations.
*/
message ReadOpChecksumInfoProto {
required ChecksumProto checksum = 1;
/**
* The offset into the block at which the first packet
* will start. This is necessary since reads will align
* backwards to a checksum chunk boundary.
*/
required uint64 chunkOffset = 2;
}
message BlockOpResponseProto {
required Status status = 1;
optional string firstBadLink = 2;
optional OpBlockChecksumResponseProto checksumResponse = 3;
optional ReadOpChecksumInfoProto readOpChecksumInfo = 4;
/** explanatory text which may be useful to log on the client side */
optional string message = 5;
/** If the server chooses to agree to the request of a client for
* short-circuit access, it will send a response message with the relevant
* file descriptors attached.
*
* In the body of the message, this version number will be set to the
* specific version number of the block data that the client is about to
* read.
*/
optional uint32 shortCircuitAccessVersion = 6;
}
/**
* Message sent from the client to the DN after reading the entire
* read request.
*/
message ClientReadStatusProto {
required Status status = 1;
}
message DNTransferAckProto {
required Status status = 1;
}
message OpBlockChecksumResponseProto {
required uint32 bytesPerCrc = 1;
required uint64 crcPerBlock = 2;
required bytes blockChecksum = 3;
optional ChecksumTypeProto crcType = 4;
optional BlockChecksumOptionsProto blockChecksumOptions = 5;
}
message OpCustomProto {
required string customId = 1;
}