zkclient::ZkNnClient Class Reference

#include <zk_nn_client.h>

Inheritance diagram for zkclient::ZkNnClient:

Public Types
enum	ListingResponse { Ok, FileDoesNotExist, FailedChildRetrieval, FileAccessRestricted }
enum	DeleteResponse {   Ok, FileDoesNotExist, FileUnderConstruction, FileIsDirectoryMismatch,   FailedChildRetrieval, FailedBlockRetrieval, FailedDataNodeRetrieval, FailedZookeeperOp,   FileAccessRestricted }
enum	GetFileInfoResponse { Ok, FileDoesNotExist, FailedReadZnode, FileAccessRestricted }
enum	MkdirResponse { Ok, FailedZnodeCreation }
enum	CreateResponse { Ok, FileAlreadyExists, FailedMkdir, FailedCreateZnode }
enum	RenameResponse {   Ok, FileDoesNotExist, RenameOpsFailed, InvalidType,   MultiOpFailed, FileAccessRestricted }
enum	ErasureCodingPoliciesResponse { Ok }
enum	ErasureCodingPolicyResponse { Ok, FileDoesNotExist }
enum	SetErasureCodingPolicyResponse { Ok, FileDoesNotExist, FailedZookeeperOp }

Public Member Functions
	ZkNnClient (std::string zkIpAndAddress)
	ZkNnClient (std::shared_ptr< ZKWrapper > zk_in, bool secureMode=false)
void	register_watches ()
uint64_t	current_time_ms ()
uint64_t	get_client_lease_timestamp (std::string client_name)
void	renew_lease (RenewLeaseRequestProto &req, RenewLeaseResponseProto &res)
void	recover_lease (RecoverLeaseRequestProto &req, RecoverLeaseResponseProto &res)
GetFileInfoResponse	get_info (GetFileInfoRequestProto &req, GetFileInfoResponseProto &res, std::string client_name="default")
CreateResponse	create_file (CreateRequestProto &request, CreateResponseProto &response)
void	get_block_locations (GetBlockLocationsRequestProto &req, GetBlockLocationsResponseProto &res, std::string client_name="default")
MkdirResponse	mkdir (MkdirsRequestProto &req, MkdirsResponseProto &res)
DeleteResponse	destroy (DeleteRequestProto &req, DeleteResponseProto &res, std::string client_name="default")
void	complete (CompleteRequestProto &req, CompleteResponseProto &res, std::string client_name="default")
RenameResponse	rename (RenameRequestProto &req, RenameResponseProto &res, std::string client_name="default")
ListingResponse	get_listing (GetListingRequestProto &req, GetListingResponseProto &res, std::string client_name="default")
void	get_content (GetContentSummaryRequestProto &req, GetContentSummaryResponseProto &res, std::string client_name="default")
void	set_file_info_content (ContentSummaryProto *status, const std::string &path, FileZNode &znode_data)
void	set_node_policy (char policy)
char	get_node_policy ()
ErasureCodingPoliciesResponse	get_erasure_coding_policies (GetErasureCodingPoliciesRequestProto &req, GetErasureCodingPoliciesResponseProto &res)
ErasureCodingPolicyResponse	get_erasure_coding_policy_of_path (GetErasureCodingPolicyRequestProto &req, GetErasureCodingPolicyResponseProto &res)
SetErasureCodingPolicyResponse	set_erasure_coding_policy_of_path (SetErasureCodingPolicyRequestProto &req, SetErasureCodingPolicyResponseProto &res)
bool	set_permission (SetPermissionRequestProto &req, SetPermissionResponseProto &res)
bool	set_owner (SetOwnerRequestProto &req, SetOwnerResponseProto &res, std::string client_name="default")
bool	add_block (AddBlockRequestProto &req, AddBlockResponseProto &res, std::string client_name="default")
bool	set_owner (SetOwnerRequestProto &req, SetOwnerResponseProto &res)
bool	add_block (const std::string &fileName, u_int64_t &block_id, std::vector< std::string > &dataNodes, uint32_t replication_factor)
bool	add_block_group (const std::string &filePath, u_int64_t &block_group_id, std::vector< std::string > &dataNodes, std::vector< char > &blockIndices, uint32_t total_num_storage_blocks)
uint32_t	get_total_num_storage_blocks (const std::string &fileName, u_int64_t &block_group_id)
u_int64_t	generate_storage_block_id (uint64_t block_group_id, uint64_t index_within_group)
u_int64_t	generate_block_group_id ()
u_int64_t	get_block_group_id (u_int64_t storage_block_id)
u_int64_t	get_index_within_block_group (u_int64_t storage_block_id)
bool	abandon_block (AbandonBlockRequestProto &req, AbandonBlockResponseProto &res, std::string client_name="default")
bool	previousBlockComplete (uint64_t prev_id)
bool	file_exists (const std::string &path)
bool	get_block_size (const u_int64_t &block_id, uint64_t &blocksize)
bool	find_live_datanodes (const uint64_t blockId, int error_code, std::vector< std::string > &live_data_nodes)
bool	find_datanode_for_block (std::vector< std::string > &datanodes, std::vector< std::string > &excluded_dns, const u_int64_t blockId, uint32_t replication_factor, uint64_t blocksize)
bool	find_all_datanodes_with_block (const uint64_t &block_uuid, std::vector< std::string > &rdatanodes, int &error_code)
bool	rename_ops_for_file (const std::string &src, const std::string &dst, std::vector< std::shared_ptr< ZooOp >> &ops)
bool	rename_ops_for_dir (const std::string &src, const std::string &dst, std::vector< std::shared_ptr< ZooOp >> &ops)
bool	check_acks ()
void	get_block_locations (const std::string &src, google::protobuf::uint64 offset, google::protobuf::uint64 length, LocatedBlocksProto *blocks, std::string client_name="default")
bool	append_file (AppendRequestProto &req, AppendResponseProto &res)
void	update_block_for_pipeline (UpdateBlockForPipelineRequestProto &req, UpdateBlockForPipelineResponseProto &res)
bool	process_request (std::string client_name, std::string file_path, AppendRequestProto &req)
bool	get_primary_block_info (std::string file_path, AppendRequestProto &req, AppendResponseProto &res)
bool	check_lease (std::string client_name, std::string file_path)
void	read_file_znode (FileZNode &znode_data, const std::string &path)
bool	cache_contains (const std::string &path)
int	cache_size ()
std::string	find_parent (const std::string &path)
Public Member Functions inherited from zkclient::ZkClientCommon
	ZkClientCommon (std::string hostAndIp)
	ZkClientCommon (std::shared_ptr< ZKWrapper > zk)
void	init ()
std::string	get_block_metadata_path (u_int64_t block_or_block_group_id)
u_int64_t	get_block_group_id (u_int64_t storage_block_id)
u_int64_t	get_index_within_block_group (u_int64_t storage_block_id)

Public Attributes
char	policy
const char *	EC_REPLICATION = "replication"
const char *	DEFAULT_EC_POLICY = "RS-6-3-1024k"
uint32_t	DEFAULT_EC_CELLSIZE = 1024*1024
uint32_t	DEFAULT_EC_ID = 1
uint32_t	REPLICATION_EC_ID = 63
const uint32_t	DEFAULT_DATA_UNITS = 6
const uint32_t	DEFAULT_PARITY_UNITS = 3
const char *	DEFAULT_EC_CODEC_NAME = "rs"
std::string	DEFAULT_STORAGE_ID = "1"
std::string	REPLICATION_STORAGE_ID = "63"
ECSchemaProto	DEFAULT_EC_SCHEMA
ErasureCodingPolicyProto	RS_SOLOMON_PROTO
ErasureCodingPolicyProto	REPLICATION_PROTO
ECSchemaProto	REPLICATION_1_2_SCHEMA
Public Attributes inherited from zkclient::ZkClientCommon
std::shared_ptr< ZKWrapper >	zk

Static Public Attributes
static const std::string	CLASS_NAME
Static Public Attributes inherited from zkclient::ZkClientCommon
static const char	NAMESPACE_PATH [] = "/fileSystem"
static const char	BLOCKS_TREE [] = "/blocks"
static const char	HEALTH [] = "/health"
static const char	HEALTH_BACKSLASH [] = "/health/"
static const char	STATS [] = "/stats"
static const char	HEARTBEAT [] = "/heartbeat"
static const char	WORK_QUEUES [] = "/work_queues/"
static const char	REPLICATE_QUEUES [] = "/work_queues/replicate/"
static const char	REPLICATE_QUEUES_NO_BACKSLASH []
static const char	DELETE_QUEUES [] = "/work_queues/delete/"
static const char	DELETE_QUEUES_NO_BACKSLASH [] = "/work_queues/delete"
static const char	WAIT_FOR_ACK [] = "wait_for_acks"
static const char	WAIT_FOR_ACK_BACKSLASH [] = "wait_for_acks/"
static const char	REPLICATE_BACKSLASH [] = "replicate/"
static const char	BLOCK_LOCATIONS [] = "/block_locations/"
static const char	BLOCK_GROUP_LOCATIONS [] = "/block_group_locations/"
static const char	BLOCKS [] = "/blocks"
static const char	EC_RECOVER_QUEUES [] = "/work_queues/ec_recover/"
static const char	EC_RECOVER_QUEUES_NO_BACKSLASH []
static const char	LEASES [] = "/leases"
static const char	CLIENTS [] = "/clients"

Additional Inherited Members
Static Public Member Functions inherited from zkclient::ZkClientCommon
static bool	is_ec_block (u_int64_t block_id)

Detailed Description

This is used by ClientNamenodeProtocolImpl to communicate the zookeeper.

Constructor & Destructor Documentation

zkclient::ZkNnClient::ZkNnClient ( std::shared_ptr< ZKWrapper >  zk_in, bool  secureMode = false  )

inlineexplicit

Use this constructor to build ZkNnClient with a custom ZKWrapper. Which will allow you to set a root directory for all operations on this client

Parameters

zk_in	shared pointer to a ZKWrapper
secureMode	boolean indicating using secure mode or not

Returns

ZkNnClient

Member Function Documentation

bool zkclient::ZkNnClient::abandon_block	(	AbandonBlockRequestProto &	req,
		AbandonBlockResponseProto &	res,
		std::string	client_name = "default"
	)

Abandons the block - basically reverses all of add block's multiops

Parameters

req	AbandonBlockRequestProtoProto
res	AbandonBlockResponseProto
client_name	client's name as string

Returns

boolean indicating whether operation succeeded or not

Since the names were a bit strange and it was a pain to go back and figure out where these were again, I'm writing what the proto fields are here.

message has: required ExtendedBlockProto b = 1; required string src = 2; required string holder = 3; optional uint64 fileId = 4 [default = 0]; // default to GRANDFATHER_INODE_ID

ExtendedBlockProto has: required string poolId = 1; // Block pool id - gloablly unique across clusters required uint64 blockId = 2; // the local id within a pool required uint64 generationStamp = 3; optional uint64 numBytes = 4 [default = 0]; // len does not belong in ebid

bool zkclient::ZkNnClient::add_block	(	AddBlockRequestProto &	req,
		AddBlockResponseProto &	res,
		std::string	client_name = "default"
	)

Adds a block by making appropriate namespace changes and returns information about the set of DataNodes that the block data should be hosted by.

Parameters

req	AddBlockRequestProto
res	AddBlockResponseProto
client_name	client's name as string

Returns

boolean indicating whether operation succeeded or not

bool zkclient::ZkNnClient::add_block	(	const std::string &	fileName,
		u_int64_t &	block_id,
		std::vector< std::string > &	dataNodes,
		uint32_t	replication_factor
	)

A helper method that achieves the above add_block method. Does 1) Creates namespace changes to the given file. 2) Generates a block id. The id is generated randomly for replication blocks and based on the hierarchical naming scheme for EC blocks. 3) Finds a set of data nodes on which to allocate the new block. In the case of replication, the set of DataNodes has primary / secondary replicas of the block. In the case of EC, each DataNode hosts a block group.

bool zkclient::ZkNnClient::add_block_group	(	const std::string &	filePath,
		u_int64_t &	block_group_id,
		std::vector< std::string > &	dataNodes,
		std::vector< char > &	blockIndices,
		uint32_t	total_num_storage_blocks
	)

Makes metadata changes required to add a new block group. This helper method is called for an EC file.

Parameters

filePath	the file specified by its path.
block_group_id	the block group id to be generated.
dataNodes	the set of data nodes on which to allocate each storage block.
blockIndices	the set of block indices within a block gorup.
total_num_storage_blocks	the number of data + parity storage blocks.

Returns

true if successful. false otherwise.

bool zkclient::ZkNnClient::append_file	(	AppendRequestProto &	req,
		AppendResponseProto &	res
	)

Main append file mechanism and associated helpers.

bool zkclient::ZkNnClient::check_acks

(

)

Look through the wait_for_acks work queue to check the replication status of the pending blocks and take an appropriate action to ensure that the blocks get replicated

Checks that each block UUID in the wait_for_acks dir:

1. has REPLICATION_FACTOR many children
2. if the block UUID was created more than ACK_TIMEOUT milliseconds ago TODO: Add to header file

   Returns

void zkclient::ZkNnClient::complete	(	CompleteRequestProto &	req,
		CompleteResponseProto &	res,
		std::string	client_name = "default"
	)

Complete the file.

Parameters

req	CompleteRequestProto
res	CompleteResponseProto
client_name	client's name as string

Returns

ZkNnClient::CreateResponse zkclient::ZkNnClient::create_file	(	CreateRequestProto &	request,
		CreateResponseProto &	response
	)

Create the file.

Parameters

req	CreateRequestProto
res	CreateResponseProto

Returns

CreateResponse

Create a new file entry in the namespace.

This will create an empty file specified by the source path, a full path originated at the root.

uint64_t zkclient::ZkNnClient::current_time_ms

(

)

Returns the current timestamp in milliseconds

ZkNnClient::DeleteResponse zkclient::ZkNnClient::destroy	(	DeleteRequestProto &	request,
		DeleteResponseProto &	response,
		std::string	client_name = "default"
	)

Destroy the file.

Parameters

req	DeleteRequestProto
res	DeleteResponseProto
client_name	client's name as string

Returns

MkdirResponse

Go down directories recursively. If a child is a file, then put its deletion on a queue. Files delete themselves, but directories are deleted by their parent (so root can't be deleted)

bool zkclient::ZkNnClient::file_exists

(

const std::string &

path

)

Information that the protocol might need to respond to individual rpc calls

std::string zkclient::ZkNnClient::find_parent

(

const std::string &

path

)

Find the parent directory of a path

Parameters

path	The path of the source file

Returns

the parent of the path

u_int64_t zkclient::ZkNnClient::generate_block_group_id

(

)

Generates the block group id.

Returns

an 64 bit unsigned integer that has bit 2 ~ bit 48 arbitrarily filled.

u_int64_t zkclient::ZkNnClient::generate_storage_block_id	(	uint64_t	block_group_id,
		uint64_t	index_within_group
	)

Given the block group id and index in the block group, returns the hierarchical block id.

Parameters

block_group_id	the id of a block group this storage block belongs to.
index_within_group	the index within the block group.

Returns

the storage block id.

u_int64_t zkclient::ZkNnClient::get_block_group_id

(

u_int64_t

storage_block_id

)

Gets the block group id from the storage block id. i.e. bit 2 ~ bit 48.

Parameters

storage_block_id

the given storage block id.

Returns

the block group id.

void zkclient::ZkNnClient::get_block_locations	(	GetBlockLocationsRequestProto &	req,
		GetBlockLocationsResponseProto &	res,
		std::string	client_name = "default"
	)

Get locations of blocks.

Parameters

req	GetBlockLocationsRequestProto
res	GetBlockLocationsResponseProto
client_name	client's name as string

Returns

bool zkclient::ZkNnClient::get_block_size	(	const u_int64_t &	block_id,
		uint64_t &	blocksize
	)

Reads the blocksize of the given block_id from zookeeper and returns

uint64_t zkclient::ZkNnClient::get_client_lease_timestamp

(

std::string

client_name

)

Returns the latest timestamp by the client

void zkclient::ZkNnClient::get_content	(	GetContentSummaryRequestProto &	req,
		GetContentSummaryResponseProto &	res,
		std::string	client_name = "default"
	)

Get content of the file.

Parameters

req	GetContentSummaryRequestProto
res	GetContentSummaryResponseProto
client_name	client's name as string

Returns

ZkNnClient::ErasureCodingPoliciesResponse zkclient::ZkNnClient::get_erasure_coding_policies	(	GetErasureCodingPoliciesRequestProto &	req,
		GetErasureCodingPoliciesResponseProto &	res
	)

Returns the erasure coding policies loaded in Namenode, excluding REPLICATION policy.

ZkNnClient::ErasureCodingPolicyResponse zkclient::ZkNnClient::get_erasure_coding_policy_of_path	(	GetErasureCodingPolicyRequestProto &	req,
		GetErasureCodingPolicyResponseProto &	res
	)

Returns the erasure coding policy of a file or a directory specified by the path.

u_int64_t zkclient::ZkNnClient::get_index_within_block_group

(

u_int64_t

storage_block_id

)

Gets the index within the block group.

Parameters

storage_block_id

the given storage block id.

Returns

the index within the block group.

ZkNnClient::GetFileInfoResponse zkclient::ZkNnClient::get_info	(	GetFileInfoRequestProto &	req,
		GetFileInfoResponseProto &	res,
		std::string	client_name = "default"
	)

Get info of the file.

Parameters

req	GetFileInfoRequestProto
res	GetFileInfoResponseProto

Returns

GetFileInfoResponse

ZkNnClient::ListingResponse zkclient::ZkNnClient::get_listing	(	GetListingRequestProto &	req,
		GetListingResponseProto &	res,
		std::string	client_name = "default"
	)

Get listing of the file.

Parameters

req	GetListingRequestProto
res	GetListingResponseProto
client_name	client's name as string

Returns

ListingResponse

uint32_t zkclient::ZkNnClient::get_total_num_storage_blocks	(	const std::string &	fileName,
		u_int64_t &	block_group_id
	)

Given a file, figure out the number of storage blocks to have within a block group.

Parameters

fileName	the file name.
block_group_id	the block group id.

Returns

the number of storage blocks to have within a block group.

ZkNnClient::MkdirResponse zkclient::ZkNnClient::mkdir	(	MkdirsRequestProto &	request,
		MkdirsResponseProto &	response
	)

Make a directory.

Parameters

req	MkdirsRequestProto
res	MkdirsResponseProto

Returns

MkdirResponse

Make a directory in zookeeper

void zkclient::ZkNnClient::read_file_znode	(	FileZNode &	znode_data,
		const std::string &	path
	)

Read a znode corresponding to a file into znode_data

ZkNnClient::RenameResponse zkclient::ZkNnClient::rename	(	RenameRequestProto &	req,
		RenameResponseProto &	res,
		std::string	client_name = "default"
	)

Rename the file.

Parameters

req	RenameRequestProto
res	RenameResponseProto
client_name	client's name as string

Returns

RenameResponse

Rename a file in the zookeeper filesystem

bool zkclient::ZkNnClient::rename_ops_for_file	(	const std::string &	src,
		const std::string &	dst,
		std::vector< std::shared_ptr< ZooOp >> &	ops
	)

Generates multiop ops for renaming src to dst

Parameters

src	The path to the source file (not znode) within the filesystem
dst	The path to the renamed destination file (not znode) within the filesystem
ops	The vector of multiops which will make up the overall atomic rename operation

Returns

Boolean indicating success or failure of the rename

void zkclient::ZkNnClient::renew_lease	(	RenewLeaseRequestProto &	req,
		RenewLeaseResponseProto &	res
	)

These methods will correspond to proto calls that the client namenode protocol handles

ZkNnClient::SetErasureCodingPolicyResponse zkclient::ZkNnClient::set_erasure_coding_policy_of_path	(	SetErasureCodingPolicyRequestProto &	req,
		SetErasureCodingPolicyResponseProto &	res
	)

Sets the erasure coding policy of a path by the given erasure coding policy name.

void zkclient::ZkNnClient::set_file_info_content	(	ContentSummaryProto *	status,
		const std::string &	path,
		FileZNode &	znode_data
	)

Sets file info content.

bool zkclient::ZkNnClient::set_owner	(	SetOwnerRequestProto &	req,
		SetOwnerResponseProto &	res,
		std::string	client_name = "default"
	)

Sets the owner of the file.

Parameters

req	SetOwnerRequestProto
res	SetOwnerResponseProto
client_name	client's name as string

Returns

boolean indicating whether operation succeeded or not

bool zkclient::ZkNnClient::set_permission	(	SetPermissionRequestProto &	req,
		SetPermissionResponseProto &	res
	)

Sets the permission of the file.

Parameters

req	SetPermissionRequestProto
res	SetPermissionResponseProto

Returns

boolean indicating whether operation succeeded or not

---

The documentation for this class was generated from the following files:

• zookeeper/include/zk_nn_client.h
• zookeeper/source/zk_nn_client.cc