#include <read-write-util.h>

Collaboration diagram for impala::ReadWriteUtil:

Public Member Functions
template<>
uint16_t	GetInt (const uint8_t *buf)

template<>
uint32_t	GetInt (const uint8_t *buf)

template<>
uint64_t	GetInt (const uint8_t *buf)

Static Public Member Functions
static int	PutZInt (int32_t integer, uint8_t *buf)
	Put a zigzag encoded integer into a buffer and return its length. More...

static int	PutZLong (int64_t longint, uint8_t *buf)
	Put a zigzag encoded long integer into a buffer and return its length. More...

template<typename T >
static T	GetInt (const uint8_t *buffer)
	Get a big endian integer from a buffer. The buffer does not have to be word aligned. More...

static int	GetVLong (uint8_t buf, int64_t vlong)

static int	GetVInt (uint8_t buf, int32_t vint)

static int64_t	PutVLong (int64_t val, uint8_t *buf)

static int64_t	PutVInt (int32_t val, uint8_t *buf)

static int	VLongRequiredBytes (int64_t val)
	returns size of the encoded long value, not including the 1 byte for length More...

static int	GetVLong (uint8_t buf, int64_t offset, int64_t vlong)

static void	PutInt (uint8_t *buf, uint16_t integer)

static void	PutInt (uint8_t *buf, uint32_t integer)

static void	PutInt (uint8_t *buf, uint64_t integer)

static std::string	HexDump (const uint8_t *buf, int64_t length)
	Dump the first length bytes of buf to a Hex string. More...

static std::string	HexDump (const char *buf, int64_t length)

static bool	IsNegativeVInt (int8_t byte)
	Determines the sign of a VInt/VLong from the first byte. More...

static int	DecodeVIntSize (int8_t byte)
	Determines the total length in bytes of a Writable VInt/VLong from the first byte. More...

static int64_t	ReadZLong (uint8_t **buf)

static int32_t	ReadZInt (uint8_t **buf)
	Read a zig-zag encoded int. More...

template<class T >
static bool	Read (uint8_t *buf, int buf_len, T val, Status status)

static bool	SkipBytes (uint8_t *buf, int buf_len, int num_bytes, Status *status)
	Skip the next num_bytes bytes. More...

Static Public Attributes
static const int	MAX_VINT_LEN = 9
	Maximum length for Writeable VInt. More...

static const int	MAX_ZINT_LEN = 5
	Maximum lengths for Zigzag encodings. More...

static const int	MAX_ZLONG_LEN = 10

Detailed Description

Class for reading and writing various data types. Note: be very careful using signed ints. Casting from a signed int to an unsigned is not a problem. However, bit shifts will do sign extension on unsigned ints, which is rarely the right thing to do for byte level operations.

Definition at line 34 of file read-write-util.h.

Member Function Documentation

int impala::ReadWriteUtil::DecodeVIntSize ( int8_t byte )

inlinestatic

Determines the total length in bytes of a Writable VInt/VLong from the first byte.

Definition at line 254 of file read-write-util.h.

Referenced by GetVLong(), and impala::ScannerContext::Stream::ReadVLong().

template<typename T >

static T impala::ReadWriteUtil::GetInt ( const uint8_t * buffer )

static

Get a big endian integer from a buffer. The buffer does not have to be word aligned.

template<>

uint16_t impala::ReadWriteUtil::GetInt ( const uint8_t * buf )

inline

Definition at line 110 of file read-write-util.h.

template<>

uint32_t impala::ReadWriteUtil::GetInt ( const uint8_t * buf )

inline

Definition at line 115 of file read-write-util.h.

template<>

uint64_t impala::ReadWriteUtil::GetInt ( const uint8_t * buf )

inline

Definition at line 120 of file read-write-util.h.

int impala::ReadWriteUtil::GetVInt	(	uint8_t *	buf,
		int32_t *	vint
	)

inlinestatic

Definition at line 141 of file read-write-util.h.

References GetVLong().

Referenced by impala::HdfsRCFileScanner::GetCurrentKeyBuffer(), and impala::HdfsRCFileScanner::ReadKeyBuffers().

int impala::ReadWriteUtil::GetVLong	(	uint8_t *	buf,
		int64_t *	vlong
	)

inlinestatic

Get a variable-length Long or int value from a byte buffer. Returns the length of the long/int If the size byte is corrupted then return -1;

Definition at line 148 of file read-write-util.h.

Referenced by impala::HdfsSequenceScanner::GetRecord(), GetVInt(), impala::HdfsRCFileScanner::NextField(), and impala::HdfsSequenceScanner::ProcessDecompressedBlock().

int impala::ReadWriteUtil::GetVLong	(	uint8_t *	buf,
		int64_t	offset,
		int64_t *	vlong
	)

inlinestatic

Read a variable-length Long value from a byte buffer starting at the specified byte offset.

Definition at line 152 of file read-write-util.h.

References DecodeVIntSize(), IsNegativeVInt(), and MAX_VINT_LEN.

string ReadWriteUtil::HexDump	(	const uint8_t *	buf,
		int64_t	length
	)

static

Dump the first length bytes of buf to a Hex string.

Definition at line 72 of file read-write-util.cc.

Referenced by HexDump(), impala::HdfsAvroScanner::ReadFileHeader(), impala::HdfsSequenceScanner::ReadFileHeader(), impala::HdfsRCFileScanner::ReadFileHeader(), and impala::BaseSequenceScanner::ReadSync().

string ReadWriteUtil::HexDump	(	const char *	buf,
		int64_t	length
	)

static

Definition at line 81 of file read-write-util.cc.

References HexDump().

bool impala::ReadWriteUtil::IsNegativeVInt ( int8_t byte )

inlinestatic

Determines the sign of a VInt/VLong from the first byte.

Definition at line 250 of file read-write-util.h.

Referenced by GetVLong(), and impala::ScannerContext::Stream::ReadVLong().

void impala::ReadWriteUtil::PutInt	(	uint8_t *	buf,
		uint16_t	integer
	)

inlinestatic

Put an Integer into a buffer in big endian order. The buffer must be big enough.

Definition at line 126 of file read-write-util.h.

Referenced by impala::HdfsSequenceTableWriter::Init(), impala::SnappyBlockCompressor::ProcessBlock(), impala::TestBigEndian(), and impala::WriteStream::WriteInt().

void impala::ReadWriteUtil::PutInt	(	uint8_t *	buf,
		uint32_t	integer
	)

inlinestatic

Definition at line 131 of file read-write-util.h.

References impala::BitUtil::ByteSwap().

void impala::ReadWriteUtil::PutInt	(	uint8_t *	buf,
		uint64_t	integer
	)

inlinestatic

Definition at line 136 of file read-write-util.h.

References impala::BitUtil::ByteSwap().

int64_t impala::ReadWriteUtil::PutVInt	(	int32_t	val,
		uint8_t *	buf
	)

inlinestatic

Definition at line 212 of file read-write-util.h.

References PutVLong().

Referenced by impala::WriteStream::WriteVInt().

int64_t impala::ReadWriteUtil::PutVLong	(	int64_t	val,
		uint8_t *	buf
	)

inlinestatic

Writes a variable-length Long or int value to a byte buffer. Returns the number of bytes written

Definition at line 191 of file read-write-util.h.

References VLongRequiredBytes().

Referenced by PutVInt(), and impala::WriteStream::WriteVLong().

int ReadWriteUtil::PutZInt	(	int32_t	integer,
		uint8_t *	buf
	)

static

Put a zigzag encoded integer into a buffer and return its length.

Definition at line 38 of file read-write-util.cc.

Referenced by impala::TestZInt(), and impala::WriteStream::WriteZInt().

int ReadWriteUtil::PutZLong	(	int64_t	longint,
		uint8_t *	buf
	)

static

Put a zigzag encoded long integer into a buffer and return its length.

Definition at line 55 of file read-write-util.cc.

Referenced by impala::TestZLong(), and impala::WriteStream::WriteZLong().

template<class T >

bool impala::ReadWriteUtil::Read	(	uint8_t **	buf,
		int *	buf_len,
		T *	val,
		Status *	status
	)

inlinestatic

The following methods read data from a buffer without assuming the buffer is long enough. If the buffer isn't long enough or another error occurs, they return false and update the status with the error. Otherwise they return true. buffer is advanced past the data read and buf_len is decremented appropriately. Read a native type T (e.g. bool, float) directly into output (i.e. input is cast directly to T and incremented by sizeof(T)).

Definition at line 222 of file read-write-util.h.

References UNLIKELY.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage().

int32_t impala::ReadWriteUtil::ReadZInt ( uint8_t ** buf )

inlinestatic

Read a zig-zag encoded int.

Definition at line 216 of file read-write-util.h.

References ReadZLong().

Referenced by impala::HdfsAvroScanner::ReadAvroInt32(), and impala::TestZInt().

int64_t ReadWriteUtil::ReadZLong ( uint8_t ** buf )

static

Read a zig-zag encoded long. This is the integer encoding defined by google.com protocol-buffers: https://developers.google.com/protocol-buffers/docs/encoding *buf is incremented past the encoded long.

Definition at line 24 of file read-write-util.cc.

Referenced by impala::HdfsAvroScanner::ReadAvroChar(), impala::HdfsAvroScanner::ReadAvroDecimal(), impala::HdfsAvroScanner::ReadAvroInt64(), impala::HdfsAvroScanner::ReadAvroString(), impala::HdfsAvroScanner::ReadAvroVarchar(), ReadZInt(), and impala::TestZLong().