Impala
Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
Public Member Functions | Protected Member Functions | Protected Attributes | Private Member Functions | Private Attributes | List of all members
impala::HdfsParquetScanner::ColumnReader< T > Class Template Reference
Inheritance diagram for impala::HdfsParquetScanner::ColumnReader< T >:
Collaboration diagram for impala::HdfsParquetScanner::ColumnReader< T >:

Public Member Functions

 ColumnReader (HdfsParquetScanner *parent, const SchemaNode &node)
 
Status Reset (const parquet::ColumnMetaData *metadata, ScannerContext::Stream *stream)
 
void Close ()
 
int64_t total_len () const
 
const SlotDescriptor * slot_desc () const
 
const parquet::SchemaElement & schema_element () const
 
int col_idx () const
 
int max_def_level () const
 
THdfsCompression::type codec () const
 
bool ReadValue (MemPool *pool, Tuple *tuple, bool *conjuncts_failed)
 

Protected Member Functions

virtual void CreateDictionaryDecoder (uint8_t *values, int size)
 
virtual Status InitDataPage (uint8_t *data, int size)
 
virtual bool ReadSlot (void *slot, MemPool *pool, bool *conjuncts_failed)
 
Status ReadDataPage ()
 
int ReadDefinitionLevel ()
 

Protected Attributes

HdfsParquetScanner * parent_
 
const SchemaNode & node_
 
const parquet::ColumnMetaData * metadata_
 
scoped_ptr< Codec > decompressor_
 
ScannerContext::Stream * stream_
 
boost::scoped_ptr< MemPool > decompressed_data_pool_
 
parquet::PageHeader current_page_header_
 
int num_buffered_values_
 
uint8_t * data_
 
RleDecoder rle_def_levels_
 
BitReader bit_packed_def_levels_
 
DictDecoderBase * dict_decoder_base_
 
int64_t num_values_read_
 
const Bitmap * bitmap_filter_
 
uint32_t hash_seed_
 
int64_t rows_returned_
 
int64_t bitmap_filter_rows_rejected_
 

Private Member Functions

void CopySlot (T *slot, MemPool *pool)
 
void ConvertSlot (const T *src, T *dst, MemPool *pool)
 
template<>
void CopySlot (StringValue *slot, MemPool *pool)
 
template<>
void ConvertSlot (const StringValue *src, StringValue *dst, MemPool *pool)
 
template<>
void ConvertSlot (const TimestampValue *src, TimestampValue *dst, MemPool *pool)
 

Private Attributes

scoped_ptr< DictDecoder< T > > dict_decoder_
 
bool needs_conversion_
 
int fixed_len_size_
 

Detailed Description

template<typename T>
class impala::HdfsParquetScanner::ColumnReader< T >

Definition at line 285 of file hdfs-parquet-scanner.cc.

Constructor & Destructor Documentation

template<typename T >
impala::HdfsParquetScanner::ColumnReader< T >::ColumnReader ( HdfsParquetScanner *  parent,
const SchemaNode &  node 
)
inline

Definition at line 287 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::FileVersion::application, impala::ParquetPlainEncoder::DecimalSize(), impala::HdfsParquetScanner::file_version_, impala::HdfsParquetScanner::ColumnReader< T >::fixed_len_size_, impala::ColumnType::len, impala::HdfsParquetScanner::ColumnReader< T >::needs_conversion_, impala::HdfsParquetScanner::BaseColumnReader::slot_desc(), impala::ColumnType::type, impala::SlotDescriptor::type(), impala::TYPE_BOOLEAN, impala::TYPE_CHAR, impala::TYPE_DECIMAL, impala::TYPE_TIMESTAMP, and impala::TYPE_VARCHAR.

Member Function Documentation

void impala::HdfsParquetScanner::BaseColumnReader::Close ( )
inlineinherited

Definition at line 163 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::BaseColumnReader::decompressor_.

THdfsCompression::type impala::HdfsParquetScanner::BaseColumnReader::codec ( ) const
inlineinherited

Definition at line 172 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::BaseColumnReader::metadata_, and impala::PARQUET_TO_IMPALA_CODEC.

int impala::HdfsParquetScanner::BaseColumnReader::col_idx ( ) const
inlineinherited

Definition at line 170 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::SchemaNode::col_idx, and impala::HdfsParquetScanner::BaseColumnReader::node_.

Referenced by impala::HdfsParquetScanner::ValidateColumn().

template<typename T >
void impala::HdfsParquetScanner::ColumnReader< T >::ConvertSlot ( const T *  src,
T *  dst,
MemPool *  pool 
)
inlineprivate

Definition at line 358 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

template<>
void impala::HdfsParquetScanner::ColumnReader< StringValue >::ConvertSlot ( const StringValue *  src,
StringValue *  dst,
MemPool *  pool 
)
private

Definition at line 382 of file hdfs-parquet-scanner.cc.

References impala::MemPool::Allocate(), impala::StringValue::len, impala::StringValue::PadWithSpaces(), impala::StringValue::ptr, and impala::TYPE_CHAR.

template<>
void impala::HdfsParquetScanner::ColumnReader< TimestampValue >::ConvertSlot ( const TimestampValue *  src,
TimestampValue *  dst,
MemPool *  pool 
)
private

Definition at line 401 of file hdfs-parquet-scanner.cc.

References impala::TimestampValue::HasDateAndTime(), and impala::TimestampValue::UtcToLocal().

template<typename T >
void impala::HdfsParquetScanner::ColumnReader< T >::CopySlot ( T *  slot,
MemPool *  pool 
)
inlineprivate

Definition at line 353 of file hdfs-parquet-scanner.cc.

template<>
void impala::HdfsParquetScanner::ColumnReader< StringValue >::CopySlot ( StringValue *  slot,
MemPool *  pool 
)
private

Definition at line 373 of file hdfs-parquet-scanner.cc.

References impala::MemPool::Allocate(), impala::StringValue::len, and impala::StringValue::ptr.

template<typename T >
virtual void impala::HdfsParquetScanner::ColumnReader< T >::CreateDictionaryDecoder ( uint8_t *  values,
int  size 
)
inlineprotectedvirtual

Implements impala::HdfsParquetScanner::BaseColumnReader.

Definition at line 306 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::ColumnReader< T >::dict_decoder_, impala::HdfsParquetScanner::BaseColumnReader::dict_decoder_base_, and impala::HdfsParquetScanner::ColumnReader< T >::fixed_len_size_.

template<typename T >
virtual Status impala::HdfsParquetScanner::ColumnReader< T >::InitDataPage ( uint8_t *  data,
int  size 
)
inlineprotectedvirtual

Implements impala::HdfsParquetScanner::BaseColumnReader.

Definition at line 311 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::BaseColumnReader::bitmap_filter_, impala::HdfsParquetScanner::BaseColumnReader::bitmap_filter_rows_rejected_, impala::HdfsParquetScanner::BaseColumnReader::current_page_header_, impala::HdfsParquetScanner::ColumnReader< T >::dict_decoder_, impala::Status::OK, and impala::HdfsParquetScanner::BaseColumnReader::rows_returned_.

int impala::HdfsParquetScanner::BaseColumnReader::max_def_level ( ) const
inlineinherited

Definition at line 171 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::SchemaNode::max_def_level, and impala::HdfsParquetScanner::BaseColumnReader::node_.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage().

Status HdfsParquetScanner::BaseColumnReader::ReadDataPage ( )
protectedinherited

Definition at line 532 of file hdfs-parquet-scanner.cc.

References impala::Status::AddDetail(), impala::HdfsScanner::AttachPool(), impala::HdfsParquetScanner::BaseColumnReader::bit_packed_def_levels_, impala::BitUtil::Ceil(), impala::HdfsParquetScanner::BaseColumnReader::CreateDictionaryDecoder(), impala::HdfsParquetScanner::BaseColumnReader::current_page_header_, impala::HdfsParquetScanner::BaseColumnReader::data_, impala::HdfsScanner::decompress_timer_, impala::HdfsParquetScanner::BaseColumnReader::decompressed_data_pool_, impala::HdfsParquetScanner::BaseColumnReader::decompressor_, impala::DeserializeThriftMsg(), impala::HdfsParquetScanner::BaseColumnReader::dict_decoder_base_, impala::HdfsParquetScanner::dictionary_pool_, impala::ScannerContext::Stream::eosr(), impala::HdfsParquetScanner::file_version_, impala::ScannerContext::Stream::GetBuffer(), impala::ScannerContext::Stream::GetBytes(), impala::HdfsParquetScanner::BaseColumnReader::InitDataPage(), impala::BitUtil::Log2(), LOG_OR_ABORT, impala::HdfsParquetScanner::BaseColumnReader::max_def_level(), MAX_PAGE_HEADER_SIZE, impala::HdfsParquetScanner::BaseColumnReader::metadata_, impala::HdfsParquetScanner::BaseColumnReader::num_buffered_values_, impala::DictDecoderBase::num_entries(), impala::HdfsScanNode::num_partition_keys(), impala::HdfsParquetScanner::BaseColumnReader::num_values_read_, impala::Status::OK, impala::Status::ok(), impala::HdfsParquetScanner::BaseColumnReader::parent_, impala::PLAIN, impala::PrettyPrinter::Print(), impala::ReadWriteUtil::Read(), impala::ScannerContext::Stream::ReadBytes(), RequiresSkippedDictionaryHeaderCheck(), RETURN_IF_ERROR, impala::HdfsParquetScanner::BaseColumnReader::rle_def_levels_, impala::HdfsScanNode::runtime_state(), impala::HdfsScanner::scan_node_, SCOPED_TIMER, impala::ScannerContext::Stream::SkipBytes(), impala::HdfsParquetScanner::BaseColumnReader::slot_desc(), impala::HdfsParquetScanner::BaseColumnReader::stream_, impala::TYPE_BOOLEAN, and VLOG_FILE.

int HdfsParquetScanner::BaseColumnReader::ReadDefinitionLevel ( )
inlineprotectedinherited

Definition at line 720 of file hdfs-parquet-scanner.cc.

template<typename T >
virtual bool impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot ( void *  slot,
MemPool *  pool,
bool *  conjuncts_failed 
)
inlineprotectedvirtual

Implements impala::HdfsParquetScanner::BaseColumnReader.

Definition at line 330 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::BaseColumnReader::bitmap_filter_, impala::HdfsParquetScanner::BaseColumnReader::bitmap_filter_rows_rejected_, impala::HdfsParquetScanner::ColumnReader< T >::ConvertSlot(), impala::HdfsParquetScanner::BaseColumnReader::current_page_header_, impala::HdfsParquetScanner::BaseColumnReader::data_, impala::HdfsParquetScanner::ColumnReader< T >::dict_decoder_, impala::HdfsParquetScanner::ColumnReader< T >::fixed_len_size_, impala::Bitmap::Get(), impala::RawValue::GetHashValue(), impala::HdfsParquetScanner::BaseColumnReader::hash_seed_, impala::HdfsParquetScanner::ColumnReader< T >::needs_conversion_, impala::PLAIN, impala::HdfsParquetScanner::BaseColumnReader::rows_returned_, and impala::HdfsParquetScanner::BaseColumnReader::slot_desc().

bool HdfsParquetScanner::BaseColumnReader::ReadValue ( MemPool *  pool,
Tuple *  tuple,
bool *  conjuncts_failed 
)
inlineinherited

Definition at line 744 of file hdfs-parquet-scanner.cc.

References impala::Tuple::GetSlot(), pool, and impala::Tuple::SetNull().

Status impala::HdfsParquetScanner::BaseColumnReader::Reset ( const parquet::ColumnMetaData *  metadata,
ScannerContext::Stream *  stream 
)
inlineinherited

Definition at line 145 of file hdfs-parquet-scanner.cc.

References impala::Codec::CreateDecompressor(), impala::HdfsParquetScanner::BaseColumnReader::data_, impala::HdfsParquetScanner::BaseColumnReader::decompressor_, impala::HdfsParquetScanner::BaseColumnReader::dict_decoder_base_, impala::HdfsParquetScanner::BaseColumnReader::metadata_, impala::HdfsParquetScanner::BaseColumnReader::num_buffered_values_, impala::HdfsParquetScanner::BaseColumnReader::num_values_read_, impala::Status::OK, impala::PARQUET_TO_IMPALA_CODEC, RETURN_IF_ERROR, and impala::HdfsParquetScanner::BaseColumnReader::stream_.

const parquet::SchemaElement& impala::HdfsParquetScanner::BaseColumnReader::schema_element ( ) const
inlineinherited

Definition at line 169 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::SchemaNode::element, and impala::HdfsParquetScanner::BaseColumnReader::node_.

Referenced by impala::HdfsParquetScanner::ValidateColumn().

const SlotDescriptor* impala::HdfsParquetScanner::BaseColumnReader::slot_desc ( ) const
inlineinherited

Definition at line 168 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::BaseColumnReader::node_, and impala::HdfsParquetScanner::SchemaNode::slot_desc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader(), impala::HdfsParquetScanner::BoolColumnReader::BoolColumnReader(), impala::HdfsParquetScanner::ColumnReader< T >::ColumnReader(), impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot(), and impala::HdfsParquetScanner::ValidateColumn().

int64_t impala::HdfsParquetScanner::BaseColumnReader::total_len ( ) const
inlineinherited

Definition at line 167 of file hdfs-parquet-scanner.cc.

References impala::HdfsParquetScanner::BaseColumnReader::metadata_.

Member Data Documentation

BitReader impala::HdfsParquetScanner::BaseColumnReader::bit_packed_def_levels_
protectedinherited

Definition at line 219 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage().

const Bitmap* impala::HdfsParquetScanner::BaseColumnReader::bitmap_filter_
protectedinherited

Definition at line 228 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader(), impala::HdfsParquetScanner::ColumnReader< T >::InitDataPage(), and impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

int64_t impala::HdfsParquetScanner::BaseColumnReader::bitmap_filter_rows_rejected_
protectedinherited

Definition at line 238 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader(), impala::HdfsParquetScanner::ColumnReader< T >::InitDataPage(), and impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

parquet::PageHeader impala::HdfsParquetScanner::BaseColumnReader::current_page_header_
protectedinherited

Definition at line 208 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::ColumnReader< T >::InitDataPage(), impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), and impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

uint8_t* impala::HdfsParquetScanner::BaseColumnReader::data_
protectedinherited

Definition at line 214 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot(), and impala::HdfsParquetScanner::BaseColumnReader::Reset().

boost::scoped_ptr<MemPool> impala::HdfsParquetScanner::BaseColumnReader::decompressed_data_pool_
protectedinherited

Definition at line 205 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage().

scoped_ptr<Codec> impala::HdfsParquetScanner::BaseColumnReader::decompressor_
protectedinherited

Definition at line 201 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::Close(), impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), and impala::HdfsParquetScanner::BaseColumnReader::Reset().

template<typename T >
scoped_ptr<DictDecoder<T> > impala::HdfsParquetScanner::ColumnReader< T >::dict_decoder_
private

Definition at line 362 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::ColumnReader< T >::CreateDictionaryDecoder(), impala::HdfsParquetScanner::ColumnReader< T >::InitDataPage(), and impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

DictDecoderBase* impala::HdfsParquetScanner::BaseColumnReader::dict_decoder_base_
protectedinherited

Definition at line 222 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::ColumnReader< T >::CreateDictionaryDecoder(), impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), and impala::HdfsParquetScanner::BaseColumnReader::Reset().

template<typename T >
int impala::HdfsParquetScanner::ColumnReader< T >::fixed_len_size_
private

Definition at line 369 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::ColumnReader< T >::ColumnReader(), impala::HdfsParquetScanner::ColumnReader< T >::CreateDictionaryDecoder(), and impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

uint32_t impala::HdfsParquetScanner::BaseColumnReader::hash_seed_
protectedinherited

Definition at line 230 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader(), and impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

const parquet::ColumnMetaData* impala::HdfsParquetScanner::BaseColumnReader::metadata_
protectedinherited

Definition at line 200 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::codec(), impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), impala::HdfsParquetScanner::BaseColumnReader::Reset(), and impala::HdfsParquetScanner::BaseColumnReader::total_len().

template<typename T >
bool impala::HdfsParquetScanner::ColumnReader< T >::needs_conversion_
private

Definition at line 365 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::ColumnReader< T >::ColumnReader(), and impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

const SchemaNode& impala::HdfsParquetScanner::BaseColumnReader::node_
protectedinherited

Definition at line 198 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::col_idx(), impala::HdfsParquetScanner::BaseColumnReader::max_def_level(), impala::HdfsParquetScanner::BaseColumnReader::schema_element(), and impala::HdfsParquetScanner::BaseColumnReader::slot_desc().

int impala::HdfsParquetScanner::BaseColumnReader::num_buffered_values_
protectedinherited

Definition at line 211 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), and impala::HdfsParquetScanner::BaseColumnReader::Reset().

int64_t impala::HdfsParquetScanner::BaseColumnReader::num_values_read_
protectedinherited

Definition at line 225 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), and impala::HdfsParquetScanner::BaseColumnReader::Reset().

HdfsParquetScanner* impala::HdfsParquetScanner::BaseColumnReader::parent_
protectedinherited

Definition at line 197 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader(), impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), and impala::HdfsParquetScanner::BoolColumnReader::ReadSlot().

RleDecoder impala::HdfsParquetScanner::BaseColumnReader::rle_def_levels_
protectedinherited

Definition at line 218 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage().

int64_t impala::HdfsParquetScanner::BaseColumnReader::rows_returned_
protectedinherited

Definition at line 237 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader(), impala::HdfsParquetScanner::ColumnReader< T >::InitDataPage(), and impala::HdfsParquetScanner::ColumnReader< T >::ReadSlot().

ScannerContext::Stream* impala::HdfsParquetScanner::BaseColumnReader::stream_
protectedinherited

Definition at line 202 of file hdfs-parquet-scanner.cc.

Referenced by impala::HdfsParquetScanner::AssembleRows(), impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage(), and impala::HdfsParquetScanner::BaseColumnReader::Reset().

The documentation for this class was generated from the following file: