impala::HBaseScanNode Class Reference

#include <hbase-scan-node.h>

Inheritance diagram for impala::HBaseScanNode:

Collaboration diagram for impala::HBaseScanNode:

Public Member Functions
	HBaseScanNode (ObjectPool *pool, const TPlanNode &tnode, const DescriptorTbl &descs)
	~HBaseScanNode ()
virtual Status	Prepare (RuntimeState *state)
virtual Status	Open (RuntimeState *state)
	Start HBase scan using hbase_scanner_. More...
virtual Status	GetNext (RuntimeState *state, RowBatch *row_batch, bool *eos)
virtual Status	Reset (RuntimeState *state)
	NYI. More...
virtual void	Close (RuntimeState *state)
	Close the hbase_scanner_, and report errors. More...
const int	suggested_max_caching () const
void	SetScanRanges (const std::vector< TScanRangeParams > &scan_range_params)
virtual bool	IsScanNode () const
RuntimeProfile::Counter *	bytes_read_counter () const
RuntimeProfile::Counter *	rows_read_counter () const
RuntimeProfile::Counter *	read_timer () const
RuntimeProfile::Counter *	total_throughput_counter () const
RuntimeProfile::Counter *	per_read_thread_throughput_counter () const
RuntimeProfile::Counter *	materialize_tuple_timer () const
RuntimeProfile::Counter *	scan_ranges_complete_counter () const
RuntimeProfile::ThreadCounters *	scanner_thread_counters () const
RuntimeProfile::Counter &	active_scanner_thread_counter ()
RuntimeProfile::Counter *	average_scanner_thread_concurrency () const
virtual Status	Init (const TPlanNode &tnode)
void	CollectNodes (TPlanNodeType::type node_type, std::vector< ExecNode * > *nodes)
void	CollectScanNodes (std::vector< ExecNode * > *nodes)
	Collect all scan node types. More...
std::string	DebugString () const
	Returns a string representation in DFS order of the plan rooted at this. More...
const std::vector< ExprContext * > &	conjunct_ctxs () const
int	id () const
TPlanNodeType::type	type () const
const RowDescriptor &	row_desc () const
int64_t	rows_returned () const
int64_t	limit () const
bool	ReachedLimit ()
RuntimeProfile *	runtime_profile ()
MemTracker *	mem_tracker ()
MemTracker *	expr_mem_tracker ()

Static Public Member Functions
static Status	CreateTree (ObjectPool *pool, const TPlan &plan, const DescriptorTbl &descs, ExecNode **root)
static void	SetDebugOptions (int node_id, TExecNodePhase::type phase, TDebugAction::type action, ExecNode *tree)
	Set debug action for node with given id in 'tree'. More...
static bool	EvalConjuncts (ExprContext *const *ctxs, int num_ctxs, TupleRow *row)
static llvm::Function *	CodegenEvalConjuncts (RuntimeState *state, const std::vector< ExprContext * > &conjunct_ctxs, const char *name="EvalConjuncts")
static int	GetNodeIdFromProfile (RuntimeProfile *p)
	Extract node id from p->name(). More...

Static Public Attributes
static const std::string	BYTES_READ_COUNTER = "BytesRead"
	names of ScanNode common counters More...
static const std::string	ROWS_READ_COUNTER = "RowsRead"
static const std::string	TOTAL_HDFS_READ_TIMER = "TotalRawHdfsReadTime(*)"
static const std::string	TOTAL_HBASE_READ_TIMER = "TotalRawHBaseReadTime(*)"
static const std::string	TOTAL_THROUGHPUT_COUNTER = "TotalReadThroughput"
static const std::string	PER_READ_THREAD_THROUGHPUT_COUNTER
static const std::string	NUM_DISKS_ACCESSED_COUNTER = "NumDisksAccessed"
static const std::string	MATERIALIZE_TUPLE_TIMER = "MaterializeTupleTime(*)"
static const std::string	SCAN_RANGES_COMPLETE_COUNTER = "ScanRangesComplete"
static const std::string	SCANNER_THREAD_COUNTERS_PREFIX = "ScannerThreads"
static const std::string	SCANNER_THREAD_TOTAL_WALLCLOCK_TIME
static const std::string	AVERAGE_SCANNER_THREAD_CONCURRENCY
static const std::string	AVERAGE_HDFS_READ_THREAD_CONCURRENCY
static const std::string	NUM_SCANNER_THREADS_STARTED
static const std::string	ROW_THROUGHPUT_COUNTER = "RowsReturnedRate"
	Names of counters shared by all exec nodes. More...

Protected Member Functions
virtual void	DebugString (int indentation_level, std::stringstream *out) const
	Write debug string of this into out. More...
ExecNode *	child (int i)
bool	is_closed ()
void	InitRuntimeProfile (const std::string &name)
Status	ExecDebugAction (TExecNodePhase::type phase, RuntimeState *state)
void	AddRuntimeExecOption (const std::string &option)
	Appends option to 'runtime_exec_options_'. More...
virtual Status	QueryMaintenance (RuntimeState *state)
void	AddExprCtxToFree (ExprContext *ctx)
void	AddExprCtxsToFree (const std::vector< ExprContext * > &ctxs)
void	AddExprCtxsToFree (const SortExecExprs &sort_exec_exprs)

Static Protected Member Functions
static Status	CreateNode (ObjectPool *pool, const TPlanNode &tnode, const DescriptorTbl &descs, ExecNode **node)
	Create a single exec node derived from thrift node; place exec node in 'pool'. More...
static Status	CreateTreeHelper (ObjectPool *pool, const std::vector< TPlanNode > &tnodes, const DescriptorTbl &descs, ExecNode *parent, int *node_idx, ExecNode **root)

Protected Attributes
const std::vector < TScanRangeParams > *	scan_range_params_
	The scan ranges this scan node is responsible for. Not owned. More...
RuntimeProfile::Counter *	bytes_read_counter_
RuntimeProfile::TimeSeriesCounter *	bytes_read_timeseries_counter_
	Time series of the bytes_read_counter_. More...
RuntimeProfile::Counter *	rows_read_counter_
	rows/tuples read from the scanner (including those discarded by EvalConjucts()) More...
RuntimeProfile::Counter *	read_timer_
RuntimeProfile::Counter *	total_throughput_counter_
	Wall based aggregate read throughput [bytes/sec]. More...
RuntimeProfile::Counter *	per_read_thread_throughput_counter_
	Per thread read throughput [bytes/sec]. More...
RuntimeProfile::Counter *	num_disks_accessed_counter_
RuntimeProfile::Counter *	materialize_tuple_timer_
RuntimeProfile::Counter *	scan_ranges_complete_counter_
RuntimeProfile::ThreadCounters *	scanner_thread_counters_
	Aggregated scanner thread counters. More...
RuntimeProfile::Counter	active_scanner_thread_counter_
	The number of active scanner threads that are not blocked by IO. More...
RuntimeProfile::Counter *	average_scanner_thread_concurrency_
RuntimeProfile::Counter	active_hdfs_read_thread_counter_
	The number of active hdfs reading threads reading for this node. More...
RuntimeProfile::Counter *	average_hdfs_read_thread_concurrency_
RuntimeProfile::Counter *	num_scanner_threads_started_counter_
std::vector < RuntimeProfile::Counter * >	hdfs_read_thread_concurrency_bucket_
int	id_
TPlanNodeType::type	type_
ObjectPool *	pool_
std::vector< ExprContext * >	conjunct_ctxs_
std::vector< ExecNode * >	children_
RowDescriptor	row_descriptor_
TExecNodePhase::type	debug_phase_
TDebugAction::type	debug_action_
int64_t	limit_
int64_t	num_rows_returned_
boost::scoped_ptr< RuntimeProfile >	runtime_profile_
RuntimeProfile::Counter *	rows_returned_counter_
RuntimeProfile::Counter *	rows_returned_rate_
boost::scoped_ptr< MemTracker >	mem_tracker_
	Account for peak memory used by this node. More...
boost::scoped_ptr< MemTracker >	expr_mem_tracker_
	MemTracker that should be used for ExprContexts. More...
boost::mutex	exec_options_lock_
std::string	runtime_exec_options_

Private Member Functions
void	WriteTextSlot (const std::string &family, const std::string &qualifier, void *value, int value_length, SlotDescriptor *slot, RuntimeState *state, bool *error_in_row)

Static Private Member Functions
static bool	CmpColPos (const SlotDescriptor *a, const SlotDescriptor *b)
	Compare two slots based on their column position, to sort them ascending. More...

Private Attributes
const std::string	table_name_
	Name of HBase table (not necessarily the table name mapped to Hive). More...
TupleId	tuple_id_
	Tuple id resolved in Prepare() to set tuple_desc_;. More...
const TupleDescriptor *	tuple_desc_
	Descriptor of tuples read from HBase table. More...
int	tuple_idx_
	Tuple index in tuple row. More...
HBaseTableScanner::ScanRangeVector	scan_range_vector_
	scan ranges of a region server More...
std::vector< THBaseFilter >	filters_
	HBase Filters to be set in HBaseTableScanner. More...
int	num_errors_
	Counts the total number of conversion errors for this table. More...
boost::scoped_ptr< MemPool >	tuple_pool_
	Pool for allocating tuple data, including all varying-length slots. More...
boost::scoped_ptr < HBaseTableScanner >	hbase_scanner_
	Jni helper for scanning an HBase table. More...
std::vector< SlotDescriptor * >	sorted_non_key_slots_
	List of non-row-key slots sorted by col_pos(). Populated in Prepare(). More...
std::vector< const HBaseTableDescriptor::HBaseColumnDescriptor * >	sorted_cols_
SlotDescriptor *	row_key_slot_
bool	row_key_binary_encoded_
	True, if row key is binary encoded. More...
int	tuple_buffer_size_
	Size of tuple buffer determined by size of tuples and capacity of row batches. More...
Tuple *	tuple_
	Current tuple. More...
boost::scoped_ptr< TextConverter >	text_converter_
	Helper class for converting text to other types;. More...
int	suggested_max_caching_

Static Private Attributes
static const int	SKIP_COLUMN = -1
static const int	ROW_KEY = 0
	Column 0 in the Impala metadata refers to the HBasw row key. More...

Detailed Description

Definition at line 29 of file hbase-scan-node.h.

Constructor & Destructor Documentation

HBaseScanNode::HBaseScanNode	(	ObjectPool *	pool,
		const TPlanNode &	tnode,
		const DescriptorTbl &	descs
	)

Definition at line 34 of file hbase-scan-node.cc.

References suggested_max_caching_.

HBaseScanNode::~HBaseScanNode

(

)

Definition at line 53 of file hbase-scan-node.cc.

Member Function Documentation

RuntimeProfile::Counter& impala::ScanNode::active_scanner_thread_counter ( )

inlineinherited

Definition at line 113 of file scan-node.h.

References impala::ScanNode::active_scanner_thread_counter_.

void impala::ExecNode::AddExprCtxsToFree ( const std::vector< ExprContext * > &  ctxs )

protectedinherited

Referenced by impala::ExecNode::AddExprCtxsToFree(), impala::UnionNode::Prepare(), impala::SortNode::Prepare(), impala::TopNNode::Prepare(), impala::ExchangeNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::ExecNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), and impala::HdfsScanNode::Prepare().

void impala::ExecNode::AddExprCtxsToFree ( const SortExecExprs &  sort_exec_exprs )

protectedinherited

Definition at line 410 of file exec-node.cc.

References impala::ExecNode::AddExprCtxsToFree(), impala::SortExecExprs::lhs_ordering_expr_ctxs(), impala::SortExecExprs::rhs_ordering_expr_ctxs(), and impala::SortExecExprs::sort_tuple_slot_expr_ctxs().

void impala::ExecNode::AddExprCtxToFree ( ExprContext *  ctx )

inlineprotectedinherited

Add an ExprContext to have its local allocations freed by QueryMaintenance(). Exprs that are evaluated in the main execution thread should be added. Exprs evaluated in a separate thread are generally not safe to add, since a local allocation may be freed while it's being used. Rather than using this mechanism, threads should call FreeLocalAllocations() on local ExprContexts periodically.

Definition at line 276 of file exec-node.h.

References impala::ExecNode::expr_ctxs_to_free_.

Referenced by impala::AnalyticEvalNode::Prepare().

void impala::ExecNode::AddRuntimeExecOption ( const std::string &  option )

protectedinherited

Appends option to 'runtime_exec_options_'.

Definition at line 188 of file exec-node.cc.

References impala::RuntimeProfile::AddInfoString(), impala::ExecNode::exec_options_lock_, impala::ExecNode::runtime_exec_options_, and impala::ExecNode::runtime_profile().

Referenced by impala::PartitionedHashJoinNode::AttachProbeFilters(), impala::HashJoinNode::ConstructBuildSide(), impala::BlockingJoinNode::Open(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::HdfsScanNode::Prepare(), and impala::HdfsScanNode::StopAndFinalizeCounters().

RuntimeProfile::Counter* impala::ScanNode::average_scanner_thread_concurrency ( ) const

inlineinherited

Definition at line 116 of file scan-node.h.

References impala::ScanNode::average_scanner_thread_concurrency_.

RuntimeProfile::Counter* impala::ScanNode::bytes_read_counter ( ) const

inlineinherited

Definition at line 95 of file scan-node.h.

References impala::ScanNode::bytes_read_counter_.

Referenced by impala::HBaseTableScanner::GetFamily(), impala::HBaseTableScanner::GetQualifier(), impala::HBaseTableScanner::GetRowKey(), impala::HBaseTableScanner::GetValue(), impala::HdfsScanNode::Open(), and impala::HdfsScanNode::StopAndFinalizeCounters().

ExecNode* impala::ExecNode::child ( int  i )

inlineprotectedinherited

Definition at line 241 of file exec-node.h.

References impala::ExecNode::children_.

Referenced by impala::CrossJoinNode::BuildListDebugString(), impala::BlockingJoinNode::BuildSideThread(), impala::HashJoinNode::CodegenCreateOutputRow(), impala::PartitionedHashJoinNode::CodegenCreateOutputRow(), impala::CrossJoinNode::ConstructBuildSide(), impala::HashJoinNode::ConstructBuildSide(), impala::PartitionedHashJoinNode::ConstructBuildSide(), impala::BlockingJoinNode::GetLeftChildRowString(), impala::SelectNode::GetNext(), impala::UnionNode::GetNext(), impala::CrossJoinNode::GetNext(), impala::HashJoinNode::GetNext(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::PartitionedAggregationNode::Partition::InitStreams(), impala::HashJoinNode::LeftJoinGetNext(), impala::PartitionedHashJoinNode::NextProbeRowBatch(), impala::SelectNode::Open(), impala::SortNode::Open(), impala::TopNNode::Open(), impala::BlockingJoinNode::Open(), impala::AggregationNode::Open(), impala::AnalyticEvalNode::Open(), impala::PartitionedAggregationNode::Open(), impala::UnionNode::OpenCurrentChild(), impala::SelectNode::Prepare(), impala::SortNode::Prepare(), impala::UnionNode::Prepare(), impala::TopNNode::Prepare(), impala::BlockingJoinNode::Prepare(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::PartitionedHashJoinNode::ProcessBuildInput(), impala::AnalyticEvalNode::ProcessChildBatches(), and impala::SortNode::SortInput().

void HBaseScanNode::Close ( RuntimeState *  state )

virtual

Close the hbase_scanner_, and report errors.

Reimplemented from impala::ExecNode.

Definition at line 277 of file hbase-scan-node.cc.

References impala::ScanNode::bytes_read_timeseries_counter_, impala::ExecNode::Close(), getJNIEnv(), hbase_scanner_, impala::ExecNode::is_closed(), num_errors_, impala::RuntimeState::ReportFileErrors(), impala::ExecNode::runtime_profile_, SCOPED_TIMER, impala::PeriodicCounterUpdater::StopRateCounter(), impala::PeriodicCounterUpdater::StopTimeSeriesCounter(), table_name_, impala::ScanNode::total_throughput_counter(), and tuple_pool_.

static bool impala::HBaseScanNode::CmpColPos ( const SlotDescriptor *  a, const SlotDescriptor *  b  )

staticprivate

Compare two slots based on their column position, to sort them ascending.

Function * impala::ExecNode::CodegenEvalConjuncts ( RuntimeState *  state, const std::vector< ExprContext * > &  conjunct_ctxs, const char *  name = "EvalConjuncts"  )

staticinherited

Returns a codegen'd version of EvalConjuncts(), or NULL if the function couldn't be codegen'd. The codegen'd version uses inlined, codegen'd GetBooleanVal() functions.

Definition at line 452 of file exec-node.cc.

References impala::LlvmCodeGen::FnPrototype::AddArgument(), impala::LlvmCodeGen::context(), impala::CodegenAnyVal::CreateCallWrapped(), impala::LlvmCodeGen::false_value(), impala::LlvmCodeGen::FinalizeFunction(), impala::RuntimeState::GetCodegen(), impala::Status::GetDetail(), impala::CodegenAnyVal::GetIsNull(), impala::LlvmCodeGen::GetType(), impala::CodegenAnyVal::GetVal(), impala::TupleRow::LLVM_CLASS_NAME, impala::ExprContext::LLVM_CLASS_NAME, impala::Status::ok(), impala::LlvmCodeGen::true_value(), impala::ExecNode::type(), impala::TYPE_BOOLEAN, impala::TYPE_INT, and VLOG_QUERY.

Referenced by impala::HdfsAvroScanner::CodegenDecodeAvroData(), impala::HashJoinNode::CodegenProcessProbeBatch(), and impala::PartitionedHashJoinNode::CodegenProcessProbeBatch().

void impala::ExecNode::CollectNodes ( TPlanNodeType::type  node_type, std::vector< ExecNode * > *  nodes  )

inherited

Collect all nodes of given 'node_type' that are part of this subtree, and return in 'nodes'.

Definition at line 359 of file exec-node.cc.

References impala::ExecNode::children_, and impala::ExecNode::type_.

Referenced by impala::ExecNode::CollectScanNodes(), and impala::PlanFragmentExecutor::Prepare().

void impala::ExecNode::CollectScanNodes ( std::vector< ExecNode * > *  nodes )

inherited

Collect all scan node types.

Definition at line 366 of file exec-node.cc.

References impala::ExecNode::CollectNodes().

Referenced by impala::PlanFragmentExecutor::Prepare().

const std::vector<ExprContext*>& impala::ExecNode::conjunct_ctxs ( ) const

inlineinherited

Definition at line 152 of file exec-node.h.

References impala::ExecNode::conjunct_ctxs_.

Referenced by impala::HdfsScanNode::ComputeSlotMaterializationOrder(), impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), impala::HashJoinNode::GetNext(), impala::PartitionedHashJoinNode::OutputUnmatchedBuild(), impala::HashJoinNode::ProcessProbeBatch(), and impala::PartitionedHashJoinNode::ProcessProbeBatch().

Status impala::ExecNode::CreateNode ( ObjectPool *  pool, const TPlanNode &  tnode, const DescriptorTbl &  descs, ExecNode **  node  )

staticprotectedinherited

Create a single exec node derived from thrift node; place exec node in 'pool'.

Definition at line 260 of file exec-node.cc.

References impala::ObjectPool::Add(), impala::Status::OK, and RETURN_IF_ERROR.

Referenced by impala::ExecNode::CreateTreeHelper().

Status impala::ExecNode::CreateTree ( ObjectPool *  pool, const TPlan &  plan, const DescriptorTbl &  descs, ExecNode **  root  )

staticinherited

Creates exec node tree from list of nodes contained in plan via depth-first traversal. All nodes are placed in pool. Returns error if 'plan' is corrupted, otherwise success.

Definition at line 199 of file exec-node.cc.

References impala::ExecNode::CreateTreeHelper(), impala::Status::OK, and impala::Status::ok().

Referenced by impala::PlanFragmentExecutor::Prepare().

Status impala::ExecNode::CreateTreeHelper ( ObjectPool *  pool, const std::vector< TPlanNode > &  tnodes, const DescriptorTbl &  descs, ExecNode *  parent, int *  node_idx, ExecNode **  root  )

staticprotectedinherited

Definition at line 218 of file exec-node.cc.

References impala::RuntimeProfile::AddChild(), impala::ExecNode::children_, impala::ExecNode::CreateNode(), impala::Status::OK, RETURN_IF_ERROR, and impala::ExecNode::runtime_profile().

Referenced by impala::ExecNode::CreateTree().

void HBaseScanNode::DebugString ( int  indentation_level, std::stringstream *  out  ) const

protectedvirtual

Write debug string of this into out.

Reimplemented from impala::ExecNode.

Definition at line 294 of file hbase-scan-node.cc.

References impala::ExecNode::children_, impala::HBaseTableScanner::ScanRange::DebugString(), scan_range_vector_, table_name_, and tuple_id_.

string impala::ExecNode::DebugString ( ) const

inherited

Returns a string representation in DFS order of the plan rooted at this.

Definition at line 345 of file exec-node.cc.

Referenced by impala::SortNode::DebugString(), impala::TopNNode::DebugString(), impala::ExchangeNode::DebugString(), impala::AggregationNode::DebugString(), impala::AnalyticEvalNode::DebugString(), impala::PartitionedAggregationNode::DebugString(), impala::BlockingJoinNode::DebugString(), and impala::PlanFragmentExecutor::Prepare().

bool impala::ExecNode::EvalConjuncts ( ExprContext *const *  ctxs, int  num_ctxs, TupleRow *  row  )

staticinherited

Evaluate ExprContexts over row. Returns true if all exprs return true. TODO: This doesn't use the vector<Expr*> signature because I haven't figured out how to deal with declaring a templated std:vector type in IR

Definition at line 393 of file exec-node.cc.

References impala::ExprContext::GetBooleanVal(), impala_udf::AnyVal::is_null, and impala_udf::BooleanVal::val.

Referenced by impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), impala::HdfsScanner::EvalConjuncts(), EvalOtherJoinConjuncts(), EvalOtherJoinConjuncts2(), impala::PartitionedHashJoinNode::EvaluateNullProbe(), GetNext(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::PartitionedHashJoinNode::OutputNullAwareProbeRows(), impala::PartitionedHashJoinNode::OutputUnmatchedBuild(), impala::CrossJoinNode::ProcessLeftChildBatch(), impala::HashJoinNode::ProcessProbeBatch(), and impala::PartitionedHashJoinNode::ProcessProbeBatch().

Status impala::ExecNode::ExecDebugAction ( TExecNodePhase::type  phase, RuntimeState *  state  )

protectedinherited

Executes debug_action_ if phase matches debug_phase_. 'phase' must not be INVALID.

Definition at line 378 of file exec-node.cc.

References impala::Status::CANCELLED, impala::ExecNode::debug_action_, impala::ExecNode::debug_phase_, impala::RuntimeState::is_cancelled(), and impala::Status::OK.

Referenced by impala::SelectNode::GetNext(), impala::SortNode::GetNext(), impala::UnionNode::GetNext(), GetNext(), impala::TopNNode::GetNext(), impala::ExchangeNode::GetNext(), impala::CrossJoinNode::GetNext(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::AnalyticEvalNode::GetNext(), impala::PartitionedHashJoinNode::GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::HdfsScanNode::GetNextInternal(), impala::ExecNode::Open(), and impala::ExecNode::Prepare().

MemTracker* impala::ExecNode::expr_mem_tracker ( )

inlineinherited

Definition at line 163 of file exec-node.h.

References impala::ExecNode::expr_mem_tracker_.

Referenced by impala::PartitionedAggregationNode::Partition::InitStreams(), impala::SortNode::Prepare(), impala::UnionNode::Prepare(), impala::TopNNode::Prepare(), impala::ExchangeNode::Prepare(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::ExecNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), and impala::HdfsScanNode::Prepare().

Status HBaseScanNode::GetNext ( RuntimeState *  state, RowBatch *  row_batch, bool *  eos  )

virtual

Fill the next row batch by calling Next() on the hbase_scanner_, converting text data in HBase cells to binary data.

Implements impala::ExecNode.

Definition at line 147 of file hbase-scan-node.cc.

References impala::RuntimeState::abort_on_error(), impala::MemPool::AcquireData(), impala::RowBatch::AddRow(), impala::RowBatch::AtCapacity(), impala::TupleDescriptor::byte_size(), impala::RowBatch::CommitLastRow(), impala::ExecNode::conjunct_ctxs_, COUNTER_ADD, COUNTER_SET, impala::Tuple::Create(), impala::RuntimeState::ErrorLog(), impala::ExecNode::EvalConjuncts(), impala::ExecNode::ExecDebugAction(), getJNIEnv(), impala::RowBatch::GetRow(), hbase_scanner_, impala::Tuple::Init(), impala::RuntimeState::LogError(), impala::RuntimeState::LogHasSpace(), impala::ScanNode::materialize_tuple_timer(), impala::RowBatch::MaxTupleBufferSize(), num_errors_, impala::ExecNode::num_rows_returned_, impala::Status::OK, impala::ExecNode::QueryMaintenance(), impala::ExecNode::ReachedLimit(), impala::RuntimeState::ReportFileErrors(), RETURN_IF_CANCELLED, RETURN_IF_ERROR, row_key_binary_encoded_, row_key_slot_, impala::ScanNode::rows_read_counter_, impala::ExecNode::rows_returned_counter_, impala::ExecNode::runtime_profile_, scan_range_vector_, impala::ScanNode::scanner_thread_counters(), SCOPED_THREAD_COUNTER_MEASUREMENT, SCOPED_TIMER, impala::Tuple::SetNull(), impala::TupleRow::SetTuple(), sorted_cols_, sorted_non_key_slots_, impala::TupleDescriptor::table_desc(), impala::HBaseTableDescriptor::table_name(), table_name_, tuple_, tuple_buffer_size_, impala::RowBatch::tuple_data_pool(), tuple_desc_, tuple_idx_, tuple_pool_, and WriteTextSlot().

int impala::ExecNode::GetNodeIdFromProfile ( RuntimeProfile *  p )

staticinherited

Extract node id from p->name().

Definition at line 62 of file exec-node.cc.

References impala::RuntimeProfile::metadata().

int impala::ExecNode::id ( ) const

inlineinherited

Definition at line 154 of file exec-node.h.

References impala::ExecNode::id_.

Referenced by impala::AnalyticEvalNode::AddResultTuple(), impala::AnalyticEvalNode::AddRow(), impala::AnalyticEvalNode::AnalyticEvalNode(), impala::AnalyticEvalNode::GetNext(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::HdfsScanner::InitializeWriteTuplesFn(), impala::HdfsAvroScanner::InitNewRange(), impala::AnalyticEvalNode::InitNextPartition(), impala::PartitionedAggregationNode::MoveHashPartitions(), impala::PartitionedHashJoinNode::NodeDebugString(), impala::AnalyticEvalNode::Open(), impala::HdfsScanNode::Open(), impala::PlanFragmentExecutor::Prepare(), impala::PartitionedHashJoinNode::ProcessBuildInput(), impala::AnalyticEvalNode::ProcessChildBatch(), impala::HdfsScanNode::ScannerThread(), impala::AnalyticEvalNode::TryAddRemainingResults(), impala::AnalyticEvalNode::TryAddResultTupleForCurrRow(), impala::AnalyticEvalNode::TryAddResultTupleForPrevRow(), and impala::AnalyticEvalNode::TryRemoveRowsBeforeWindow().

Status impala::ExecNode::Init ( const TPlanNode &  tnode )

virtualinherited

Initializes this object from the thrift tnode desc. The subclass should do any initialization that can fail in Init() rather than the ctor. If overridden in subclass, must first call superclass's Init().

Reimplemented in impala::PartitionedAggregationNode, impala::AnalyticEvalNode, impala::PartitionedHashJoinNode, impala::AggregationNode, impala::HashJoinNode, impala::BlockingJoinNode, impala::ExchangeNode, impala::TopNNode, impala::SortNode, and impala::UnionNode.

Definition at line 124 of file exec-node.cc.

References impala::ExecNode::conjunct_ctxs_, impala::Expr::CreateExprTrees(), impala::Status::OK, impala::ExecNode::pool_, and RETURN_IF_ERROR.

Referenced by impala::SortNode::Init(), impala::UnionNode::Init(), impala::TopNNode::Init(), impala::ExchangeNode::Init(), impala::BlockingJoinNode::Init(), impala::AggregationNode::Init(), impala::AnalyticEvalNode::Init(), and impala::PartitionedAggregationNode::Init().

void impala::ExecNode::InitRuntimeProfile ( const std::string &  name )

protectedinherited

Definition at line 371 of file exec-node.cc.

References impala::ExecNode::id_, impala::ExecNode::pool_, and impala::ExecNode::runtime_profile_.

Referenced by impala::ExecNode::ExecNode().

bool impala::ExecNode::is_closed ( )

inlineprotectedinherited

Definition at line 242 of file exec-node.h.

References impala::ExecNode::is_closed_.

Referenced by impala::SelectNode::Close(), impala::SortNode::Close(), impala::UnionNode::Close(), impala::TopNNode::Close(), impala::ExchangeNode::Close(), Close(), impala::CrossJoinNode::Close(), impala::HashJoinNode::Close(), impala::AggregationNode::Close(), impala::BlockingJoinNode::Close(), impala::AnalyticEvalNode::Close(), impala::PartitionedHashJoinNode::Close(), impala::PartitionedAggregationNode::Close(), impala::HdfsScanNode::Close(), impala::PartitionedAggregationNode::Partition::Close(), impala::PartitionedHashJoinNode::Partition::Close(), impala::PartitionedHashJoinNode::ReserveTupleStreamBlocks(), impala::PartitionedHashJoinNode::SpillPartition(), impala::PartitionedAggregationNode::SpillPartition(), and impala::PartitionedHashJoinNode::Partition::~Partition().

virtual bool impala::ScanNode::IsScanNode ( ) const

inlinevirtualinherited

Reimplemented from impala::ExecNode.

Definition at line 93 of file scan-node.h.

int64_t impala::ExecNode::limit ( ) const

inlineinherited

Definition at line 158 of file exec-node.h.

References impala::ExecNode::limit_.

Referenced by impala::CrossJoinNode::GetNext(), and impala::HashJoinNode::LeftJoinGetNext().

RuntimeProfile::Counter* impala::ScanNode::materialize_tuple_timer ( ) const

inlineinherited

Definition at line 104 of file scan-node.h.

References impala::ScanNode::materialize_tuple_timer_.

Referenced by GetNext(), impala::HdfsSequenceScanner::ProcessDecompressedBlock(), impala::HdfsTextScanner::ProcessRange(), impala::HdfsAvroScanner::ProcessRange(), impala::HdfsSequenceScanner::ProcessRange(), impala::HdfsRCFileScanner::ProcessRange(), and impala::HdfsTextScanner::WriteFields().

MemTracker* impala::ExecNode::mem_tracker ( )

inlineinherited

Definition at line 162 of file exec-node.h.

References impala::ExecNode::mem_tracker_.

Referenced by impala::ExecNode::Close(), impala::CrossJoinNode::ConstructBuildSide(), impala::HashJoinNode::ConstructBuildSide(), impala::HdfsScanNode::EnoughMemoryForScannerThread(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::SortNode::Open(), impala::TopNNode::Open(), impala::AggregationNode::Open(), impala::AnalyticEvalNode::Open(), impala::PartitionedAggregationNode::Open(), impala::HdfsScanNode::Open(), impala::UnionNode::OpenCurrentChild(), impala::SelectNode::Prepare(), Prepare(), impala::TopNNode::Prepare(), impala::BlockingJoinNode::Prepare(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::HdfsScanNode::Prepare(), impala::PartitionedHashJoinNode::PrepareNextPartition(), impala::PartitionedHashJoinNode::ProcessBuildInput(), impala::PartitionedAggregationNode::ProcessStream(), impala::HdfsRCFileScanner::ReadRowGroup(), impala::HdfsAvroScanner::ResolveSchemas(), impala::SortNode::SortInput(), and impala::HdfsScanner::StartNewRowBatch().

Status HBaseScanNode::Open ( RuntimeState *  state )

virtual

Start HBase scan using hbase_scanner_.

Reimplemented from impala::ExecNode.

Definition at line 117 of file hbase-scan-node.cc.

References filters_, getJNIEnv(), hbase_scanner_, impala::Status::OK, impala::ExecNode::Open(), impala::ExecNode::QueryMaintenance(), RETURN_IF_CANCELLED, RETURN_IF_ERROR, impala::ExecNode::runtime_profile_, scan_range_vector_, SCOPED_TIMER, and tuple_desc_.

RuntimeProfile::Counter* impala::ScanNode::per_read_thread_throughput_counter ( ) const

inlineinherited

Definition at line 101 of file scan-node.h.

References impala::ScanNode::per_read_thread_throughput_counter_.

Status HBaseScanNode::Prepare ( RuntimeState *  state )

virtual

Prepare conjuncts, create HBase columns to slots mapping, initialize hbase_scanner_, and create text_converter_.

Reimplemented from impala::ScanNode.

Definition at line 56 of file hbase-scan-node.cc.

References ADD_TIMER, impala::SlotDescriptor::ColPathLessThan(), impala::RuntimeState::desc_tbl(), impala::DescriptorTbl::GetTupleDescriptor(), hbase_scanner_, impala::RuntimeState::htable_factory(), impala::ExecNode::mem_tracker(), impala::Status::OK, impala::ScanNode::Prepare(), impala::ScanNode::read_timer_, RETURN_IF_ERROR, ROW_KEY, row_key_binary_encoded_, row_key_slot_, impala::ExecNode::runtime_profile(), impala::ScanNode::scan_range_params_, scan_range_vector_, impala::HBaseTableScanner::ScanRange::set_start_key(), impala::HBaseTableScanner::ScanRange::set_stop_key(), impala::TupleDescriptor::slots(), sorted_cols_, sorted_non_key_slots_, impala::TupleDescriptor::table_desc(), impala::ScanNode::TOTAL_HBASE_READ_TIMER, tuple_desc_, tuple_id_, tuple_idx_, and tuple_pool_.

Status impala::ExecNode::QueryMaintenance ( RuntimeState *  state )

protectedvirtualinherited

Frees any local allocations made by expr_ctxs_to_free_ and returns the result of state->CheckQueryState(). Nodes should call this periodically, e.g. once per input row batch. This should not be called outside the main execution thread. Nodes may override this to add extra periodic cleanup, e.g. freeing other local allocations. ExecNodes overriding this function should return ExecNode::QueryMaintenance().

Reimplemented in impala::PartitionedAggregationNode, and impala::AnalyticEvalNode.

Definition at line 401 of file exec-node.cc.

References impala::RuntimeState::CheckQueryState(), impala::ExecNode::expr_ctxs_to_free_, and impala::ExprContext::FreeLocalAllocations().

Referenced by impala::CrossJoinNode::ConstructBuildSide(), impala::HashJoinNode::ConstructBuildSide(), impala::SelectNode::GetNext(), impala::SortNode::GetNext(), impala::UnionNode::GetNext(), GetNext(), impala::TopNNode::GetNext(), impala::ExchangeNode::GetNext(), impala::CrossJoinNode::GetNext(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::PartitionedHashJoinNode::GetNext(), impala::HdfsScanNode::GetNextInternal(), Open(), impala::SortNode::Open(), impala::TopNNode::Open(), impala::BlockingJoinNode::Open(), impala::AggregationNode::Open(), impala::PartitionedHashJoinNode::ProcessBuildInput(), impala::AnalyticEvalNode::QueryMaintenance(), impala::PartitionedAggregationNode::QueryMaintenance(), and impala::SortNode::SortInput().

bool impala::ExecNode::ReachedLimit ( )

inlineinherited

Definition at line 159 of file exec-node.h.

References impala::ExecNode::limit_, and impala::ExecNode::num_rows_returned_.

Referenced by impala::HdfsParquetScanner::AssembleRows(), impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), impala::HdfsTextScanner::FinishScanRange(), impala::SelectNode::GetNext(), impala::UnionNode::GetNext(), impala::SortNode::GetNext(), GetNext(), impala::ExchangeNode::GetNext(), impala::CrossJoinNode::GetNext(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::PartitionedHashJoinNode::GetNext(), impala::AnalyticEvalNode::GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::HdfsScanNode::GetNextInternal(), impala::ExchangeNode::GetNextMerging(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::HashJoinNode::LeftJoinGetNext(), impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange(), impala::HdfsTextScanner::ProcessRange(), impala::HdfsAvroScanner::ProcessRange(), impala::HdfsSequenceScanner::ProcessRange(), impala::HdfsRCFileScanner::ProcessRange(), and impala::PlanFragmentExecutor::ReachedLimit().

RuntimeProfile::Counter* impala::ScanNode::read_timer ( ) const

inlineinherited

Definition at line 97 of file scan-node.h.

References impala::ScanNode::read_timer_.

Referenced by impala::HBaseTableScanner::Next(), and impala::HdfsScanNode::Open().

Status HBaseScanNode::Reset ( RuntimeState *  state )

virtual

NYI.

Reimplemented from impala::ExecNode.

Definition at line 272 of file hbase-scan-node.cc.

const RowDescriptor& impala::ExecNode::row_desc ( ) const

inlineinherited

Definition at line 156 of file exec-node.h.

References impala::ExecNode::row_descriptor_.

Referenced by impala::CrossJoinNode::BuildListDebugString(), impala::HashJoinNode::CodegenCreateOutputRow(), impala::PartitionedHashJoinNode::CodegenCreateOutputRow(), impala::CrossJoinNode::ConstructBuildSide(), impala::HashJoinNode::ConstructBuildSide(), impala::BlockingJoinNode::GetLeftChildRowString(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::PartitionedAggregationNode::Partition::InitStreams(), impala::TopNNode::Open(), impala::AggregationNode::Open(), impala::AnalyticEvalNode::Open(), impala::PartitionedAggregationNode::Open(), impala::UnionNode::OpenCurrentChild(), impala::SelectNode::Prepare(), impala::SortNode::Prepare(), impala::UnionNode::Prepare(), impala::TopNNode::Prepare(), impala::BlockingJoinNode::Prepare(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::ExecNode::Prepare(), impala::PlanFragmentExecutor::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::HdfsScanNode::Prepare(), impala::PartitionedHashJoinNode::ProcessBuildInput(), impala::PartitionedAggregationNode::ProcessStream(), impala::PlanFragmentExecutor::row_desc(), impala::SortNode::SortInput(), and impala::HdfsScanner::StartNewRowBatch().

RuntimeProfile::Counter* impala::ScanNode::rows_read_counter ( ) const

inlineinherited

Definition at line 96 of file scan-node.h.

References impala::ScanNode::rows_read_counter_.

Referenced by impala::HdfsParquetScanner::AssembleRows(), impala::HdfsTextScanner::FinishScanRange(), impala::HdfsSequenceScanner::ProcessDecompressedBlock(), impala::HdfsParquetScanner::ProcessFooter(), impala::HdfsTextScanner::ProcessRange(), impala::HdfsAvroScanner::ProcessRange(), impala::HdfsSequenceScanner::ProcessRange(), and impala::HdfsRCFileScanner::ProcessRange().

int64_t impala::ExecNode::rows_returned ( ) const

inlineinherited

Definition at line 157 of file exec-node.h.

References impala::ExecNode::num_rows_returned_.

Referenced by impala::CrossJoinNode::GetNext(), impala::AnalyticEvalNode::GetNext(), impala::HashJoinNode::LeftJoinGetNext(), impala::PartitionedAggregationNode::Open(), impala::HdfsSequenceScanner::ProcessDecompressedBlock(), impala::CrossJoinNode::ProcessLeftChildBatch(), impala::HashJoinNode::ProcessProbeBatch(), and impala::HdfsTextScanner::WriteFields().

RuntimeProfile* impala::ExecNode::runtime_profile ( )

inlineinherited

Definition at line 161 of file exec-node.h.

References impala::ExecNode::runtime_profile_.

Referenced by impala::ExecNode::AddRuntimeExecOption(), impala::BlockingJoinNode::BuildSideThread(), impala::ExecNode::CreateTreeHelper(), impala::PartitionedAggregationNode::Partition::InitStreams(), impala::SortNode::Open(), impala::AnalyticEvalNode::Open(), impala::PartitionedAggregationNode::Open(), impala::HdfsScanNode::Open(), impala::HdfsTextScanner::Prepare(), Prepare(), impala::BaseSequenceScanner::Prepare(), impala::ExchangeNode::Prepare(), impala::HdfsParquetScanner::Prepare(), impala::BlockingJoinNode::Prepare(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::ExecNode::Prepare(), impala::ScanNode::Prepare(), impala::PlanFragmentExecutor::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::HdfsScanner::Prepare(), impala::HdfsScanNode::Prepare(), and impala::PartitionedHashJoinNode::ProcessBuildInput().

RuntimeProfile::Counter* impala::ScanNode::scan_ranges_complete_counter ( ) const

inlineinherited

Definition at line 107 of file scan-node.h.

References impala::ScanNode::scan_ranges_complete_counter_.

RuntimeProfile::ThreadCounters* impala::ScanNode::scanner_thread_counters ( ) const

inlineinherited

Definition at line 110 of file scan-node.h.

References impala::ScanNode::scanner_thread_counters_.

Referenced by GetNext(), and impala::HdfsScanNode::ScannerThread().

void impala::ExecNode::SetDebugOptions ( int  node_id, TExecNodePhase::type  phase, TDebugAction::type  action, ExecNode *  tree  )

staticinherited

Set debug action for node with given id in 'tree'.

Definition at line 332 of file exec-node.cc.

References impala::ExecNode::children_, impala::ExecNode::debug_action_, impala::ExecNode::debug_phase_, and impala::ExecNode::id_.

Referenced by impala::PlanFragmentExecutor::Prepare().

void impala::ScanNode::SetScanRanges ( const std::vector< TScanRangeParams > &  scan_range_params )

inlineinherited

This should be called before Prepare(), and the argument must be not destroyed until after Prepare().

Definition at line 89 of file scan-node.h.

References impala::ScanNode::scan_range_params_.

Referenced by impala::PlanFragmentExecutor::Prepare().

const int impala::HBaseScanNode::suggested_max_caching ( ) const

inline

Definition at line 52 of file hbase-scan-node.h.

References suggested_max_caching_.

Referenced by impala::HBaseTableScanner::HBaseTableScanner().

RuntimeProfile::Counter* impala::ScanNode::total_throughput_counter ( ) const

inlineinherited

Definition at line 98 of file scan-node.h.

References impala::ScanNode::total_throughput_counter_.

Referenced by Close(), and impala::HdfsScanNode::StopAndFinalizeCounters().

TPlanNodeType::type impala::ExecNode::type ( ) const

inlineinherited

Definition at line 155 of file exec-node.h.

References impala::ExecNode::type_.

Referenced by impala::ExecNode::CodegenEvalConjuncts(), impala::PartitionedAggregationNode::CodegenUpdateTuple(), and impala::PlanFragmentExecutor::Prepare().

void HBaseScanNode::WriteTextSlot ( const std::string &  family, const std::string &  qualifier, void *  value, int  value_length, SlotDescriptor *  slot, RuntimeState *  state, bool *  error_in_row  )

private

Writes a slot in tuple_ from an HBase value containing text data. The HBase value is converted into the appropriate target type.

Definition at line 129 of file hbase-scan-node.cc.

References impala::RuntimeState::LogError(), impala::RuntimeState::LogHasSpace(), text_converter_, tuple_, tuple_pool_, and impala::SlotDescriptor::type().

Referenced by GetNext().

Member Data Documentation

RuntimeProfile::Counter impala::ScanNode::active_hdfs_read_thread_counter_

protectedinherited

The number of active hdfs reading threads reading for this node.

Definition at line 164 of file scan-node.h.

Referenced by impala::HdfsScanNode::Close(), and impala::HdfsScanNode::Open().

RuntimeProfile::Counter impala::ScanNode::active_scanner_thread_counter_

protectedinherited

The number of active scanner threads that are not blocked by IO.

Definition at line 157 of file scan-node.h.

Referenced by impala::ScanNode::active_scanner_thread_counter(), impala::HdfsScanNode::Close(), impala::HdfsScanNode::EnoughMemoryForScannerThread(), impala::HdfsScanNode::Open(), impala::HdfsScanNode::ScannerThread(), and impala::HdfsScanNode::ThreadTokenAvailableCb().

const string impala::ScanNode::AVERAGE_HDFS_READ_THREAD_CONCURRENCY

staticinherited

Initial value:

=
    "AverageHdfsReadThreadConcurrency"

Definition at line 133 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open().

RuntimeProfile::Counter* impala::ScanNode::average_hdfs_read_thread_concurrency_

protectedinherited

Average number of active hdfs reading threads This should be created in Open and stopped when all the scanner threads are done.

Definition at line 168 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open(), and impala::HdfsScanNode::StopAndFinalizeCounters().

const string impala::ScanNode::AVERAGE_SCANNER_THREAD_CONCURRENCY

staticinherited

Initial value:

=
    "AverageScannerThreadConcurrency"

Definition at line 132 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open().

RuntimeProfile::Counter* impala::ScanNode::average_scanner_thread_concurrency_

protectedinherited

Average number of active scanner threads This should be created in Open and stopped when all the scanner threads are done.

Definition at line 161 of file scan-node.h.

Referenced by impala::ScanNode::average_scanner_thread_concurrency(), impala::HdfsScanNode::Open(), and impala::HdfsScanNode::StopAndFinalizeCounters().

const string impala::ScanNode::BYTES_READ_COUNTER = "BytesRead"

staticinherited

names of ScanNode common counters

Definition at line 121 of file scan-node.h.

Referenced by impala::ScanNode::Prepare().

RuntimeProfile::Counter* impala::ScanNode::bytes_read_counter_

protectedinherited

Definition at line 140 of file scan-node.h.

Referenced by impala::ScanNode::bytes_read_counter(), impala::HdfsScanNode::Open(), and impala::ScanNode::Prepare().

RuntimeProfile::TimeSeriesCounter* impala::ScanNode::bytes_read_timeseries_counter_

protectedinherited

Time series of the bytes_read_counter_.

Definition at line 142 of file scan-node.h.

Referenced by Close(), impala::ScanNode::Prepare(), and impala::HdfsScanNode::StopAndFinalizeCounters().

std::vector<ExecNode*> impala::ExecNode::children_

protectedinherited

Definition at line 214 of file exec-node.h.

Referenced by impala::ExecNode::child(), impala::ExecNode::Close(), impala::ExecNode::CollectNodes(), impala::ExecNode::CreateTreeHelper(), DebugString(), impala::UnionNode::GetNext(), impala::UnionNode::Open(), impala::AggregationNode::Open(), impala::PartitionedAggregationNode::Open(), impala::UnionNode::OpenCurrentChild(), impala::ExecNode::Prepare(), impala::PartitionedAggregationNode::ProcessStream(), impala::ExecNode::Reset(), and impala::ExecNode::SetDebugOptions().

std::vector<ExprContext*> impala::ExecNode::conjunct_ctxs_

protectedinherited

Definition at line 212 of file exec-node.h.

Referenced by impala::ExecNode::Close(), impala::HashJoinNode::CodegenProcessProbeBatch(), impala::PartitionedHashJoinNode::CodegenProcessProbeBatch(), impala::ExecNode::conjunct_ctxs(), impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), GetNext(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::TopNNode::Init(), impala::ExecNode::Init(), impala::PartitionedHashJoinNode::Init(), impala::ExecNode::Open(), impala::PartitionedHashJoinNode::OutputUnmatchedBuild(), impala::ExecNode::Prepare(), impala::CrossJoinNode::ProcessLeftChildBatch(), impala::HashJoinNode::ProcessProbeBatch(), and impala::PartitionedHashJoinNode::ProcessProbeBatch().

TDebugAction::type impala::ExecNode::debug_action_

protectedinherited

Definition at line 220 of file exec-node.h.

Referenced by impala::ExecNode::ExecDebugAction(), and impala::ExecNode::SetDebugOptions().

TExecNodePhase::type impala::ExecNode::debug_phase_

protectedinherited

debug-only: if debug_action_ is not INVALID, node will perform action in debug_phase_

Definition at line 219 of file exec-node.h.

Referenced by impala::ExecNode::ExecDebugAction(), and impala::ExecNode::SetDebugOptions().

boost::mutex impala::ExecNode::exec_options_lock_

protectedinherited

Execution options that are determined at runtime. This is added to the runtime profile at Close(). Examples for options logged here would be "Codegen Enabled"

Definition at line 238 of file exec-node.h.

Referenced by impala::ExecNode::AddRuntimeExecOption().

boost::scoped_ptr<MemTracker> impala::ExecNode::expr_mem_tracker_

protectedinherited

MemTracker that should be used for ExprContexts.

Definition at line 233 of file exec-node.h.

Referenced by impala::ExecNode::expr_mem_tracker(), and impala::ExecNode::Prepare().

std::vector<THBaseFilter> impala::HBaseScanNode::filters_

private

HBase Filters to be set in HBaseTableScanner.

Definition at line 82 of file hbase-scan-node.h.

Referenced by Open().

boost::scoped_ptr<HBaseTableScanner> impala::HBaseScanNode::hbase_scanner_

private

Jni helper for scanning an HBase table.

Definition at line 91 of file hbase-scan-node.h.

Referenced by Close(), GetNext(), Open(), and Prepare().

std::vector<RuntimeProfile::Counter*> impala::ScanNode::hdfs_read_thread_concurrency_bucket_

protectedinherited

HDFS read thread concurrency bucket: bucket[i] refers to the number of sample taken where there are i concurrent hdfs read thread running

Definition at line 174 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open(), and impala::HdfsScanNode::StopAndFinalizeCounters().

int impala::ExecNode::id_

protectedinherited

Definition at line 209 of file exec-node.h.

Referenced by impala::PartitionedAggregationNode::CreateHashPartitions(), impala::ExecNode::id(), impala::ExecNode::InitRuntimeProfile(), impala::PartitionedAggregationNode::NextPartition(), impala::ExchangeNode::Prepare(), impala::PartitionedHashJoinNode::PrepareNextPartition(), impala::PartitionedHashJoinNode::ProcessBuildInput(), and impala::ExecNode::SetDebugOptions().

int64_t impala::ExecNode::limit_

protectedinherited

Definition at line 222 of file exec-node.h.

Referenced by impala::SortNode::GetNext(), impala::HdfsScanNode::GetNextInternal(), impala::ExchangeNode::GetNextMerging(), impala::TopNNode::InsertTupleRow(), impala::HdfsScanNode::limit(), impala::ExecNode::limit(), impala::TopNNode::Open(), and impala::ExecNode::ReachedLimit().

const string impala::ScanNode::MATERIALIZE_TUPLE_TIMER = "MaterializeTupleTime(*)"

staticinherited

Definition at line 128 of file scan-node.h.

Referenced by impala::ScanNode::Prepare().

RuntimeProfile::Counter* impala::ScanNode::materialize_tuple_timer_

protectedinherited

Definition at line 151 of file scan-node.h.

Referenced by impala::ScanNode::materialize_tuple_timer(), and impala::ScanNode::Prepare().

boost::scoped_ptr<MemTracker> impala::ExecNode::mem_tracker_

protectedinherited

Account for peak memory used by this node.

Definition at line 230 of file exec-node.h.

Referenced by impala::ExecNode::mem_tracker(), and impala::ExecNode::Prepare().

const string impala::ScanNode::NUM_DISKS_ACCESSED_COUNTER = "NumDisksAccessed"

staticinherited

Definition at line 127 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open().

RuntimeProfile::Counter* impala::ScanNode::num_disks_accessed_counter_

protectedinherited

Definition at line 150 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open(), and impala::HdfsScanNode::StopAndFinalizeCounters().

int impala::HBaseScanNode::num_errors_

private

Counts the total number of conversion errors for this table.

Definition at line 85 of file hbase-scan-node.h.

Referenced by Close(), and GetNext().

int64_t impala::ExecNode::num_rows_returned_

protectedinherited

Definition at line 223 of file exec-node.h.

Referenced by impala::ExecNode::Close(), impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), impala::SortNode::GetNext(), GetNext(), impala::TopNNode::GetNext(), impala::ExchangeNode::GetNext(), impala::CrossJoinNode::GetNext(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::PartitionedHashJoinNode::GetNext(), impala::AnalyticEvalNode::GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::HdfsScanNode::GetNextInternal(), impala::ExchangeNode::GetNextMerging(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::HashJoinNode::LeftJoinGetNext(), impala::PartitionedHashJoinNode::OutputUnmatchedBuild(), impala::ExecNode::ReachedLimit(), and impala::ExecNode::rows_returned().

const string impala::ScanNode::NUM_SCANNER_THREADS_STARTED

staticinherited

Initial value:

=
    "NumScannerThreadsStarted"

Definition at line 134 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open().

RuntimeProfile::Counter* impala::ScanNode::num_scanner_threads_started_counter_

protectedinherited

Definition at line 170 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open(), and impala::HdfsScanNode::ThreadTokenAvailableCb().

const string impala::ScanNode::PER_READ_THREAD_THROUGHPUT_COUNTER

staticinherited

Initial value:

=
    "PerReadThreadRawHdfsThroughput"

Definition at line 126 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open().

RuntimeProfile::Counter* impala::ScanNode::per_read_thread_throughput_counter_

protectedinherited

Per thread read throughput [bytes/sec].

Definition at line 149 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open(), and impala::ScanNode::per_read_thread_throughput_counter().

ObjectPool* impala::ExecNode::pool_

protectedinherited

Definition at line 211 of file exec-node.h.

Referenced by impala::SortNode::Init(), impala::UnionNode::Init(), impala::TopNNode::Init(), impala::ExchangeNode::Init(), impala::HashJoinNode::Init(), impala::AggregationNode::Init(), impala::ExecNode::Init(), impala::PartitionedHashJoinNode::Init(), impala::AnalyticEvalNode::Init(), impala::PartitionedAggregationNode::Init(), impala::ExecNode::InitRuntimeProfile(), impala::HdfsScanNode::Open(), impala::PartitionedHashJoinNode::Prepare(), and impala::PartitionedHashJoinNode::ProcessBuildInput().

RuntimeProfile::Counter* impala::ScanNode::read_timer_

protectedinherited

Definition at line 145 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open(), Prepare(), and impala::ScanNode::read_timer().

RowDescriptor impala::ExecNode::row_descriptor_

protectedinherited

Definition at line 215 of file exec-node.h.

Referenced by impala::SortNode::Open(), impala::SortNode::Prepare(), impala::TopNNode::Prepare(), impala::ExchangeNode::Prepare(), and impala::ExecNode::row_desc().

const int impala::HBaseScanNode::ROW_KEY = 0

staticprivate

Column 0 in the Impala metadata refers to the HBasw row key.

Definition at line 61 of file hbase-scan-node.h.

Referenced by Prepare().

bool impala::HBaseScanNode::row_key_binary_encoded_

private

True, if row key is binary encoded.

Definition at line 107 of file hbase-scan-node.h.

Referenced by GetNext(), and Prepare().

SlotDescriptor* impala::HBaseScanNode::row_key_slot_

private

Slot into which the HBase row key is written. NULL if row key is not requested.

Definition at line 104 of file hbase-scan-node.h.

Referenced by GetNext(), and Prepare().

const string impala::ExecNode::ROW_THROUGHPUT_COUNTER = "RowsReturnedRate"

staticinherited

Names of counters shared by all exec nodes.

Definition at line 169 of file exec-node.h.

Referenced by impala::ExecNode::Prepare().

const string impala::ScanNode::ROWS_READ_COUNTER = "RowsRead"

staticinherited

Definition at line 122 of file scan-node.h.

Referenced by impala::ScanNode::Prepare().

RuntimeProfile::Counter* impala::ScanNode::rows_read_counter_

protectedinherited

rows/tuples read from the scanner (including those discarded by EvalConjucts())

Definition at line 144 of file scan-node.h.

Referenced by GetNext(), impala::ScanNode::Prepare(), and impala::ScanNode::rows_read_counter().

RuntimeProfile::Counter* impala::ExecNode::rows_returned_counter_

protectedinherited

Definition at line 226 of file exec-node.h.

Referenced by impala::ExecNode::Close(), impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), impala::SortNode::GetNext(), GetNext(), impala::TopNNode::GetNext(), impala::ExchangeNode::GetNext(), impala::CrossJoinNode::GetNext(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::AnalyticEvalNode::GetNext(), impala::PartitionedHashJoinNode::GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::HdfsScanNode::GetNextInternal(), impala::ExchangeNode::GetNextMerging(), impala::HashJoinNode::LeftJoinGetNext(), impala::PartitionedHashJoinNode::OutputUnmatchedBuild(), and impala::ExecNode::Prepare().

RuntimeProfile::Counter* impala::ExecNode::rows_returned_rate_

protectedinherited

Definition at line 227 of file exec-node.h.

Referenced by impala::ExecNode::Prepare().

std::string impala::ExecNode::runtime_exec_options_

protectedinherited

Definition at line 239 of file exec-node.h.

Referenced by impala::ExecNode::AddRuntimeExecOption().

boost::scoped_ptr<RuntimeProfile> impala::ExecNode::runtime_profile_

protectedinherited

Definition at line 225 of file exec-node.h.

Referenced by Close(), impala::SelectNode::GetNext(), impala::UnionNode::GetNext(), impala::SortNode::GetNext(), GetNext(), impala::TopNNode::GetNext(), impala::ExchangeNode::GetNext(), impala::CrossJoinNode::GetNext(), impala::HashJoinNode::GetNext(), impala::AggregationNode::GetNext(), impala::AnalyticEvalNode::GetNext(), impala::PartitionedHashJoinNode::GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::HdfsScanNode::GetNext(), impala::ExecNode::InitRuntimeProfile(), impala::SelectNode::Open(), Open(), impala::UnionNode::Open(), impala::SortNode::Open(), impala::TopNNode::Open(), impala::ExchangeNode::Open(), impala::DataSourceScanNode::Open(), impala::BlockingJoinNode::Open(), impala::AggregationNode::Open(), impala::AnalyticEvalNode::Open(), impala::PartitionedAggregationNode::Open(), impala::SelectNode::Prepare(), impala::UnionNode::Prepare(), impala::SortNode::Prepare(), impala::TopNNode::Prepare(), impala::BlockingJoinNode::Prepare(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::ExecNode::Prepare(), impala::ScanNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::HdfsScanNode::Prepare(), impala::ExecNode::runtime_profile(), and impala::HdfsScanNode::StopAndFinalizeCounters().

const std::vector<TScanRangeParams>* impala::ScanNode::scan_range_params_

protectedinherited

The scan ranges this scan node is responsible for. Not owned.

Definition at line 138 of file scan-node.h.

Referenced by Prepare(), impala::HdfsScanNode::Prepare(), and impala::ScanNode::SetScanRanges().

HBaseTableScanner::ScanRangeVector impala::HBaseScanNode::scan_range_vector_

private

scan ranges of a region server

Definition at line 79 of file hbase-scan-node.h.

Referenced by DebugString(), GetNext(), Open(), and Prepare().

const string impala::ScanNode::SCAN_RANGES_COMPLETE_COUNTER = "ScanRangesComplete"

staticinherited

Definition at line 129 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open().

RuntimeProfile::Counter* impala::ScanNode::scan_ranges_complete_counter_

protectedinherited

Definition at line 152 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open(), and impala::ScanNode::scan_ranges_complete_counter().

RuntimeProfile::ThreadCounters* impala::ScanNode::scanner_thread_counters_

protectedinherited

Aggregated scanner thread counters.

Definition at line 154 of file scan-node.h.

Referenced by impala::ScanNode::Prepare(), and impala::ScanNode::scanner_thread_counters().

const string impala::ScanNode::SCANNER_THREAD_COUNTERS_PREFIX = "ScannerThreads"

staticinherited

Definition at line 130 of file scan-node.h.

Referenced by impala::ScanNode::Prepare().

const string impala::ScanNode::SCANNER_THREAD_TOTAL_WALLCLOCK_TIME

staticinherited

Initial value:

=
    "ScannerThreadsTotalWallClockTime"

Definition at line 131 of file scan-node.h.

Referenced by impala::HdfsTextScanner::Prepare(), and impala::ScanNode::Prepare().

const int impala::HBaseScanNode::SKIP_COLUMN = -1

staticprivate

Definition at line 59 of file hbase-scan-node.h.

std::vector<const HBaseTableDescriptor::HBaseColumnDescriptor* > impala::HBaseScanNode::sorted_cols_

private

List of pointers to family/qualifier/binary encoding in same sort order as sorted_non_key_slots_. The memory pointed to by the list-elements is owned by the corresponding HBaseTableDescriptor.

Definition at line 100 of file hbase-scan-node.h.

Referenced by GetNext(), and Prepare().

std::vector<SlotDescriptor*> impala::HBaseScanNode::sorted_non_key_slots_

private

List of non-row-key slots sorted by col_pos(). Populated in Prepare().

Definition at line 94 of file hbase-scan-node.h.

Referenced by GetNext(), and Prepare().

int impala::HBaseScanNode::suggested_max_caching_

private

Max value for "setCaching" suggested by FE. If no value was suggested by the FE, this will be 0.

Definition at line 120 of file hbase-scan-node.h.

Referenced by HBaseScanNode(), and suggested_max_caching().

const std::string impala::HBaseScanNode::table_name_

private

Name of HBase table (not necessarily the table name mapped to Hive).

Definition at line 67 of file hbase-scan-node.h.

Referenced by Close(), DebugString(), and GetNext().

boost::scoped_ptr<TextConverter> impala::HBaseScanNode::text_converter_

private

Helper class for converting text to other types;.

Definition at line 116 of file hbase-scan-node.h.

Referenced by WriteTextSlot().

const string impala::ScanNode::TOTAL_HBASE_READ_TIMER = "TotalRawHBaseReadTime(*)"

staticinherited

Definition at line 124 of file scan-node.h.

Referenced by Prepare().

const string impala::ScanNode::TOTAL_HDFS_READ_TIMER = "TotalRawHdfsReadTime(*)"

staticinherited

Definition at line 123 of file scan-node.h.

Referenced by impala::HdfsScanNode::Open().

const string impala::ScanNode::TOTAL_THROUGHPUT_COUNTER = "TotalReadThroughput"

staticinherited

Definition at line 125 of file scan-node.h.

Referenced by impala::ScanNode::Prepare().

RuntimeProfile::Counter* impala::ScanNode::total_throughput_counter_

protectedinherited

Wall based aggregate read throughput [bytes/sec].

Definition at line 147 of file scan-node.h.

Referenced by impala::ScanNode::Prepare(), and impala::ScanNode::total_throughput_counter().

Tuple* impala::HBaseScanNode::tuple_

private

Current tuple.

Definition at line 113 of file hbase-scan-node.h.

Referenced by GetNext(), and WriteTextSlot().

int impala::HBaseScanNode::tuple_buffer_size_

private

Size of tuple buffer determined by size of tuples and capacity of row batches.

Definition at line 110 of file hbase-scan-node.h.

Referenced by GetNext().

const TupleDescriptor* impala::HBaseScanNode::tuple_desc_

private

Descriptor of tuples read from HBase table.

Definition at line 73 of file hbase-scan-node.h.

Referenced by GetNext(), Open(), and Prepare().

TupleId impala::HBaseScanNode::tuple_id_

private

Tuple id resolved in Prepare() to set tuple_desc_;.

Definition at line 70 of file hbase-scan-node.h.

Referenced by DebugString(), and Prepare().

int impala::HBaseScanNode::tuple_idx_

private

Tuple index in tuple row.

Definition at line 76 of file hbase-scan-node.h.

Referenced by GetNext(), and Prepare().

boost::scoped_ptr<MemPool> impala::HBaseScanNode::tuple_pool_

private

Pool for allocating tuple data, including all varying-length slots.

Definition at line 88 of file hbase-scan-node.h.

Referenced by Close(), GetNext(), Prepare(), and WriteTextSlot().

TPlanNodeType::type impala::ExecNode::type_

protectedinherited

Definition at line 210 of file exec-node.h.

Referenced by impala::ExecNode::CollectNodes(), and impala::ExecNode::type().

---

The documentation for this class was generated from the following files:

• be/src/exec/hbase-scan-node.h
• be/src/exec/hbase-scan-node.cc