#include <aggregation-node.h>

Inheritance diagram for impala::AggregationNode:

Collaboration diagram for impala::AggregationNode:

Public Member Functions
	AggregationNode (ObjectPool *pool, const TPlanNode &tnode, const DescriptorTbl &descs)

virtual Status	Init (const TPlanNode &tnode)

virtual Status	Prepare (RuntimeState *state)

virtual Status	Open (RuntimeState *state)

virtual Status	GetNext (RuntimeState state, RowBatch row_batch, bool *eos)

virtual Status	Reset (RuntimeState *state)

virtual void	Close (RuntimeState *state)

void	CollectNodes (TPlanNodeType::type node_type, std::vector< ExecNode * > *nodes)

void	CollectScanNodes (std::vector< ExecNode * > *nodes)
	Collect all scan node types. More...

std::string	DebugString () const
	Returns a string representation in DFS order of the plan rooted at this. More...

const std::vector< ExprContext * > &	conjunct_ctxs () const

int	id () const

TPlanNodeType::type	type () const

const RowDescriptor &	row_desc () const

int64_t	rows_returned () const

int64_t	limit () const

bool	ReachedLimit ()

RuntimeProfile *	runtime_profile ()

MemTracker *	mem_tracker ()

MemTracker *	expr_mem_tracker ()

Static Public Member Functions
static Status	CreateTree (ObjectPool pool, const TPlan &plan, const DescriptorTbl &descs, ExecNode *root)

static void	SetDebugOptions (int node_id, TExecNodePhase::type phase, TDebugAction::type action, ExecNode *tree)
	Set debug action for node with given id in 'tree'. More...

static bool	EvalConjuncts (ExprContext const ctxs, int num_ctxs, TupleRow *row)

static llvm::Function *	CodegenEvalConjuncts (RuntimeState state, const std::vector< ExprContext > &conjunct_ctxs, const char *name="EvalConjuncts")

static int	GetNodeIdFromProfile (RuntimeProfile *p)
	Extract node id from p->name(). More...

Static Public Attributes
static const char *	LLVM_CLASS_NAME = "class.impala::AggregationNode"

static const std::string	ROW_THROUGHPUT_COUNTER = "RowsReturnedRate"
	Names of counters shared by all exec nodes. More...

Protected Member Functions
virtual void	DebugString (int indentation_level, std::stringstream *out) const

ExecNode *	child (int i)

bool	is_closed ()

virtual bool	IsScanNode () const

void	InitRuntimeProfile (const std::string &name)

Status	ExecDebugAction (TExecNodePhase::type phase, RuntimeState *state)

void	AddRuntimeExecOption (const std::string &option)
	Appends option to 'runtime_exec_options_'. More...

virtual Status	QueryMaintenance (RuntimeState *state)

void	AddExprCtxToFree (ExprContext *ctx)

void	AddExprCtxsToFree (const std::vector< ExprContext * > &ctxs)

void	AddExprCtxsToFree (const SortExecExprs &sort_exec_exprs)

Static Protected Member Functions
static Status	CreateNode (ObjectPool pool, const TPlanNode &tnode, const DescriptorTbl &descs, ExecNode *node)
	Create a single exec node derived from thrift node; place exec node in 'pool'. More...

static Status	CreateTreeHelper (ObjectPool pool, const std::vector< TPlanNode > &tnodes, const DescriptorTbl &descs, ExecNode parent, int node_idx, ExecNode *root)

Protected Attributes
int	id_

TPlanNodeType::type	type_

ObjectPool *	pool_

std::vector< ExprContext * >	conjunct_ctxs_

std::vector< ExecNode * >	children_

RowDescriptor	row_descriptor_

TExecNodePhase::type	debug_phase_

TDebugAction::type	debug_action_

int64_t	limit_

int64_t	num_rows_returned_

boost::scoped_ptr< RuntimeProfile >	runtime_profile_

RuntimeProfile::Counter *	rows_returned_counter_

RuntimeProfile::Counter *	rows_returned_rate_

boost::scoped_ptr< MemTracker >	mem_tracker_
	Account for peak memory used by this node. More...

boost::scoped_ptr< MemTracker >	expr_mem_tracker_
	MemTracker that should be used for ExprContexts. More...

boost::mutex	exec_options_lock_

std::string	runtime_exec_options_

Private Types
typedef void(*	ProcessRowBatchFn )(AggregationNode , RowBatch )

Private Member Functions
Tuple *	ConstructIntermediateTuple ()

void	UpdateTuple (Tuple tuple, TupleRow row)

Tuple *	FinalizeTuple (Tuple tuple, MemPool pool)

void	ProcessRowBatchNoGrouping (RowBatch *batch)
	Do the aggregation for all tuple rows in the batch. More...

void	ProcessRowBatchWithGrouping (RowBatch *batch)

llvm::Function *	CodegenProcessRowBatch (RuntimeState state, llvm::Function update_tuple_fn)

llvm::Function *	CodegenUpdateSlot (RuntimeState state, AggFnEvaluator evaluator, SlotDescriptor *slot_desc)

llvm::Function *	CodegenUpdateTuple (RuntimeState *state)
	Codegen UpdateTuple(). Returns NULL if codegen is unsuccessful. More...

Private Attributes
boost::scoped_ptr< OldHashTable >	hash_tbl_

OldHashTable::Iterator	output_iterator_

std::vector< AggFnEvaluator * >	aggregate_evaluators_

std::vector < impala_udf::FunctionContext * >	agg_fn_ctxs_
	FunctionContext for each agg fn and backing pool. More...

boost::scoped_ptr< MemPool >	agg_fn_pool_

std::vector< ExprContext * >	probe_expr_ctxs_
	Exprs used to evaluate input rows. More...

std::vector< ExprContext * >	build_expr_ctxs_

TupleId	intermediate_tuple_id_
	Tuple into which Update()/Merge()/Serialize() results are stored. More...

TupleDescriptor *	intermediate_tuple_desc_

TupleId	output_tuple_id_

TupleDescriptor *	output_tuple_desc_

Tuple *	singleton_intermediate_tuple_

boost::scoped_ptr< MemPool >	tuple_pool_

llvm::Function *	codegen_process_row_batch_fn_
	IR for process row batch. NULL if codegen is disabled. More...

ProcessRowBatchFn	process_row_batch_fn_
	Jitted ProcessRowBatch function pointer. Null if codegen is disabled. More...

bool	needs_finalize_

RuntimeProfile::Counter *	build_timer_
	Time spent processing the child rows. More...

RuntimeProfile::Counter *	get_results_timer_
	Time spent returning the aggregated rows. More...

RuntimeProfile::Counter *	hash_table_buckets_counter_
	Num buckets in hash table. More...

RuntimeProfile::Counter *	hash_table_load_factor_counter_
	Load factor in hash table. More...

Detailed Description

Node for in-memory hash aggregation. The node creates a hash set of aggregation intermediate tuples, which contain slots for all grouping and aggregation exprs (the grouping slots precede the aggregation expr slots in the output tuple descriptor). TODO: codegen cross-compiled UDAs and get rid of handcrafted IR. TODO: investigate high compile time for wide tables

Definition at line 50 of file aggregation-node.h.

Member Typedef Documentation

typedef void(* impala::AggregationNode::ProcessRowBatchFn)(AggregationNode *, RowBatch *)

private

Definition at line 100 of file aggregation-node.h.

Constructor & Destructor Documentation

impala::AggregationNode::AggregationNode	(	ObjectPool *	pool,
		const TPlanNode &	tnode,
		const DescriptorTbl &	descs
	)

Definition at line 56 of file aggregation-node.cc.

Member Function Documentation

void impala::ExecNode::AddExprCtxsToFree ( const std::vector< ExprContext * > & ctxs )

protectedinherited

Referenced by impala::ExecNode::AddExprCtxsToFree(), impala::UnionNode::Prepare(), impala::SortNode::Prepare(), impala::TopNNode::Prepare(), impala::ExchangeNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::ExecNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), and impala::HdfsScanNode::Prepare().

void impala::ExecNode::AddExprCtxsToFree ( const SortExecExprs & sort_exec_exprs )

protectedinherited

Definition at line 410 of file exec-node.cc.

References impala::ExecNode::AddExprCtxsToFree(), impala::SortExecExprs::lhs_ordering_expr_ctxs(), impala::SortExecExprs::rhs_ordering_expr_ctxs(), and impala::SortExecExprs::sort_tuple_slot_expr_ctxs().

void impala::ExecNode::AddExprCtxToFree ( ExprContext * ctx )

inlineprotectedinherited

Add an ExprContext to have its local allocations freed by QueryMaintenance(). Exprs that are evaluated in the main execution thread should be added. Exprs evaluated in a separate thread are generally not safe to add, since a local allocation may be freed while it's being used. Rather than using this mechanism, threads should call FreeLocalAllocations() on local ExprContexts periodically.

Definition at line 276 of file exec-node.h.

References impala::ExecNode::expr_ctxs_to_free_.

Referenced by impala::AnalyticEvalNode::Prepare().

void impala::ExecNode::AddRuntimeExecOption ( const std::string & option )

protectedinherited

Appends option to 'runtime_exec_options_'.

Definition at line 188 of file exec-node.cc.

References impala::RuntimeProfile::AddInfoString(), impala::ExecNode::exec_options_lock_, impala::ExecNode::runtime_exec_options_, and impala::ExecNode::runtime_profile().

Referenced by impala::PartitionedHashJoinNode::AttachProbeFilters(), impala::HashJoinNode::ConstructBuildSide(), impala::BlockingJoinNode::Open(), impala::HashJoinNode::Prepare(), Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::HdfsScanNode::Prepare(), and impala::HdfsScanNode::StopAndFinalizeCounters().

ExecNode* impala::ExecNode::child ( int i )

inlineprotectedinherited

Definition at line 241 of file exec-node.h.

References impala::ExecNode::children_.

void impala::AggregationNode::Close ( RuntimeState * state )

virtual

Close() will get called for every exec node, regardless of what else is called and the status of these calls (i.e. Prepare() may never have been called, or Prepare()/Open()/GetNext() returned with an error). Close() releases all resources that were allocated in Open()/GetNext(), even if the latter ended with an error. Close() can be called if the node has been prepared or the node is closed. The default implementation updates runtime profile counters and calls Close() on the children. Subclasses should check if the node has already been closed (is_closed()), then close themselves, then call the base Close(). Nodes that are using tuples returned by a child may call Close() on their children before their own Close() if the child node has returned eos. It is only safe to call Close() on the child node while the parent node is still returning rows if the parent node fully materializes the child's input.

Reimplemented from impala::ExecNode.

Definition at line 278 of file aggregation-node.cc.

References agg_fn_ctxs_, agg_fn_pool_, aggregate_evaluators_, impala::OldHashTable::Iterator::AtEnd(), build_expr_ctxs_, impala::TupleDescriptor::byte_size(), impala::ExecNode::Close(), impala::Expr::Close(), impala::Tuple::Create(), impala::AggFnEvaluator::Finalize(), impala::OldHashTable::Iterator::GetTuple(), hash_tbl_, impala::ExecNode::is_closed(), needs_finalize_, impala::OldHashTable::Iterator::Next(), output_iterator_, output_tuple_desc_, probe_expr_ctxs_, impala::AggFnEvaluator::Serialize(), and tuple_pool_.

Function * impala::ExecNode::CodegenEvalConjuncts	(	RuntimeState *	state,
		const std::vector< ExprContext * > &	conjunct_ctxs,
		const char *	name = `"EvalConjuncts"`
	)

staticinherited

Returns a codegen'd version of EvalConjuncts(), or NULL if the function couldn't be codegen'd. The codegen'd version uses inlined, codegen'd GetBooleanVal() functions.

Definition at line 452 of file exec-node.cc.

References impala::LlvmCodeGen::FnPrototype::AddArgument(), impala::LlvmCodeGen::context(), impala::CodegenAnyVal::CreateCallWrapped(), impala::LlvmCodeGen::false_value(), impala::LlvmCodeGen::FinalizeFunction(), impala::RuntimeState::GetCodegen(), impala::Status::GetDetail(), impala::CodegenAnyVal::GetIsNull(), impala::LlvmCodeGen::GetType(), impala::CodegenAnyVal::GetVal(), impala::TupleRow::LLVM_CLASS_NAME, impala::ExprContext::LLVM_CLASS_NAME, impala::Status::ok(), impala::LlvmCodeGen::true_value(), impala::ExecNode::type(), impala::TYPE_BOOLEAN, impala::TYPE_INT, and VLOG_QUERY.

Referenced by impala::HdfsAvroScanner::CodegenDecodeAvroData(), impala::HashJoinNode::CodegenProcessProbeBatch(), and impala::PartitionedHashJoinNode::CodegenProcessProbeBatch().

Function * impala::AggregationNode::CodegenProcessRowBatch	(	RuntimeState *	state,
		llvm::Function *	update_tuple_fn
	)

private

Codegen the process row batch loop. The loop has already been compiled to IR and loaded into the codegen object. UpdateAggTuple has also been codegen'd to IR. This function will modify the loop subsituting the UpdateAggTuple function call with the (inlined) codegen'd 'update_tuple_fn'.

Definition at line 782 of file aggregation-node.cc.

References impala::LlvmCodeGen::codegen_timer(), impala::RuntimeState::GetCodegen(), impala::LlvmCodeGen::GetFunction(), hash_tbl_, impala::Status::ok(), impala::LlvmCodeGen::OptimizeFunctionWithExprs(), probe_expr_ctxs_, impala::LlvmCodeGen::ReplaceCallSites(), and SCOPED_TIMER.

Referenced by Prepare().

llvm::Function * impala::AggregationNode::CodegenUpdateSlot	(	RuntimeState *	state,
		AggFnEvaluator *	evaluator,
		SlotDescriptor *	slot_desc
	)

private

Codegen for updating aggregate_exprs at slot_idx. Returns NULL if unsuccessful. slot_idx is the idx into aggregate_exprs_ (does not include grouping exprs).

Definition at line 506 of file aggregation-node.cc.

References impala::LlvmCodeGen::FnPrototype::AddArgument(), impala::AggFnEvaluator::agg_op(), impala::SlotDescriptor::CodegenUpdateNull(), impala::AggFnEvaluator::COUNT, impala::CodegenAnyVal::CreateCallWrapped(), impala::SlotDescriptor::field_idx(), impala::TupleDescriptor::GenerateLlvmStruct(), impala::RuntimeState::GetCodegen(), impala::Expr::GetCodegendComputeFn(), impala::Status::GetDetail(), impala::GetHllUpdateFunction2(), impala::CodegenAnyVal::GetIsNull(), impala::CodegenAnyVal::GetNonNullVal(), impala::CodegenAnyVal::GetUnloweredType(), impala::CodegenAnyVal::GetVal(), impala::AggFnEvaluator::input_expr_ctxs(), intermediate_tuple_desc_, impala::SlotDescriptor::is_materialized(), impala::AggFnEvaluator::is_merge(), impala::SlotDescriptor::is_nullable(), impala::TupleRow::LLVM_CLASS_NAME, impala::ExprContext::LLVM_CLASS_NAME, impala::FunctionContextImpl::LLVM_FUNCTIONCONTEXT_NAME, impala::AggFnEvaluator::MAX, impala::AggFnEvaluator::MIN, impala::AggFnEvaluator::NDV, impala::Status::ok(), impala::CodegenAnyVal::SetFromRawValue(), impala::AggFnEvaluator::SUM, impala::CodegenAnyVal::ToNativeValue(), impala::ColumnType::type, impala::SlotDescriptor::type(), impala::Expr::type(), impala::TYPE_BIGINT, impala::TYPE_DOUBLE, impala::TYPE_FLOAT, impala::TYPE_STRING, impala::TYPE_TIMESTAMP, impala::CodegenAnyVal::value(), and VLOG_QUERY.

Referenced by CodegenUpdateTuple().

Function * impala::AggregationNode::CodegenUpdateTuple ( RuntimeState * state )

private

Codegen UpdateTuple(). Returns NULL if codegen is unsuccessful.

Definition at line 678 of file aggregation-node.cc.

Referenced by Prepare().

void impala::ExecNode::CollectNodes	(	TPlanNodeType::type	node_type,
		std::vector< ExecNode * > *	nodes
	)

inherited

Collect all nodes of given 'node_type' that are part of this subtree, and return in 'nodes'.

Definition at line 359 of file exec-node.cc.

References impala::ExecNode::children_, and impala::ExecNode::type_.

Referenced by impala::ExecNode::CollectScanNodes(), and impala::PlanFragmentExecutor::Prepare().

void impala::ExecNode::CollectScanNodes ( std::vector< ExecNode * > * nodes )

inherited

Collect all scan node types.

Definition at line 366 of file exec-node.cc.

References impala::ExecNode::CollectNodes().

Referenced by impala::PlanFragmentExecutor::Prepare().

const std::vector<ExprContext*>& impala::ExecNode::conjunct_ctxs ( ) const

inlineinherited

Definition at line 152 of file exec-node.h.

References impala::ExecNode::conjunct_ctxs_.

Referenced by impala::HdfsScanNode::ComputeSlotMaterializationOrder(), impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), impala::HashJoinNode::GetNext(), impala::PartitionedHashJoinNode::OutputUnmatchedBuild(), impala::HashJoinNode::ProcessProbeBatch(), and impala::PartitionedHashJoinNode::ProcessProbeBatch().

Tuple * impala::AggregationNode::ConstructIntermediateTuple ( )

private

Constructs a new aggregation intermediate tuple (allocated from tuple_pool_), initialized to grouping values computed over 'current_row_'. Aggregation expr slots are set to their initial values.

Definition at line 314 of file aggregation-node.cc.

References agg_fn_ctxs_, impala::AggFnEvaluator::agg_op(), aggregate_evaluators_, impala::TupleDescriptor::byte_size(), impala::Tuple::Create(), impala::Tuple::GetSlot(), hash_tbl_, impala::AggFnEvaluator::Init(), intermediate_tuple_desc_, impala::AggFnEvaluator::MAX, impala::AggFnEvaluator::MIN, probe_expr_ctxs_, impala::Tuple::SetNull(), impala::ExprValue::SetToMax(), impala::ExprValue::SetToMin(), impala::TupleDescriptor::slots(), tuple_pool_, impala::TYPE_CHAR, impala::TYPE_DECIMAL, impala::TYPE_STRING, impala::TYPE_TIMESTAMP, impala::TYPE_VARCHAR, and impala::RawValue::Write().

Referenced by Prepare(), and ProcessRowBatchWithGrouping().

Status impala::ExecNode::CreateNode	(	ObjectPool *	pool,
		const TPlanNode &	tnode,
		const DescriptorTbl &	descs,
		ExecNode **	node
	)

staticprotectedinherited

Create a single exec node derived from thrift node; place exec node in 'pool'.

Definition at line 260 of file exec-node.cc.

References impala::ObjectPool::Add(), impala::Status::OK, and RETURN_IF_ERROR.

Referenced by impala::ExecNode::CreateTreeHelper().

Status impala::ExecNode::CreateTree	(	ObjectPool *	pool,
		const TPlan &	plan,
		const DescriptorTbl &	descs,
		ExecNode **	root
	)

staticinherited

Creates exec node tree from list of nodes contained in plan via depth-first traversal. All nodes are placed in pool. Returns error if 'plan' is corrupted, otherwise success.

Definition at line 199 of file exec-node.cc.

References impala::ExecNode::CreateTreeHelper(), impala::Status::OK, and impala::Status::ok().

Referenced by impala::PlanFragmentExecutor::Prepare().

Status impala::ExecNode::CreateTreeHelper	(	ObjectPool *	pool,
		const std::vector< TPlanNode > &	tnodes,
		const DescriptorTbl &	descs,
		ExecNode *	parent,
		int *	node_idx,
		ExecNode **	root
	)

staticprotectedinherited

Definition at line 218 of file exec-node.cc.

References impala::RuntimeProfile::AddChild(), impala::ExecNode::children_, impala::ExecNode::CreateNode(), impala::Status::OK, RETURN_IF_ERROR, and impala::ExecNode::runtime_profile().

Referenced by impala::ExecNode::CreateTree().

void impala::AggregationNode::DebugString	(	int	indentation_level,
		std::stringstream *	out
	)		const

protectedvirtual

Recursive helper method for generating a string for DebugString(). Implementations should call DebugString(int, std::stringstream) on their children. Input parameters: indentation_level: Current level in plan tree. Output parameters: out: Stream to accumulate debug string.

Reimplemented from impala::ExecNode.

Definition at line 401 of file aggregation-node.cc.

References aggregate_evaluators_, impala::AggFnEvaluator::DebugString(), impala::ExecNode::DebugString(), impala::Expr::DebugString(), intermediate_tuple_id_, needs_finalize_, output_tuple_id_, and probe_expr_ctxs_.

string impala::ExecNode::DebugString ( ) const

inherited

Returns a string representation in DFS order of the plan rooted at this.

Definition at line 345 of file exec-node.cc.

Referenced by impala::SortNode::DebugString(), impala::TopNNode::DebugString(), impala::ExchangeNode::DebugString(), DebugString(), impala::AnalyticEvalNode::DebugString(), impala::PartitionedAggregationNode::DebugString(), impala::BlockingJoinNode::DebugString(), and impala::PlanFragmentExecutor::Prepare().

bool impala::ExecNode::EvalConjuncts	(	ExprContext const	ctxs,
		int	num_ctxs,
		TupleRow *	row
	)

staticinherited

Evaluate ExprContexts over row. Returns true if all exprs return true. TODO: This doesn't use the vector<Expr*> signature because I haven't figured out how to deal with declaring a templated std:vector type in IR

Definition at line 393 of file exec-node.cc.

References impala::ExprContext::GetBooleanVal(), impala_udf::AnyVal::is_null, and impala_udf::BooleanVal::val.

Referenced by impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), impala::HdfsScanner::EvalConjuncts(), EvalOtherJoinConjuncts(), EvalOtherJoinConjuncts2(), impala::PartitionedHashJoinNode::EvaluateNullProbe(), impala::HBaseScanNode::GetNext(), impala::HashJoinNode::GetNext(), GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::PartitionedHashJoinNode::OutputNullAwareProbeRows(), impala::PartitionedHashJoinNode::OutputUnmatchedBuild(), impala::CrossJoinNode::ProcessLeftChildBatch(), impala::HashJoinNode::ProcessProbeBatch(), and impala::PartitionedHashJoinNode::ProcessProbeBatch().

Status impala::ExecNode::ExecDebugAction	(	TExecNodePhase::type	phase,
		RuntimeState *	state
	)

protectedinherited

Executes debug_action_ if phase matches debug_phase_. 'phase' must not be INVALID.

Definition at line 378 of file exec-node.cc.

References impala::Status::CANCELLED, impala::ExecNode::debug_action_, impala::ExecNode::debug_phase_, impala::RuntimeState::is_cancelled(), and impala::Status::OK.

MemTracker* impala::ExecNode::expr_mem_tracker ( )

inlineinherited

Definition at line 163 of file exec-node.h.

References impala::ExecNode::expr_mem_tracker_.

Referenced by impala::PartitionedAggregationNode::Partition::InitStreams(), impala::SortNode::Prepare(), impala::UnionNode::Prepare(), impala::TopNNode::Prepare(), impala::ExchangeNode::Prepare(), impala::HashJoinNode::Prepare(), Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::ExecNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), and impala::HdfsScanNode::Prepare().

Tuple * impala::AggregationNode::FinalizeTuple	(	Tuple *	tuple,
		MemPool *	pool
	)

private

Called on the intermediate tuple of each group after all input rows have been consumed and aggregated. Computes the final aggregate values to be returned in GetNext() using the agg fn evaluators' Serialize() or Finalize(). For the Finalize() case if the output tuple is different from the intermediate tuple, then a new tuple is allocated from 'pool' to hold the final result. Returns the tuple holding the final aggregate values.

Definition at line 374 of file aggregation-node.cc.

References agg_fn_ctxs_, aggregate_evaluators_, impala::TupleDescriptor::byte_size(), impala::Tuple::Create(), impala::AggFnEvaluator::Finalize(), impala::Tuple::GetSlot(), intermediate_tuple_desc_, intermediate_tuple_id_, impala::Tuple::IsNull(), needs_finalize_, impala::SlotDescriptor::null_indicator_offset(), output_tuple_desc_, output_tuple_id_, pool, probe_expr_ctxs_, impala::AggFnEvaluator::Serialize(), impala::TupleDescriptor::slots(), impala::SlotDescriptor::tuple_offset(), and impala::RawValue::Write().

Referenced by GetNext().

Status impala::AggregationNode::GetNext	(	RuntimeState *	state,
		RowBatch *	row_batch,
		bool *	eos
	)

virtual

Retrieves rows and returns them via row_batch. Sets eos to true if subsequent calls will not retrieve any more rows. Data referenced by any tuples returned in row_batch must not be overwritten by the callee until Close() is called. The memory holding that data can be returned via row_batch's tuple_data_pool (in which case it may be deleted by the caller) or held on to by the callee. The row_batch, including its tuple_data_pool, will be destroyed by the caller at some point prior to the final Close() call. In other words, if the memory holding the tuple data will be referenced by the callee in subsequent GetNext() calls, it must not be attached to the row_batch's tuple_data_pool. Caller must not be holding any io buffers. This will cause deadlock. TODO: AggregationNode and HashJoinNode cannot be "re-opened" yet.

Implements impala::ExecNode.

Definition at line 230 of file aggregation-node.cc.

Referenced by Open().

int impala::ExecNode::GetNodeIdFromProfile ( RuntimeProfile * p )

staticinherited

Extract node id from p->name().

Definition at line 62 of file exec-node.cc.

References impala::RuntimeProfile::metadata().

int impala::ExecNode::id ( ) const

inlineinherited

Definition at line 154 of file exec-node.h.

References impala::ExecNode::id_.

Status impala::AggregationNode::Init ( const TPlanNode & tnode )

virtual

Initializes this object from the thrift tnode desc. The subclass should do any initialization that can fail in Init() rather than the ctor. If overridden in subclass, must first call superclass's Init().

Reimplemented from impala::ExecNode.

Definition at line 72 of file aggregation-node.cc.

References aggregate_evaluators_, impala::AggFnEvaluator::Create(), impala::Expr::CreateExprTrees(), impala::ExecNode::Init(), impala::Status::OK, impala::ExecNode::pool_, probe_expr_ctxs_, and RETURN_IF_ERROR.

void impala::ExecNode::InitRuntimeProfile ( const std::string & name )

protectedinherited

Definition at line 371 of file exec-node.cc.

References impala::ExecNode::id_, impala::ExecNode::pool_, and impala::ExecNode::runtime_profile_.

Referenced by impala::ExecNode::ExecNode().

bool impala::ExecNode::is_closed ( )

inlineprotectedinherited

Definition at line 242 of file exec-node.h.

References impala::ExecNode::is_closed_.

virtual bool impala::ExecNode::IsScanNode ( ) const

inlineprotectedvirtualinherited

Reimplemented in impala::ScanNode.

Definition at line 251 of file exec-node.h.

int64_t impala::ExecNode::limit ( ) const

inlineinherited

Definition at line 158 of file exec-node.h.

References impala::ExecNode::limit_.

Referenced by impala::CrossJoinNode::GetNext(), and impala::HashJoinNode::LeftJoinGetNext().

MemTracker* impala::ExecNode::mem_tracker ( )

inlineinherited

Definition at line 162 of file exec-node.h.

References impala::ExecNode::mem_tracker_.

Status impala::AggregationNode::Open ( RuntimeState * state )

virtual

Performs any preparatory work prior to calling GetNext(). Caller must not be holding any io buffers. This will cause deadlock. If overridden in subclass, must first call superclass's Open(). If a parent exec node adds slot filters (see RuntimeState::AddBitmapFilter()), they need to be added before calling Open() on the child that will consume them.

Reimplemented from impala::ExecNode.

Definition at line 174 of file aggregation-node.cc.

Status impala::AggregationNode::Prepare ( RuntimeState * state )

virtual

Sets up internal structures, etc., without doing any actual work. Must be called prior to Open(). Will only be called once in this node's lifetime. All code generation (adding functions to the LlvmCodeGen object) must happen in Prepare(). Retrieving the jit compiled function pointer must happen in Open(). If overridden in subclass, must first call superclass's Prepare().

Reimplemented from impala::ExecNode.

Definition at line 85 of file aggregation-node.cc.

References impala::ObjectPool::Add(), ADD_COUNTER, ADD_TIMER, impala::LlvmCodeGen::AddFunctionToJit(), impala::ExecNode::AddRuntimeExecOption(), agg_fn_ctxs_, agg_fn_pool_, aggregate_evaluators_, build_expr_ctxs_, build_timer_, impala::ExecNode::child(), impala::RuntimeState::codegen_enabled(), codegen_process_row_batch_fn_, CodegenProcessRowBatch(), CodegenUpdateTuple(), ConstructIntermediateTuple(), impala::RuntimeState::desc_tbl(), impala::ExecNode::expr_mem_tracker(), get_results_timer_, impala::RuntimeState::GetCodegen(), impala::DescriptorTbl::GetTupleDescriptor(), hash_table_buckets_counter_, hash_table_load_factor_counter_, hash_tbl_, intermediate_tuple_desc_, intermediate_tuple_id_, impala::ExecNode::mem_tracker(), impala::RuntimeState::obj_pool(), impala::Status::OK, output_iterator_, output_tuple_desc_, output_tuple_id_, impala::ExecNode::Prepare(), impala::Expr::Prepare(), probe_expr_ctxs_, process_row_batch_fn_, RETURN_IF_ERROR, impala::ExecNode::row_desc(), impala::ExecNode::runtime_profile(), impala::ExecNode::runtime_profile_, SCOPED_TIMER, singleton_intermediate_tuple_, impala::TupleDescriptor::slots(), tuple_pool_, impala::ColumnType::type, impala::SlotDescriptor::type(), impala::TYPE_BOOLEAN, and impala::TYPE_NULL.

void AggregationNode::ProcessRowBatchNoGrouping ( RowBatch * batch )

private

Do the aggregation for all tuple rows in the batch.

Definition at line 28 of file aggregation-node-ir.cc.

References impala::RowBatch::GetRow(), impala::RowBatch::num_rows(), singleton_intermediate_tuple_, and UpdateTuple().

Referenced by Open().

void AggregationNode::ProcessRowBatchWithGrouping ( RowBatch * batch )

private

Definition at line 34 of file aggregation-node-ir.cc.

References impala::OldHashTable::Iterator::AtEnd(), ConstructIntermediateTuple(), impala::RowBatch::GetRow(), impala::OldHashTable::Iterator::GetTuple(), hash_tbl_, impala::RowBatch::num_rows(), and UpdateTuple().

Referenced by Open().

Status impala::ExecNode::QueryMaintenance ( RuntimeState * state )

protectedvirtualinherited

Frees any local allocations made by expr_ctxs_to_free_ and returns the result of state->CheckQueryState(). Nodes should call this periodically, e.g. once per input row batch. This should not be called outside the main execution thread. Nodes may override this to add extra periodic cleanup, e.g. freeing other local allocations. ExecNodes overriding this function should return ExecNode::QueryMaintenance().

Reimplemented in impala::PartitionedAggregationNode, and impala::AnalyticEvalNode.

Definition at line 401 of file exec-node.cc.

References impala::RuntimeState::CheckQueryState(), impala::ExecNode::expr_ctxs_to_free_, and impala::ExprContext::FreeLocalAllocations().

Referenced by impala::CrossJoinNode::ConstructBuildSide(), impala::HashJoinNode::ConstructBuildSide(), impala::SelectNode::GetNext(), impala::SortNode::GetNext(), impala::UnionNode::GetNext(), impala::HBaseScanNode::GetNext(), impala::TopNNode::GetNext(), impala::ExchangeNode::GetNext(), impala::CrossJoinNode::GetNext(), impala::HashJoinNode::GetNext(), GetNext(), impala::PartitionedHashJoinNode::GetNext(), impala::HdfsScanNode::GetNextInternal(), impala::HBaseScanNode::Open(), impala::SortNode::Open(), impala::TopNNode::Open(), impala::BlockingJoinNode::Open(), Open(), impala::PartitionedHashJoinNode::ProcessBuildInput(), impala::AnalyticEvalNode::QueryMaintenance(), impala::PartitionedAggregationNode::QueryMaintenance(), and impala::SortNode::SortInput().

bool impala::ExecNode::ReachedLimit ( )

inlineinherited

Definition at line 159 of file exec-node.h.

References impala::ExecNode::limit_, and impala::ExecNode::num_rows_returned_.

Status impala::AggregationNode::Reset ( RuntimeState * state )

virtual

Resets all data-specific state, returning this node to the state it was in after calling Prepare() and before calling Open(). Prepare() must have already been called before calling Reset(). Open() and GetNext() may have optionally been called. Close() must not have been called. If overridden in a subclass, must call superclass's Reset() at the end. The default implementation calls Reset() on children. Note that this function may be called many times, so should be fast. For example, accumulated memory does not need to be freed on every call if it's expensive.

Reimplemented from impala::ExecNode.

Definition at line 273 of file aggregation-node.cc.

const RowDescriptor& impala::ExecNode::row_desc ( ) const

inlineinherited

Definition at line 156 of file exec-node.h.

References impala::ExecNode::row_descriptor_.

int64_t impala::ExecNode::rows_returned ( ) const

inlineinherited

Definition at line 157 of file exec-node.h.

References impala::ExecNode::num_rows_returned_.

Referenced by impala::CrossJoinNode::GetNext(), impala::AnalyticEvalNode::GetNext(), impala::HashJoinNode::LeftJoinGetNext(), impala::PartitionedAggregationNode::Open(), impala::HdfsSequenceScanner::ProcessDecompressedBlock(), impala::CrossJoinNode::ProcessLeftChildBatch(), impala::HashJoinNode::ProcessProbeBatch(), and impala::HdfsTextScanner::WriteFields().

RuntimeProfile* impala::ExecNode::runtime_profile ( )

inlineinherited

Definition at line 161 of file exec-node.h.

References impala::ExecNode::runtime_profile_.

void impala::ExecNode::SetDebugOptions	(	int	node_id,
		TExecNodePhase::type	phase,
		TDebugAction::type	action,
		ExecNode *	tree
	)

staticinherited

Set debug action for node with given id in 'tree'.

Definition at line 332 of file exec-node.cc.

References impala::ExecNode::children_, impala::ExecNode::debug_action_, impala::ExecNode::debug_phase_, and impala::ExecNode::id_.

Referenced by impala::PlanFragmentExecutor::Prepare().

TPlanNodeType::type impala::ExecNode::type ( ) const

inlineinherited

Definition at line 155 of file exec-node.h.

References impala::ExecNode::type_.

Referenced by impala::ExecNode::CodegenEvalConjuncts(), impala::PartitionedAggregationNode::CodegenUpdateTuple(), and impala::PlanFragmentExecutor::Prepare().

void impala::AggregationNode::UpdateTuple	(	Tuple *	tuple,
		TupleRow *	row
	)

private

Updates the aggregation intermediate tuple 'tuple' with aggregation values computed over 'row'.

Definition at line 369 of file aggregation-node.cc.

References impala::AggFnEvaluator::Add(), agg_fn_ctxs_, and aggregate_evaluators_.

Referenced by ProcessRowBatchNoGrouping(), and ProcessRowBatchWithGrouping().

Member Data Documentation

std::vector<impala_udf::FunctionContext*> impala::AggregationNode::agg_fn_ctxs_

private

FunctionContext for each agg fn and backing pool.

Definition at line 73 of file aggregation-node.h.

Referenced by Close(), CodegenUpdateTuple(), ConstructIntermediateTuple(), FinalizeTuple(), Open(), Prepare(), and UpdateTuple().

boost::scoped_ptr<MemPool> impala::AggregationNode::agg_fn_pool_

private

Definition at line 74 of file aggregation-node.h.

Referenced by Close(), and Prepare().

std::vector<AggFnEvaluator*> impala::AggregationNode::aggregate_evaluators_

private

Definition at line 70 of file aggregation-node.h.

Referenced by Close(), CodegenUpdateTuple(), ConstructIntermediateTuple(), DebugString(), FinalizeTuple(), Init(), Open(), Prepare(), and UpdateTuple().

std::vector<ExprContext*> impala::AggregationNode::build_expr_ctxs_

private

Exprs used to insert constructed aggregation tuple into the hash table. All the exprs are simply SlotRefs for the intermediate tuple.

Definition at line 80 of file aggregation-node.h.

Referenced by Close(), Open(), and Prepare().

RuntimeProfile::Counter* impala::AggregationNode::build_timer_

private

Time spent processing the child rows.

Definition at line 111 of file aggregation-node.h.

Referenced by Open(), and Prepare().

std::vector<ExecNode*> impala::ExecNode::children_

protectedinherited

Definition at line 214 of file exec-node.h.

Referenced by impala::ExecNode::child(), impala::ExecNode::Close(), impala::ExecNode::CollectNodes(), impala::ExecNode::CreateTreeHelper(), impala::HBaseScanNode::DebugString(), impala::UnionNode::GetNext(), impala::UnionNode::Open(), Open(), impala::PartitionedAggregationNode::Open(), impala::UnionNode::OpenCurrentChild(), impala::ExecNode::Prepare(), impala::PartitionedAggregationNode::ProcessStream(), impala::ExecNode::Reset(), and impala::ExecNode::SetDebugOptions().

llvm::Function* impala::AggregationNode::codegen_process_row_batch_fn_

private

IR for process row batch. NULL if codegen is disabled.

Definition at line 98 of file aggregation-node.h.

Referenced by Prepare().

std::vector<ExprContext*> impala::ExecNode::conjunct_ctxs_

protectedinherited

Definition at line 212 of file exec-node.h.

Referenced by impala::ExecNode::Close(), impala::HashJoinNode::CodegenProcessProbeBatch(), impala::PartitionedHashJoinNode::CodegenProcessProbeBatch(), impala::ExecNode::conjunct_ctxs(), impala::SelectNode::CopyRows(), impala::UnionNode::EvalAndMaterializeExprs(), impala::HBaseScanNode::GetNext(), impala::HashJoinNode::GetNext(), GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::AnalyticEvalNode::GetNextOutputBatch(), impala::TopNNode::Init(), impala::ExecNode::Init(), impala::PartitionedHashJoinNode::Init(), impala::ExecNode::Open(), impala::PartitionedHashJoinNode::OutputUnmatchedBuild(), impala::ExecNode::Prepare(), impala::CrossJoinNode::ProcessLeftChildBatch(), impala::HashJoinNode::ProcessProbeBatch(), and impala::PartitionedHashJoinNode::ProcessProbeBatch().

TDebugAction::type impala::ExecNode::debug_action_

protectedinherited

Definition at line 220 of file exec-node.h.

Referenced by impala::ExecNode::ExecDebugAction(), and impala::ExecNode::SetDebugOptions().

TExecNodePhase::type impala::ExecNode::debug_phase_

protectedinherited

debug-only: if debug_action_ is not INVALID, node will perform action in debug_phase_

Definition at line 219 of file exec-node.h.

Referenced by impala::ExecNode::ExecDebugAction(), and impala::ExecNode::SetDebugOptions().

boost::mutex impala::ExecNode::exec_options_lock_

protectedinherited

Execution options that are determined at runtime. This is added to the runtime profile at Close(). Examples for options logged here would be "Codegen Enabled"

Definition at line 238 of file exec-node.h.

Referenced by impala::ExecNode::AddRuntimeExecOption().

boost::scoped_ptr<MemTracker> impala::ExecNode::expr_mem_tracker_

protectedinherited

MemTracker that should be used for ExprContexts.

Definition at line 233 of file exec-node.h.

Referenced by impala::ExecNode::expr_mem_tracker(), and impala::ExecNode::Prepare().

RuntimeProfile::Counter* impala::AggregationNode::get_results_timer_

private

Time spent returning the aggregated rows.

Definition at line 113 of file aggregation-node.h.

Referenced by GetNext(), and Prepare().

RuntimeProfile::Counter* impala::AggregationNode::hash_table_buckets_counter_

private

Num buckets in hash table.

Definition at line 115 of file aggregation-node.h.

Referenced by Open(), and Prepare().

RuntimeProfile::Counter* impala::AggregationNode::hash_table_load_factor_counter_

private

Load factor in hash table.

Definition at line 117 of file aggregation-node.h.

Referenced by Open(), and Prepare().

boost::scoped_ptr<OldHashTable> impala::AggregationNode::hash_tbl_

private

Definition at line 67 of file aggregation-node.h.

Referenced by Close(), CodegenProcessRowBatch(), ConstructIntermediateTuple(), Open(), Prepare(), and ProcessRowBatchWithGrouping().

int impala::ExecNode::id_

protectedinherited

Definition at line 209 of file exec-node.h.

Referenced by impala::PartitionedAggregationNode::CreateHashPartitions(), impala::ExecNode::id(), impala::ExecNode::InitRuntimeProfile(), impala::PartitionedAggregationNode::NextPartition(), impala::ExchangeNode::Prepare(), impala::PartitionedHashJoinNode::PrepareNextPartition(), impala::PartitionedHashJoinNode::ProcessBuildInput(), and impala::ExecNode::SetDebugOptions().

TupleDescriptor* impala::AggregationNode::intermediate_tuple_desc_

private

Definition at line 84 of file aggregation-node.h.

Referenced by CodegenUpdateSlot(), CodegenUpdateTuple(), ConstructIntermediateTuple(), FinalizeTuple(), and Prepare().

TupleId impala::AggregationNode::intermediate_tuple_id_

private

Tuple into which Update()/Merge()/Serialize() results are stored.

Definition at line 83 of file aggregation-node.h.

Referenced by DebugString(), FinalizeTuple(), and Prepare().

int64_t impala::ExecNode::limit_

protectedinherited

Definition at line 222 of file exec-node.h.

Referenced by impala::SortNode::GetNext(), impala::HdfsScanNode::GetNextInternal(), impala::ExchangeNode::GetNextMerging(), impala::TopNNode::InsertTupleRow(), impala::HdfsScanNode::limit(), impala::ExecNode::limit(), impala::TopNNode::Open(), and impala::ExecNode::ReachedLimit().

const char * impala::AggregationNode::LLVM_CLASS_NAME = "class.impala::AggregationNode"

static

Definition at line 61 of file aggregation-node.h.

Referenced by CodegenUpdateTuple().

boost::scoped_ptr<MemTracker> impala::ExecNode::mem_tracker_

protectedinherited

Account for peak memory used by this node.

Definition at line 230 of file exec-node.h.

Referenced by impala::ExecNode::mem_tracker(), and impala::ExecNode::Prepare().

bool impala::AggregationNode::needs_finalize_

private

Certain aggregates require a finalize step, which is the final step of the aggregate after consuming all input rows. The finalize step converts the aggregate value into its final form. This is true if this node contains aggregate that requires a finalize step.

Definition at line 108 of file aggregation-node.h.

Referenced by Close(), DebugString(), and FinalizeTuple().

int64_t impala::ExecNode::num_rows_returned_

protectedinherited