#include <runtime-state.h>

Collaboration diagram for impala::RuntimeState:

Public Member Functions
	RuntimeState (const TPlanFragmentInstanceCtx &fragment_instance_ctx, const std::string &cgroup, ExecEnv *exec_env)

	RuntimeState (const TQueryCtx &query_ctx)
	RuntimeState for executing expr in fe-support. More...

	~RuntimeState ()
	Empty d'tor to avoid issues with scoped_ptr. More...

void	InitMemTrackers (const TUniqueId &query_id, const std::string *request_pool, int64_t query_bytes_limit, int64_t query_rm_reservation_limit_bytes=-1)

Status	CreateBlockMgr ()
	Gets/Creates the query wide block mgr. More...

ObjectPool *	obj_pool () const

const DescriptorTbl &	desc_tbl () const

void	set_desc_tbl (DescriptorTbl *desc_tbl)

const TQueryOptions &	query_options () const

int	batch_size () const

bool	abort_on_error () const

bool	abort_on_default_limit_exceeded () const

int	max_errors () const

const TQueryCtx &	query_ctx () const

const TPlanFragmentInstanceCtx &	fragment_ctx () const

const std::string &	effective_user () const

const std::string &	do_as_user () const

const std::string &	connected_user () const

const TimestampValue *	now () const

void	set_now (const TimestampValue *now)

const ErrorLogMap &	error_log () const

const std::vector< std::pair < std::string, int > > &	file_errors () const

const TUniqueId &	query_id () const

const TUniqueId &	fragment_instance_id () const

const std::string &	cgroup () const

ExecEnv *	exec_env ()

DataStreamMgr *	stream_mgr ()

HBaseTableFactory *	htable_factory ()

ImpalaInternalServiceClientCache *	impalad_client_cache ()

CatalogServiceClientCache *	catalogd_client_cache ()

DiskIoMgr *	io_mgr ()

MemTracker *	instance_mem_tracker ()

MemTracker *	query_mem_tracker ()

ThreadResourceMgr::ResourcePool *	resource_pool ()

FileMoveMap *	hdfs_files_to_move ()

std::vector < DiskIoMgr::RequestContext * > *	reader_contexts ()

void	set_fragment_root_id (PlanNodeId id)

uint32_t	fragment_hash_seed () const

uint32_t	slot_filter_bitmap_size () const

void	AddBitmapFilter (SlotId slot, Bitmap bitmap, bool acquired_ownership)

const Bitmap *	GetBitmapFilter (SlotId slot)

PartitionStatusMap *	per_partition_status ()

RuntimeProfile *	runtime_profile ()
	Returns runtime state profile. More...

bool	codegen_enabled () const
	Returns true if codegen is enabled for this query. More...

bool	codegen_created () const

Status	GetCodegen (LlvmCodeGen **codegen, bool initialize=true)

BufferedBlockMgr *	block_mgr ()

Status	query_status ()

bool	LogError (const ErrorMsg &msg)

bool	LogHasSpace ()
	Returns true if the error log has not reached max_errors_. More...

void	ReportFileErrors (const std::string &file_name, int num_errors)
	Report that num_errors occurred while parsing file_name. More...

void	ClearFileErrors ()
	Clear the file errors. More...

bool	ErrorLogIsEmpty ()
	Return true if error log is empty. More...

std::string	ErrorLog ()
	Returns the error log lines as a string joined with ' '. More...

void	GetUnreportedErrors (ErrorLogMap *new_errors)

std::string	FileErrors ()
	Returns a string representation of the file_errors_. More...

bool	is_cancelled () const

void	set_is_cancelled (bool v)

RuntimeProfile::Counter *	total_cpu_timer ()

RuntimeProfile::Counter *	total_storage_wait_timer ()

RuntimeProfile::Counter *	total_network_send_timer ()

RuntimeProfile::Counter *	total_network_receive_timer ()

void	set_query_status (const std::string &err_msg)
	Sets query_status_ with err_msg if no error has been set yet. More...

Status	SetMemLimitExceeded (MemTracker *tracker=NULL, int64_t failed_allocation_size=0)

Status	CheckQueryState ()

QueryResourceMgr *	query_resource_mgr () const

void	SetQueryResourceMgr (QueryResourceMgr *res_mgr)

Private Member Functions
Status	Init (ExecEnv *exec_env)
	Set per-fragment state. More...

Status	CreateCodegen ()

	RuntimeState (const RuntimeState &)
	prohibit copies More...

Private Attributes
DescriptorTbl *	desc_tbl_

boost::scoped_ptr< ObjectPool >	obj_pool_

SpinLock	error_log_lock_
	Lock protecting error_log_ and unreported_error_idx_. More...

ErrorLogMap	error_log_
	Logs error messages. More...

SpinLock	file_errors_lock_
	Lock protecting file_errors_. More...

std::vector< std::pair < std::string, int > >	file_errors_
	Stores the number of parse errors per file. More...

TPlanFragmentInstanceCtx	fragment_instance_ctx_

boost::scoped_ptr< TimestampValue >	now_

std::string	cgroup_

ExecEnv *	exec_env_

boost::scoped_ptr< LlvmCodeGen >	codegen_

ThreadResourceMgr::ResourcePool *	resource_pool_

FileMoveMap	hdfs_files_to_move_

PartitionStatusMap	per_partition_status_
	Records summary statistics for the results of inserts into Hdfs partitions. More...

RuntimeProfile	profile_

RuntimeProfile::Counter *	total_cpu_timer_
	Total CPU time (across all threads), including all wait times. More...

RuntimeProfile::Counter *	total_storage_wait_timer_
	Total time waiting in storage (across all threads) More...

RuntimeProfile::Counter *	total_network_send_timer_
	Total time spent sending over the network (across all threads) More...

RuntimeProfile::Counter *	total_network_receive_timer_
	Total time spent receiving over the network (across all threads) More...

boost::shared_ptr< MemTracker >	query_mem_tracker_

boost::scoped_ptr< MemTracker >	instance_mem_tracker_
	Memory usage of this fragment instance. More...

bool	is_cancelled_
	if true, execution should stop with a CANCELLED status More...

SpinLock	query_status_lock_

Status	query_status_

QueryResourceMgr *	query_resource_mgr_

std::vector < DiskIoMgr::RequestContext * >	reader_contexts_
	Reader contexts that need to be closed when the fragment is closed. More...

boost::shared_ptr < BufferedBlockMgr >	block_mgr_

PlanNodeId	root_node_id_

SpinLock	bitmap_lock_
	Lock protecting slot_bitmap_filters_. More...

boost::unordered_map< SlotId, Bitmap * >	slot_bitmap_filters_

Static Private Attributes
static const int	DEFAULT_BATCH_SIZE = 1024

Detailed Description

A collection of items that are part of the global state of a query and shared across all execution nodes of that query.

Definition at line 69 of file runtime-state.h.

Constructor & Destructor Documentation

impala::RuntimeState::RuntimeState	(	const TPlanFragmentInstanceCtx &	fragment_instance_ctx,
		const std::string &	cgroup,
		ExecEnv *	exec_env
	)

impala::RuntimeState::RuntimeState ( const TQueryCtx & query_ctx )

RuntimeState for executing expr in fe-support.

Definition at line 80 of file runtime-state.cc.

References DEFAULT_BATCH_SIZE, and fragment_instance_ctx_.

impala::RuntimeState::~RuntimeState ( )

Empty d'tor to avoid issues with scoped_ptr.

Definition at line 94 of file runtime-state.cc.

References block_mgr_, instance_mem_tracker_, query_mem_tracker_, and slot_bitmap_filters_.

impala::RuntimeState::RuntimeState ( const RuntimeState & )

private

prohibit copies

Member Function Documentation

bool impala::RuntimeState::abort_on_default_limit_exceeded ( ) const

inline

Definition at line 102 of file runtime-state.h.

References query_ctx().

bool impala::RuntimeState::abort_on_error ( ) const

inline

Definition at line 99 of file runtime-state.h.

References query_ctx().

Referenced by impala::FunctionContextImpl::Close(), impala::HdfsTextScanner::FillByteBufferGzip(), impala::HdfsTextScanner::FinishScanRange(), impala::HBaseScanNode::GetNext(), impala::HdfsRCFileScanner::ProcessRange(), impala::BaseSequenceScanner::ProcessSplit(), impala::HdfsScanner::ReportColumnParseError(), impala::HdfsScanner::ReportTupleParseError(), and impala::HdfsTextScanner::WriteFields().

void impala::RuntimeState::AddBitmapFilter	(	SlotId	slot,
		Bitmap *	bitmap,
		bool *	acquired_ownership
	)

Adds a bitmap filter on slot 'slot'. If hash(slot) % bitmap.Size() is false, this value can be filtered out. Multiple bitmap filters can be added to a single slot. If it is the first call to add a bitmap filter for the specific slot, indicated by 'acquired_ownership', then the passed bitmap should not be deleted by the caller. Thread safe.

Definition at line 294 of file runtime-state.cc.

References impala::Bitmap::And(), bitmap_lock_, and slot_bitmap_filters_.

Referenced by impala::OldHashTable::AddBitmapFilters(), and impala::PartitionedHashJoinNode::AttachProbeFilters().

int impala::RuntimeState::batch_size ( ) const

inline

Definition at line 98 of file runtime-state.h.

References query_ctx().

Referenced by impala::PartitionedHashJoinNode::Partition::BuildHashTableInternal(), impala::CrossJoinNode::ConstructBuildSide(), impala::HashJoinNode::ConstructBuildSide(), impala::AggregationNode::GetNext(), impala::PartitionedAggregationNode::GetNext(), impala::Sorter::MergeIntermediateRuns(), impala::TopNNode::Open(), impala::AggregationNode::Open(), impala::AnalyticEvalNode::Open(), impala::PartitionedAggregationNode::Open(), impala::UnionNode::OpenCurrentChild(), impala::HdfsTextScanner::Prepare(), impala::SelectNode::Prepare(), impala::BlockingJoinNode::Prepare(), impala::HdfsSequenceScanner::Prepare(), impala::PartitionedHashJoinNode::ProcessBuildInput(), impala::AnalyticEvalNode::ProcessChildBatches(), impala::PartitionedAggregationNode::ProcessStream(), impala::SortNode::SortInput(), and impala::HdfsScanner::StartNewRowBatch().

BufferedBlockMgr* impala::RuntimeState::block_mgr ( )

inline

Definition at line 194 of file runtime-state.h.

References block_mgr_.

CatalogServiceClientCache* impala::RuntimeState::catalogd_client_cache ( )

inline

Definition at line 136 of file runtime-state.h.

References impala::ExecEnv::catalogd_client_cache(), and exec_env_.

const std::string& impala::RuntimeState::cgroup ( ) const

inline

Definition at line 129 of file runtime-state.h.

References cgroup_.

Referenced by impala::BlockingJoinNode::Open(), and impala::HdfsScanNode::Prepare().

Status impala::RuntimeState::CheckQueryState ( )

Returns a non-OK status if query execution should stop (e.g., the query was cancelled or a mem limit was exceeded). Exec nodes should check this periodically so execution doesn't continue if the query terminates abnormally.

Definition at line 286 of file runtime-state.cc.

References instance_mem_tracker_, query_status_, query_status_lock_, and SetMemLimitExceeded().

Referenced by impala::HdfsScanner::CommitRows(), impala::ExecNode::QueryMaintenance(), impala::HBaseTableSink::Send(), impala::DataStreamSender::Send(), and impala::HdfsTableSink::Send().

void impala::RuntimeState::ClearFileErrors ( )

inline

Clear the file errors.

Definition at line 220 of file runtime-state.h.

References file_errors_.

bool impala::RuntimeState::codegen_created ( ) const

inline

Returns true if the codegen object has been created. Note that this may return false even when codegen is enabled if nothing has been codegen'd.

Definition at line 187 of file runtime-state.h.

References codegen_.

Referenced by Java_com_cloudera_impala_service_FeSupport_NativeEvalConstExprs(), and impala::ScalarFnCall::Prepare().

bool impala::RuntimeState::codegen_enabled ( ) const

inline

Returns true if codegen is enabled for this query.

Definition at line 183 of file runtime-state.h.

References query_options().

Referenced by impala::HdfsTextScanner::Codegen(), impala::HdfsAvroScanner::Codegen(), impala::HdfsSequenceScanner::Codegen(), impala::ScalarFnCall::GetCodegendComputeFn(), impala::ScalarFnCall::GetUdf(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), and impala::PartitionedAggregationNode::Prepare().

const std::string& impala::RuntimeState::connected_user ( ) const

inline

Definition at line 116 of file runtime-state.h.

References query_ctx().

Referenced by effective_user().

Status impala::RuntimeState::CreateBlockMgr ( )

Gets/Creates the query wide block mgr.

Definition at line 166 of file runtime-state.cc.

References block_mgr_, BLOCK_MGR_MEM_FRACTION, BLOCK_MGR_MEM_MIN_REMAINING, impala::BufferedBlockMgr::Create(), io_mgr(), impala::Status::OK, impala::PrettyPrinter::Print(), query_mem_tracker(), query_mem_tracker_, query_options(), RETURN_IF_ERROR, and runtime_profile().

Status impala::RuntimeState::CreateCodegen ( )

private

Create a codegen object in codegen_. No-op if it has already been called. This is created on first use.

Definition at line 188 of file runtime-state.cc.

References impala::RuntimeProfile::AddChild(), codegen_, fragment_instance_id(), impala::LlvmCodeGen::LoadImpalaIR(), obj_pool_, impala::Status::OK, impala::PrintId(), profile_, and RETURN_IF_ERROR.

Referenced by GetCodegen().

const DescriptorTbl& impala::RuntimeState::desc_tbl ( ) const

inline

Definition at line 93 of file runtime-state.h.

References desc_tbl_.

Referenced by impala::HdfsScanNode::ComputeSlotMaterializationOrder(), impala::SlotRef::Prepare(), impala::HBaseScanNode::Prepare(), impala::HBaseTableSink::Prepare(), impala::UnionNode::Prepare(), impala::DataSourceScanNode::Prepare(), impala::AggregationNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::HdfsScanNode::Prepare(), impala::HdfsTableSink::Prepare(), and set_desc_tbl().

const std::string& impala::RuntimeState::do_as_user ( ) const

inline

Definition at line 115 of file runtime-state.h.

References query_ctx().

Referenced by effective_user().

const std::string& impala::RuntimeState::effective_user ( ) const

inline

Definition at line 108 of file runtime-state.h.

References connected_user(), do_as_user(), and query_ctx().

Referenced by impala::DataSourceScanNode::Open().

const ErrorLogMap& impala::RuntimeState::error_log ( ) const

inline

Definition at line 121 of file runtime-state.h.

References error_log_.

Referenced by impala::HdfsScanNode::ScannerThread().

string impala::RuntimeState::ErrorLog ( )

Returns the error log lines as a string joined with '
'.

Definition at line 203 of file runtime-state.cc.

References error_log_, error_log_lock_, and impala::PrintErrorMapToString().

Referenced by impala::HBaseScanNode::GetNext(), impala::HdfsRCFileScanner::ProcessRange(), impala::HdfsScanNode::ScannerThread(), and impala::HdfsTextScanner::WriteFields().

bool impala::RuntimeState::ErrorLogIsEmpty ( )

Return true if error log is empty.

Definition at line 198 of file runtime-state.cc.

References error_log_, and error_log_lock_.

ExecEnv* impala::RuntimeState::exec_env ( )

inline

Definition at line 130 of file runtime-state.h.

References exec_env_.

Referenced by Init(), impala::BlockingJoinNode::Open(), and impala::HdfsScanNode::Prepare().

const std::vector<std::pair<std::string, int> >& impala::RuntimeState::file_errors ( ) const

inline

Definition at line 122 of file runtime-state.h.

References file_errors_.

string impala::RuntimeState::FileErrors ( )

Returns a string representation of the file_errors_.

Definition at line 208 of file runtime-state.cc.

References file_errors_, and file_errors_lock_.

const TPlanFragmentInstanceCtx& impala::RuntimeState::fragment_ctx ( ) const

inline

Definition at line 107 of file runtime-state.h.

References fragment_instance_ctx_.

Referenced by impala::UnionNode::GetNext().

uint32_t impala::RuntimeState::fragment_hash_seed ( ) const

inline

The seed value to use when hashing tuples. See comment on root_node_id_. We add one to prevent having a hash seed of 0.

Definition at line 154 of file runtime-state.h.

References root_node_id_.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader(), impala::HashJoinNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), and impala::PartitionedAggregationNode::Prepare().

const TUniqueId& impala::RuntimeState::fragment_instance_id ( ) const

inline

Definition at line 126 of file runtime-state.h.

References fragment_instance_ctx_.

Referenced by CreateCodegen(), impala::ExchangeNode::FillInputRowBatch(), impala::ExchangeNode::Prepare(), and impala::HdfsTableSink::Prepare().

const Bitmap* impala::RuntimeState::GetBitmapFilter ( SlotId slot )

inline

Returns bitmap filter on 'slot'. Returns NULL if there are no bitmap filters on this slot. It is not safe to concurrently call AddBitmapFilter() and GetBitmapFilter(). All calls to AddBitmapFilter() should happen before.

Definition at line 172 of file runtime-state.h.

References slot_bitmap_filters_.

Referenced by impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader().

Status impala::RuntimeState::GetCodegen	(	LlvmCodeGen **	codegen,
		bool	initialize = `true`
	)

Returns codegen_ in 'codegen'. If 'initialize' is true, codegen_ will be created if it has not been initialized by a previous call already. If 'initialize' is false, 'codegen' will be set to NULL if codegen_ has not been initialized.

Definition at line 312 of file runtime-state.cc.

References codegen_, CreateCodegen(), impala::Status::OK, and RETURN_IF_ERROR.

Referenced by impala::HdfsTextScanner::Codegen(), impala::HdfsAvroScanner::Codegen(), impala::HdfsSequenceScanner::Codegen(), impala::CompoundPredicate::CodegenComputeFn(), impala::HdfsAvroScanner::CodegenDecodeAvroData(), impala::HashTableCtx::CodegenEquals(), impala::OldHashTable::CodegenEquals(), impala::ExecNode::CodegenEvalConjuncts(), impala::HashTableCtx::CodegenEvalRow(), impala::OldHashTable::CodegenEvalTupleRow(), impala::HashTableCtx::CodegenHashCurrentRow(), impala::OldHashTable::CodegenHashCurrentRow(), impala::PartitionedAggregationNode::CodegenProcessBatch(), impala::HashJoinNode::CodegenProcessBuildBatch(), impala::PartitionedHashJoinNode::CodegenProcessBuildBatch(), impala::HashJoinNode::CodegenProcessProbeBatch(), impala::PartitionedHashJoinNode::CodegenProcessProbeBatch(), impala::AggregationNode::CodegenProcessRowBatch(), impala::AggregationNode::CodegenUpdateSlot(), impala::PartitionedAggregationNode::CodegenUpdateSlot(), impala::AggregationNode::CodegenUpdateTuple(), impala::PartitionedAggregationNode::CodegenUpdateTuple(), impala::NullLiteral::GetCodegendComputeFn(), impala::CaseExpr::GetCodegendComputeFn(), impala::SlotRef::GetCodegendComputeFn(), impala::Literal::GetCodegendComputeFn(), impala::ScalarFnCall::GetCodegendComputeFn(), impala::Expr::GetCodegendComputeFnWrapper(), impala::ScalarFnCall::GetFunction(), impala::ScalarFnCall::GetUdf(), Java_com_cloudera_impala_service_FeSupport_NativeEvalConstExprs(), impala::HashJoinNode::Prepare(), impala::AggregationNode::Prepare(), impala::ScalarFnCall::Prepare(), impala::PartitionedHashJoinNode::Prepare(), and impala::PartitionedAggregationNode::Prepare().

void impala::RuntimeState::GetUnreportedErrors ( ErrorLogMap * new_errors )

Append all accumulated errors since the last call to this function to new_errors to be sent back to the coordinator

Definition at line 236 of file runtime-state.cc.

References error_log_, and error_log_lock_.

Referenced by impala::FragmentMgr::FragmentExecState::ReportStatusCb().

FileMoveMap* impala::RuntimeState::hdfs_files_to_move ( )

inline

Definition at line 144 of file runtime-state.h.

References hdfs_files_to_move_.

Referenced by impala::HdfsTableSink::CreateNewTmpFile(), impala::HdfsTableSink::GetOutputPartition(), and impala::FragmentMgr::FragmentExecState::ReportStatusCb().

HBaseTableFactory* impala::RuntimeState::htable_factory ( )

inline

Definition at line 132 of file runtime-state.h.

References exec_env_, and impala::ExecEnv::htable_factory().

Referenced by impala::HBaseTableWriter::Init(), and impala::HBaseScanNode::Prepare().

ImpalaInternalServiceClientCache* impala::RuntimeState::impalad_client_cache ( )

inline

Definition at line 133 of file runtime-state.h.

References exec_env_, and impala::ExecEnv::impalad_client_cache().

Referenced by impala::DataStreamSender::Channel::Init().

Status impala::RuntimeState::Init ( ExecEnv * exec_env )

private

Set per-fragment state.

Definition at line 120 of file runtime-state.cc.

References ADD_TIMER, DEFAULT_BATCH_SIZE, exec_env(), exec_env_, fragment_instance_ctx_, impala::Status::OK, profile_, query_options(), impala::ThreadResourceMgr::RegisterPool(), resource_pool_, runtime_profile(), SCOPED_TIMER, impala::ExecEnv::thread_mgr(), total_cpu_timer_, total_network_receive_timer_, total_network_send_timer_, total_storage_wait_timer_, and impala::RuntimeProfile::total_time_counter().

void impala::RuntimeState::InitMemTrackers	(	const TUniqueId &	query_id,
		const std::string *	request_pool,
		int64_t	query_bytes_limit,
		int64_t	query_rm_reservation_limit_bytes = `-1`
	)

Set up five-level hierarchy of mem trackers: process, pool, query, fragment instance. The instance tracker is tied to our profile. Specific parts of the fragment (i.e. exec nodes, sinks, data stream senders, etc) will add a fifth level when they are initialized. This function also initializes a user function mem tracker (in the fifth level). If 'request_pool' is null, no request pool mem tracker is set up, i.e. query pools will have the process mem pool as the parent.

Definition at line 152 of file runtime-state.cc.

References exec_env_, impala::MemTracker::GetQueryMemTracker(), impala::MemTracker::GetRequestPoolMemTracker(), instance_mem_tracker_, impala::name, impala::ExecEnv::process_mem_tracker(), query_mem_tracker_, query_resource_mgr(), and runtime_profile().

Referenced by Java_com_cloudera_impala_service_FeSupport_NativeEvalConstExprs(), and impala::DataStreamTest::Sender().

MemTracker* impala::RuntimeState::instance_mem_tracker ( )

inline

Definition at line 140 of file runtime-state.h.

References instance_mem_tracker_.

Referenced by impala::ExecNode::Close(), impala::DataStreamMgr::CreateRecvr(), impala::DataSink::Prepare(), impala::DataStreamSender::Prepare(), impala::ExecNode::Prepare(), impala::HdfsTableSink::Prepare(), and impala::HdfsPartitionDescriptor::PrepareExprs().

DiskIoMgr* impala::RuntimeState::io_mgr ( )

inline

Definition at line 139 of file runtime-state.h.

References impala::ExecEnv::disk_io_mgr(), and exec_env_.

Referenced by impala::HdfsScanNode::AddDiskIoRanges(), impala::HdfsScanNode::AllocateScanRange(), impala::HdfsScanNode::Close(), impala::BufferedBlockMgr::Create(), CreateBlockMgr(), impala::HdfsParquetScanner::InitColumns(), impala::HdfsScanNode::Open(), impala::HdfsParquetScanner::ProcessFooter(), impala::BaseSequenceScanner::ReadPastSize(), impala::HdfsScanNode::ScannerThread(), impala::HdfsScanNode::SetDone(), and impala::HdfsScanNode::StopAndFinalizeCounters().

bool impala::RuntimeState::is_cancelled ( ) const

inline

Definition at line 235 of file runtime-state.h.

References is_cancelled_.

Referenced by impala::ExecNode::ExecDebugAction(), impala::Sorter::TupleSorter::SortHelper(), and impala::BufferedBlockMgr::WriteComplete().

bool impala::RuntimeState::LogError ( const ErrorMsg & msg )

Log an error that will be sent back to the coordinator based on an instance of the ErrorMsg class. The runtime state aggregates log messages based on type with one exception: messages with the GENERAL type are not aggregated but are kept individually.

Definition at line 224 of file runtime-state.cc.

References impala::AppendError(), error_log_, error_log_lock_, impala::ErrorCount(), max_errors(), impala::ErrorMsg::msg(), query_id(), query_options(), and VLOG_QUERY.

Referenced by impala::HBaseTable::Close(), impala::HBaseTableWriter::Close(), impala::DataSourceScanNode::Close(), impala::DataStreamSender::Channel::Close(), impala::HdfsTableSink::ClosePartitionFile(), impala::OldHashTable::CodegenEquals(), impala::OldHashTable::CodegenEvalTupleRow(), impala::HdfsAvroScanner::CodegenMaterializeTuple(), impala::BufferedBlockMgr::ConsumeMemory(), impala::HdfsTextScanner::FillByteBufferGzip(), impala::HdfsTextScanner::FinishScanRange(), impala::HBaseScanNode::GetNext(), impala::HdfsTextScanner::IssueInitialRanges(), impala::HdfsParquetScanner::IssueInitialRanges(), impala::BlockingJoinNode::Open(), impala::HdfsScanNode::Prepare(), impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange(), impala::HdfsRCFileScanner::ProcessRange(), impala::BaseSequenceScanner::ProcessSplit(), impala::HdfsSequenceScanner::ReadCompressedBlock(), impala::HdfsScanner::ReportColumnParseError(), impala::HdfsScanner::ReportTupleParseError(), impala::HdfsAvroScanner::ResolveSchemas(), SetMemLimitExceeded(), impala::HdfsScanNode::StopAndFinalizeCounters(), impala::BufferedBlockMgr::WriteComplete(), impala::HdfsTextScanner::WriteFields(), and impala::HBaseScanNode::WriteTextSlot().

bool impala::RuntimeState::LogHasSpace ( )

inline

Returns true if the error log has not reached max_errors_.

Definition at line 211 of file runtime-state.h.

References error_log_, error_log_lock_, and query_options().

Referenced by impala::HdfsTextScanner::FillByteBufferGzip(), impala::HdfsTextScanner::FinishScanRange(), impala::HBaseScanNode::GetNext(), impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange(), impala::HdfsRCFileScanner::ProcessRange(), impala::HdfsSequenceScanner::ReadCompressedBlock(), impala::HdfsScanner::ReportColumnParseError(), impala::HdfsScanner::ReportTupleParseError(), impala::HdfsTextScanner::WriteFields(), and impala::HBaseScanNode::WriteTextSlot().

int impala::RuntimeState::max_errors ( ) const

inline

Definition at line 105 of file runtime-state.h.

References query_options().

Referenced by LogError().

const TimestampValue* impala::RuntimeState::now ( ) const

inline

Definition at line 119 of file runtime-state.h.

References now_.

Referenced by impala::TimestampFunctions::Now(), and impala::TimestampFunctions::Unix().

ObjectPool* impala::RuntimeState::obj_pool ( ) const

inline

Definition at line 92 of file runtime-state.h.

References obj_pool_.

Referenced by impala::ScannerContext::AddStream(), impala::HdfsScanNode::AllocateScanRange(), impala::ExprContext::Clone(), impala::HdfsScanNode::CreateAndPrepareScanner(), impala::PartitionedAggregationNode::CreateHashPartitions(), impala::HdfsParquetScanner::CreateReader(), impala::HdfsTableSink::GetOutputPartition(), impala::HdfsParquetTableWriter::Init(), impala::PartitionedAggregationNode::Partition::InitStreams(), impala::CaseExpr::Open(), impala::ScalarFnCall::Open(), impala::HBaseTableSink::Prepare(), impala::AggregationNode::Prepare(), impala::PartitionedHashJoinNode::Prepare(), impala::AnalyticEvalNode::Prepare(), impala::AggFnEvaluator::Prepare(), impala::PartitionedAggregationNode::Prepare(), impala::HdfsTableSink::Prepare(), impala::HBaseTableSink::PrepareExprs(), impala::BaseSequenceScanner::ProcessSplit(), and impala::HdfsScanNode::ScannerThread().

PartitionStatusMap* impala::RuntimeState::per_partition_status ( )

inline

Definition at line 177 of file runtime-state.h.

References per_partition_status_.

Referenced by impala::HdfsTableSink::FinalizePartitionFile(), impala::HdfsTableSink::GetOutputPartition(), impala::HBaseTableSink::Prepare(), impala::FragmentMgr::FragmentExecState::ReportStatusCb(), and impala::HBaseTableSink::Send().

const TQueryCtx& impala::RuntimeState::query_ctx ( ) const

inline

Definition at line 106 of file runtime-state.h.

References fragment_instance_ctx_.

Referenced by abort_on_default_limit_exceeded(), abort_on_error(), batch_size(), connected_user(), impala::UtilityFunctions::CurrentDatabase(), do_as_user(), effective_user(), impala::UtilityFunctions::Pid(), query_id(), query_options(), and SetMemLimitExceeded().

const TUniqueId& impala::RuntimeState::query_id ( ) const

inline

Definition at line 125 of file runtime-state.h.

References query_ctx().

Referenced by impala::ExecNode::Close(), impala::BufferedBlockMgr::Create(), LogError(), impala::DataSourceScanNode::Open(), and impala::HdfsTableSink::Prepare().

MemTracker* impala::RuntimeState::query_mem_tracker ( )

inline

Definition at line 141 of file runtime-state.h.

References query_mem_tracker_.

Referenced by CreateBlockMgr(), and Java_com_cloudera_impala_service_FeSupport_NativeEvalConstExprs().

const TQueryOptions& impala::RuntimeState::query_options ( ) const

inline

Definition at line 95 of file runtime-state.h.

References query_ctx().

Referenced by codegen_enabled(), CreateBlockMgr(), impala::HdfsParquetTableWriter::default_block_size(), impala::HBaseTableScanner::HBaseTableScanner(), impala::HdfsSequenceTableWriter::Init(), impala::HdfsTextTableWriter::Init(), impala::HdfsParquetTableWriter::Init(), impala::HdfsAvroTableWriter::Init(), Init(), impala::HdfsTableSink::InitOutputPartition(), LogError(), LogHasSpace(), max_errors(), and impala::HdfsScanNode::Open().

QueryResourceMgr* impala::RuntimeState::query_resource_mgr ( ) const

inline

Definition at line 269 of file runtime-state.h.

References query_resource_mgr_.

Referenced by impala::HdfsScanNode::Close(), InitMemTrackers(), impala::HdfsScanNode::Open(), impala::HdfsScanNode::ScannerThread(), and impala::HdfsScanNode::ThreadTokenAvailableCb().

Status impala::RuntimeState::query_status ( )

inline

Definition at line 199 of file runtime-state.h.

References query_status_, and query_status_lock_.

std::vector<DiskIoMgr::RequestContext*>* impala::RuntimeState::reader_contexts ( )

inline

Definition at line 145 of file runtime-state.h.

References reader_contexts_.

Referenced by impala::HdfsScanNode::Close().

void impala::RuntimeState::ReportFileErrors	(	const std::string &	file_name,
		int	num_errors
	)

Report that num_errors occurred while parsing file_name.

Definition at line 219 of file runtime-state.cc.

References file_errors_, and file_errors_lock_.

Referenced by impala::HBaseScanNode::Close(), impala::HBaseScanNode::GetNext(), impala::HdfsRCFileScanner::ProcessRange(), and impala::HdfsScanner::ReportTupleParseError().

ThreadResourceMgr::ResourcePool* impala::RuntimeState::resource_pool ( )

inline

Definition at line 142 of file runtime-state.h.

References resource_pool_.

Referenced by impala::HdfsScanNode::AddDiskIoRanges(), impala::BlockingJoinNode::BuildSideThread(), impala::HdfsScanNode::Close(), impala::BlockingJoinNode::Open(), impala::HdfsScanNode::Open(), and impala::HdfsScanNode::ScannerThread().

RuntimeProfile* impala::RuntimeState::runtime_profile ( )

inline

Returns runtime state profile.

Definition at line 180 of file runtime-state.h.

References profile_.

Referenced by CreateBlockMgr(), Init(), and InitMemTrackers().

void impala::RuntimeState::set_desc_tbl ( DescriptorTbl * desc_tbl )

inline

Definition at line 94 of file runtime-state.h.

References desc_tbl(), and desc_tbl_.

Referenced by impala::DataStreamTest::Sender().

void impala::RuntimeState::set_fragment_root_id ( PlanNodeId id )

inline

Definition at line 147 of file runtime-state.h.

References root_node_id_.

void impala::RuntimeState::set_is_cancelled ( bool v )

inline

Definition at line 236 of file runtime-state.h.

References is_cancelled_.

void impala::RuntimeState::set_now ( const TimestampValue * now )

void impala::RuntimeState::set_query_status ( const std::string & err_msg )

inline

Sets query_status_ with err_msg if no error has been set yet.

Definition at line 250 of file runtime-state.h.

References impala::Status::ok(), query_status_, and query_status_lock_.

Status impala::RuntimeState::SetMemLimitExceeded	(	MemTracker *	tracker = `NULL`,
		int64_t	failed_allocation_size = `0`
	)

Sets query_status_ to MEM_LIMIT_EXCEEDED and logs all the registered trackers. Subsequent calls to this will be no-ops. If failed_allocation_size is not 0, then it is the size of the allocation (in bytes) that would have exceeded the limit allocated for 'tracker'. This value and tracker are only used for error reporting.

Definition at line 247 of file runtime-state.cc.

References exec_env_, impala::GetTablesMissingStatsWarning(), impala::Status::IsMemLimitExceeded(), impala::MemTracker::label(), impala::MemTracker::LimitExceeded(), LogError(), impala::MemTracker::LogUsage(), impala::Status::MEM_LIMIT_EXCEEDED, impala::Status::ok(), impala::PrettyPrinter::Print(), impala::ExecEnv::process_mem_tracker(), query_ctx(), query_mem_tracker_, query_status_, and query_status_lock_.

Referenced by CheckQueryState(), impala::PartitionedAggregationNode::CreateHashPartitions(), impala::HdfsScanNode::GetNext(), impala::OldHashTable::MemLimitExceeded(), impala::PartitionedAggregationNode::NextPartition(), impala::PartitionedHashJoinNode::PrepareNextPartition(), impala::PartitionedHashJoinNode::ProcessBuildInput(), impala::HdfsRCFileScanner::ReadRowGroup(), and impala::HdfsScanNode::ScannerThread().

void impala::RuntimeState::SetQueryResourceMgr ( QueryResourceMgr * res_mgr )

inline

Definition at line 270 of file runtime-state.h.

References query_resource_mgr_.

uint32_t impala::RuntimeState::slot_filter_bitmap_size ( ) const

inline

Size to use when building bitmap filters. This is a prime number which reduces collisions and the resulting bitmap is just under 4Kb. Having all bitmaps be the same size allows us to combine (i.e. AND) bitmaps.

Definition at line 159 of file runtime-state.h.

Referenced by impala::OldHashTable::AddBitmapFilters(), impala::PartitionedHashJoinNode::AllocateProbeFilters(), impala::PartitionedHashJoinNode::BuildHashTables(), and impala::HashJoinNode::ConstructBuildSide().

DataStreamMgr* impala::RuntimeState::stream_mgr ( )

inline

Definition at line 131 of file runtime-state.h.

References exec_env_, and impala::ExecEnv::stream_mgr().

RuntimeProfile::Counter* impala::RuntimeState::total_cpu_timer ( )

inline

Definition at line 238 of file runtime-state.h.

References total_cpu_timer_.

Referenced by impala::BlockingJoinNode::BuildSideThread(), and impala::HdfsScanNode::ScannerThread().

RuntimeProfile::Counter* impala::RuntimeState::total_network_receive_timer ( )

inline

Definition at line 245 of file runtime-state.h.

References total_network_receive_timer_.

Referenced by impala::ExchangeNode::FillInputRowBatch().

RuntimeProfile::Counter* impala::RuntimeState::total_network_send_timer ( )

inline

Definition at line 242 of file runtime-state.h.

References total_network_send_timer_.

RuntimeProfile::Counter* impala::RuntimeState::total_storage_wait_timer ( )

inline

Definition at line 239 of file runtime-state.h.

References total_storage_wait_timer_.

Member Data Documentation

SpinLock impala::RuntimeState::bitmap_lock_

private

Lock protecting slot_bitmap_filters_.

Definition at line 374 of file runtime-state.h.

Referenced by AddBitmapFilter().

boost::shared_ptr<BufferedBlockMgr> impala::RuntimeState::block_mgr_

private

BufferedBlockMgr object used to allocate and manage blocks of input data in memory with a fixed memory budget. The block mgr is shared by all fragments for this query.

Definition at line 362 of file runtime-state.h.

Referenced by block_mgr(), CreateBlockMgr(), and ~RuntimeState().

std::string impala::RuntimeState::cgroup_

private

The Impala-internal cgroup into which execution threads are assigned. If empty, no RM is enabled.

Definition at line 307 of file runtime-state.h.

Referenced by cgroup().

boost::scoped_ptr<LlvmCodeGen> impala::RuntimeState::codegen_

private

Definition at line 309 of file runtime-state.h.

Referenced by codegen_created(), CreateCodegen(), and GetCodegen().

const int impala::RuntimeState::DEFAULT_BATCH_SIZE = 1024

staticprivate

Definition at line 280 of file runtime-state.h.

Referenced by Init(), and RuntimeState().

DescriptorTbl* impala::RuntimeState::desc_tbl_

private

Definition at line 282 of file runtime-state.h.

Referenced by desc_tbl(), and set_desc_tbl().

ErrorLogMap impala::RuntimeState::error_log_

private

Logs error messages.

Definition at line 289 of file runtime-state.h.

Referenced by error_log(), ErrorLog(), ErrorLogIsEmpty(), GetUnreportedErrors(), LogError(), and LogHasSpace().

SpinLock impala::RuntimeState::error_log_lock_

private

Lock protecting error_log_ and unreported_error_idx_.

Definition at line 286 of file runtime-state.h.

Referenced by ErrorLog(), ErrorLogIsEmpty(), GetUnreportedErrors(), LogError(), and LogHasSpace().

ExecEnv* impala::RuntimeState::exec_env_

private

Definition at line 308 of file runtime-state.h.

Referenced by catalogd_client_cache(), exec_env(), htable_factory(), impalad_client_cache(), Init(), InitMemTrackers(), io_mgr(), SetMemLimitExceeded(), and stream_mgr().

std::vector<std::pair<std::string, int> > impala::RuntimeState::file_errors_

private

Stores the number of parse errors per file.

Definition at line 295 of file runtime-state.h.

Referenced by ClearFileErrors(), file_errors(), FileErrors(), and ReportFileErrors().

SpinLock impala::RuntimeState::file_errors_lock_

private

Lock protecting file_errors_.

Definition at line 292 of file runtime-state.h.

Referenced by FileErrors(), and ReportFileErrors().

TPlanFragmentInstanceCtx impala::RuntimeState::fragment_instance_ctx_

private

Context of this fragment instance, including its unique id, the total number of fragment instances, the query context, the coordinator address, etc.

Definition at line 299 of file runtime-state.h.

Referenced by fragment_ctx(), fragment_instance_id(), Init(), query_ctx(), and RuntimeState().

FileMoveMap impala::RuntimeState::hdfs_files_to_move_

private

Temporary Hdfs files created, and where they should be moved to ultimately. Mapping a filename to a blank destination causes it to be deleted.

Definition at line 317 of file runtime-state.h.

Referenced by hdfs_files_to_move().

boost::scoped_ptr<MemTracker> impala::RuntimeState::instance_mem_tracker_

private

Memory usage of this fragment instance.

Definition at line 341 of file runtime-state.h.

Referenced by CheckQueryState(), InitMemTrackers(), instance_mem_tracker(), and ~RuntimeState().

bool impala::RuntimeState::is_cancelled_

private

if true, execution should stop with a CANCELLED status

Definition at line 344 of file runtime-state.h.

Referenced by is_cancelled(), and set_is_cancelled().

boost::scoped_ptr<TimestampValue> impala::RuntimeState::now_

private

Query-global timestamp, e.g., for implementing now(). Set from query_globals_. Use pointer to avoid inclusion of timestampvalue.h and avoid clang issues.

Definition at line 303 of file runtime-state.h.

Referenced by now().

boost::scoped_ptr<ObjectPool> impala::RuntimeState::obj_pool_

private

Definition at line 283 of file runtime-state.h.

Referenced by CreateCodegen(), and obj_pool().

PartitionStatusMap impala::RuntimeState::per_partition_status_

private

Records summary statistics for the results of inserts into Hdfs partitions.

Definition at line 320 of file runtime-state.h.

Referenced by per_partition_status().

RuntimeProfile impala::RuntimeState::profile_

private

Definition at line 322 of file runtime-state.h.

Referenced by CreateCodegen(), Init(), and runtime_profile().

boost::shared_ptr<MemTracker> impala::RuntimeState::query_mem_tracker_

private

MemTracker that is shared by all fragment instances running on this host. The query mem tracker must be released after the instance_mem_tracker_.

Definition at line 338 of file runtime-state.h.

Referenced by CreateBlockMgr(), InitMemTrackers(), query_mem_tracker(), SetMemLimitExceeded(), and ~RuntimeState().

QueryResourceMgr* impala::RuntimeState::query_resource_mgr_

private

Query-wide resource manager for resource expansion etc. Not owned by us; owned by the ResourceBroker instead.

Definition at line 354 of file runtime-state.h.

Referenced by query_resource_mgr(), and SetQueryResourceMgr().

Status impala::RuntimeState::query_status_

private

Definition at line 350 of file runtime-state.h.

Referenced by CheckQueryState(), query_status(), set_query_status(), and SetMemLimitExceeded().

SpinLock impala::RuntimeState::query_status_lock_

private

Non-OK if an error has occurred and query execution should abort. Used only for asynchronously reporting such errors (e.g., when a UDF reports an error), so this will not necessarily be set in all error cases.

Definition at line 349 of file runtime-state.h.

Referenced by CheckQueryState(), query_status(), set_query_status(), and SetMemLimitExceeded().

std::vector<DiskIoMgr::RequestContext*> impala::RuntimeState::reader_contexts_

private

Reader contexts that need to be closed when the fragment is closed.

Definition at line 357 of file runtime-state.h.

Referenced by reader_contexts().

ThreadResourceMgr::ResourcePool* impala::RuntimeState::resource_pool_

private

Thread resource management object for this fragment's execution. The runtime state is responsible for returning this pool to the thread mgr.

Definition at line 313 of file runtime-state.h.

Referenced by Init(), and resource_pool().

PlanNodeId impala::RuntimeState::root_node_id_

private

This is the node id of the root node for this plan fragment. This is used as the hash seed and has two useful properties: 1) It is the same for all exec nodes in a fragment, so the resulting hash values can be shared (i.e. for slot_bitmap_filters_). 2) It is different between different fragments, so we do not run into hash collisions after data partitioning (across fragments). See IMPALA-219 for more details.

Definition at line 371 of file runtime-state.h.

Referenced by fragment_hash_seed(), and set_fragment_root_id().

boost::unordered_map<SlotId, Bitmap*> impala::RuntimeState::slot_bitmap_filters_

private

Bitmap filter on the hash for 'SlotId'. If bitmap[hash(slot]] is unset, this value can be filtered out. These filters are generated during the query execution.

Definition at line 378 of file runtime-state.h.

Referenced by AddBitmapFilter(), GetBitmapFilter(), and ~RuntimeState().

RuntimeProfile::Counter* impala::RuntimeState::total_cpu_timer_

private

Total CPU time (across all threads), including all wait times.

Definition at line 325 of file runtime-state.h.

Referenced by Init(), and total_cpu_timer().

RuntimeProfile::Counter* impala::RuntimeState::total_network_receive_timer_

private

Total time spent receiving over the network (across all threads)

Definition at line 334 of file runtime-state.h.

Referenced by Init(), and total_network_receive_timer().