#include "service/impala-server.h"
#include <algorithm>
#include <exception>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/filesystem.hpp>
#include <boost/date_time/posix_time/posix_time_types.hpp>
#include <boost/unordered_set.hpp>
#include <boost/foreach.hpp>
#include <boost/bind.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
#include <google/malloc_extension.h>
#include <gutil/strings/substitute.h>
#include <openssl/evp.h>
#include <openssl/err.h>
#include <rapidjson/rapidjson.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <unistd.h>
#include "catalog/catalog-server.h"
#include "catalog/catalog-util.h"
#include "common/logging.h"
#include "common/version.h"
#include "rpc/authentication.h"
#include "rpc/thrift-util.h"
#include "rpc/thrift-thread.h"
#include "rpc/rpc-trace.h"
#include "runtime/client-cache.h"
#include "runtime/data-stream-mgr.h"
#include "runtime/exec-env.h"
#include "runtime/lib-cache.h"
#include "runtime/timestamp-value.h"
#include "runtime/tmp-file-mgr.h"
#include "service/fragment-exec-state.h"
#include "service/impala-internal-service.h"
#include "service/query-exec-state.h"
#include "service/query-options.h"
#include "statestore/simple-scheduler.h"
#include "util/bit-util.h"
#include "util/cgroups-mgr.h"
#include "util/container-util.h"
#include "util/debug-util.h"
#include "util/error-util.h"
#include "util/impalad-metrics.h"
#include "util/network-util.h"
#include "util/parse-util.h"
#include "util/redactor.h"
#include "util/string-parser.h"
#include "util/summary-util.h"
#include "util/uid-util.h"
#include "gen-cpp/Types_types.h"
#include "gen-cpp/ImpalaService.h"
#include "gen-cpp/DataSinks_types.h"
#include "gen-cpp/ImpalaService_types.h"
#include "gen-cpp/ImpalaInternalService.h"
#include "common/names.h"

Classes
class	impala::CancellationWork

Namespaces
	impala
	This file contains type definitions that are used throughout the code base.

Functions
	DECLARE_int32 (be_port)

	DECLARE_string (nn)

	DECLARE_int32 (nn_port)

	DECLARE_string (authorized_proxy_user_config)

	DECLARE_bool (abort_on_config_error)

	DECLARE_bool (disk_spill_encryption)

	DEFINE_int32 (beeswax_port, 21000,"port on which Beeswax client requests are served")

	DEFINE_int32 (hs2_port, 21050,"port on which HiveServer2 client requests are served")

	DEFINE_int32 (fe_service_threads, 64,"number of threads available to serve client requests")

	DEFINE_int32 (be_service_threads, 64,"(Advanced) number of threads available to serve backend execution requests")

	DEFINE_string (default_query_options,"","key=value pair of default query options for"" impalad, separated by ','")

	DEFINE_int32 (query_log_size, 25,"Number of queries to retain in the query log. If -1, ""the query log has unbounded size.")

	DEFINE_bool (log_query_to_file, true,"if true, logs completed query profiles to file.")

	DEFINE_int64 (max_result_cache_size, 100000L,"Maximum number of query results a client ""may request to be cached on a per-query basis to support restarting fetches. This ""option guards against unreasonably large result caches requested by clients. ""Requests exceeding this maximum will be rejected.")

	DEFINE_int32 (max_audit_event_log_file_size, 5000,"The maximum size (in queries) of the ""audit event log file before a new one is created (if event logging is enabled)")

	DEFINE_string (audit_event_log_dir,"","The directory in which audit event log files are ""written. Setting this flag will enable audit event logging.")

	DEFINE_bool (abort_on_failed_audit_event, true,"Shutdown Impala if there is a problem ""recording an audit event.")

	DEFINE_int32 (max_lineage_log_file_size, 5000,"The maximum size (in queries) of ""the lineage event log file before a new one is created (if lineage logging is ""enabled)")

	DEFINE_string (lineage_event_log_dir,"","The directory in which lineage event log ""files are written. Setting this flag with enable lineage logging.")

	DEFINE_bool (abort_on_failed_lineage_event, true,"Shutdown Impala if there is a problem ""recording a lineage record.")

	DEFINE_string (profile_log_dir,"","The directory in which profile log files are"" written. If blank, defaults to <log_file_dir>/profiles")

	DEFINE_int32 (max_profile_log_file_size, 5000,"The maximum size (in queries) of the ""profile log file before a new one is created")

	DEFINE_int32 (cancellation_thread_pool_size, 5,"(Advanced) Size of the thread-pool processing cancellations due to node failure")

	DEFINE_string (ssl_server_certificate,"","The full path to the SSL certificate file used"" to authenticate Impala to clients. If set, both Beeswax and HiveServer2 ports will ""only accept SSL connections")

	DEFINE_string (ssl_private_key,"","The full path to the private key used as a ""counterpart to the public key contained in --ssl_server_certificate. If ""--ssl_server_certificate is set, this option must be set as well.")

	DEFINE_string (ssl_client_ca_certificate,"","(Advanced) The full path to a certificate ""used by Thrift clients to check the validity of a server certificate. May either be ""a certificate for a third-party Certificate Authority, or a copy of the certificate ""the client expects to receive from the server.")

	DEFINE_int32 (idle_session_timeout, 0,"The time, in seconds, that a session may be idle"" for before it is closed (and all running queries cancelled) by Impala. If 0, idle"" sessions are never expired.")

	DEFINE_int32 (idle_query_timeout, 0,"The time, in seconds, that a query may be idle for"" (i.e. no processing work is done and no updates are received from the client) ""before it is cancelled. If 0, idle queries are never expired. The query option ""QUERY_TIMEOUT_S overrides this setting, but, if set, --idle_query_timeout represents"" the maximum allowable timeout.")

	DEFINE_string (local_nodemanager_url,"","The URL of the local Yarn Node Manager's HTTP ""interface, used to detect if the Node Manager fails")

	DECLARE_bool (enable_rm)

	DECLARE_bool (compact_catalog_topic)

Status	impala::CreateImpalaServer (ExecEnv exec_env, int beeswax_port, int hs2_port, int be_port, ThriftServer beeswax_server, ThriftServer hs2_server, ThriftServer be_server, ImpalaServer *impala_server)

Variables
const string	impala::PROFILE_LOG_FILE_PREFIX = "impala_profile_log_1.1-"

const string	impala::AUDIT_EVENT_LOG_FILE_PREFIX = "impala_audit_event_log_1.0-"

const string	impala::LINEAGE_LOG_FILE_PREFIX = "impala_lineage_log_1.0-"

const uint32_t	impala::MAX_CANCELLATION_QUEUE_SIZE = 65536

const string	impala::BEESWAX_SERVER_NAME = "beeswax-frontend"

const string	impala::HS2_SERVER_NAME = "hiveserver2-frontend"

const int	impala::MAX_NM_MISSED_HEARTBEATS = 5

Function Documentation

DECLARE_bool ( abort_on_config_error )

DECLARE_bool ( disk_spill_encryption )

DECLARE_bool ( enable_rm )

DECLARE_bool ( compact_catalog_topic )

DECLARE_int32 ( be_port )

DECLARE_int32 ( nn_port )

DECLARE_string ( nn )

DECLARE_string ( authorized_proxy_user_config )

DEFINE_bool	(	log_query_to_file	,
		true	,
		"if	true,
		logs completed query profiles to file."
	)

DEFINE_bool	(	abort_on_failed_audit_event	,
		true	,
		"Shutdown Impala if there is a problem ""recording an audit event."
	)

DEFINE_bool	(	abort_on_failed_lineage_event	,
		true	,
		"Shutdown Impala if there is a problem ""recording a lineage record."
	)

DEFINE_int32	(	beeswax_port	,
		21000	,
		"port on which Beeswax client requests are served"
	)

DEFINE_int32	(	hs2_port	,
		21050	,
		"port on which HiveServer2 client requests are served"
	)

DEFINE_int32	(	fe_service_threads	,
		64	,
		"number of threads available to serve client requests"
	)

DEFINE_int32	(	be_service_threads	,
		64	,
		"(Advanced) number of threads available to serve backend execution requests"
	)

DEFINE_int32	(	query_log_size	,
		25	,
		"Number of queries to retain in the query log. If -	1,
		""the query log has unbounded size."
	)

DEFINE_int32	(	max_audit_event_log_file_size	,
		5000	,
		"The maximum size (in queries) of the ""audit event log file before a new one is created (if event logging is enabled)"
	)

DEFINE_int32	(	max_lineage_log_file_size	,
		5000	,
		"The maximum size (in queries) of ""the lineage event log file before a new one is created (if lineage logging is ""enabled)"
	)

DEFINE_int32	(	max_profile_log_file_size	,
		5000	,
		"The maximum size (in queries) of the ""profile log file before a new one is created"
	)

DEFINE_int32	(	cancellation_thread_pool_size	,
		5	,
		"(Advanced) Size of the thread-pool processing cancellations due to node failure"
	)

DEFINE_int32	(	idle_session_timeout	,
		0	,
		"The	time,
		in	seconds,
		that a session may be idle""for before it is closed(and all running queries cancelled) by Impala.If	0,
		idle""sessions are never expired."
	)

DEFINE_int32	(	idle_query_timeout	,
		0	,
		"The	time,
		in	seconds,
		that a query may be idle for""(i.e.no processing work is done and no updates are received from the client)""before it is cancelled.If	0,
		idle queries are never expired.The query option""QUERY_TIMEOUT_S overrides this	setting,
		but	,
		if	set,
		--idle_query_timeout represents""the maximum allowable timeout."
	)

DEFINE_int64	(	max_result_cache_size	,
		100000L	,
		"Maximum number of query results a client ""may request to be cached on a per-query basis to support restarting fetches. This ""option guards against unreasonably large result caches requested by clients. ""Requests exceeding this maximum will be rejected."
	)

DEFINE_string	(	default_query_options	,
		""	,
		"	key = `value pair of default query options for"" impalad`,
		separated by '	,
		'"
	)

DEFINE_string	(	audit_event_log_dir	,
		""	,
		"The directory in which audit event log files are ""written. Setting this flag will enable audit event logging."
	)

DEFINE_string	(	lineage_event_log_dir	,
		""	,
		"The directory in which lineage event log ""files are written. Setting this flag with enable lineage logging."
	)

DEFINE_string	(	profile_log_dir	,
		""	,
		"The directory in which profile log files are"" written. If	blank,
		defaults to< log_file_dir >/profiles"
	)

DEFINE_string	(	ssl_server_certificate	,
		""	,
		"The full path to the SSL certificate file used"" to authenticate Impala to clients. If	set,
		both Beeswax and HiveServer2 ports will""only accept SSL connections"
	)

DEFINE_string	(	ssl_private_key	,
		""	,
		"The full path to the private key used as a ""counterpart to the public key contained in --ssl_server_certificate. If ""--ssl_server_certificate is	set,
		this option must be set as well."
	)

DEFINE_string	(	ssl_client_ca_certificate	,
		""	,
		"(Advanced) The full path to a certificate ""used by Thrift clients to check the validity of a server certificate. May either be ""a certificate for a third-party Certificate	Authority,
		or a copy of the certificate""the client expects to receive from the server."
	)

DEFINE_string	(	local_nodemanager_url	,
		""	,
		"The URL of the local Yarn Node Manager's HTTP ""	interface,
		used to detect if the Node Manager fails"
	)

Classes

Namespaces

Functions

Variables

Function Documentation