|
Impala
Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.
|
|
Impala
Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.
|
#include <impala-server.h>
Classes | |
| class | AsciiQueryResultSet |
| struct | CatalogUpdateVersionInfo |
| Contains details on the version information of a catalog update. More... | |
| struct | ExpirationEventComparator |
| Comparator that breaks ties when two queries have identical expiration deadlines. More... | |
| class | HS2ColumnarResultSet |
| class | HS2RowOrientedResultSet |
| class | QueryExecState |
| class | QueryResultSet |
| struct | QueryStateRecord |
| Snapshot of a query's state, archived in the query log. More... | |
| class | ScopedSessionState |
| struct | SessionState |
Public Member Functions | |
| ImpalaServer (ExecEnv *exec_env) | |
| ~ImpalaServer () | |
| virtual void | query (beeswax::QueryHandle &query_handle, const beeswax::Query &query) |
| ImpalaService rpcs: Beeswax API (implemented in impala-beeswax-server.cc) More... | |
| virtual void | executeAndWait (beeswax::QueryHandle &query_handle, const beeswax::Query &query, const beeswax::LogContextId &client_ctx) |
| virtual void | explain (beeswax::QueryExplanation &query_explanation, const beeswax::Query &query) |
| virtual void | fetch (beeswax::Results &query_results, const beeswax::QueryHandle &query_handle, const bool start_over, const int32_t fetch_size) |
| virtual void | get_results_metadata (beeswax::ResultsMetadata &results_metadata, const beeswax::QueryHandle &handle) |
| virtual void | close (const beeswax::QueryHandle &handle) |
| virtual beeswax::QueryState::type | get_state (const beeswax::QueryHandle &handle) |
| virtual void | echo (std::string &echo_string, const std::string &input_string) |
| virtual void | clean (const beeswax::LogContextId &log_context) |
| virtual void | get_log (std::string &log, const beeswax::LogContextId &context) |
| virtual void | get_default_configuration (std::vector< beeswax::ConfigVariable > &configurations, const bool include_hadoop) |
| virtual void | dump_config (std::string &config) |
| virtual void | Cancel (impala::TStatus &status, const beeswax::QueryHandle &query_id) |
| ImpalaService rpcs: extensions over Beeswax (implemented in impala-beeswax-server.cc) More... | |
| virtual void | CloseInsert (impala::TInsertResult &insert_result, const beeswax::QueryHandle &query_handle) |
| virtual void | PingImpalaService (TPingImpalaServiceResp &return_val) |
| Pings the Impala service and gets the server version string. More... | |
| virtual void | GetRuntimeProfile (std::string &profile_output, const beeswax::QueryHandle &query_id) |
| virtual void | GetExecSummary (impala::TExecSummary &result, const beeswax::QueryHandle &query_id) |
| virtual void | ResetCatalog (impala::TStatus &status) |
| Performs a full catalog metadata reset, invalidating all table and database metadata. More... | |
| virtual void | ResetTable (impala::TStatus &status, const TResetTableReq &request) |
| virtual void | OpenSession (apache::hive::service::cli::thrift::TOpenSessionResp &return_val, const apache::hive::service::cli::thrift::TOpenSessionReq &request) |
| virtual void | CloseSession (apache::hive::service::cli::thrift::TCloseSessionResp &return_val, const apache::hive::service::cli::thrift::TCloseSessionReq &request) |
| virtual void | GetInfo (apache::hive::service::cli::thrift::TGetInfoResp &return_val, const apache::hive::service::cli::thrift::TGetInfoReq &request) |
| virtual void | ExecuteStatement (apache::hive::service::cli::thrift::TExecuteStatementResp &return_val, const apache::hive::service::cli::thrift::TExecuteStatementReq &request) |
| virtual void | GetTypeInfo (apache::hive::service::cli::thrift::TGetTypeInfoResp &return_val, const apache::hive::service::cli::thrift::TGetTypeInfoReq &request) |
| virtual void | GetCatalogs (apache::hive::service::cli::thrift::TGetCatalogsResp &return_val, const apache::hive::service::cli::thrift::TGetCatalogsReq &request) |
| virtual void | GetSchemas (apache::hive::service::cli::thrift::TGetSchemasResp &return_val, const apache::hive::service::cli::thrift::TGetSchemasReq &request) |
| virtual void | GetTables (apache::hive::service::cli::thrift::TGetTablesResp &return_val, const apache::hive::service::cli::thrift::TGetTablesReq &request) |
| virtual void | GetTableTypes (apache::hive::service::cli::thrift::TGetTableTypesResp &return_val, const apache::hive::service::cli::thrift::TGetTableTypesReq &request) |
| virtual void | GetColumns (apache::hive::service::cli::thrift::TGetColumnsResp &return_val, const apache::hive::service::cli::thrift::TGetColumnsReq &request) |
| virtual void | GetFunctions (apache::hive::service::cli::thrift::TGetFunctionsResp &return_val, const apache::hive::service::cli::thrift::TGetFunctionsReq &request) |
| virtual void | GetOperationStatus (apache::hive::service::cli::thrift::TGetOperationStatusResp &return_val, const apache::hive::service::cli::thrift::TGetOperationStatusReq &request) |
| virtual void | CancelOperation (apache::hive::service::cli::thrift::TCancelOperationResp &return_val, const apache::hive::service::cli::thrift::TCancelOperationReq &request) |
| virtual void | CloseOperation (apache::hive::service::cli::thrift::TCloseOperationResp &return_val, const apache::hive::service::cli::thrift::TCloseOperationReq &request) |
| virtual void | GetResultSetMetadata (apache::hive::service::cli::thrift::TGetResultSetMetadataResp &return_val, const apache::hive::service::cli::thrift::TGetResultSetMetadataReq &request) |
| virtual void | FetchResults (apache::hive::service::cli::thrift::TFetchResultsResp &return_val, const apache::hive::service::cli::thrift::TFetchResultsReq &request) |
| virtual void | GetLog (apache::hive::service::cli::thrift::TGetLogResp &return_val, const apache::hive::service::cli::thrift::TGetLogReq &request) |
| virtual void | GetExecSummary (TGetExecSummaryResp &return_val, const TGetExecSummaryReq &request) |
| virtual void | GetRuntimeProfile (TGetRuntimeProfileResp &return_val, const TGetRuntimeProfileReq &request) |
| virtual void | GetDelegationToken (apache::hive::service::cli::thrift::TGetDelegationTokenResp &return_val, const apache::hive::service::cli::thrift::TGetDelegationTokenReq &req) |
| virtual void | CancelDelegationToken (apache::hive::service::cli::thrift::TCancelDelegationTokenResp &return_val, const apache::hive::service::cli::thrift::TCancelDelegationTokenReq &req) |
| virtual void | RenewDelegationToken (apache::hive::service::cli::thrift::TRenewDelegationTokenResp &return_val, const apache::hive::service::cli::thrift::TRenewDelegationTokenReq &req) |
| void | ReportExecStatus (TReportExecStatusResult &return_val, const TReportExecStatusParams ¶ms) |
| void | TransmitData (TTransmitDataResult &return_val, const TTransmitDataParams ¶ms) |
| virtual void | ConnectionStart (const ThriftServer::ConnectionContext &session_context) |
| SessionHandlerIf methods. More... | |
| virtual void | ConnectionEnd (const ThriftServer::ConnectionContext &session_context) |
| void | MembershipCallback (const StatestoreSubscriber::TopicDeltaMap &incoming_topic_deltas, std::vector< TTopicDelta > *subscriber_topic_updates) |
| void | CatalogUpdateCallback (const StatestoreSubscriber::TopicDeltaMap &topic_deltas, std::vector< TTopicDelta > *topic_updates) |
| bool | IsOffline () |
| Returns true if Impala is offline (and not accepting queries), false otherwise. More... | |
| bool | IsLineageLoggingEnabled () |
| Returns true if lineage logging is enabled, false otherwise. More... | |
Static Public Member Functions | |
| static void | PrepareQueryContext (TQueryCtx *query_ctx) |
Private Types | |
| typedef std::list < QueryStateRecord > | QueryLog |
| FIFO list of query records, which are written after the query finishes executing. More... | |
| typedef boost::unordered_map < TUniqueId, QueryLog::iterator > | QueryLogIndex |
| Index that allows lookup via TUniqueId into the query log. More... | |
| typedef boost::unordered_map < TUniqueId, boost::shared_ptr < QueryExecState > > | QueryExecStateMap |
| typedef boost::unordered_map < TUniqueId, boost::shared_ptr < SessionState > > | SessionStateMap |
| A map from session identifier to a structure containing per-session information. More... | |
| typedef boost::unordered_map < TUniqueId, std::vector < TUniqueId > > | ConnectionToSessionMap |
| typedef boost::unordered_map < TNetworkAddress, boost::unordered_set < TUniqueId > > | QueryLocations |
| A map from backend to the list of queries currently running there. More... | |
| typedef boost::unordered_map < std::string, TNetworkAddress > | BackendAddressMap |
| typedef boost::unordered_map < std::string, boost::unordered_set < std::string > > | ProxyUserMap |
| typedef std::pair< int64_t, TUniqueId > | ExpirationEvent |
| typedef std::set < ExpirationEvent, ExpirationEventComparator > | ExpirationQueue |
Private Member Functions | |
| QueryResultSet * | CreateHS2ResultSet (apache::hive::service::cli::thrift::TProtocolVersion::type version, const TResultSetMetadata &metadata, apache::hive::service::cli::thrift::TRowSet *rowset=NULL) |
| boost::shared_ptr< QueryExecState > | GetQueryExecState (const TUniqueId &query_id, bool lock) |
| bool | GetSessionIdForQuery (const TUniqueId &query_id, TUniqueId *session_id) |
| Status | UpdateCatalogMetrics () |
| Updates the number of databases / tables metrics from the FE catalog. More... | |
| Status | Execute (TQueryCtx *query_ctx, boost::shared_ptr< SessionState > session_state, boost::shared_ptr< QueryExecState > *exec_state) |
| Status | ExecuteInternal (const TQueryCtx &query_ctx, boost::shared_ptr< SessionState > session_state, bool *registered_exec_state, boost::shared_ptr< QueryExecState > *exec_state) |
| Implements Execute() logic, but doesn't unregister query on error. More... | |
| Status | RegisterQuery (boost::shared_ptr< SessionState > session_state, const boost::shared_ptr< QueryExecState > &exec_state) |
| Status | SetQueryInflight (boost::shared_ptr< SessionState > session_state, const boost::shared_ptr< QueryExecState > &exec_state) |
| Status | UnregisterQuery (const TUniqueId &query_id, bool check_inflight, const Status *cause=NULL) |
| Status | CancelInternal (const TUniqueId &query_id, bool check_inflight, const Status *cause=NULL) |
| Status | CloseSessionInternal (const TUniqueId &session_id, bool ignore_if_absent) |
| Status | GetRuntimeProfileStr (const TUniqueId &query_id, bool base64_encoded, std::stringstream *output) |
| Status | GetExecSummary (const TUniqueId &query_id, TExecSummary *result) |
| Returns the exec summary for this query. More... | |
| void | HadoopVarzUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *document) |
| void | QueryStateUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *document) |
| void | QueryProfileUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *document) |
| void | QuerySummaryCallback (bool include_plan_json, bool include_summary, const Webserver::ArgumentMap &args, rapidjson::Document *document) |
| void | CancelQueryUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *document) |
| Webserver callback. Cancels an in-flight query and writes the result to 'contents'. More... | |
| void | QueryProfileEncodedUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *document) |
| void | InflightQueryIdsUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *document) |
| void | SessionsUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *document) |
| void | CatalogUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *output) |
| Webserver callback that prints a list of all known databases and tables. More... | |
| void | CatalogObjectsUrlCallback (const Webserver::ArgumentMap &args, rapidjson::Document *output) |
| Webserver callback that allows for dumping information on objects in the catalog. More... | |
| void | InitializeConfigVariables () |
| void | RegisterWebserverCallbacks (Webserver *webserver) |
| Registers all the per-Impalad webserver callbacks. More... | |
| Status | InitProfileLogging () |
| Status | InitAuditEventLogging () |
| Status | InitLineageLogging () |
| bool | IsAuditEventLoggingEnabled () |
| Returns true if audit event logging is enabled, false otherwise. More... | |
| Status | LogAuditRecord (const QueryExecState &exec_state, const TExecRequest &request) |
| Status | LogLineageRecord (const TExecRequest &request) |
| void | LogQueryEvents (const QueryExecState &exec_state) |
| Log audit and column lineage events. More... | |
| void | LogFileFlushThread () |
| Runs once every 5s to flush the profile log file to disk. More... | |
| void | AuditEventLoggerFlushThread () |
| Runs once every 5s to flush the audit log file to disk. More... | |
| void | LineageLoggerFlushThread () |
| Runs once every 5s to flush the lineage log file to disk. More... | |
| void | ArchiveQuery (const QueryExecState &query) |
| Status | AuthorizeProxyUser (const std::string &user, const std::string &do_as_user) |
| void | QueryStateToJson (const ImpalaServer::QueryStateRecord &record, rapidjson::Value *value, rapidjson::Document *document) |
| Status | QueryToTQueryContext (const beeswax::Query &query, TQueryCtx *query_ctx) |
| Beeswax private methods. More... | |
| void | TUniqueIdToQueryHandle (const TUniqueId &query_id, beeswax::QueryHandle *handle) |
| void | QueryHandleToTUniqueId (const beeswax::QueryHandle &handle, TUniqueId *query_id) |
| void | RaiseBeeswaxException (const std::string &msg, const char *sql_state) |
| Helper function to raise BeeswaxException. More... | |
| Status | FetchInternal (const TUniqueId &query_id, bool start_over, int32_t fetch_size, beeswax::Results *query_results) |
| Executes the fetch logic. Doesn't clean up the exec state if an error occurs. More... | |
| Status | CloseInsertInternal (const TUniqueId &query_id, TInsertResult *insert_result) |
| void | ExecuteMetadataOp (const apache::hive::service::cli::thrift::THandleIdentifier &session_handle, TMetadataOpRequest *request, apache::hive::service::cli::thrift::TOperationHandle *handle, apache::hive::service::cli::thrift::TStatus *status) |
| HiveServer2 private methods (implemented in impala-hs2-server.cc) More... | |
| Status | FetchInternal (const TUniqueId &query_id, int32_t fetch_size, bool fetch_first, apache::hive::service::cli::thrift::TFetchResultsResp *fetch_results) |
| Status | TExecuteStatementReqToTQueryContext (const apache::hive::service::cli::thrift::TExecuteStatementReq execute_request, TQueryCtx *query_ctx) |
| void | CancelFromThreadPool (uint32_t thread_id, const CancellationWork &cancellation_work) |
| Status | ProcessCatalogUpdateResult (const TCatalogUpdateResult &catalog_update_result, bool wait_for_all_subscribers) |
| void | ExpireSessions () |
| void | ExpireQueries () |
| void | DetectNmFailures () |
| void | SetOffline (bool offline) |
| Set is_offline_ to the argument's value. More... | |
| Status | GetSessionState (const TUniqueId &session_id, boost::shared_ptr< SessionState > *session_state, bool mark_active=false) |
| void | MarkSessionInactive (boost::shared_ptr< SessionState > session) |
Static Private Member Functions | |
| static Status | InitLoggingDir (const std::string &log_dir) |
| static Status | THandleIdentifierToTUniqueId (const apache::hive::service::cli::thrift::THandleIdentifier &handle, TUniqueId *unique_id, TUniqueId *secret) |
| Helper functions to translate between HiveServer2 and Impala structs. More... | |
| static void | TUniqueIdToTHandleIdentifier (const TUniqueId &unique_id, const TUniqueId &secret, apache::hive::service::cli::thrift::THandleIdentifier *handle) |
Private Attributes | |
| boost::mutex | query_log_lock_ |
| Guards query_log_ and query_log_index_. More... | |
| QueryLog | query_log_ |
| QueryLogIndex | query_log_index_ |
| boost::scoped_ptr< SimpleLogger > | profile_logger_ |
| boost::scoped_ptr< SimpleLogger > | audit_event_logger_ |
| boost::scoped_ptr< SimpleLogger > | lineage_logger_ |
| boost::scoped_ptr< Thread > | profile_log_file_flush_thread_ |
| If profile logging is enabled, wakes once every 5s to flush query profiles to disk. More... | |
| boost::scoped_ptr< Thread > | audit_event_logger_flush_thread_ |
| If audit event logging is enabled, wakes once every 5s to flush audit events to disk. More... | |
| boost::scoped_ptr< Thread > | lineage_logger_flush_thread_ |
| If lineage logging is enabled, wakes once every 5s to flush lineage events to disk. More... | |
| ExecEnv * | exec_env_ |
| global, per-server state More... | |
| boost::scoped_ptr< ThreadPool < CancellationWork > > | cancellation_thread_pool_ |
| boost::scoped_ptr< Thread > | session_timeout_thread_ |
| Thread that runs ExpireSessions if FLAGS_idle_session_timeout > 0. More... | |
| QueryExecStateMap | query_exec_state_map_ |
| boost::mutex | query_exec_state_map_lock_ |
| TQueryOptions | default_query_options_ |
| Default query options in the form of TQueryOptions and beeswax::ConfigVariable. More... | |
| std::vector < beeswax::ConfigVariable > | default_configs_ |
| boost::mutex | session_state_map_lock_ |
| SessionStateMap | session_state_map_ |
| boost::mutex | connection_to_sessions_map_lock_ |
| Protects connection_to_sessions_map_. May be taken before session_state_map_lock_. More... | |
| ConnectionToSessionMap | connection_to_sessions_map_ |
| boost::mutex | query_locations_lock_ |
| QueryLocations | query_locations_ |
| BackendAddressMap | known_backends_ |
| boost::uuids::random_generator | uuid_generator_ |
| Generate unique session id for HiveServer2 session. More... | |
| boost::mutex | uuid_lock_ |
| Lock to protect uuid_generator. More... | |
| boost::mutex | catalog_version_lock_ |
| boost::condition_variable | catalog_version_update_cv_ |
| Variable to signal when the catalog version has been modified. More... | |
| CatalogUpdateVersionInfo | catalog_update_info_ |
| The version information from the last successfull call to UpdateCatalog(). More... | |
| int64_t | min_subscriber_catalog_topic_version_ |
| ProxyUserMap | authorized_proxy_user_config_ |
| boost::mutex | query_expiration_lock_ |
| Guards queries_by_timestamp_. Must not be acquired before a session state lock. More... | |
| ExpirationQueue | queries_by_timestamp_ |
| boost::scoped_ptr< Thread > | query_expiration_thread_ |
| Container for a thread that runs ExpireQueries() if FLAGS_idle_query_timeout is set. More... | |
| boost::scoped_ptr< Thread > | nm_failure_detection_thread_ |
| Container thread for DetectNmFailures(). More... | |
| boost::mutex | is_offline_lock_ |
| Protects is_offline_. More... | |
| bool | is_offline_ |
| True if Impala server is offline, false otherwise. More... | |
Static Private Attributes | |
| static const char * | SQLSTATE_SYNTAX_ERROR_OR_ACCESS_VIOLATION = "42000" |
| static const char * | SQLSTATE_GENERAL_ERROR = "HY000" |
| static const char * | SQLSTATE_OPTIONAL_FEATURE_NOT_IMPLEMENTED = "HYC00" |
| static const int | ASCII_PRECISION = 16 |
| Ascii output precision for double/float. More... | |
Friends | |
| class | ChildQuery |
| class | ScopedSessionState |
| For access to GetSessionState() / MarkSessionInactive() More... | |
An ImpalaServer contains both frontend and backend functionality; it implements ImpalaService (Beeswax), ImpalaHiveServer2Service (HiveServer2) and ImpalaInternalService APIs. This class is partially thread-safe. To ensure freedom from deadlock, locks on the maps are obtained before locks on the items contained in the maps. TODO: The state of a running query is currently not cleaned up if the query doesn't experience any errors at runtime and close() doesn't get called. The solution is to have a separate thread that cleans up orphaned query execution states after a timeout period. TODO: The same doesn't apply to the execution state of an individual plan fragment: the originating coordinator might die, but we can get notified of that via the statestore. This still needs to be implemented.
Definition at line 82 of file impala-server.h.
|
private |
A map from unique backend ID to the corresponding TNetworkAddress of that backend. Used to track membership updates from the statestore so queries can be cancelled when a backend is removed. It's not enough to just cancel fragments that are running based on the deletions mentioned in the most recent statestore heartbeat; sometimes cancellations are skipped and the statestore, at its discretion, may send only a delta of the current membership so we need to compute any deletions. TODO: Currently there are multiple locations where cluster membership is tracked, here and in the scheduler. This should be consolidated so there is a single component (the scheduler?) that tracks this information and calls other interested components.
Definition at line 930 of file impala-server.h.
|
private |
Map from a connection ID to the associated list of sessions so that all can be closed when the connection ends. HS2 allows for multiplexing several sessions across a single connection. If a session has already been closed (only possible via HS2) it is not removed from this map to avoid the cost of looking it up.
Definition at line 893 of file impala-server.h.
|
private |
Describes a query expiration event (t, q) where t is the expiration deadline in seconds, and q is the query ID.
Definition at line 981 of file impala-server.h.
|
private |
Ordered set of (expiration_time, query_id) pairs. This queue is updated either by RegisterQuery(), which adds a new query to the set, or by ExpireQueries(), which updates the entries as it iterates over the set. Therefore, it is not directly accessed during normal query execution and the benefit of that is there is no competition for locks when ExpireQueries() walks this list to find expired queries. In order to make the query expiration algorithm work, the following condition always holds: * For any pair (t, q) in the set, t is always a lower bound on the true expiration time of the query q (in q->idle_time()). Therefore we can bound the expiration error by sleeping to no more than t + d for some small d (in our implementation, d is 1s). The reason that the expiration time saved in each entry here may not exactly match the true expiration time of a query is because we wish to avoid accessing (and therefore locking) this structure on every query activity. Instead, queries maintain an accurate expiration time, and this structure guarantees that we will always (modulo scheduling delays out of our control) read the expiration time before it has passed.
Definition at line 1011 of file impala-server.h.
|
private |
Map of short usernames of authorized proxy users to the set of user(s) they are allowed to delegate to. Populated by parsing the –authorized_proxy_users_config flag.
Definition at line 973 of file impala-server.h.
|
private |
map from query id to exec state; QueryExecState is owned by us and referenced as a shared_ptr to allow asynchronous deletion
Definition at line 765 of file impala-server.h.
|
private |
A map from backend to the list of queries currently running there.
Definition at line 918 of file impala-server.h.
|
private |
FIFO list of query records, which are written after the query finishes executing.
Definition at line 724 of file impala-server.h.
|
private |
Index that allows lookup via TUniqueId into the query log.
Definition at line 728 of file impala-server.h.
|
private |
A map from session identifier to a structure containing per-session information.
Definition at line 882 of file impala-server.h.
| impala::ImpalaServer::ImpalaServer | ( | ExecEnv * | exec_env | ) |
Definition at line 226 of file impala-server.cc.
References impala::StatestoreSubscriber::AddTopic(), authorized_proxy_user_config_, CancelFromThreadPool(), cancellation_thread_pool_, CatalogUpdateCallback(), impala::ImpaladMetrics::CreateMetrics(), impala_udf::DebugString(), DetectNmFailures(), exec_env_, EXIT_IF_ERROR, ExpireQueries(), ExpireSessions(), impala::ExecEnv::frontend(), impala::MetricGroup::GetChildGroup(), impala::Status::GetDetail(), impala::CatalogServer::IMPALA_CATALOG_TOPIC, impala::SimpleScheduler::IMPALA_MEMBERSHIP_TOPIC, impala::ImpaladMetrics::IMPALA_SERVER_START_TIME, impala::TmpFileMgr::Init(), InitAuditEventLogging(), InitializeConfigVariables(), InitLineageLogging(), InitProfileLogging(), is_offline_, impala::TimestampValue::LocalTime(), impala::MAX_CANCELLATION_QUEUE_SIZE, MembershipCallback(), impala::ExecEnv::metrics(), nm_failure_detection_thread_, impala::Status::ok(), query_expiration_thread_, RegisterWebserverCallbacks(), session_timeout_thread_, impala::ExecEnv::SetImpalaServer(), impala::ExecEnv::subscriber(), UpdateCatalogMetrics(), impala::Frontend::ValidateSettings(), and impala::ExecEnv::webserver().
Referenced by impala::CreateImpalaServer().
| impala::ImpalaServer::~ImpalaServer | ( | ) |
Definition at line 678 of file impala-server.cc.
|
private |
Copies a query's state into the query log. Called immediately prior to a QueryExecState's deletion. Also writes the query profile to the profile log on disk. Must be called with query_exec_state_map_lock_ held
Definition at line 641 of file impala-server.cc.
References impala::ImpalaServer::QueryExecState::coord(), impala::ImpalaServer::QueryStateRecord::exec_summary, impala::Status::GetDetail(), impala::Status::ok(), impala::ImpalaServer::QueryExecState::profile(), profile_logger_, impala::ImpalaServer::QueryExecState::query_id(), query_log_, query_log_index_, query_log_lock_, impala::RuntimeProfile::SerializeToArchiveString(), and impala::UnixMillis().
Referenced by UnregisterQuery().
|
private |
Runs once every 5s to flush the audit log file to disk.
Definition at line 611 of file impala-server.cc.
References audit_event_logger_, impala::Status::GetDetail(), and impala::Status::ok().
Referenced by InitAuditEventLogging().
|
private |
Checks whether the given user is allowed to delegate as the specified do_as_user. Returns OK if the authorization suceeds, otherwise returns an status with details on why the failure occurred.
Definition at line 1102 of file impala-server.cc.
References authorized_proxy_user_config_, and impala::Status::OK.
|
virtual |
ImpalaService rpcs: extensions over Beeswax (implemented in impala-beeswax-server.cc)
Definition at line 435 of file impala-beeswax-server.cc.
References impala::CancelInternal(), impala::OK, impala::query_id(), RAISE_IF_ERROR, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 1133 of file impala-hs2-server.cc.
|
private |
Helper method to process cancellations that result from failed backends, called from the cancellation thread pool. The cancellation_work contains the query id to cancel and a cause listing the failed backends that led to cancellation. Calls CancelInternal directly, but has a signature compatible with the thread pool.
Definition at line 1083 of file impala-server.cc.
References CancelInternal(), impala::CancellationWork::cause(), impala::Status::GetDetail(), impala::Status::ok(), impala::CancellationWork::query_id(), impala::CancellationWork::unregister(), UnregisterQuery(), and VLOG_QUERY.
Referenced by ImpalaServer().
|
private |
Initiates query cancellation reporting the given cause as the query status. Assumes deliberate cancellation by the user if the cause is NULL. Returns an error if query_id is not found. If check_inflight is true, then return an error if the query is not yet in-flight. Otherwise, returns OK. Queries still need to be unregistered, after cancellation. Caller should not hold any locks when calling this function.
Definition at line 907 of file impala-server.cc.
References GetQueryExecState(), impala::Status::OK, impala::PrintId(), and VLOG_QUERY.
Referenced by CancelFromThreadPool(), and UnregisterQuery().
|
virtual |
Definition at line 916 of file impala-hs2-server.cc.
References impala::CancelInternal(), HS2_RETURN_ERROR, HS2_RETURN_IF_ERROR, impala::PrintId(), impala::query_id(), VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
private |
Webserver callback. Cancels an in-flight query and writes the result to 'contents'.
Definition at line 139 of file impala-server-callbacks.cc.
References impala::Status::GetDetail(), impala::Status::ok(), and ParseQueryId().
|
private |
Webserver callback that allows for dumping information on objects in the catalog.
Definition at line 417 of file impala-server-callbacks.cc.
References impala::exec_env_, impala::ExecEnv::frontend(), impala::Frontend::GetCatalogObject(), impala::Status::GetDetail(), impala::Status::ok(), impala::TCatalogObjectFromObjectName(), and impala::TCatalogObjectTypeFromName().
| void impala::ImpalaServer::CatalogUpdateCallback | ( | const StatestoreSubscriber::TopicDeltaMap & | topic_deltas, |
| std::vector< TTopicDelta > * | topic_updates | ||
| ) |
Definition at line 1137 of file impala-server.cc.
References impala::ImpaladMetrics::CATALOG_READY, impala::ImpalaServer::CatalogUpdateVersionInfo::catalog_service_id, impala::ImpalaServer::CatalogUpdateVersionInfo::catalog_topic_version, catalog_update_info_, impala::ImpalaServer::CatalogUpdateVersionInfo::catalog_version, catalog_version_lock_, catalog_version_update_cv_, impala::DeserializeThriftMsg(), impala::LibCache::DropCache(), exec_env_, impala::ExecEnv::frontend(), impala::Frontend::GetCatalogObject(), impala::Status::GetDetail(), impala::CatalogServer::IMPALA_CATALOG_TOPIC, impala::LibCache::instance(), min_subscriber_catalog_topic_version_, impala::Status::ok(), impala::LibCache::RemoveEntry(), impala::LibCache::SetNeedsRefresh(), impala::TCatalogObjectFromEntryKey(), impala::Frontend::UpdateCatalogCache(), and UpdateCatalogMetrics().
Referenced by ImpalaServer().
|
private |
Webserver callback that prints a list of all known databases and tables.
Definition at line 376 of file impala-server-callbacks.cc.
References impala::exec_env_, impala::ExecEnv::frontend(), impala::Frontend::GetDbNames(), impala::Status::GetDetail(), impala::Frontend::GetTableNames(), and impala::Status::ok().
|
virtual |
Definition at line 382 of file impala-beeswax-server.cc.
|
virtual |
Definition at line 343 of file impala-beeswax-server.cc.
References impala::PrintId(), impala::query_id(), RAISE_IF_ERROR, VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
Referenced by DetectNmFailures().
|
virtual |
Definition at line 447 of file impala-beeswax-server.cc.
References impala::Status::GetDetail(), impala::Status::ok(), impala::PrintId(), impala::query_id(), RAISE_IF_ERROR, VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
private |
Populate insert_result and clean up exec state. If the query status is an error, insert_result is not populated and the status is returned.
Definition at line 621 of file impala-beeswax-server.cc.
References RETURN_IF_ERROR.
|
virtual |
Definition at line 937 of file impala-hs2-server.cc.
References HS2_RETURN_ERROR, HS2_RETURN_IF_ERROR, impala::PrintId(), impala::query_id(), VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 642 of file impala-hs2-server.cc.
References HS2_RETURN_IF_ERROR, and VLOG_QUERY.
|
private |
Close the session and release all resource used by this session. Caller should not hold any locks when calling this function. If ignore_if_absent is true, returns OK even if a session with the supplied ID does not exist.
Definition at line 925 of file impala-server.cc.
References impala::ImpaladMetrics::IMPALA_SERVER_NUM_OPEN_BEESWAX_SESSIONS, impala::ImpaladMetrics::IMPALA_SERVER_NUM_OPEN_HS2_SESSIONS, impala::Status::OK, impala::query_id(), session_state_map_, session_state_map_lock_, and UnregisterQuery().
Referenced by ConnectionEnd().
|
virtual |
Called when a Beeswax or HS2 connection terminates. Unregisters all sessions associated with the closed connection.
Implements impala::ThriftServer::ConnectionHandlerIf.
Definition at line 1495 of file impala-server.cc.
References CloseSessionInternal(), impala::ThriftServer::ConnectionContext::connection_id, connection_to_sessions_map_, connection_to_sessions_map_lock_, impala::Status::GetDetail(), impala::ThriftServer::ConnectionContext::network_address, and impala::Status::ok().
|
virtual |
SessionHandlerIf methods.
Called when a Beeswax or HS2 connection starts. For Beeswax, registers a new SessionState associated with the new connection. For HS2, this is a no-op (HS2 has an explicit CreateSession RPC).
Implements impala::ThriftServer::ConnectionHandlerIf.
Definition at line 1460 of file impala-server.cc.
References impala::BEESWAX_SERVER_NAME, impala::ThriftServer::ConnectionContext::connection_id, connection_to_sessions_map_, connection_to_sessions_map_lock_, default_query_options_, impala::ImpaladMetrics::IMPALA_SERVER_NUM_OPEN_BEESWAX_SESSIONS, impala::TimestampValue::LocalTime(), impala::ThriftServer::ConnectionContext::network_address, impala::ThriftServer::ConnectionContext::server_name, session_state_map_, session_state_map_lock_, impala::UnixMillis(), and impala::ThriftServer::ConnectionContext::username.
|
private |
Definition at line 388 of file impala-hs2-server.cc.
|
private |
Periodically opens a socket to FLAGS_local_nodemanager_url to check if the Yarn Node Manager is running. If not, this method calls SetOffline(true), and when the NM recovers, calls SetOffline(false). Only called (in nm_failure_detection_thread_) if FLAGS_enable_rm is true.
Definition at line 1743 of file impala-server.cc.
References close(), impala::FailureDetector::FAILED, impala::MissedHeartbeatFailureDetector::GetPeerState(), impala::GetStrErrMsg(), impala::MakeNetworkAddress(), impala::MAX_NM_MISSED_HEARTBEATS, SetOffline(), impala::SleepForMs(), and impala::MissedHeartbeatFailureDetector::UpdateHeartbeat().
Referenced by ImpalaServer().
|
virtual |
ImpalaService rpcs: unimplemented parts of Beeswax API. These APIs will not be implemented because ODBC driver does not use them.
Definition at line 428 of file impala-beeswax-server.cc.
References RAISE_IF_ERROR, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 375 of file impala-beeswax-server.cc.
References RAISE_IF_ERROR, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
private |
Starts asynchronous execution of query. Creates QueryExecState (returned in exec_state), registers it and calls Coordinator::Execute(). If it returns with an error status, exec_state will be NULL and nothing will have been registered in query_exec_state_map_. session_state is a ptr to the session running this query and must have been checked out. query_session_state is a snapshot of session state that changes when the query was run. (e.g. default database).
Definition at line 680 of file impala-server.cc.
References ExecuteInternal(), impala::ImpaladMetrics::IMPALA_SERVER_NUM_QUERIES, impala::Status::ok(), PrepareQueryContext(), impala::Redact(), and UnregisterQuery().
|
virtual |
Definition at line 202 of file impala-beeswax-server.cc.
References impala::Status::GetDetail(), impala::Status::ok(), RAISE_IF_ERROR, VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
private |
Implements Execute() logic, but doesn't unregister query on error.
Definition at line 700 of file impala-server.cc.
References exec_env_, impala::ExecEnv::frontend(), impala::Status::GetDetail(), impala::Frontend::GetExecRequest(), IsOffline(), impala::Status::OK, impala::Status::ok(), query_locations_, query_locations_lock_, RegisterQuery(), RETURN_IF_ERROR, UpdateCatalogMetrics(), and VLOG_QUERY.
Referenced by Execute().
|
private |
HiveServer2 private methods (implemented in impala-hs2-server.cc)
Starts the synchronous execution of a HiverServer2 metadata operation. If the execution succeeds, an QueryExecState will be created and registered in query_exec_state_map_. Otherwise, nothing will be registered in query_exec_state_map_ and an error status will be returned. As part of this call, the TMetadataOpRequest struct will be populated with the requesting user's session state. Returns a TOperationHandle and TStatus.
Definition at line 398 of file impala-hs2-server.cc.
References impala::exec_env_, impala::ExecEnv::frontend(), impala::Status::GetDetail(), impala::Status::ok(), and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 683 of file impala-hs2-server.cc.
References impala::Status::GetDetail(), HS2_RETURN_ERROR, HS2_RETURN_IF_ERROR, impala::IMPALA_RESULT_CACHING_OPT, impala::Status::ok(), VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
private |
Runs forever, walking queries_by_timestamp_ and expiring any queries that have been idle (i.e. no client input and no time spent processing locally) for FLAGS_idle_query_timeout seconds.
Definition at line 1555 of file impala-server.cc.
References cancellation_thread_pool_, impala_udf::DebugString(), GetQueryExecState(), impala::ImpaladMetrics::NUM_QUERIES_EXPIRED, impala::PrettyPrinter::Print(), impala::PrintId(), queries_by_timestamp_, query_expiration_lock_, impala::query_id(), impala::SleepForMs(), impala::UnixMillis(), and VLOG_QUERY.
Referenced by ImpalaServer().
|
private |
To be run in a thread. Every FLAGS_idle_session_timeout / 2 seconds, wakes up and checks all sessions for their last-idle time. Those that have been idle for longer than FLAGS_idle_session_timeout are 'expired': they will no longer accept queries and any running queries associated with those sessions are cancelled.
Definition at line 1517 of file impala-server.cc.
References cancellation_thread_pool_, impala::TimestampValue::DebugString(), impala::ImpaladMetrics::NUM_SESSIONS_EXPIRED, impala::query_id(), session_state_map_, session_state_map_lock_, impala::SleepForMs(), and impala::UnixMillis().
Referenced by ImpalaServer().
|
virtual |
Definition at line 253 of file impala-beeswax-server.cc.
References impala::exec_env_, impala::ExecEnv::frontend(), impala::Frontend::GetExplainPlan(), RAISE_IF_ERROR, VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 272 of file impala-beeswax-server.cc.
References impala::Status::GetDetail(), impala::Status::ok(), impala::PrintId(), impala::query_id(), RAISE_IF_ERROR, VLOG_ROW, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
private |
Executes the fetch logic. Doesn't clean up the exec state if an error occurs.
Definition at line 563 of file impala-beeswax-server.cc.
References RETURN_IF_ERROR, impala::ThriftToType(), and impala::TypeToOdbcString().
|
private |
Executes the fetch logic for HiveServer2 FetchResults. If fetch_first is true, then the query's state should be reset to fetch from the beginning of the result set. Doesn't clean up the exec state if an error occurs.
|
virtual |
Definition at line 1011 of file impala-hs2-server.cc.
References impala::Status::GetDetail(), HS2_RETURN_ERROR, HS2_RETURN_IF_ERROR, impala::Status::IsRecoverableError(), impala::Status::ok(), impala::PrintId(), impala::query_id(), and VLOG_ROW.
|
virtual |
Return ImpalaQueryOptions default values and "support_start_over/false" to indicate that Impala does not support start over in the fetch call. Hue relies on this not to issue a "start_over" fetch call. "include_hadoop" is not applicable.
Definition at line 419 of file impala-beeswax-server.cc.
References RAISE_IF_ERROR, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 385 of file impala-beeswax-server.cc.
References impala::query_id(), RAISE_IF_ERROR, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 298 of file impala-beeswax-server.cc.
References impala::PrintId(), impala::query_id(), RAISE_IF_ERROR, impala::ThriftToType(), impala::TypeToOdbcString(), VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 355 of file impala-beeswax-server.cc.
References impala::PrintId(), impala::query_id(), RAISE_IF_ERROR, VLOG_QUERY, VLOG_ROW, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 776 of file impala-hs2-server.cc.
References VLOG_QUERY.
|
virtual |
Definition at line 849 of file impala-hs2-server.cc.
References VLOG_QUERY.
|
virtual |
Definition at line 1127 of file impala-hs2-server.cc.
|
virtual |
Definition at line 482 of file impala-beeswax-server.cc.
References impala::Status::GetDetail(), impala::Status::ok(), impala::PrintId(), impala::query_id(), RAISE_IF_ERROR, VLOG_RPC, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 1083 of file impala-hs2-server.cc.
References HS2_RETURN_IF_ERROR, impala::query_id(), and impala::ImpalaServer::ScopedSessionState::WithSession().
|
private |
Returns the exec summary for this query.
Definition at line 576 of file impala-server.cc.
References GetQueryExecState(), impala::Status::OK, impala::PrintId(), query_log_index_, and query_log_lock_.
|
virtual |
Definition at line 867 of file impala-hs2-server.cc.
References VLOG_QUERY.
|
virtual |
Definition at line 655 of file impala-hs2-server.cc.
References HS2_RETURN_ERROR, HS2_RETURN_IF_ERROR, IMPALA_BUILD_VERSION, VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 1049 of file impala-hs2-server.cc.
References HS2_RETURN_ERROR, HS2_RETURN_IF_ERROR, impala::query_id(), and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 885 of file impala-hs2-server.cc.
References HS2_RETURN_ERROR, HS2_RETURN_IF_ERROR, impala::PrintId(), impala::query_id(), impala::QueryStateToTOperationState(), and VLOG_ROW.
|
private |
Return exec state for given query_id, or NULL if not found. If 'lock' is true, the returned exec state's lock() will be acquired before the query_exec_state_map_lock_ is released.
Definition at line 1725 of file impala-server.cc.
References query_exec_state_map_, and query_exec_state_map_lock_.
Referenced by CancelInternal(), ExpireQueries(), GetExecSummary(), and ReportExecStatus().
|
virtual |
Definition at line 960 of file impala-hs2-server.cc.
References HS2_RETURN_ERROR, HS2_RETURN_IF_ERROR, impala::PrintId(), impala::query_id(), impala::ColumnType::ToHs2Type(), VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
|
virtual |
Definition at line 1105 of file impala-hs2-server.cc.
References HS2_RETURN_IF_ERROR, impala::query_id(), and impala::ImpalaServer::ScopedSessionState::WithSession().
|
private |
Gets the runtime profile string for a given query_id and writes it to the output stream. First searches for the query id in the map of in-flight queries. If no match is found there, the query log is searched. Returns OK if the profile was found, otherwise a Status object with an error message will be returned. The output stream will not be modified on error. If base64_encoded, outputs the base64 encoded profile output, otherwise the human readable string.
Definition at line 541 of file impala-server.cc.
References impala::Status::OK, impala::PrintId(), query_exec_state_map_, query_exec_state_map_lock_, query_log_index_, and query_log_lock_.
|
virtual |
Definition at line 794 of file impala-hs2-server.cc.
References VLOG_QUERY.
|
private |
Writes the session id, if found, for the given query to the output parameter. Returns false if no query with the given ID is found.
Definition at line 1712 of file impala-server.cc.
References query_exec_state_map_, and query_exec_state_map_lock_.
|
private |
Returns session state for given session_id. If not found, session_state will be NULL and an error status will be returned. If mark_active is true, also checks if the session is expired or closed and increments the session's reference counter if it is still alive.
Definition at line 965 of file impala-server.cc.
References impala::TimestampValue::DebugString(), impala::Status::OK, session_state_map_, and session_state_map_lock_.
Referenced by impala::ImpalaServer::ScopedSessionState::WithSession().
|
virtual |
Definition at line 812 of file impala-hs2-server.cc.
References VLOG_QUERY.
|
virtual |
Definition at line 830 of file impala-hs2-server.cc.
References VLOG_QUERY.
|
virtual |
Definition at line 758 of file impala-hs2-server.cc.
References VLOG_QUERY.
|
private |
Json callback for /hadoop-varz. Produces Json with a list, 'configs', of (key, value) pairs, one for each Hadoop configuration value.
Definition at line 108 of file impala-server-callbacks.cc.
References impala::exec_env_, impala::ExecEnv::frontend(), impala::Frontend::GetAllHadoopConfigs(), and impala::Status::ok().
|
private |
Webserver callback. Produces a list of inflight query IDs printed as text in 'contents'.
Definition at line 202 of file impala-server-callbacks.cc.
|
private |
Checks settings for audit event logging, including whether the output directory exists and is writeable, and initialises the first log file. Returns OK unless there is some problem preventing audit event log files from being written. If an error is returned, impalad startup will be aborted.
Definition at line 470 of file impala-server.cc.
References impala::AUDIT_EVENT_LOG_FILE_PREFIX, audit_event_logger_, audit_event_logger_flush_thread_, AuditEventLoggerFlushThread(), IsAuditEventLoggingEnabled(), impala::Status::OK, and RETURN_IF_ERROR.
Referenced by ImpalaServer().
|
private |
Initialize "default_configs_" to show the default values for ImpalaQueryOptions and "support_start_over/false" to indicate that Impala does not support start over in the fetch call.
Definition at line 1045 of file impala-server.cc.
References default_configs_, default_query_options_, impala::Status::GetDetail(), impala::Status::ok(), impala::ParseQueryOptions(), and impala::TQueryOptionsToMap().
Referenced by ImpalaServer().
|
private |
Checks settings for lineage logging, including whether the output directory exists and is writeable, and initialises the first log file. Returns OK unless there is some problem preventing lineage log files from being written. If an error is returned, impalad startup will be aborted.
Definition at line 378 of file impala-server.cc.
References IsLineageLoggingEnabled(), impala::LINEAGE_LOG_FILE_PREFIX, lineage_logger_, lineage_logger_flush_thread_, LineageLoggerFlushThread(), impala::Status::OK, and RETURN_IF_ERROR.
Referenced by ImpalaServer().
|
staticprivate |
Initializes a logging directory, creating the directory if it does not already exist. If there is any error creating the directory an error will be returned.
|
private |
Checks settings for profile logging, including whether the output directory exists and is writeable, and initialises the first log file. Returns OK unless there is some problem preventing profile log files from being written. If an error is returned, the constructor will disable profile logging.
Definition at line 524 of file impala-server.cc.
References LogFileFlushThread(), impala::Status::OK, profile_log_file_flush_thread_, impala::PROFILE_LOG_FILE_PREFIX, profile_logger_, and RETURN_IF_ERROR.
Referenced by ImpalaServer().
|
private |
Returns true if audit event logging is enabled, false otherwise.
Definition at line 466 of file impala-server.cc.
Referenced by InitAuditEventLogging(), and LogQueryEvents().
| bool impala::ImpalaServer::IsLineageLoggingEnabled | ( | ) |
Returns true if lineage logging is enabled, false otherwise.
Definition at line 374 of file impala-server.cc.
Referenced by InitLineageLogging(), Java_com_cloudera_impala_service_FeSupport_NativeGetStartupOptions(), and LogQueryEvents().
|
inline |
Returns true if Impala is offline (and not accepting queries), false otherwise.
Definition at line 239 of file impala-server.h.
References is_offline_, and is_offline_lock_.
Referenced by ExecuteInternal().
|
private |
Runs once every 5s to flush the lineage log file to disk.
Definition at line 626 of file impala-server.cc.
References impala::Status::GetDetail(), lineage_logger_, and impala::Status::ok().
Referenced by InitLineageLogging().
|
private |
Definition at line 391 of file impala-server.cc.
References audit_event_logger_, impala::ImpalaServer::QueryExecState::connected_user(), impala::TimestampValue::DebugString(), impala::ImpalaServer::QueryExecState::do_as_user(), impala::ImpalaServer::QueryExecState::effective_user(), impala::Status::GetDetail(), impala::Frontend::IsAuthorizationError(), impala::ImpalaServer::SessionState::network_address, impala::Status::ok(), impala::PrintId(), impala::PrintTCatalogObjectType(), impala::PrintTCatalogOpType(), impala::PrintTDdlType(), impala::PrintTStmtType(), impala::ImpalaServer::QueryExecState::query_id(), impala::ImpalaServer::QueryExecState::query_status(), impala::Redact(), impala::ImpalaServer::QueryExecState::session(), impala::ImpalaServer::QueryExecState::session_id(), impala::ImpalaServer::QueryExecState::sql_stmt(), impala::ImpalaServer::QueryExecState::start_time(), and impala::UnixMillis().
Referenced by LogQueryEvents().
|
private |
Runs once every 5s to flush the profile log file to disk.
Definition at line 604 of file impala-server.cc.
References profile_logger_.
Referenced by InitProfileLogging().
|
private |
Definition at line 348 of file impala-server.cc.
References lineage_logger_, and impala::Status::OK.
Referenced by LogQueryEvents().
|
private |
Log audit and column lineage events.
Definition at line 483 of file impala-server.cc.
References impala::ImpalaServer::QueryExecState::catalog_op_type(), impala::ImpalaServer::QueryExecState::exec_request(), impala::ImpalaServer::QueryExecState::fetched_rows(), IsAuditEventLoggingEnabled(), impala::Frontend::IsAuthorizationError(), IsLineageLoggingEnabled(), LogAuditRecord(), LogLineageRecord(), impala::Status::ok(), impala::ImpalaServer::QueryExecState::query_status(), and impala::ImpalaServer::QueryExecState::stmt_type().
Referenced by UnregisterQuery().
|
inlineprivate |
Decrement the session's reference counter and mark last_accessed_ms so that state expiration can proceed.
Definition at line 905 of file impala-server.h.
References impala::UnixMillis().
Referenced by impala::ImpalaServer::ScopedSessionState::~ScopedSessionState().
| void impala::ImpalaServer::MembershipCallback | ( | const StatestoreSubscriber::TopicDeltaMap & | incoming_topic_deltas, |
| std::vector< TTopicDelta > * | subscriber_topic_updates | ||
| ) |
Called when a membership update is received from the statestore. Looks for active nodes that have failed, and cancels any queries running on them.
Definition at line 1317 of file impala-server.cc.
References cancellation_thread_pool_, impala::ClientCache< T >::CloseConnections(), impala::DeserializeThriftMsg(), exec_env_, impala::SimpleScheduler::IMPALA_MEMBERSHIP_TOPIC, impala::ExecEnv::impalad_client_cache(), known_backends_, impala::MAX_CANCELLATION_QUEUE_SIZE, impala::Status::ok(), impala::query_id(), query_locations_, and query_locations_lock_.
Referenced by ImpalaServer().
|
virtual |
ImpalaHiveServer2Service rpcs: HiveServer2 API (implemented in impala-hs2-server.cc) TODO: Migrate existing extra ImpalaServer RPCs to ImpalaHiveServer2Service.
Definition at line 559 of file impala-hs2-server.cc.
References HS2_RETURN_IF_ERROR, MAX_SUPPORTED_HS2_VERSION, impala::SetQueryOption(), impala::TQueryOptionsToMap(), impala::UnixMillis(), uuid_generator_, impala::UUIDToTUniqueId(), and VLOG_QUERY.
|
virtual |
Pings the Impala service and gets the server version string.
Definition at line 494 of file impala-beeswax-server.cc.
References impala::GetVersionString(), RAISE_IF_ERROR, VLOG_RPC, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
static |
Generates a unique id for this query and sets it in the given query context. Prepares the given query context by populating fields required for evaluating certain expressions, such as now(), pid(), etc. Should be called before handing the query context to the frontend for query compilation.
Definition at line 769 of file impala-server.cc.
References impala_udf::DebugString(), impala::TimestampValue::LocalTime(), impala::MakeNetworkAddress(), and impala::UUIDToTUniqueId().
Referenced by Execute().
|
private |
Processes a CatalogUpdateResult returned from the CatalogServer and ensures the update has been applied to the local impalad's catalog cache. If wait_for_all_subscribers is true, this function will also wait until all catalog topic subscribers have processed the update. Called from QueryExecState after executing any statement that modifies the catalog. If wait_for_all_subscribers is false AND if the TCatalogUpdateResult contains TCatalogObject(s) to add and/or remove, this function will update the local cache by directly calling UpdateCatalog() with the TCatalogObject results. Otherwise this function will wait until the local impalad's catalog cache has been updated from a statestore heartbeat that includes this catalog update's catalog version. If wait_for_all_subscribers is true, this function also wait all other catalog topic subscribers to process this update by checking the current min_subscriber_topic_version included in each state store heartbeat.
Definition at line 1262 of file impala-server.cc.
References impala::ImpalaServer::CatalogUpdateVersionInfo::catalog_service_id, impala::ImpalaServer::CatalogUpdateVersionInfo::catalog_topic_version, catalog_update_info_, impala::ImpalaServer::CatalogUpdateVersionInfo::catalog_version, catalog_version_lock_, catalog_version_update_cv_, exec_env_, impala::ExecEnv::frontend(), impala::Status::GetDetail(), min_subscriber_catalog_topic_version_, impala::Status::OK, impala::Status::ok(), RETURN_IF_ERROR, impala::Frontend::UpdateCatalogCache(), and VLOG_QUERY.
|
virtual |
ImpalaService rpcs: Beeswax API (implemented in impala-beeswax-server.cc)
Definition at line 171 of file impala-beeswax-server.cc.
References impala::Status::GetDetail(), impala::Status::ok(), RAISE_IF_ERROR, VLOG_QUERY, and impala::ImpalaServer::ScopedSessionState::WithSession().
|
inlineprivate |
Definition at line 551 of file impala-beeswax-server.cc.
References impala::ParseId().
|
private |
Webserver callback. Upon return, 'document' will contain the query profile as a base64 encoded object in 'contents'.
Definition at line 183 of file impala-server-callbacks.cc.
References impala::Status::GetDetail(), impala::Status::ok(), and ParseQueryId().
|
private |
Json callback for /query_profile. Expects query_id as an argument, produces Json with 'profile' set to the profile string, and 'query_id' set to the query ID.
Definition at line 159 of file impala-server-callbacks.cc.
References impala::Status::GetDetail(), impala::Status::ok(), and ParseQueryId().
|
private |
Helper method to render a single QueryStateRecord as a Json object Used by QueryStateUrlCallback().
Definition at line 214 of file impala-server-callbacks.cc.
References impala::TimestampValue::DebugString(), impala::ImpalaServer::QueryStateRecord::default_db, impala::ImpalaServer::QueryStateRecord::effective_user, impala::ImpalaServer::QueryStateRecord::end_time, impala::ImpalaServer::QueryStateRecord::event_sequence, impala::ImpalaServer::QueryStateRecord::has_coord, impala::TimestampValue::HasDate(), impala::ImpalaServer::QueryStateRecord::id, impala::ImpalaServer::QueryStateRecord::num_complete_fragments, impala::ImpalaServer::QueryStateRecord::num_rows_fetched, impala::PrintId(), impala::progress(), impala::query_id(), impala::ImpalaServer::QueryStateRecord::query_state, impala::RedactCopy(), impala::ImpalaServer::QueryStateRecord::start_time, impala::ImpalaServer::QueryStateRecord::stmt, impala::ImpalaServer::QueryStateRecord::stmt_type, impala::TimestampValue::ToSubsecondUnixTime(), and impala::ImpalaServer::QueryStateRecord::total_fragments.
|
private |
Webserver callback. Returns two sorted lists of queries, one in-flight and one completed, as well as a list of active backends and their plan-fragment count. "in_flight_queries": [], "num_in_flight_queries": 0, "completed_queries": [ { "effective_user": "henry", "default_db": "default", "stmt": "select sleep(10000)", "stmt_type": "QUERY", "start_time": "2014-08-07 18:37:47.923614000", "end_time": "2014-08-07 18:37:58.146494000", "progress": "0 / 0 (0%)", "state": "FINISHED", "rows_fetched": 1, "query_id": "7c459a59fb8cefe3:8b7042d55bf19887" } ], "completed_log_size": 25, "query_locations": [ { "location": "henry-impala:22000", "count": 0 } ]
Definition at line 276 of file impala-server-callbacks.cc.
|
private |
Webserver callback. Produces a Json structure with query summary information. Example: { "summary": <....>, "plan": <....>, "stmt": "select count(*) from functional.alltypes" "id": <...>, "state": "FINISHED" } If include_plan_json is true, 'plan_json' will be set to a JSON representation of the query plan. If include_summary is true, 'summary' will be a text rendering of the query summary.
Definition at line 565 of file impala-server-callbacks.cc.
References impala::Status::GetDetail(), impala::Status::ok(), ParseQueryId(), PlanToJson(), PrintExecSummary(), impala::PrintId(), impala::query_id(), and impala::RedactCopy().
|
private |
Beeswax private methods.
Helper functions to translate between Beeswax and Impala structs
Definition at line 512 of file impala-beeswax-server.cc.
References impala::OK, impala::ParseQueryOptions(), RETURN_IF_ERROR, and VLOG_QUERY.
|
private |
Helper function to raise BeeswaxException.
Definition at line 556 of file impala-beeswax-server.cc.
|
private |
Registers the query exec state with query_exec_state_map_ using the globally unique query_id and add the query id to session state's open query list. The caller must have checked out the session state.
Definition at line 783 of file impala-server.cc.
References impala::Status::OK, impala::PrintId(), query_exec_state_map_, query_exec_state_map_lock_, and impala::query_id().
Referenced by ExecuteInternal().
|
private |
Registers all the per-Impalad webserver callbacks.
Definition at line 42 of file impala-server-callbacks.cc.
References impala::Webserver::RegisterUrlCallback().
Referenced by ImpalaServer().
|
virtual |
Definition at line 1139 of file impala-hs2-server.cc.
| void impala::ImpalaServer::ReportExecStatus | ( | TReportExecStatusResult & | return_val, |
| const TReportExecStatusParams & | params | ||
| ) |
ImpalaService common extensions (implemented in impala-server.cc) ImpalaInternalService rpcs
Definition at line 990 of file impala-server.cc.
References GetQueryExecState(), impala::PrintId(), VLOG_FILE, and VLOG_QUERY.
|
virtual |
Performs a full catalog metadata reset, invalidating all table and database metadata.
Definition at line 504 of file impala-beeswax-server.cc.
|
virtual |
Resets the specified table's catalog metadata, forcing a reload on the next access. Returns an error if the table or database was not found in the catalog.
Definition at line 508 of file impala-beeswax-server.cc.
|
private |
Json callback for /sessions, which prints a table of active client sessions. "sessions": [ { "type": "BEESWAX", "num_queries": 0, "user": "", "delegated_user": "", "session_id": "6242f69b02e4d609:ac84df1fbb0e16a3", "network_address": "127.0.0.1:46484", "default_database": "default", "start_time": "2014-08-07 22:50:49", "last_accessed": "2014-08-07 22:50:49", "expired": false, "closed": false, "ref_count": 0 } ], "num_sessions": 1
Definition at line 328 of file impala-server-callbacks.cc.
References impala::TimestampValue::DebugString(), impala::PrintId(), and impala::PrintTSessionType().
|
private |
Set is_offline_ to the argument's value.
Definition at line 1737 of file impala-server.cc.
References impala::ImpaladMetrics::IMPALA_SERVER_READY, is_offline_, and is_offline_lock_.
Referenced by DetectNmFailures().
|
private |
Adds the query to the set of in-flight queries for the session. The query remains in-flight until the query is unregistered. Until a query is in-flight, an attempt to cancel or close the query by the user will return an error status. If the session is closed before a query is in-flight, then the query cancellation is deferred until after the issuing path has completed initializing the query. Once a query is in-flight, it can be cancelled/closed asynchronously by the user (e.g. via an RPC) and the session close path can close (cancel and unregister) it. The query must have already been registered using RegisterQuery(). The caller must have checked out the session state.
Definition at line 807 of file impala-server.cc.
References impala::Status::OK, impala::PrettyPrinter::Print(), impala::PrintId(), queries_by_timestamp_, query_expiration_lock_, impala::query_id(), impala::UnixMillis(), and VLOG_QUERY.
|
private |
Definition at line 522 of file impala-hs2-server.cc.
References impala::IMPALA_RESULT_CACHING_OPT, impala::OK, impala::ParseId(), RETURN_IF_ERROR, impala::SetQueryOption(), and VLOG_QUERY.
|
inlinestaticprivate |
Helper functions to translate between HiveServer2 and Impala structs.
Returns !ok() if handle.guid.size() or handle.secret.size() != 16
Definition at line 23 of file impala-server.inline.h.
References impala::Status::OK.
| void impala::ImpalaServer::TransmitData | ( | TTransmitDataResult & | return_val, |
| const TTransmitDataParams & | params | ||
| ) |
Definition at line 1018 of file impala-server.cc.
References impala::DataStreamMgr::AddData(), impala::DataStreamMgr::CloseSender(), exec_env_, impala::Status::ok(), impala::Status::SetTStatus(), impala::ExecEnv::stream_mgr(), and VLOG_ROW.
|
inlineprivate |
Definition at line 544 of file impala-beeswax-server.cc.
References impala::PrintId().
|
inlinestaticprivate |
Definition at line 40 of file impala-server.inline.h.
Referenced by impala::ChildQuery::ExecAndFetch().
|
private |
Unregister the query by cancelling it, removing exec_state from query_exec_state_map_, and removing the query id from session state's in-flight query list. If check_inflight is true, then return an error if the query is not yet in-flight. Otherwise, proceed even if the query isn't yet in-flight (for cleaning up after an error on the query issuing path).
Definition at line 838 of file impala-server.cc.
References ArchiveQuery(), CancelInternal(), impala::exec_summary(), LogQueryEvents(), impala::Status::OK, impala::PrintExecSummary(), query_exec_state_map_, query_exec_state_map_lock_, impala::query_id(), query_locations_, query_locations_lock_, RETURN_IF_ERROR, and VLOG_QUERY.
Referenced by CancelFromThreadPool(), CloseSessionInternal(), and Execute().
|
private |
Updates the number of databases / tables metrics from the FE catalog.
Definition at line 893 of file impala-server.cc.
References impala::ImpaladMetrics::CATALOG_NUM_DBS, impala::ImpaladMetrics::CATALOG_NUM_TABLES, exec_env_, impala::ExecEnv::frontend(), impala::Frontend::GetDbNames(), impala::Frontend::GetTableNames(), impala::Status::OK, and RETURN_IF_ERROR.
Referenced by CatalogUpdateCallback(), ExecuteInternal(), and ImpalaServer().
|
friend |
Definition at line 248 of file impala-server.h.
Referenced by impala::ImpalaServer::QueryExecState::ExecDdlRequest().
|
friend |
For access to GetSessionState() / MarkSessionInactive()
Definition at line 873 of file impala-server.h.
|
staticprivate |
Ascii output precision for double/float.
Definition at line 301 of file impala-server.h.
|
private |
Logger for writing audit events, one per line with the format: "<current timestamp>" : { JSON object }
Definition at line 737 of file impala-server.h.
Referenced by AuditEventLoggerFlushThread(), InitAuditEventLogging(), and LogAuditRecord().
|
private |
If audit event logging is enabled, wakes once every 5s to flush audit events to disk.
Definition at line 747 of file impala-server.h.
Referenced by InitAuditEventLogging().
|
private |
Definition at line 974 of file impala-server.h.
Referenced by AuthorizeProxyUser(), and ImpalaServer().
|
private |
Thread pool to process cancellation requests that come from failed Impala demons to avoid blocking the statestore callback.
Definition at line 757 of file impala-server.h.
Referenced by ExpireQueries(), ExpireSessions(), ImpalaServer(), and MembershipCallback().
|
private |
The version information from the last successfull call to UpdateCatalog().
Definition at line 962 of file impala-server.h.
Referenced by CatalogUpdateCallback(), and ProcessCatalogUpdateResult().
|
private |
Lock for catalog_update_version_info_, min_subscriber_catalog_topic_version_, and catalog_version_update_cv_
Definition at line 941 of file impala-server.h.
Referenced by CatalogUpdateCallback(), and ProcessCatalogUpdateResult().
|
private |
Variable to signal when the catalog version has been modified.
Definition at line 944 of file impala-server.h.
Referenced by CatalogUpdateCallback(), and ProcessCatalogUpdateResult().
|
private |
Definition at line 894 of file impala-server.h.
Referenced by ConnectionEnd(), and ConnectionStart().
|
private |
Protects connection_to_sessions_map_. May be taken before session_state_map_lock_.
Definition at line 886 of file impala-server.h.
Referenced by ConnectionEnd(), and ConnectionStart().
|
private |
Definition at line 771 of file impala-server.h.
Referenced by InitializeConfigVariables().
|
private |
Default query options in the form of TQueryOptions and beeswax::ConfigVariable.
Definition at line 770 of file impala-server.h.
Referenced by ConnectionStart(), and InitializeConfigVariables().
|
private |
global, per-server state
Definition at line 753 of file impala-server.h.
Referenced by CatalogUpdateCallback(), impala::ImpalaServer::QueryExecState::Done(), impala::ImpalaServer::QueryExecState::Exec(), impala::ImpalaServer::QueryExecState::ExecDdlRequest(), impala::ImpalaServer::QueryExecState::ExecLocalCatalogOp(), impala::ImpalaServer::QueryExecState::ExecQueryOrDmlRequest(), ExecuteInternal(), ImpalaServer(), MembershipCallback(), ProcessCatalogUpdateResult(), TransmitData(), impala::ImpalaServer::QueryExecState::UpdateCatalog(), UpdateCatalogMetrics(), and impala::ImpalaServer::QueryExecState::UpdateTableAndColumnStats().
|
private |
True if Impala server is offline, false otherwise.
Definition at line 1024 of file impala-server.h.
Referenced by ImpalaServer(), IsOffline(), and SetOffline().
|
private |
Protects is_offline_.
Definition at line 1021 of file impala-server.h.
Referenced by IsOffline(), and SetOffline().
|
private |
Definition at line 931 of file impala-server.h.
Referenced by MembershipCallback().
|
private |
Logger for writing lineage events, one per line with the format: { JSON object }
Definition at line 741 of file impala-server.h.
Referenced by InitLineageLogging(), LineageLoggerFlushThread(), and LogLineageRecord().
|
private |
If lineage logging is enabled, wakes once every 5s to flush lineage events to disk.
Definition at line 750 of file impala-server.h.
Referenced by InitLineageLogging().
|
private |
The current minimum topic version processed across all subscribers of the catalog topic. Used to determine when other nodes have successfully processed a catalog update. Updated with each catalog topic heartbeat from the statestore.
Definition at line 967 of file impala-server.h.
Referenced by CatalogUpdateCallback(), and ProcessCatalogUpdateResult().
|
private |
Container thread for DetectNmFailures().
Definition at line 1018 of file impala-server.h.
Referenced by ImpalaServer().
|
private |
If profile logging is enabled, wakes once every 5s to flush query profiles to disk.
Definition at line 744 of file impala-server.h.
Referenced by InitProfileLogging().
|
private |
Logger for writing encoded query profiles, one per line with the following format: <ms-since-epoch> <query-id> <thrift query="" profile="" url="" encoded="" and="" gzipped>="">
Definition at line 733 of file impala-server.h.
Referenced by ArchiveQuery(), InitProfileLogging(), and LogFileFlushThread().
|
private |
Definition at line 1012 of file impala-server.h.
Referenced by ExpireQueries(), and SetQueryInflight().
|
private |
Definition at line 766 of file impala-server.h.
Referenced by GetQueryExecState(), GetRuntimeProfileStr(), GetSessionIdForQuery(), RegisterQuery(), and UnregisterQuery().
|
private |
Definition at line 767 of file impala-server.h.
Referenced by GetQueryExecState(), GetRuntimeProfileStr(), GetSessionIdForQuery(), RegisterQuery(), and UnregisterQuery().
|
private |
Guards queries_by_timestamp_. Must not be acquired before a session state lock.
Definition at line 977 of file impala-server.h.
Referenced by ExpireQueries(), and SetQueryInflight().
|
private |
Container for a thread that runs ExpireQueries() if FLAGS_idle_query_timeout is set.
Definition at line 1015 of file impala-server.h.
Referenced by ImpalaServer().
|
private |
Definition at line 919 of file impala-server.h.
Referenced by ExecuteInternal(), MembershipCallback(), and UnregisterQuery().
|
private |
protects query_locations_. Must always be taken after query_exec_state_map_lock_ if both are required.
Definition at line 914 of file impala-server.h.
Referenced by ExecuteInternal(), MembershipCallback(), and UnregisterQuery().
|
private |
Definition at line 725 of file impala-server.h.
Referenced by ArchiveQuery().
|
private |
Definition at line 729 of file impala-server.h.
Referenced by ArchiveQuery(), GetExecSummary(), and GetRuntimeProfileStr().
|
private |
Guards query_log_ and query_log_index_.
Definition at line 721 of file impala-server.h.
Referenced by ArchiveQuery(), GetExecSummary(), and GetRuntimeProfileStr().
|
private |
Definition at line 883 of file impala-server.h.
Referenced by CloseSessionInternal(), ConnectionStart(), ExpireSessions(), and GetSessionState().
|
private |
Protects session_state_map_. Should be taken before any query exec-state locks, including query_exec_state_map_lock_. Should be taken before individual session-state locks.
Definition at line 878 of file impala-server.h.
Referenced by CloseSessionInternal(), ConnectionStart(), ExpireSessions(), and GetSessionState().
|
private |
Thread that runs ExpireSessions if FLAGS_idle_session_timeout > 0.
Definition at line 760 of file impala-server.h.
Referenced by ImpalaServer().
|
staticprivate |
Definition at line 297 of file impala-server.h.
|
staticprivate |
Definition at line 298 of file impala-server.h.
|
staticprivate |
Relevant ODBC SQL State code; for more info, goto http://msdn.microsoft.com/en-us/library/ms714687.aspx
Definition at line 292 of file impala-server.h.
|
private |
Generate unique session id for HiveServer2 session.
Definition at line 934 of file impala-server.h.
|
private |
Lock to protect uuid_generator.
Definition at line 937 of file impala-server.h.
1.8.6 
