com.cloudera.impala.catalog.HdfsTable Class Reference

Inheritance diagram for com.cloudera.impala.catalog.HdfsTable:

Collaboration diagram for com.cloudera.impala.catalog.HdfsTable:

Classes
class	FileBlocksInfo
class	FsKey

Public Member Functions
Map< String, List < FileDescriptor > >	getFileDescMap ()
boolean	spansMultipleFileSystems ()
	HdfsTable (TableId id, org.apache.hadoop.hive.metastore.api.Table msTbl, Db db, String name, String owner)
TCatalogObjectType	getCatalogObjectType ()
List< HdfsPartition >	getPartitions ()
boolean	isMarkedCached ()
HashMap< Long, HdfsPartition >	getPartitionMap ()
HashSet< Long >	getNullPartitionIds (int i)
HashSet< Long >	getPartitionIds ()
TreeMap< LiteralExpr, HashSet < Long > >	getPartitionValueMap (int i)
String	getNullPartitionKeyValue ()
String	getNullColumnValue ()
String	getLocation ()
List< FieldSchema >	getFieldSchemas ()
List< FieldSchema >	getNonPartitionFieldSchemas ()
boolean	hasWriteAccess ()
String	getFirstLocationWithoutWriteAccess ()
HdfsPartition	getPartition (List< PartitionKeyValue > partitionSpec)
HdfsPartition	getPartitionFromThriftPartitionSpec (List< TPartitionKeyValue > partitionSpec)
HdfsPartition	createPartition (StorageDescriptor storageDescriptor, org.apache.hadoop.hive.metastore.api.Partition msPartition) throws CatalogException
void	addPartition (HdfsPartition partition)
HdfsPartition	dropPartition (List< TPartitionKeyValue > partitionSpec)
void	load (Table cachedEntry, HiveMetaStoreClient client, org.apache.hadoop.hive.metastore.api.Table msTbl) throws TableLoadingException
TTableDescriptor	toThriftDescriptor (Set< Long > referencedPartitions)
TTable	toThrift ()
long	getNumHdfsFiles ()
long	getTotalHdfsBytes ()
String	getHdfsBaseDir ()
boolean	isAvroTable ()
int	getNumNodes ()
ListMap< TNetworkAddress >	getHostIndex ()
HdfsFileFormat	getMajorityFormat ()
TResultSet	getTableStats ()
TResultSet	getFiles (List< TPartitionKeyValue > partitionSpec) throws CatalogException
void	addColumn (Column col)
void	clearColumns ()
void	updateLastDdlTime (long ddlTime)
void	validate () throws TableLoadingException
TCatalogObject	toTCatalogObject ()
Db	getDb ()
String	getName ()
String	getFullName ()
TableName	getTableName ()
String	getOwner ()
ArrayList< Column >	getColumns ()
List< String >	getColumnNames ()
String	getStorageHandlerClassName ()
ArrayList< Column >	getColumnsInHiveOrder ()
List< Column >	getNonClusteringColumns ()
Column	getColumn (String name)
org.apache.hadoop.hive.metastore.api.Table	getMetaStoreTable ()
int	getNumClusteringCols ()
TableId	getId ()
long	getNumRows ()
ArrayType	getType ()
long	getCatalogVersion ()
void	setCatalogVersion (long catalogVersion)
boolean	isLoaded ()

Static Public Member Functions
static String	getAvroSchema (List< Map< String, String >> schemaSearchLocations, String tableName) throws TableLoadingException
static Table	fromMetastoreTable (TableId id, Db db, org.apache.hadoop.hive.metastore.api.Table msTbl)
static Table	fromThrift (Db parentDb, TTable thriftTable) throws TableLoadingException

Protected Member Functions
List< String >	getColumnNamesWithHmsStats ()
void	loadFromThrift (TTable thriftTable) throws TableLoadingException
void	loadAllColumnStats (HiveMetaStoreClient client)
Type	parseColumnType (FieldSchema fs) throws TableLoadingException

Static Protected Member Functions
static long	getRowCount (Map< String, String > parameters)

Protected Attributes
String	hdfsBaseDir_
final org.apache.hadoop.hive.metastore.api.Table	msTable_
final TableId	id_
final Db	db_
final String	name_
final String	owner_
TTableDescriptor	tableDesc_
List< FieldSchema >	fields_
TAccessLevel	accessLevel_ = TAccessLevel.READ_WRITE
int	numClusteringCols_
long	numRows_ = -1
final ArrayType	type_ = new ArrayType(new StructType())
long	lastDdlTime_

Static Protected Attributes
static EnumSet< TableType >	SUPPORTED_TABLE_TYPES

Static Package Functions
	[static initializer]

Private Member Functions
void	loadBlockMetadata (FileSystem fs, FileStatus file, FileDescriptor fd, HdfsFileFormat fileFormat, Map< FsKey, FileBlocksInfo > perFsFileBlocks)
void	synthesizeBlockMetadata (FileSystem fs, FileDescriptor fd, HdfsFileFormat fileFormat)
void	loadDiskIds (Map< FsKey, FileBlocksInfo > perFsFileBlocks)
void	loadColumns (List< FieldSchema > fieldSchemas, HiveMetaStoreClient client) throws TableLoadingException
void	populatePartitionMd ()
void	resetPartitionMd ()
void	loadPartitions (List< org.apache.hadoop.hive.metastore.api.Partition > msPartitions, org.apache.hadoop.hive.metastore.api.Table msTbl, Map< String, List< FileDescriptor >> oldFileDescMap) throws IOException, CatalogException
TAccessLevel	getAvailableAccessLevel (FileSystem fs, Path location) throws IOException
HdfsPartition	createPartition (StorageDescriptor storageDescriptor, org.apache.hadoop.hive.metastore.api.Partition msPartition, Map< String, List< FileDescriptor >> oldFileDescMap, Map< FsKey, FileBlocksInfo > perFsFileBlocks) throws CatalogException
void	addPerFsFileBlocks (Map< FsKey, FileBlocksInfo > fsToBlocks, FileSystem fs, List< THdfsFileBlock > blocks, List< BlockLocation > locations)
void	updatePartitionMdAndColStats (HdfsPartition partition)
void	addDefaultPartition (StorageDescriptor storageDescriptor) throws CatalogException
THdfsTable	getTHdfsTable (boolean includeFileDesc, Set< Long > refPartitions)

Static Private Member Functions
static int	getDiskId (VolumeId hdfsVolumeId)

Private Attributes
String	nullColumnValue_
String	nullPartitionKeyValue_
String	avroSchema_ = null
boolean	isMarkedCached_ = false
final List< HdfsPartition >	partitions_
final ArrayList< TreeMap < LiteralExpr, HashSet< Long > > >	partitionValuesMap_
final ArrayList< HashSet< Long > >	nullPartitionIds_ = Lists.newArrayList()
final HashMap< Long, HdfsPartition >	partitionMap_ = Maps.newHashMap()
final HashSet< Long >	partitionIds_ = Sets.newHashSet()
boolean	hasPartitionMd_ = false
final ListMap< TNetworkAddress >	hostIndex_ = new ListMap<TNetworkAddress>()
final Map< String, List < FileDescriptor > >	fileDescMap_ = Maps.newHashMap()
long	numHdfsFiles_
long	totalHdfsBytes_
boolean	multipleFileSystems_ = false

Static Private Attributes
static final String	DEFAULT_NULL_COLUMN_VALUE = "\\N"
static final int	NUM_PARTITION_FETCH_RETRIES = 5
static final TNetworkAddress	REMOTE_NETWORK_ADDRESS
static final long	MIN_SYNTHETIC_BLOCK_SIZE = 1024 * 1024
static boolean	hasLoggedDiskIdFormatWarning_ = false
static final Logger	LOG = LoggerFactory.getLogger(HdfsTable.class)
static final Configuration	CONF = new Configuration()
static final boolean	SUPPORTS_VOLUME_ID

Detailed Description

Internal representation of table-related metadata of a file-resident table on a Hadoop filesystem. The table data can be accessed through libHDFS (which is more of an abstraction over Hadoop's FileSystem class rather than DFS specifically). A partitioned table can even span multiple filesystems.

Owned by Catalog instance. The partition keys constitute the clustering columns.

This class is not thread-safe due to the static counter variable inside HdfsPartition. Also not thread safe because of possibility of concurrent modifications to the list of partitions in methods addPartition and dropPartition.

Definition at line 104 of file HdfsTable.java.

Constructor & Destructor Documentation

com.cloudera.impala.catalog.HdfsTable.HdfsTable ( TableId  id, org.apache.hadoop.hive.metastore.api.Table  msTbl, Db  db, String  name, String  owner  )

inline

Definition at line 421 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.load().

Member Function Documentation

com.cloudera.impala.catalog.HdfsTable.[static initializer] ( )

inlinestaticpackage

void com.cloudera.impala.catalog.Table.addColumn ( Column  col )

inlineinherited

Definition at line 114 of file Table.java.

References com.cloudera.impala.catalog.ArrayType.getItemType(), and com.cloudera.impala.catalog.Table.type_.

Referenced by com.cloudera.impala.catalog.View.load(), com.cloudera.impala.catalog.HBaseTable.load(), com.cloudera.impala.catalog.DataSourceTable.loadColumns(), and com.cloudera.impala.catalog.HdfsTable.loadColumns().

void com.cloudera.impala.catalog.HdfsTable.addDefaultPartition ( StorageDescriptor  storageDescriptor ) throws CatalogException

inlineprivate

Definition at line 977 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.loadPartitions().

void com.cloudera.impala.catalog.HdfsTable.addPartition ( HdfsPartition  partition )

inline

Adds the partition to the HdfsTable.

Note: This method is not thread safe because it modifies the list of partitions and the HdfsTable's partition metadata.

Definition at line 892 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.partitions_, com.cloudera.impala.catalog.HdfsTable.totalHdfsBytes_, and com.cloudera.impala.catalog.HdfsTable.updatePartitionMdAndColStats().

Referenced by com.cloudera.impala.catalog.HdfsTable.loadPartitions().

void com.cloudera.impala.catalog.HdfsTable.addPerFsFileBlocks ( Map< FsKey, FileBlocksInfo >  fsToBlocks, FileSystem  fs, List< THdfsFileBlock >  blocks, List< BlockLocation >  locations  )

inlineprivate

Add the given THdfsFileBlocks and BlockLocations to the FileBlockInfo for the given filesystem.

Definition at line 875 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.loadBlockMetadata().

void com.cloudera.impala.catalog.Table.clearColumns ( )

inlineinherited

Definition at line 121 of file Table.java.

References com.cloudera.impala.catalog.ArrayType.getItemType(), and com.cloudera.impala.catalog.Table.type_.

Referenced by com.cloudera.impala.catalog.HBaseTable.load().

HdfsPartition com.cloudera.impala.catalog.HdfsTable.createPartition ( StorageDescriptor  storageDescriptor, org.apache.hadoop.hive.metastore.api.Partition  msPartition  ) throws CatalogException

inline

Creates a new HdfsPartition object to be added to HdfsTable's partition list. Partitions may be empty, or may not even exist in the filesystem (a partition's location may have been changed to a new path that is about to be created by an INSERT). Also loads the block metadata for this partition. Returns new partition if successful or null if none was added. Separated from addPartition to reduce the number of operations done while holding the lock on HdfsTable.

Exceptions

CatalogException

if the supplied storage descriptor contains metadata that Impala can't understand.

Definition at line 735 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.fileDescMap_, and com.cloudera.impala.catalog.HdfsTable.loadDiskIds().

Referenced by com.cloudera.impala.catalog.HdfsTable.loadPartitions().

HdfsPartition com.cloudera.impala.catalog.HdfsTable.createPartition ( StorageDescriptor  storageDescriptor, org.apache.hadoop.hive.metastore.api.Partition  msPartition, Map< String, List< FileDescriptor >>  oldFileDescMap, Map< FsKey, FileBlocksInfo >  perFsFileBlocks  ) throws CatalogException

inlineprivate

Creates a new HdfsPartition object to be added to the internal partition list. Populates with file format information and file locations. Partitions may be empty, or may not even exist on the filesystem (a partition's location may have been changed to a new path that is about to be created by an INSERT). For unchanged files (indicated by unchanged mtime), reuses the FileDescriptor from the oldFileDescMap. The one exception is if the partition is marked as cached in which case the block metadata cannot be reused. Otherwise, creates a new FileDescriptor for each modified or new file and adds it to newFileDescMap. Both old and newFileDescMap are Maps of parent directory (partition location) to list of files (FileDescriptors) under that directory. Returns new partition if successful or null if none was added. Separated from addPartition to reduce the number of operations done while holding the lock on the hdfs table.

Exceptions

CatalogException

if the supplied storage descriptor contains metadata that Impala can't understand.

Definition at line 764 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.CONF, com.cloudera.impala.catalog.HdfsCompression.fromFileName(), com.cloudera.impala.catalog.HdfsTable.getAvailableAccessLevel(), com.cloudera.impala.catalog.Table.getColumns(), com.cloudera.impala.catalog.HdfsStorageDescriptor.getFileFormat(), com.cloudera.impala.catalog.HdfsTable.getLocation(), com.cloudera.impala.catalog.Table.getName(), com.cloudera.impala.catalog.HdfsTable.isMarkedCached(), com.cloudera.impala.catalog.HdfsTable.isMarkedCached_, com.cloudera.impala.catalog.HdfsTable.loadBlockMetadata(), com.cloudera.impala.catalog.HdfsCompression.LZO_INDEX, com.cloudera.impala.catalog.HdfsTable.multipleFileSystems_, com.cloudera.impala.catalog.HdfsTable.nullPartitionKeyValue_, and com.cloudera.impala.catalog.HdfsTable.numHdfsFiles_.

HdfsPartition com.cloudera.impala.catalog.HdfsTable.dropPartition ( List< TPartitionKeyValue >  partitionSpec )

inline

Drops the partition having the given partition spec from HdfsTable. Cleans up its metadata from all the mappings used to speed up partition pruning/lookup. Also updates partition column statistics. Given partitionSpec must match exactly one partition. Returns the HdfsPartition that was dropped. If the partition does not exist, returns null.

Note: This method is not thread safe because it modifies the list of partitions and the HdfsTable's partition metadata.

Definition at line 942 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.getColumns(), com.cloudera.impala.catalog.HdfsTable.getPartitionFromThriftPartitionSpec(), com.cloudera.impala.catalog.HdfsTable.nullPartitionIds_, com.cloudera.impala.catalog.Table.numClusteringCols_, com.cloudera.impala.catalog.HdfsTable.partitions_, and com.cloudera.impala.catalog.HdfsTable.totalHdfsBytes_.

static Table com.cloudera.impala.catalog.Table.fromMetastoreTable ( TableId  id, Db  db, org.apache.hadoop.hive.metastore.api.Table  msTbl  )

inlinestaticinherited

Creates a table of the appropriate type based on the given hive.metastore.api.Table object.

Definition at line 207 of file Table.java.

References com.cloudera.impala.catalog.DataSourceTable.isDataSourceTable(), com.cloudera.impala.catalog.HBaseTable.isHBaseTable(), and com.cloudera.impala.catalog.HdfsFileFormat.isHdfsFormatClass().

static Table com.cloudera.impala.catalog.Table.fromThrift ( Db  parentDb, TTable  thriftTable  ) throws TableLoadingException

inlinestaticinherited

Factory method that creates a new Table from its Thrift representation. Determines the type of table to create based on the Thrift table provided.

Definition at line 231 of file Table.java.

TAccessLevel com.cloudera.impala.catalog.HdfsTable.getAvailableAccessLevel ( FileSystem  fs, Path  location  ) throws IOException

inlineprivate

Gets the AccessLevel that is available for Impala for this table based on the permissions Impala has on the given path. If the path does not exist, recurses up the path until a existing parent directory is found, and inherit access permissions from that.

Definition at line 699 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), and com.cloudera.impala.catalog.HdfsTable.loadPartitions().

static String com.cloudera.impala.catalog.HdfsTable.getAvroSchema ( List< Map< String, String >>  schemaSearchLocations, String  tableName  ) throws TableLoadingException

inlinestatic

Gets an Avro table's JSON schema from the list of given table property search locations. The schema may be specified as a string literal or provided as a Hadoop FileSystem or http URL that points to the schema. This function does not perform any validation on the returned string (e.g., it may not be a valid schema). If the schema was found to be specified as a SCHEMA_URL, this function will attempt to download the schema from the given URL. Throws a TableLoadingException if no schema is found or if there was any error extracting the schema.

Definition at line 1177 of file HdfsTable.java.

References com.cloudera.impala.common.FileSystemUtil.isPathReachable(), and path().

TCatalogObjectType com.cloudera.impala.catalog.HdfsTable.getCatalogObjectType ( )

inline

Implements com.cloudera.impala.catalog.CatalogObject.

Definition at line 428 of file HdfsTable.java.

long com.cloudera.impala.catalog.Table.getCatalogVersion ( )

inlineinherited

Implements com.cloudera.impala.catalog.CatalogObject.

Definition at line 408 of file Table.java.

References com.cloudera.impala.catalog.Table.catalogVersion_.

Referenced by com.cloudera.impala.catalog.CatalogServiceCatalog.dropPartition(), com.cloudera.impala.catalog.CatalogServiceCatalog.getCatalogObjects(), com.cloudera.impala.catalog.ImpaladCatalog.removeTable(), com.cloudera.impala.catalog.CatalogServiceCatalog.replaceTableIfUnchanged(), and com.cloudera.impala.catalog.Table.toTCatalogObject().

Column com.cloudera.impala.catalog.Table.getColumn ( String  name )

inlineinherited

Case-insensitive lookup.

Definition at line 392 of file Table.java.

Referenced by com.cloudera.impala.catalog.Table.loadAllColumnStats(), and com.cloudera.impala.catalog.CatalogTest.testColStatsColTypeMismatch().

List<String> com.cloudera.impala.catalog.Table.getColumnNames ( )

inlineinherited

Returns a list of the column names ordered by position.

Definition at line 354 of file Table.java.

References com.cloudera.impala.catalog.Table.colsByPos_.

Referenced by com.cloudera.impala.analysis.ColumnLineageGraph.addTargetColumnLabels(), com.cloudera.impala.catalog.HdfsTable.getTHdfsTable(), com.cloudera.impala.catalog.DataSourceTable.toThriftDescriptor(), com.cloudera.impala.catalog.HBaseTable.toThriftDescriptor(), and com.cloudera.impala.catalog.HdfsTable.toThriftDescriptor().

List<String> com.cloudera.impala.catalog.HdfsTable.getColumnNamesWithHmsStats ( )

inlineprotected

Definition at line 1233 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.getColumns(), and com.cloudera.impala.catalog.Table.numClusteringCols_.

ArrayList<Column> com.cloudera.impala.catalog.Table.getColumns ( )

inlineinherited

Definition at line 349 of file Table.java.

References com.cloudera.impala.catalog.Table.colsByPos_.

Referenced by com.cloudera.impala.service.CatalogOpExecutor.alterTableOrViewRename(), com.cloudera.impala.analysis.InsertStmt.checkColumnCoverage(), com.cloudera.impala.catalog.HdfsTable.createPartition(), com.cloudera.impala.catalog.HdfsTable.dropPartition(), com.cloudera.impala.catalog.HdfsTable.getColumnNamesWithHmsStats(), com.cloudera.impala.catalog.HBaseTable.getColumnsInHiveOrder(), com.cloudera.impala.catalog.HdfsTable.getTableStats(), com.cloudera.impala.catalog.HBaseTable.getTHBaseTable(), com.cloudera.impala.analysis.InsertStmt.prepareExpressions(), com.cloudera.impala.catalog.HdfsTable.resetPartitionMd(), com.cloudera.impala.catalog.DataSourceTable.toThriftDescriptor(), com.cloudera.impala.catalog.HBaseTable.toThriftDescriptor(), com.cloudera.impala.catalog.HdfsTable.toThriftDescriptor(), and com.cloudera.impala.catalog.HdfsTable.updatePartitionMdAndColStats().

ArrayList<Column> com.cloudera.impala.catalog.Table.getColumnsInHiveOrder ( )

inlineinherited

Returns the list of all columns, but with partition columns at the end of the list rather than the beginning. This is equivalent to the order in which Hive enumerates columns.

Definition at line 373 of file Table.java.

References com.cloudera.impala.catalog.Table.colsByPos_, com.cloudera.impala.catalog.Table.getNonClusteringColumns(), and com.cloudera.impala.catalog.Table.numClusteringCols_.

Referenced by com.cloudera.impala.service.DescribeResultFactory.describeTableMinimal(), com.cloudera.impala.service.Frontend.getColumnStats(), and com.cloudera.impala.analysis.InsertStmt.prepareExpressions().

Db com.cloudera.impala.catalog.Table.getDb ( )

inlineinherited

Definition at line 341 of file Table.java.

References com.cloudera.impala.catalog.Table.db_.

Referenced by com.cloudera.impala.service.CatalogOpExecutor.alterTableUpdateStats(), com.cloudera.impala.service.CatalogOpExecutor.createHiveColStats(), com.cloudera.impala.analysis.Analyzer.getTable(), com.cloudera.impala.analysis.InsertStmt.setTargetTable(), com.cloudera.impala.analysis.ShowStatsStmt.toThrift(), com.cloudera.impala.analysis.ComputeStatsStmt.toThrift(), and com.cloudera.impala.service.CatalogOpExecutor.updateCatalog().

static int com.cloudera.impala.catalog.HdfsTable.getDiskId ( VolumeId  hdfsVolumeId )

inlinestaticprivate

Returns a disk id (0-based) index from the Hdfs VolumeId object. There is currently no public API to get at the volume id. We'll have to get it by accessing the internals.

Definition at line 243 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.hasLoggedDiskIdFormatWarning_.

Referenced by com.cloudera.impala.catalog.HdfsTable.loadDiskIds().

List<FieldSchema> com.cloudera.impala.catalog.HdfsTable.getFieldSchemas ( )

inline

Definition at line 453 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.fields_.

Map<String, List<FileDescriptor> > com.cloudera.impala.catalog.HdfsTable.getFileDescMap ( )

inline

Definition at line 265 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.fileDescMap_.

TResultSet com.cloudera.impala.catalog.HdfsTable.getFiles ( List< TPartitionKeyValue >  partitionSpec ) throws CatalogException

inline

Returns files info for the given dbname/tableName and partition spec. Returns files info for all partitions if partition spec is null.

Definition at line 1456 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.getPartitionFromThriftPartitionSpec(), com.cloudera.impala.catalog.HdfsTable.partitions_, com.cloudera.impala.catalog.Type.STRING, and com.cloudera.impala.catalog.ScalarType.toThrift().

String com.cloudera.impala.catalog.HdfsTable.getFirstLocationWithoutWriteAccess ( )

inline

Returns the first location (HDFS path) that Impala does not have WRITE access to, or an null if none is found. For an unpartitioned table, this just checks the hdfsBaseDir. For a partitioned table it checks all partition directories.

Definition at line 470 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.accessLevel_, com.cloudera.impala.catalog.Table.getMetaStoreTable(), com.cloudera.impala.catalog.HdfsTable.hdfsBaseDir_, com.cloudera.impala.util.TAccessLevelUtil.impliesWriteAccess(), and com.cloudera.impala.catalog.HdfsTable.partitions_.

Referenced by com.cloudera.impala.analysis.InsertStmt.setTargetTable().

String com.cloudera.impala.catalog.Table.getFullName ( )

inlineinherited

Definition at line 343 of file Table.java.

References com.cloudera.impala.catalog.Table.db_, com.cloudera.impala.catalog.Db.getName(), and com.cloudera.impala.catalog.Table.name_.

Referenced by com.cloudera.impala.analysis.ShowPartitionsStmt.analyze(), com.cloudera.impala.analysis.ShowStatsStmt.analyze(), com.cloudera.impala.analysis.ShowFilesStmt.analyze(), com.cloudera.impala.analysis.AlterTableStmt.analyze(), com.cloudera.impala.analysis.TableRef.analyze(), com.cloudera.impala.analysis.InsertStmt.checkColumnCoverage(), com.cloudera.impala.planner.DataSourceScanNode.getNodeExplainString(), com.cloudera.impala.planner.HBaseScanNode.getNodeExplainString(), com.cloudera.impala.analysis.Analyzer.getTable(), com.cloudera.impala.catalog.HdfsTable.load(), com.cloudera.impala.catalog.Table.loadAllColumnStats(), com.cloudera.impala.catalog.HdfsTable.loadColumns(), com.cloudera.impala.catalog.HdfsTable.loadDiskIds(), and com.cloudera.impala.analysis.InsertStmt.setTargetTable().

String com.cloudera.impala.catalog.HdfsTable.getHdfsBaseDir ( )

inline

Definition at line 1319 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.hdfsBaseDir_.

ListMap<TNetworkAddress> com.cloudera.impala.catalog.HdfsTable.getHostIndex ( )

inline

Get the index of hosts that store replicas of blocks of this table.

Definition at line 1328 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.hostIndex_.

TableId com.cloudera.impala.catalog.Table.getId ( )

inlineinherited

Definition at line 403 of file Table.java.

References com.cloudera.impala.catalog.Table.id_.

Referenced by com.cloudera.impala.planner.HBaseTableSink.toThrift(), and com.cloudera.impala.planner.HdfsTableSink.toThrift().

String com.cloudera.impala.catalog.HdfsTable.getLocation ( )

inline

Definition at line 451 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), com.cloudera.impala.catalog.HdfsPartition.HdfsPartition(), com.cloudera.impala.catalog.HdfsTable.loadPartitions(), and com.cloudera.impala.analysis.InsertStmt.setTargetTable().

HdfsFileFormat com.cloudera.impala.catalog.HdfsTable.getMajorityFormat ( )

inline

Returns the file format that the majority of partitions are stored in.

Definition at line 1333 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.partitions_.

Referenced by com.cloudera.impala.planner.PlanFragment.computeCanAddSlotFilters(), and com.cloudera.impala.planner.HdfsScanNode.computeCosts().

org.apache.hadoop.hive.metastore.api.Table com.cloudera.impala.catalog.Table.getMetaStoreTable ( )

inlineinherited

Returns the metastore.api.Table object this Table was created from. Returns null if the derived Table object was not created from a metastore Table (ex. InlineViews).

Definition at line 398 of file Table.java.

References com.cloudera.impala.catalog.Table.msTable_.

Referenced by com.cloudera.impala.catalog.HdfsTable.getFirstLocationWithoutWriteAccess(), com.cloudera.impala.catalog.HdfsTable.getPartitionFromThriftPartitionSpec(), com.cloudera.impala.catalog.HdfsTable.getTableStats(), com.cloudera.impala.catalog.View.init(), com.cloudera.impala.catalog.DataSourceTable.load(), com.cloudera.impala.catalog.HBaseTable.load(), com.cloudera.impala.catalog.HdfsTable.load(), com.cloudera.impala.catalog.HBaseTable.loadFromThrift(), com.cloudera.impala.catalog.HdfsTable.loadFromThrift(), and com.cloudera.impala.catalog.Table.toThrift().

String com.cloudera.impala.catalog.Table.getName ( )

inlineinherited

Implements com.cloudera.impala.catalog.CatalogObject.

Definition at line 342 of file Table.java.

References com.cloudera.impala.catalog.Table.name_.

Referenced by com.cloudera.impala.service.CatalogOpExecutor.alterTableUpdateStats(), com.cloudera.impala.catalog.CatalogTest.checkTableCols(), com.cloudera.impala.service.CatalogOpExecutor.createHiveColStats(), com.cloudera.impala.catalog.HdfsTable.createPartition(), com.cloudera.impala.catalog.HdfsPartition.fromThrift(), com.cloudera.impala.analysis.Analyzer.getTable(), com.cloudera.impala.analysis.InlineViewRef.InlineViewRef(), com.cloudera.impala.catalog.Table.parseColumnType(), com.cloudera.impala.analysis.InsertStmt.setTargetTable(), com.cloudera.impala.analysis.ShowStatsStmt.toThrift(), com.cloudera.impala.analysis.ComputeStatsStmt.toThrift(), com.cloudera.impala.service.CatalogOpExecutor.updateCatalog(), and com.cloudera.impala.analysis.WithClause.WithClause().

List<Column> com.cloudera.impala.catalog.Table.getNonClusteringColumns ( )

inlineinherited

Returns the list of all columns excluding any partition columns.

Definition at line 385 of file Table.java.

References com.cloudera.impala.catalog.Table.numClusteringCols_.

Referenced by com.cloudera.impala.analysis.ComputeStatsStmt.analyze(), and com.cloudera.impala.catalog.Table.getColumnsInHiveOrder().

List<FieldSchema> com.cloudera.impala.catalog.HdfsTable.getNonPartitionFieldSchemas ( )

inline

Definition at line 454 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.getNumClusteringCols().

String com.cloudera.impala.catalog.HdfsTable.getNullColumnValue ( )

inline

Definition at line 446 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.nullColumnValue_.

HashSet<Long> com.cloudera.impala.catalog.HdfsTable.getNullPartitionIds ( int  i )

inline

Definition at line 435 of file HdfsTable.java.

String com.cloudera.impala.catalog.HdfsTable.getNullPartitionKeyValue ( )

inline

Returns the value Hive is configured to use for NULL partition key values. Set during load.

Definition at line 445 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.nullPartitionKeyValue_.

Referenced by com.cloudera.impala.catalog.HdfsTable.getPartition(), and com.cloudera.impala.catalog.HdfsTable.getPartitionFromThriftPartitionSpec().

int com.cloudera.impala.catalog.Table.getNumClusteringCols ( )

inlineinherited

Definition at line 402 of file Table.java.

References com.cloudera.impala.catalog.Table.numClusteringCols_.

Referenced by com.cloudera.impala.analysis.ShowPartitionsStmt.analyze(), com.cloudera.impala.analysis.ComputeStatsStmt.analyze(), com.cloudera.impala.catalog.CatalogTest.checkTableCols(), com.cloudera.impala.service.CatalogOpExecutor.dropTableOrView(), com.cloudera.impala.analysis.ComputeStatsStmt.getBaseColumnStatsQuerySelectList(), com.cloudera.impala.planner.HdfsScanNode.getNodeExplainString(), com.cloudera.impala.catalog.HdfsTable.getNonPartitionFieldSchemas(), com.cloudera.impala.planner.HdfsScanNode.prunePartitions(), and com.cloudera.impala.service.CatalogOpExecutor.updateCatalog().

long com.cloudera.impala.catalog.HdfsTable.getNumHdfsFiles ( )

inline

Definition at line 1317 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.numHdfsFiles_.

int com.cloudera.impala.catalog.HdfsTable.getNumNodes ( )

inline

Definition at line 1323 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.loadDiskIds().

long com.cloudera.impala.catalog.Table.getNumRows ( )

inlineinherited

Definition at line 404 of file Table.java.

References com.cloudera.impala.catalog.Table.numRows_.

Referenced by com.cloudera.impala.analysis.IsNullPredicate.analyze().

String com.cloudera.impala.catalog.Table.getOwner ( )

inlineinherited

Definition at line 348 of file Table.java.

References com.cloudera.impala.catalog.Table.owner_.

HdfsPartition com.cloudera.impala.catalog.HdfsTable.getPartition ( List< PartitionKeyValue >  partitionSpec )

inline

Gets the HdfsPartition matching the given partition spec. Returns null if no match was found.

Definition at line 491 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.getNullPartitionKeyValue(), and com.cloudera.impala.catalog.HdfsTable.getPartitionFromThriftPartitionSpec().

HdfsPartition com.cloudera.impala.catalog.HdfsTable.getPartitionFromThriftPartitionSpec ( List< TPartitionKeyValue >  partitionSpec )

inline

Gets the HdfsPartition matching the Thrift version of the partition spec. Returns null if no match was found.

Definition at line 505 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.getMetaStoreTable(), com.cloudera.impala.catalog.HdfsTable.getNullPartitionKeyValue(), and com.cloudera.impala.catalog.HdfsTable.getPartitions().

Referenced by com.cloudera.impala.catalog.HdfsTable.dropPartition(), com.cloudera.impala.catalog.HdfsTable.getFiles(), and com.cloudera.impala.catalog.HdfsTable.getPartition().

HashSet<Long> com.cloudera.impala.catalog.HdfsTable.getPartitionIds ( )

inline

Definition at line 436 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.partitionIds_.

HashMap<Long, HdfsPartition> com.cloudera.impala.catalog.HdfsTable.getPartitionMap ( )

inline

Definition at line 434 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.partitionMap_.

List<HdfsPartition> com.cloudera.impala.catalog.HdfsTable.getPartitions ( )

inline

Definition at line 429 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.partitions_.

Referenced by com.cloudera.impala.service.CatalogOpExecutor.alterTableSetCached(), com.cloudera.impala.service.CatalogOpExecutor.alterTableUpdateStats(), com.cloudera.impala.analysis.ComputeStatsStmt.analyze(), com.cloudera.impala.planner.HdfsScanNode.computeStats(), com.cloudera.impala.service.CatalogOpExecutor.dropTableOrView(), com.cloudera.impala.service.CatalogOpExecutor.dropTableStats(), com.cloudera.impala.catalog.HdfsTable.getPartitionFromThriftPartitionSpec(), com.cloudera.impala.testutil.BlockIdGenerator.main(), and com.cloudera.impala.catalog.CatalogObjectToFromThriftTest.TestPartitionedTable().

TreeMap<LiteralExpr, HashSet<Long> > com.cloudera.impala.catalog.HdfsTable.getPartitionValueMap ( int  i )

inline

Definition at line 437 of file HdfsTable.java.

static long com.cloudera.impala.catalog.Table.getRowCount ( Map< String, String >  parameters )

inlinestaticprotectedinherited

Returns the value of the ROW_COUNT constant, or -1 if not found.

Definition at line 191 of file Table.java.

Referenced by com.cloudera.impala.catalog.DataSourceTable.load(), com.cloudera.impala.catalog.HBaseTable.load(), com.cloudera.impala.catalog.HdfsTable.load(), and com.cloudera.impala.catalog.HdfsTable.loadPartitions().

String com.cloudera.impala.catalog.Table.getStorageHandlerClassName ( )

inlineinherited

Subclasses should override this if they provide a storage handler class. Currently only HBase tables need to provide a storage handler.

Definition at line 366 of file Table.java.

TableName com.cloudera.impala.catalog.Table.getTableName ( )

inlineinherited

Definition at line 344 of file Table.java.

References com.cloudera.impala.catalog.Table.db_, com.cloudera.impala.catalog.Db.getName(), and com.cloudera.impala.catalog.Table.name_.

Referenced by com.cloudera.impala.analysis.ComputeStatsStmt.analyze(), com.cloudera.impala.service.CatalogOpExecutor.dropStats(), and com.cloudera.impala.analysis.InlineViewRef.InlineViewRef().

TResultSet com.cloudera.impala.catalog.HdfsTable.getTableStats ( )

inline

Returns statistics on this table as a tabular result set. Used for the SHOW TABLE STATS statement. The schema of the returned TResultSet is set inside this method.

Definition at line 1363 of file HdfsTable.java.

References com.cloudera.impala.catalog.Type.BIGINT, com.cloudera.impala.catalog.Table.getColumns(), com.cloudera.impala.catalog.Table.getMetaStoreTable(), com.cloudera.impala.catalog.Column.getName(), com.cloudera.impala.catalog.Table.numClusteringCols_, com.cloudera.impala.catalog.HdfsTable.numHdfsFiles_, com.cloudera.impala.catalog.Table.numRows_, com.cloudera.impala.catalog.HdfsTable.partitions_, com.cloudera.impala.catalog.Type.STRING, com.cloudera.impala.catalog.HdfsTable.totalHdfsBytes_, and com.cloudera.impala.catalog.ScalarType.toThrift().

THdfsTable com.cloudera.impala.catalog.HdfsTable.getTHdfsTable ( boolean  includeFileDesc, Set< Long >  refPartitions  )

inlineprivate

Create a THdfsTable corresponding to this HdfsTable. If includeFileDesc is true, then then all partitions and THdfsFileDescs of each partition should be included. Otherwise, don't include any THdfsFileDescs, and include only those partitions in the refPartitions set (the backend doesn't need metadata for unreferenced partitions).

Definition at line 1295 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.avroSchema_, com.cloudera.impala.catalog.Table.getColumnNames(), com.cloudera.impala.catalog.HdfsTable.hdfsBaseDir_, com.cloudera.impala.catalog.HdfsTable.multipleFileSystems_, com.cloudera.impala.catalog.HdfsTable.nullColumnValue_, com.cloudera.impala.catalog.HdfsTable.nullPartitionKeyValue_, and com.cloudera.impala.catalog.HdfsTable.partitions_.

Referenced by com.cloudera.impala.catalog.HdfsTable.toThrift(), and com.cloudera.impala.catalog.HdfsTable.toThriftDescriptor().

long com.cloudera.impala.catalog.HdfsTable.getTotalHdfsBytes ( )

inline

Definition at line 1318 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.totalHdfsBytes_.

ArrayType com.cloudera.impala.catalog.Table.getType ( )

inlineinherited

Definition at line 405 of file Table.java.

References com.cloudera.impala.catalog.Table.type_.

Referenced by com.cloudera.impala.catalog.HdfsTable.load().

boolean com.cloudera.impala.catalog.HdfsTable.hasWriteAccess ( )

inline

Definition at line 461 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.accessLevel_.

Referenced by com.cloudera.impala.analysis.InsertStmt.setTargetTable().

boolean com.cloudera.impala.catalog.HdfsTable.isAvroTable ( )

inline

Definition at line 1320 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.avroSchema_.

Referenced by com.cloudera.impala.analysis.ComputeStatsStmt.analyze().

boolean com.cloudera.impala.catalog.Table.isLoaded ( )

inlineinherited

Implements com.cloudera.impala.catalog.CatalogObject.

Definition at line 416 of file Table.java.

Referenced by com.cloudera.impala.service.CatalogOpExecutor.getExistingTable(), com.cloudera.impala.service.Frontend.getMissingTbls(), com.cloudera.impala.catalog.CatalogServiceCatalog.getOrLoadTable(), com.cloudera.impala.testutil.ImpaladTestCatalog.getTable(), com.cloudera.impala.catalog.ImpaladCatalog.getTable(), and com.cloudera.impala.analysis.Analyzer.getTable().

boolean com.cloudera.impala.catalog.HdfsTable.isMarkedCached ( )

inline

Definition at line 432 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.isMarkedCached_.

Referenced by com.cloudera.impala.analysis.AlterTableSetLocationStmt.analyze(), com.cloudera.impala.catalog.HdfsTable.createPartition(), and com.cloudera.impala.service.CatalogOpExecutor.dropTableOrView().

void com.cloudera.impala.catalog.HdfsTable.load ( Table  cachedEntry, HiveMetaStoreClient  client, org.apache.hadoop.hive.metastore.api.Table  msTbl  ) throws TableLoadingException

inline

Load the table metadata and reuse metadata to speed up metadata loading. If the lastDdlTime has not been changed, that means the Hive metastore metadata has not been changed. Reuses the old Hive partition metadata from cachedEntry. To speed up Hdfs metadata loading, if a file's mtime has not been changed, reuses the old file block metadata from old value.

There are several cases where the cachedEntry might be reused incorrectly:

1. an ALTER TABLE ADD PARTITION or dynamic partition insert is executed through Hive. This does not update the lastDdlTime.
2. Hdfs rebalancer is executed. This changes the block locations but won't update the mtime (file modification time). If any of these occurs, user has to execute "invalidate metadata" to invalidate the metadata cache of the table to trigger a fresh load.

Definition at line 1003 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsFileFormat.AVRO, com.cloudera.impala.catalog.HdfsTable.avroSchema_, com.cloudera.impala.catalog.Table.db_, com.cloudera.impala.catalog.HdfsTable.DEFAULT_NULL_COLUMN_VALUE, com.cloudera.impala.catalog.HdfsFileFormat.fromJavaClassName(), com.cloudera.impala.catalog.Table.getFullName(), com.cloudera.impala.catalog.Table.getMetaStoreTable(), com.cloudera.impala.catalog.Db.getName(), com.cloudera.impala.catalog.Table.getRowCount(), com.cloudera.impala.catalog.Table.getType(), com.cloudera.impala.catalog.HdfsTable.HdfsTable(), com.cloudera.impala.catalog.Table.lastDdlTime_, com.cloudera.impala.catalog.HdfsTable.loadColumns(), com.cloudera.impala.catalog.HdfsTable.loadPartitions(), com.cloudera.impala.catalog.Table.name_, com.cloudera.impala.catalog.HdfsTable.nullColumnValue_, com.cloudera.impala.catalog.HdfsTable.nullPartitionKeyValue_, com.cloudera.impala.catalog.HdfsTable.NUM_PARTITION_FETCH_RETRIES, com.cloudera.impala.catalog.Table.numClusteringCols_, com.cloudera.impala.catalog.HdfsTable.numHdfsFiles_, com.cloudera.impala.catalog.Table.numRows_, com.cloudera.impala.catalog.HdfsTable.partitions_, and com.cloudera.impala.catalog.HdfsTable.totalHdfsBytes_.

void com.cloudera.impala.catalog.Table.loadAllColumnStats ( HiveMetaStoreClient  client )

inlineprotectedinherited

Loads column statistics for all columns in this table from the Hive metastore. Any errors are logged and ignored, since the absence of column stats is not critical to the correctness of the system.

Definition at line 155 of file Table.java.

References com.cloudera.impala.catalog.Table.getColumn(), com.cloudera.impala.catalog.Table.getColumnNamesWithHmsStats(), com.cloudera.impala.catalog.Table.getFullName(), com.cloudera.impala.catalog.Column.getType(), com.cloudera.impala.catalog.ColumnStats.isSupportedColType(), com.cloudera.impala.catalog.Table.name_, and com.cloudera.impala.catalog.Column.updateStats().

Referenced by com.cloudera.impala.catalog.HBaseTable.load(), and com.cloudera.impala.catalog.HdfsTable.loadColumns().

void com.cloudera.impala.catalog.HdfsTable.loadBlockMetadata ( FileSystem  fs, FileStatus  file, FileDescriptor  fd, HdfsFileFormat  fileFormat, Map< FsKey, FileBlocksInfo >  perFsFileBlocks  )

inlineprivate

Queries the filesystem to load the file block metadata (e.g. DFS blocks) for the given file. Adds the newly created block metadata and block location to the perFsFileBlocks, so that the disk IDs for each block can be retrieved with one call to DFS.

Definition at line 275 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.addPerFsFileBlocks(), com.cloudera.impala.catalog.HdfsPartition.FileDescriptor.getFileBlocks(), com.cloudera.impala.common.FileSystemUtil.hasGetFileBlockLocations(), com.cloudera.impala.catalog.HdfsTable.hostIndex_, com.cloudera.impala.catalog.Table.name_, and com.cloudera.impala.catalog.HdfsTable.synthesizeBlockMetadata().

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition().

void com.cloudera.impala.catalog.HdfsTable.loadColumns ( List< FieldSchema >  fieldSchemas, HiveMetaStoreClient  client  ) throws TableLoadingException

inlineprivate

Create columns corresponding to fieldSchemas, including column statistics. Throws a TableLoadingException if the metadata is incompatible with what we support.

Definition at line 567 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.addColumn(), com.cloudera.impala.catalog.Table.fields_, com.cloudera.impala.catalog.Table.getFullName(), com.cloudera.impala.catalog.Table.loadAllColumnStats(), com.cloudera.impala.catalog.Table.numClusteringCols_, com.cloudera.impala.catalog.Table.parseColumnType(), and com.cloudera.impala.catalog.Type.supportsTablePartitioning().

Referenced by com.cloudera.impala.catalog.HdfsTable.load().

void com.cloudera.impala.catalog.HdfsTable.loadDiskIds ( Map< FsKey, FileBlocksInfo >  perFsFileBlocks )

inlineprivate

Populates disk/volume ID metadata inside the newly created THdfsFileBlocks. perFsFileBlocks maps from each filesystem to a FileBLocksInfo. The first list contains the newly created THdfsFileBlocks and the second contains the corresponding BlockLocations.

Definition at line 363 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.getDiskId(), com.cloudera.impala.catalog.Table.getFullName(), com.cloudera.impala.catalog.HdfsTable.getNumNodes(), com.cloudera.impala.catalog.HdfsTable.FileBlocksInfo.locations, and com.cloudera.impala.catalog.HdfsTable.SUPPORTS_VOLUME_ID.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), and com.cloudera.impala.catalog.HdfsTable.loadPartitions().

void com.cloudera.impala.catalog.HdfsTable.loadFromThrift ( TTable  thriftTable ) throws TableLoadingException

inlineprotected

Definition at line 1243 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.avroSchema_, com.cloudera.impala.catalog.Table.getMetaStoreTable(), com.cloudera.impala.catalog.HdfsTable.hdfsBaseDir_, com.cloudera.impala.catalog.HdfsTable.isMarkedCached_, com.cloudera.impala.catalog.HdfsTable.multipleFileSystems_, com.cloudera.impala.catalog.HdfsTable.nullColumnValue_, com.cloudera.impala.catalog.HdfsTable.nullPartitionKeyValue_, com.cloudera.impala.catalog.HdfsTable.numHdfsFiles_, com.cloudera.impala.catalog.HdfsTable.populatePartitionMd(), com.cloudera.impala.catalog.HdfsTable.resetPartitionMd(), and com.cloudera.impala.catalog.HdfsTable.totalHdfsBytes_.

void com.cloudera.impala.catalog.HdfsTable.loadPartitions ( List< org.apache.hadoop.hive.metastore.api.Partition >  msPartitions, org.apache.hadoop.hive.metastore.api.Table  msTbl, Map< String, List< FileDescriptor >>  oldFileDescMap  ) throws IOException, CatalogException

inlineprivate

Create HdfsPartition objects corresponding to 'partitions'.

If there are no partitions in the Hive metadata, a single partition is added with no partition keys.

For files that have not been changed, reuses file descriptors from oldFileDescMap.

TODO: If any partition fails to load, the entire table will fail to load. Instead, we should consider skipping partitions that cannot be loaded and raise a warning whenever the table is accessed.

Definition at line 629 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.accessLevel_, com.cloudera.impala.catalog.HdfsTable.addDefaultPartition(), com.cloudera.impala.catalog.HdfsTable.addPartition(), com.cloudera.impala.catalog.HdfsTable.CONF, com.cloudera.impala.catalog.HdfsTable.createPartition(), com.cloudera.impala.catalog.HdfsPartition.getAccessLevel(), com.cloudera.impala.catalog.HdfsTable.getAvailableAccessLevel(), com.cloudera.impala.catalog.HdfsTable.getLocation(), com.cloudera.impala.catalog.Table.getRowCount(), com.cloudera.impala.catalog.HdfsTable.hdfsBaseDir_, com.cloudera.impala.util.TAccessLevelUtil.impliesWriteAccess(), com.cloudera.impala.catalog.HdfsTable.isMarkedCached_, com.cloudera.impala.catalog.HdfsTable.loadDiskIds(), and com.cloudera.impala.catalog.HdfsTable.resetPartitionMd().

Referenced by com.cloudera.impala.catalog.HdfsTable.load().

Type com.cloudera.impala.catalog.Table.parseColumnType ( FieldSchema  fs ) throws TableLoadingException

inlineprotectedinherited

Gets the ColumnType from the given FieldSchema by using Impala's SqlParser. Throws a TableLoadingException if the FieldSchema could not be parsed. The type can either be:

• Supported by Impala, in which case the type is returned.
• A type Impala understands but is not yet implemented (e.g. date), the type is returned but type.IsSupported() returns false.
• A type Impala can't understand at all, and a TableLoadingException is thrown.

Definition at line 331 of file Table.java.

References com.cloudera.impala.catalog.Table.getName().

Referenced by com.cloudera.impala.catalog.View.load(), com.cloudera.impala.catalog.HBaseTable.load(), com.cloudera.impala.catalog.DataSourceTable.loadColumns(), com.cloudera.impala.catalog.HdfsTable.loadColumns(), and com.cloudera.impala.catalog.HBaseTable.supportsBinaryEncoding().

void com.cloudera.impala.catalog.HdfsTable.populatePartitionMd ( )

inlineprivate

Populate the partition metadata of an HdfsTable.

Definition at line 591 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.hasPartitionMd_, com.cloudera.impala.catalog.HdfsTable.partitions_, and com.cloudera.impala.catalog.HdfsTable.updatePartitionMdAndColStats().

Referenced by com.cloudera.impala.catalog.HdfsTable.loadFromThrift().

void com.cloudera.impala.catalog.HdfsTable.resetPartitionMd ( )

inlineprivate

Clear the partition metadata of an HdfsTable including column stats.

Definition at line 602 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.getColumns(), com.cloudera.impala.catalog.HdfsTable.hasPartitionMd_, and com.cloudera.impala.catalog.Table.numClusteringCols_.

Referenced by com.cloudera.impala.catalog.HdfsTable.loadFromThrift(), and com.cloudera.impala.catalog.HdfsTable.loadPartitions().

void com.cloudera.impala.catalog.Table.setCatalogVersion ( long  catalogVersion )

inlineinherited

Implements com.cloudera.impala.catalog.CatalogObject.

Definition at line 411 of file Table.java.

References com.cloudera.impala.catalog.Table.catalogVersion_.

boolean com.cloudera.impala.catalog.HdfsTable.spansMultipleFileSystems ( )

inline

Definition at line 267 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.multipleFileSystems_.

Referenced by com.cloudera.impala.analysis.InsertStmt.setTargetTable().

void com.cloudera.impala.catalog.HdfsTable.synthesizeBlockMetadata ( FileSystem  fs, FileDescriptor  fd, HdfsFileFormat  fileFormat  )

inlineprivate

For filesystems that don't override getFileBlockLocations, synthesize file blocks by manually splitting the file range into fixed-size blocks. That way, scan ranges can be derived from file blocks as usual. All synthesized blocks are given an invalid network address so that the scheduler will treat them as remote.

Definition at line 335 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsCompression.fromFileName(), com.cloudera.impala.catalog.HdfsPartition.FileDescriptor.getFileName(), com.cloudera.impala.catalog.HdfsTable.hostIndex_, com.cloudera.impala.catalog.HdfsFileFormat.isSplittable(), com.cloudera.impala.catalog.HdfsTable.MIN_SYNTHETIC_BLOCK_SIZE, and com.cloudera.impala.catalog.HdfsTable.REMOTE_NETWORK_ADDRESS.

Referenced by com.cloudera.impala.catalog.HdfsTable.loadBlockMetadata().

TCatalogObject com.cloudera.impala.catalog.Table.toTCatalogObject ( )

inlineinherited

Definition at line 314 of file Table.java.

References com.cloudera.impala.catalog.Table.getCatalogObjectType(), com.cloudera.impala.catalog.Table.getCatalogVersion(), and com.cloudera.impala.catalog.Table.toThrift().

TTable com.cloudera.impala.catalog.HdfsTable.toThrift ( )

inline

Definition at line 1280 of file HdfsTable.java.

References com.cloudera.impala.catalog.HdfsTable.getTHdfsTable().

TTableDescriptor com.cloudera.impala.catalog.HdfsTable.toThriftDescriptor ( Set< Long >  referencedPartitions )

inline

Definition at line 1269 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.db_, com.cloudera.impala.catalog.Table.getColumnNames(), com.cloudera.impala.catalog.Table.getColumns(), com.cloudera.impala.catalog.Db.getName(), com.cloudera.impala.catalog.HdfsTable.getTHdfsTable(), com.cloudera.impala.catalog.Table.id_, com.cloudera.impala.catalog.Table.name_, and com.cloudera.impala.catalog.Table.numClusteringCols_.

void com.cloudera.impala.catalog.Table.updateLastDdlTime ( long  ddlTime )

inlineinherited

Updates the lastDdlTime for this Table, if the new value is greater than the existing value. Does nothing if the new value is less than or equal to the existing value.

Definition at line 132 of file Table.java.

References com.cloudera.impala.catalog.Table.lastDdlTime_.

void com.cloudera.impala.catalog.HdfsTable.updatePartitionMdAndColStats ( HdfsPartition  partition )

inlineprivate

Updates the HdfsTable's partition metadata, i.e. adds the id to the HdfsTable and populates structures used for speeding up partition pruning/lookup. Also updates column stats.

Definition at line 904 of file HdfsTable.java.

References com.cloudera.impala.catalog.Table.getColumns(), com.cloudera.impala.catalog.HdfsPartition.getId(), com.cloudera.impala.catalog.HdfsPartition.getPartitionValues(), com.cloudera.impala.catalog.HdfsTable.nullPartitionIds_, and com.cloudera.impala.catalog.Table.numClusteringCols_.

Referenced by com.cloudera.impala.catalog.HdfsTable.addPartition(), and com.cloudera.impala.catalog.HdfsTable.populatePartitionMd().

void com.cloudera.impala.catalog.Table.validate ( ) throws TableLoadingException

inlineinherited

Checks preconditions for this table to function as expected. Currently only checks that all entries in colsByName_ use lower case keys.

Definition at line 279 of file Table.java.

References com.cloudera.impala.catalog.Table.colsByName_.

Member Data Documentation

TAccessLevel com.cloudera.impala.catalog.Table.accessLevel_ = TAccessLevel.READ_WRITE

protectedinherited

Definition at line 66 of file Table.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getFirstLocationWithoutWriteAccess(), com.cloudera.impala.catalog.HdfsTable.hasWriteAccess(), com.cloudera.impala.catalog.Table.loadFromThrift(), com.cloudera.impala.catalog.HdfsTable.loadPartitions(), and com.cloudera.impala.catalog.Table.toThrift().

String com.cloudera.impala.catalog.HdfsTable.avroSchema_ = null

private

Definition at line 126 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getTHdfsTable(), com.cloudera.impala.catalog.HdfsTable.isAvroTable(), com.cloudera.impala.catalog.HdfsTable.load(), and com.cloudera.impala.catalog.HdfsTable.loadFromThrift().

final Configuration com.cloudera.impala.catalog.HdfsTable.CONF = new Configuration()

staticprivate

Definition at line 188 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), and com.cloudera.impala.catalog.HdfsTable.loadPartitions().

final Db com.cloudera.impala.catalog.Table.db_

protectedinherited

Definition at line 61 of file Table.java.

Referenced by com.cloudera.impala.catalog.Table.getDb(), com.cloudera.impala.catalog.Table.getFullName(), com.cloudera.impala.catalog.Table.getTableName(), com.cloudera.impala.catalog.View.init(), com.cloudera.impala.catalog.HdfsTable.load(), com.cloudera.impala.catalog.Table.Table(), com.cloudera.impala.catalog.IncompleteTable.toThrift(), com.cloudera.impala.catalog.Table.toThrift(), com.cloudera.impala.catalog.DataSourceTable.toThriftDescriptor(), com.cloudera.impala.catalog.HBaseTable.toThriftDescriptor(), and com.cloudera.impala.catalog.HdfsTable.toThriftDescriptor().

final String com.cloudera.impala.catalog.HdfsTable.DEFAULT_NULL_COLUMN_VALUE = "\\N"

staticprivate

Definition at line 106 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.load().

List<FieldSchema> com.cloudera.impala.catalog.Table.fields_

protectedinherited

Definition at line 65 of file Table.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getFieldSchemas(), com.cloudera.impala.catalog.HdfsTable.loadColumns(), and com.cloudera.impala.catalog.Table.loadFromThrift().

final Map<String, List<FileDescriptor> > com.cloudera.impala.catalog.HdfsTable.fileDescMap_ = Maps.newHashMap()

private

Definition at line 165 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), and com.cloudera.impala.catalog.HdfsTable.getFileDescMap().

boolean com.cloudera.impala.catalog.HdfsTable.hasLoggedDiskIdFormatWarning_ = false

staticprivate

Definition at line 132 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getDiskId().

boolean com.cloudera.impala.catalog.HdfsTable.hasPartitionMd_ = false

private

Definition at line 153 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.populatePartitionMd(), and com.cloudera.impala.catalog.HdfsTable.resetPartitionMd().

String com.cloudera.impala.catalog.HdfsTable.hdfsBaseDir_

protected

Definition at line 180 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getFirstLocationWithoutWriteAccess(), com.cloudera.impala.catalog.HdfsTable.getHdfsBaseDir(), com.cloudera.impala.catalog.HdfsTable.getTHdfsTable(), com.cloudera.impala.catalog.HdfsTable.loadFromThrift(), and com.cloudera.impala.catalog.HdfsTable.loadPartitions().

final ListMap<TNetworkAddress> com.cloudera.impala.catalog.HdfsTable.hostIndex_ = new ListMap<TNetworkAddress>()

private

Definition at line 161 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getHostIndex(), com.cloudera.impala.catalog.HdfsTable.loadBlockMetadata(), and com.cloudera.impala.catalog.HdfsTable.synthesizeBlockMetadata().

final TableId com.cloudera.impala.catalog.Table.id_

protectedinherited

Definition at line 60 of file Table.java.

Referenced by com.cloudera.impala.catalog.Table.getId(), com.cloudera.impala.catalog.Table.Table(), com.cloudera.impala.catalog.DataSourceTable.toThriftDescriptor(), com.cloudera.impala.catalog.HBaseTable.toThriftDescriptor(), and com.cloudera.impala.catalog.HdfsTable.toThriftDescriptor().

boolean com.cloudera.impala.catalog.HdfsTable.isMarkedCached_ = false

private

Definition at line 130 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), com.cloudera.impala.catalog.HdfsTable.isMarkedCached(), com.cloudera.impala.catalog.HdfsTable.loadFromThrift(), and com.cloudera.impala.catalog.HdfsTable.loadPartitions().

long com.cloudera.impala.catalog.Table.lastDdlTime_

protectedinherited

Definition at line 85 of file Table.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.load(), com.cloudera.impala.catalog.Table.Table(), and com.cloudera.impala.catalog.Table.updateLastDdlTime().

final Logger com.cloudera.impala.catalog.HdfsTable.LOG = LoggerFactory.getLogger(HdfsTable.class)

staticprivate

Definition at line 182 of file HdfsTable.java.

final long com.cloudera.impala.catalog.HdfsTable.MIN_SYNTHETIC_BLOCK_SIZE = 1024 * 1024

staticprivate

Definition at line 117 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.synthesizeBlockMetadata().

final org.apache.hadoop.hive.metastore.api.Table com.cloudera.impala.catalog.Table.msTable_

protectedinherited

Definition at line 58 of file Table.java.

Referenced by com.cloudera.impala.catalog.Table.getMetaStoreTable(), and com.cloudera.impala.catalog.Table.Table().

boolean com.cloudera.impala.catalog.HdfsTable.multipleFileSystems_ = false

private

Definition at line 174 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), com.cloudera.impala.catalog.HdfsTable.getTHdfsTable(), com.cloudera.impala.catalog.HdfsTable.loadFromThrift(), and com.cloudera.impala.catalog.HdfsTable.spansMultipleFileSystems().

final String com.cloudera.impala.catalog.Table.name_

protectedinherited

Definition at line 62 of file Table.java.

Referenced by com.cloudera.impala.catalog.Table.getFullName(), com.cloudera.impala.catalog.Table.getName(), com.cloudera.impala.catalog.DataSourceTable.getRequiredTableProperty(), com.cloudera.impala.catalog.Table.getTableName(), com.cloudera.impala.catalog.View.init(), com.cloudera.impala.catalog.View.load(), com.cloudera.impala.catalog.DataSourceTable.load(), com.cloudera.impala.catalog.HBaseTable.load(), com.cloudera.impala.catalog.HdfsTable.load(), com.cloudera.impala.catalog.Table.loadAllColumnStats(), com.cloudera.impala.catalog.HdfsTable.loadBlockMetadata(), com.cloudera.impala.catalog.HBaseTable.loadFromThrift(), com.cloudera.impala.catalog.Table.Table(), com.cloudera.impala.catalog.IncompleteTable.toThrift(), com.cloudera.impala.catalog.Table.toThrift(), com.cloudera.impala.catalog.DataSourceTable.toThriftDescriptor(), and com.cloudera.impala.catalog.HdfsTable.toThriftDescriptor().

String com.cloudera.impala.catalog.HdfsTable.nullColumnValue_

private

Definition at line 120 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getNullColumnValue(), com.cloudera.impala.catalog.HdfsTable.getTHdfsTable(), com.cloudera.impala.catalog.HdfsTable.load(), and com.cloudera.impala.catalog.HdfsTable.loadFromThrift().

final ArrayList<HashSet<Long> > com.cloudera.impala.catalog.HdfsTable.nullPartitionIds_ = Lists.newArrayList()

private

Definition at line 143 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.dropPartition(), and com.cloudera.impala.catalog.HdfsTable.updatePartitionMdAndColStats().

String com.cloudera.impala.catalog.HdfsTable.nullPartitionKeyValue_

private

Definition at line 123 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), com.cloudera.impala.catalog.HdfsTable.getNullPartitionKeyValue(), com.cloudera.impala.catalog.HdfsTable.getTHdfsTable(), com.cloudera.impala.catalog.HdfsTable.load(), and com.cloudera.impala.catalog.HdfsTable.loadFromThrift().

final int com.cloudera.impala.catalog.HdfsTable.NUM_PARTITION_FETCH_RETRIES = 5

staticprivate

Definition at line 109 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.load().

int com.cloudera.impala.catalog.Table.numClusteringCols_

protectedinherited

Definition at line 69 of file Table.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.dropPartition(), com.cloudera.impala.catalog.HdfsTable.getColumnNamesWithHmsStats(), com.cloudera.impala.catalog.Table.getColumnsInHiveOrder(), com.cloudera.impala.catalog.Table.getNonClusteringColumns(), com.cloudera.impala.catalog.Table.getNumClusteringCols(), com.cloudera.impala.catalog.HdfsTable.getTableStats(), com.cloudera.impala.catalog.View.load(), com.cloudera.impala.catalog.DataSourceTable.load(), com.cloudera.impala.catalog.HBaseTable.load(), com.cloudera.impala.catalog.HdfsTable.load(), com.cloudera.impala.catalog.HdfsTable.loadColumns(), com.cloudera.impala.catalog.Table.loadFromThrift(), com.cloudera.impala.catalog.HdfsTable.resetPartitionMd(), com.cloudera.impala.catalog.Table.toThrift(), com.cloudera.impala.catalog.DataSourceTable.toThriftDescriptor(), com.cloudera.impala.catalog.HBaseTable.toThriftDescriptor(), com.cloudera.impala.catalog.HdfsTable.toThriftDescriptor(), and com.cloudera.impala.catalog.HdfsTable.updatePartitionMdAndColStats().

long com.cloudera.impala.catalog.HdfsTable.numHdfsFiles_

private

Definition at line 168 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.createPartition(), com.cloudera.impala.catalog.HdfsTable.getNumHdfsFiles(), com.cloudera.impala.catalog.HdfsTable.getTableStats(), com.cloudera.impala.catalog.HdfsTable.load(), and com.cloudera.impala.catalog.HdfsTable.loadFromThrift().

long com.cloudera.impala.catalog.Table.numRows_ = -1

protectedinherited

Definition at line 72 of file Table.java.

Referenced by com.cloudera.impala.catalog.Table.getNumRows(), com.cloudera.impala.catalog.DataSourceTable.getTableStats(), com.cloudera.impala.catalog.HdfsTable.getTableStats(), com.cloudera.impala.catalog.View.load(), com.cloudera.impala.catalog.DataSourceTable.load(), com.cloudera.impala.catalog.HBaseTable.load(), com.cloudera.impala.catalog.HdfsTable.load(), com.cloudera.impala.catalog.Table.loadFromThrift(), com.cloudera.impala.catalog.CatalogObjectToFromThriftTest.TestPartitionedTable(), and com.cloudera.impala.catalog.Table.toThrift().

final String com.cloudera.impala.catalog.Table.owner_

protectedinherited

Definition at line 63 of file Table.java.

Referenced by com.cloudera.impala.catalog.Table.getOwner(), and com.cloudera.impala.catalog.Table.Table().

final HashSet<Long> com.cloudera.impala.catalog.HdfsTable.partitionIds_ = Sets.newHashSet()

private

Definition at line 150 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getPartitionIds().

final HashMap<Long, HdfsPartition> com.cloudera.impala.catalog.HdfsTable.partitionMap_ = Maps.newHashMap()

private

Definition at line 147 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.getPartitionMap().

final List<HdfsPartition> com.cloudera.impala.catalog.HdfsTable.partitions_

private

Definition at line 134 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.addPartition(), com.cloudera.impala.catalog.HdfsTable.dropPartition(), com.cloudera.impala.catalog.HdfsTable.getFiles(), com.cloudera.impala.catalog.HdfsTable.getFirstLocationWithoutWriteAccess(), com.cloudera.impala.catalog.HdfsTable.getMajorityFormat(), com.cloudera.impala.catalog.HdfsTable.getPartitions(), com.cloudera.impala.catalog.HdfsTable.getTableStats(), com.cloudera.impala.catalog.HdfsTable.getTHdfsTable(), com.cloudera.impala.catalog.HdfsTable.load(), and com.cloudera.impala.catalog.HdfsTable.populatePartitionMd().

final ArrayList<TreeMap<LiteralExpr, HashSet<Long> > > com.cloudera.impala.catalog.HdfsTable.partitionValuesMap_

private

Initial value:

=
      Lists.newArrayList()

Definition at line 138 of file HdfsTable.java.

final TNetworkAddress com.cloudera.impala.catalog.HdfsTable.REMOTE_NETWORK_ADDRESS

staticprivate

Initial value:

=
      new TNetworkAddress("remote*addr", 0)

Definition at line 112 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.synthesizeBlockMetadata().

EnumSet<TableType> com.cloudera.impala.catalog.Table.SUPPORTED_TABLE_TYPES

staticprotectedinherited

Initial value:

= EnumSet.of(
      TableType.EXTERNAL_TABLE, TableType.MANAGED_TABLE, TableType.VIRTUAL_VIEW)

Definition at line 88 of file Table.java.

final boolean com.cloudera.impala.catalog.HdfsTable.SUPPORTS_VOLUME_ID

staticprivate

Definition at line 190 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.loadDiskIds().

TTableDescriptor com.cloudera.impala.catalog.Table.tableDesc_

protectedinherited

Definition at line 64 of file Table.java.

long com.cloudera.impala.catalog.HdfsTable.totalHdfsBytes_

private

Definition at line 171 of file HdfsTable.java.

Referenced by com.cloudera.impala.catalog.HdfsTable.addPartition(), com.cloudera.impala.catalog.HdfsTable.dropPartition(), com.cloudera.impala.catalog.HdfsTable.getTableStats(), com.cloudera.impala.catalog.HdfsTable.getTotalHdfsBytes(), com.cloudera.impala.catalog.HdfsTable.load(), and com.cloudera.impala.catalog.HdfsTable.loadFromThrift().

final ArrayType com.cloudera.impala.catalog.Table.type_ = new ArrayType(new StructType())

protectedinherited

Definition at line 82 of file Table.java.

Referenced by com.cloudera.impala.catalog.Table.addColumn(), com.cloudera.impala.catalog.Table.clearColumns(), com.cloudera.impala.catalog.Table.getType(), and com.cloudera.impala.catalog.Table.loadFromThrift().

---

The documentation for this class was generated from the following file:

• fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java