Impala
Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
Type.java
Go to the documentation of this file.
1 // Copyright 2012 Cloudera Inc.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 package com.cloudera.impala.catalog;
16 
17 import java.io.StringReader;
18 import java.util.ArrayList;
19 import java.util.List;
20 
21 import org.apache.hadoop.hive.metastore.api.FieldSchema;
22 
24 import com.cloudera.impala.analysis.SqlParser;
25 import com.cloudera.impala.analysis.SqlScanner;
27 import com.cloudera.impala.common.Pair;
28 import com.cloudera.impala.thrift.TColumnType;
29 import com.cloudera.impala.thrift.TPrimitiveType;
30 import com.cloudera.impala.thrift.TScalarType;
31 import com.cloudera.impala.thrift.TStructField;
32 import com.cloudera.impala.thrift.TTypeNode;
33 import com.cloudera.impala.thrift.TTypeNodeType;
34 import com.google.common.base.Preconditions;
35 import com.google.common.collect.Lists;
36 
42 public abstract class Type {
43  // Static constant types for scalar types that don't require additional information.
45  public static final ScalarType NULL = new ScalarType(PrimitiveType.NULL_TYPE);
46  public static final ScalarType BOOLEAN = new ScalarType(PrimitiveType.BOOLEAN);
47  public static final ScalarType TINYINT = new ScalarType(PrimitiveType.TINYINT);
48  public static final ScalarType SMALLINT = new ScalarType(PrimitiveType.SMALLINT);
49  public static final ScalarType INT = new ScalarType(PrimitiveType.INT);
50  public static final ScalarType BIGINT = new ScalarType(PrimitiveType.BIGINT);
51  public static final ScalarType FLOAT = new ScalarType(PrimitiveType.FLOAT);
52  public static final ScalarType DOUBLE = new ScalarType(PrimitiveType.DOUBLE);
53  public static final ScalarType STRING = new ScalarType(PrimitiveType.STRING);
54  public static final ScalarType BINARY = new ScalarType(PrimitiveType.BINARY);
56  public static final ScalarType DATE = new ScalarType(PrimitiveType.DATE);
57  public static final ScalarType DATETIME = new ScalarType(PrimitiveType.DATETIME);
58  public static final ScalarType DEFAULT_DECIMAL = (ScalarType)
61  public static final ScalarType DECIMAL =
63  public static final ScalarType DEFAULT_VARCHAR = ScalarType.createVarcharType(-1);
64  public static final ScalarType VARCHAR = ScalarType.createVarcharType(-1);
65  public static final ScalarType CHAR = (ScalarType) ScalarType.createCharType(-1);
66 
67  private static ArrayList<ScalarType> integerTypes;
68  private static ArrayList<ScalarType> numericTypes;
69  private static ArrayList<ScalarType> supportedTypes;
70 
71  static {
72  integerTypes = Lists.newArrayList();
73  integerTypes.add(TINYINT);
74  integerTypes.add(SMALLINT);
75  integerTypes.add(INT);
76  integerTypes.add(BIGINT);
77 
78  numericTypes = Lists.newArrayList();
79  numericTypes.add(TINYINT);
80  numericTypes.add(SMALLINT);
81  numericTypes.add(INT);
82  numericTypes.add(BIGINT);
83  numericTypes.add(FLOAT);
84  numericTypes.add(DOUBLE);
85  numericTypes.add(DECIMAL);
86 
87  supportedTypes = Lists.newArrayList();
88  supportedTypes.add(NULL);
89  supportedTypes.add(BOOLEAN);
90  supportedTypes.add(TINYINT);
91  supportedTypes.add(SMALLINT);
92  supportedTypes.add(INT);
93  supportedTypes.add(BIGINT);
94  supportedTypes.add(FLOAT);
95  supportedTypes.add(DOUBLE);
96  supportedTypes.add(STRING);
97  supportedTypes.add(VARCHAR);
98  supportedTypes.add(CHAR);
99  supportedTypes.add(TIMESTAMP);
100  supportedTypes.add(DECIMAL);
101  }
102 
103  public static ArrayList<ScalarType> getIntegerTypes() {
104  return integerTypes;
105  }
106  public static ArrayList<ScalarType> getNumericTypes() {
107  return numericTypes;
108  }
109  public static ArrayList<ScalarType> getSupportedTypes() {
110  return supportedTypes;
111  }
112 
119  public abstract String toSql();
120 
121  public boolean isInvalid() { return isScalarType(PrimitiveType.INVALID_TYPE); }
122  public boolean isValid() { return !isInvalid(); }
123  public boolean isNull() { return isScalarType(PrimitiveType.NULL_TYPE); }
124  public boolean isBoolean() { return isScalarType(PrimitiveType.BOOLEAN); }
125  public boolean isTimestamp() { return isScalarType(PrimitiveType.TIMESTAMP); }
126  public boolean isDecimal() { return isScalarType(PrimitiveType.DECIMAL); }
127  public boolean isDecimalOrNull() { return isDecimal() || isNull(); }
128  public boolean isFullySpecifiedDecimal() { return false; }
129  public boolean isWildcardDecimal() { return false; }
130  public boolean isWildcardVarchar() { return false; }
131  public boolean isWildcardChar() { return false; }
132 
133  public boolean isStringType() {
136  }
137 
138  public boolean isScalarType() { return this instanceof ScalarType; }
139  public boolean isScalarType(PrimitiveType t) {
140  return isScalarType() && ((ScalarType) this).getPrimitiveType() == t;
141  }
142 
143  public boolean isFixedPointType() {
147  }
148 
149  public boolean isFloatingPointType() {
151  }
152 
153  public boolean isIntegerType() {
156  }
157 
158  // TODO: Handle complex types properly. Some instances may be fixed length.
159  public boolean isFixedLengthType() { return false; }
160 
161  public boolean isNumericType() {
163  }
164 
165  public boolean isDateType() {
168  }
169 
170  public boolean isComplexType() { return isStructType() || isCollectionType(); }
171  public boolean isCollectionType() { return isMapType() || isArrayType(); }
172  public boolean isMapType() { return this instanceof MapType; }
173  public boolean isArrayType() { return this instanceof ArrayType; }
174  public boolean isStructType() { return this instanceof StructType; }
175 
181  public boolean isSupported() { return true; }
182 
186  public boolean supportsTablePartitioning() { return false; }
187 
189 
194  public int getSlotSize() {
195  // 8-byte pointer and 4-byte length indicator (12 bytes total).
196  // Aligning to 8 bytes so 16 total.
197  if (isCollectionType()) return 16;
198  throw new IllegalStateException("getSlotSize() not implemented for type " + toSql());
199  }
200 
201  public TColumnType toThrift() {
202  TColumnType container = new TColumnType();
203  container.setTypes(new ArrayList<TTypeNode>());
204  toThrift(container);
205  return container;
206  }
207 
211  public abstract void toThrift(TColumnType container);
212 
218  public boolean matchesType(Type t) { return false; }
219 
229  public static Type parseColumnType(FieldSchema fs) {
230  // Wrap the type string in a CREATE TABLE stmt and use Impala's Parser
231  // to get the ColumnType.
232  // Pick a table name that can't be used.
233  String stmt = String.format("CREATE TABLE $DUMMY ($DUMMY %s)", fs.getType());
234  SqlScanner input = new SqlScanner(new StringReader(stmt));
235  SqlParser parser = new SqlParser(input);
236  CreateTableStmt createTableStmt;
237  try {
238  Object o = parser.parse().value;
239  if (!(o instanceof CreateTableStmt)) {
240  // Should never get here.
241  throw new IllegalStateException("Couldn't parse create table stmt.");
242  }
243  createTableStmt = (CreateTableStmt) o;
244  if (createTableStmt.getColumnDefs().isEmpty()) {
245  // Should never get here.
246  throw new IllegalStateException("Invalid create table stmt.");
247  }
248  } catch (Exception e) {
249  return null;
250  }
251  TypeDef typeDef = createTableStmt.getColumnDefs().get(0).getTypeDef();
252  return typeDef.getType();
253  }
254 
259  public static boolean isImplicitlyCastable(Type t1, Type t2) {
260  if (t1.isScalarType() && t1.isScalarType()) {
261  return ScalarType.isImplicitlyCastable(
262  (ScalarType) t1, (ScalarType) t2);
263  }
264  return false;
265  }
266 
273  public static Type getAssignmentCompatibleType(Type t1, Type t2) {
274  if (t1.isScalarType() && t2.isScalarType()) {
275  return ScalarType.getAssignmentCompatibleType(
276  (ScalarType) t1, (ScalarType) t2);
277  }
278  return ScalarType.INVALID;
279  }
280 
281  public static List<TColumnType> toThrift(Type[] types) {
282  return toThrift(Lists.newArrayList(types));
283  }
284 
285  public static List<TColumnType> toThrift(ArrayList<Type> types) {
286  ArrayList<TColumnType> result = Lists.newArrayList();
287  for (Type t: types) {
288  result.add(t.toThrift());
289  }
290  return result;
291  }
292 
293  public static Type fromThrift(TColumnType thrift) {
294  Preconditions.checkState(thrift.types.size() > 0);
295  Pair<Type, Integer> t = fromThrift(thrift, 0);
296  Preconditions.checkState(t.second.equals(thrift.getTypesSize()));
297  return t.first;
298  }
299 
305  protected static Pair<Type, Integer> fromThrift(TColumnType col, int nodeIdx) {
306  TTypeNode node = col.getTypes().get(nodeIdx);
307  Type type = null;
308  switch (node.getType()) {
309  case SCALAR: {
310  Preconditions.checkState(node.isSetScalar_type());
311  TScalarType scalarType = node.getScalar_type();
312  if (scalarType.getType() == TPrimitiveType.CHAR) {
313  Preconditions.checkState(scalarType.isSetLen());
314  type = ScalarType.createCharType(scalarType.getLen());
315  } else if (scalarType.getType() == TPrimitiveType.VARCHAR) {
316  Preconditions.checkState(scalarType.isSetLen());
317  type = ScalarType.createVarcharType(scalarType.getLen());
318  } else if (scalarType.getType() == TPrimitiveType.DECIMAL) {
319  Preconditions.checkState(scalarType.isSetPrecision()
320  && scalarType.isSetPrecision());
321  type = ScalarType.createDecimalType(scalarType.getPrecision(),
322  scalarType.getScale());
323  } else {
324  type = ScalarType.createType(
325  PrimitiveType.fromThrift(scalarType.getType()));
326  }
327  ++nodeIdx;
328  break;
329  }
330  case ARRAY: {
331  Preconditions.checkState(nodeIdx + 1 < col.getTypesSize());
332  Pair<Type, Integer> childType = fromThrift(col, nodeIdx + 1);
333  type = new ArrayType(childType.first);
334  nodeIdx = childType.second;
335  break;
336  }
337  case MAP: {
338  Preconditions.checkState(nodeIdx + 2 < col.getTypesSize());
339  Pair<Type, Integer> keyType = fromThrift(col, nodeIdx + 1);
340  Pair<Type, Integer> valueType = fromThrift(col, keyType.second);
341  type = new MapType(keyType.first, valueType.first);
342  nodeIdx = valueType.second;
343  break;
344  }
345  case STRUCT: {
346  Preconditions.checkState(nodeIdx + node.getStruct_fieldsSize() < col.getTypesSize());
347  ArrayList<StructField> structFields = Lists.newArrayList();
348  ++nodeIdx;
349  for (int i = 0; i < node.getStruct_fieldsSize(); ++i) {
350  TStructField thriftField = node.getStruct_fields().get(i);
351  String name = thriftField.getName();
352  String comment = null;
353  if (thriftField.isSetComment()) comment = thriftField.getComment();
354  Pair<Type, Integer> res = fromThrift(col, nodeIdx);
355  nodeIdx = res.second.intValue();
356  structFields.add(new StructField(name, res.first, comment));
357  }
358  type = new StructType(structFields);
359  break;
360  }
361  }
362  return new Pair<Type, Integer>(type, nodeIdx);
363  }
364 
369  public TPrimitiveType getTPrimitiveType(TColumnType ttype) {
370  Preconditions.checkState(ttype.getTypesSize() == 1);
371  Preconditions.checkState(ttype.types.get(0).getType() == TTypeNodeType.SCALAR);
372  return ttype.types.get(0).scalar_type.getType();
373  }
374 
385  public Integer getColumnSize() {
386  if (!isScalarType()) return null;
387  if (isNumericType()) return getPrecision();
388  ScalarType t = (ScalarType) this;
389  switch (t.getPrimitiveType()) {
390  case STRING:
391  return Integer.MAX_VALUE;
392  case TIMESTAMP:
393  return 29;
394  case CHAR:
395  case VARCHAR:
396  return t.getLength();
397  default:
398  return null;
399  }
400  }
401 
407  public Integer getPrecision() {
408  if (!isScalarType()) return null;
409  ScalarType t = (ScalarType) this;
410  switch (t.getPrimitiveType()) {
411  case TINYINT:
412  return 3;
413  case SMALLINT:
414  return 5;
415  case INT:
416  return 10;
417  case BIGINT:
418  return 19;
419  case FLOAT:
420  return 7;
421  case DOUBLE:
422  return 15;
423  case DECIMAL:
424  return t.decimalPrecision();
425  default:
426  return null;
427  }
428  }
429 
436  public Integer getDecimalDigits() {
437  if (!isScalarType()) return null;
438  ScalarType t = (ScalarType) this;
439  switch (t.getPrimitiveType()) {
440  case BOOLEAN:
441  case TINYINT:
442  case SMALLINT:
443  case INT:
444  case BIGINT:
445  return 0;
446  case FLOAT:
447  return 7;
448  case DOUBLE:
449  return 15;
450  case TIMESTAMP:
451  return 9;
452  case DECIMAL:
453  return t.decimalScale();
454  default:
455  return null;
456  }
457  }
458 
471  public Integer getNumPrecRadix() {
472  if (!isScalarType()) return null;
473  ScalarType t = (ScalarType) this;
474  switch (t.getPrimitiveType()) {
475  case TINYINT:
476  case SMALLINT:
477  case INT:
478  case BIGINT:
479  case FLOAT:
480  case DOUBLE:
481  case DECIMAL:
482  return 10;
483  default:
484  // everything else (including boolean and string) is null
485  return null;
486  }
487  }
488 
493  public int getJavaSqlType() {
494  if (!isScalarType()) {
495  Preconditions.checkArgument(false, "Invalid non-scalar type " + toSql());
496  return 0;
497  }
498  ScalarType t = (ScalarType) this;
499  switch (t.getPrimitiveType()) {
500  case NULL_TYPE: return java.sql.Types.NULL;
501  case BOOLEAN: return java.sql.Types.BOOLEAN;
502  case TINYINT: return java.sql.Types.TINYINT;
503  case SMALLINT: return java.sql.Types.SMALLINT;
504  case INT: return java.sql.Types.INTEGER;
505  case BIGINT: return java.sql.Types.BIGINT;
506  case FLOAT: return java.sql.Types.FLOAT;
507  case DOUBLE: return java.sql.Types.DOUBLE;
508  case TIMESTAMP: return java.sql.Types.TIMESTAMP;
509  case STRING: return java.sql.Types.VARCHAR;
510  case CHAR: return java.sql.Types.CHAR;
511  case VARCHAR: return java.sql.Types.VARCHAR;
512  case BINARY: return java.sql.Types.BINARY;
513  case DECIMAL: return java.sql.Types.DECIMAL;
514  default:
515  Preconditions.checkArgument(false, "Invalid primitive type " +
516  t.getPrimitiveType().name());
517  return 0;
518  }
519  }
520 
534  protected static PrimitiveType[][] compatibilityMatrix;
535  static {
536  compatibilityMatrix = new
537  PrimitiveType[CHAR.ordinal() + 1][CHAR.ordinal() + 1];
538 
539  // NULL_TYPE is compatible with any type and results in the non-null type.
544  compatibilityMatrix[NULL.ordinal()][INT.ordinal()] = PrimitiveType.INT;
548  compatibilityMatrix[NULL.ordinal()][DATE.ordinal()] = PrimitiveType.DATE;
553  compatibilityMatrix[NULL.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR;
554 
558  compatibilityMatrix[BOOLEAN.ordinal()][INT.ordinal()] = PrimitiveType.INT;
559  compatibilityMatrix[BOOLEAN.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
560  compatibilityMatrix[BOOLEAN.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
561  compatibilityMatrix[BOOLEAN.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
563  compatibilityMatrix[BOOLEAN.ordinal()][DATETIME.ordinal()] =
565  compatibilityMatrix[BOOLEAN.ordinal()][TIMESTAMP.ordinal()] =
570 
573  compatibilityMatrix[TINYINT.ordinal()][INT.ordinal()] = PrimitiveType.INT;
574  compatibilityMatrix[TINYINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
575  compatibilityMatrix[TINYINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
576  compatibilityMatrix[TINYINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
578  compatibilityMatrix[TINYINT.ordinal()][DATETIME.ordinal()] =
580  compatibilityMatrix[TINYINT.ordinal()][TIMESTAMP.ordinal()] =
585 
587  compatibilityMatrix[SMALLINT.ordinal()][INT.ordinal()] = PrimitiveType.INT;
588  compatibilityMatrix[SMALLINT.ordinal()][BIGINT.ordinal()] = PrimitiveType.BIGINT;
589  compatibilityMatrix[SMALLINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
590  compatibilityMatrix[SMALLINT.ordinal()][DOUBLE.ordinal()] = PrimitiveType.DOUBLE;
592  compatibilityMatrix[SMALLINT.ordinal()][DATETIME.ordinal()] =
594  compatibilityMatrix[SMALLINT.ordinal()][TIMESTAMP.ordinal()] =
596  compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] =
598  compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] =
601 
602  compatibilityMatrix[INT.ordinal()][INT.ordinal()] = PrimitiveType.INT;
612 
614  compatibilityMatrix[BIGINT.ordinal()][FLOAT.ordinal()] = PrimitiveType.FLOAT;
617  compatibilityMatrix[BIGINT.ordinal()][DATETIME.ordinal()] =
619  compatibilityMatrix[BIGINT.ordinal()][TIMESTAMP.ordinal()] =
624 
629  compatibilityMatrix[FLOAT.ordinal()][TIMESTAMP.ordinal()] =
634 
637  compatibilityMatrix[DOUBLE.ordinal()][DATETIME.ordinal()] =
639  compatibilityMatrix[DOUBLE.ordinal()][TIMESTAMP.ordinal()] =
644 
645  compatibilityMatrix[DATE.ordinal()][DATE.ordinal()] = PrimitiveType.DATE;
651 
653  compatibilityMatrix[DATETIME.ordinal()][TIMESTAMP.ordinal()] =
655  compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] =
657  compatibilityMatrix[DATETIME.ordinal()][VARCHAR.ordinal()] =
659 
660  compatibilityMatrix[TIMESTAMP.ordinal()][TIMESTAMP.ordinal()] =
662  compatibilityMatrix[TIMESTAMP.ordinal()][STRING.ordinal()] =
664  compatibilityMatrix[TIMESTAMP.ordinal()][VARCHAR.ordinal()] =
667 
670  compatibilityMatrix[STRING.ordinal()][CHAR.ordinal()] = PrimitiveType.STRING;
671 
674 
675  compatibilityMatrix[CHAR.ordinal()][CHAR.ordinal()] = PrimitiveType.CHAR;
676  }
677 }
static List< TColumnType > toThrift(Type[] types)
Definition: Type.java:281
static ArrayList< ScalarType > getIntegerTypes()
Definition: Type.java:103
static final ScalarType NULL
Definition: Type.java:45
static final ScalarType BIGINT
Definition: Type.java:50
static final ScalarType DATE
Definition: Type.java:56
static final ScalarType CHAR
Definition: Type.java:65
static final ScalarType VARCHAR
Definition: Type.java:64
static final ScalarType STRING
Definition: Type.java:53
static ScalarType createDecimalTypeInternal(int precision, int scale)
PrimitiveType getPrimitiveType()
Definition: Type.java:188
static Type parseColumnType(FieldSchema fs)
Definition: Type.java:229
static List< TColumnType > toThrift(ArrayList< Type > types)
Definition: Type.java:285
static ArrayList< ScalarType > numericTypes
Definition: Type.java:68
static final ScalarType BOOLEAN
Definition: Type.java:46
static final ScalarType DEFAULT_DECIMAL
Definition: Type.java:58
static ArrayList< ScalarType > integerTypes
Definition: Type.java:67
static Type fromThrift(TColumnType thrift)
Definition: Type.java:293
boolean matchesType(Type t)
Definition: Type.java:218
static boolean isImplicitlyCastable(Type t1, Type t2)
Definition: Type.java:259
static final ScalarType SMALLINT
Definition: Type.java:48
static ScalarType createCharType(int len)
Definition: ScalarType.java:93
static final ScalarType FLOAT
Definition: Type.java:51
static ArrayList< ScalarType > supportedTypes
Definition: Type.java:69
static PrimitiveType[][] compatibilityMatrix
Definition: Type.java:534
TPrimitiveType getTPrimitiveType(TColumnType ttype)
Definition: Type.java:369
static Pair< Type, Integer > fromThrift(TColumnType col, int nodeIdx)
Definition: Type.java:305
static final ScalarType DOUBLE
Definition: Type.java:52
static final ScalarType TINYINT
Definition: Type.java:47
static ArrayList< ScalarType > getNumericTypes()
Definition: Type.java:106
static final ScalarType INT
Definition: Type.java:49
static final ScalarType DECIMAL
Definition: Type.java:61
static final ScalarType DEFAULT_VARCHAR
Definition: Type.java:63
static final ScalarType DATETIME
Definition: Type.java:57
string name
Definition: cpu-info.cc:50
static final ScalarType INVALID
Definition: Type.java:44
static final ScalarType BINARY
Definition: Type.java:54
static final ScalarType TIMESTAMP
Definition: Type.java:55
boolean isScalarType(PrimitiveType t)
Definition: Type.java:139
static ArrayList< ScalarType > getSupportedTypes()
Definition: Type.java:109
static Type getAssignmentCompatibleType(Type t1, Type t2)
Definition: Type.java:273