descriptors.h

Go to the documentation of this file.

// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef IMPALA_RUNTIME_DESCRIPTORS_H
#define IMPALA_RUNTIME_DESCRIPTORS_H

#include <vector>
#include <tr1/unordered_map>
#include <vector>
#include <boost/scoped_ptr.hpp>
#include <ostream>

#include "common/status.h"
#include "common/global-types.h"
#include "runtime/types.h"

#include "gen-cpp/Descriptors_types.h"  // for TTupleId
#include "gen-cpp/Types_types.h"

namespace llvm {
  class Function;
  class PointerType;
  class StructType;
};

namespace impala {

class LlvmCodeGen;
class ObjectPool;
class TDescriptorTable;
class TSlotDescriptor;
class TTable;
class TTupleDescriptor;
class Expr;
class ExprContext;
class RuntimeState;

struct LlvmTupleStruct {
  llvm::StructType* tuple_struct;
  llvm::PointerType* tuple_ptr;
  std::vector<int> indices;
};

struct NullIndicatorOffset {
  int byte_offset;
  uint8_t bit_mask;  // to extract null indicator

  NullIndicatorOffset(int byte_offset, int bit_offset)
    : byte_offset(byte_offset),
      bit_mask(bit_offset == -1 ? 0 : 1 << bit_offset) {
  }

  std::string DebugString() const;
};

std::ostream& operator<<(std::ostream& os, const NullIndicatorOffset& null_indicator);

class SlotDescriptor {
 public:
  SlotId id() const { return id_; }
  const ColumnType& type() const { return type_; }
  TupleId parent() const { return parent_; }
  int col_pos() const { return col_path_[0]; }
  const std::vector<int>& col_path() const { return col_path_; }
  int field_idx() const { return field_idx_; }
  int tuple_offset() const { return tuple_offset_; }
  const NullIndicatorOffset& null_indicator_offset() const {
    return null_indicator_offset_;
  }
  bool is_materialized() const { return is_materialized_; }
  bool is_nullable() const { return null_indicator_offset_.bit_mask != 0; }
  int slot_size() const { return slot_size_; }

  static bool ColPathLessThan(const SlotDescriptor* a, const SlotDescriptor* b);

  std::string DebugString() const;

  llvm::Function* CodegenIsNull(LlvmCodeGen*, llvm::StructType* tuple);

  llvm::Function* CodegenUpdateNull(LlvmCodeGen*, llvm::StructType* tuple, bool set_null);

 private:
  friend class DescriptorTbl;
  friend class TupleDescriptor;

  const SlotId id_;
  const ColumnType type_;
  const TupleId parent_;
  const std::vector<int> col_path_;
  const int tuple_offset_;
  const NullIndicatorOffset null_indicator_offset_;

  const int slot_idx_;

  const int slot_size_;

  int field_idx_;

  const bool is_materialized_;

  llvm::Function* is_null_fn_;
  llvm::Function* set_not_null_fn_;
  llvm::Function* set_null_fn_;

  SlotDescriptor(const TSlotDescriptor& tdesc);
};

class TableDescriptor {
 public:
  TableDescriptor(const TTableDescriptor& tdesc);
  virtual ~TableDescriptor() {}
  int num_cols() const { return num_cols_; }
  int num_clustering_cols() const { return num_clustering_cols_; }
  virtual std::string DebugString() const;

  bool IsClusteringCol(const SlotDescriptor* slot_desc) const {
    return slot_desc->col_path().size() == 1 &&
        slot_desc->col_path()[0] < num_clustering_cols_;
  }

  const std::string& name() const { return name_; }
  const std::string& database() const { return database_; }
  const std::vector<std::string>& col_names() const { return col_names_; }

 protected:
  std::string name_;
  std::string database_;
  TableId id_;
  int num_cols_;
  int num_clustering_cols_;
  std::vector<std::string> col_names_;
};

class HdfsPartitionDescriptor {
 public:
  HdfsPartitionDescriptor(const THdfsPartition& thrift_partition, ObjectPool* pool);
  char line_delim() const { return line_delim_; }
  char field_delim() const { return field_delim_; }
  char collection_delim() const { return collection_delim_; }
  char escape_char() const { return escape_char_; }
  THdfsFileFormat::type file_format() const { return file_format_; }
  const std::vector<ExprContext*>& partition_key_value_ctxs() const {
    return partition_key_value_ctxs_;
  }
  int block_size() const { return block_size_; }
  const std::string& location() const { return location_; }
  int64_t id() const { return id_; }

  Status PrepareExprs(RuntimeState* state);
  Status OpenExprs(RuntimeState* state);
  void CloseExprs(RuntimeState* state);

  std::string DebugString() const;

 private:
  char line_delim_;
  char field_delim_;
  char collection_delim_;
  char escape_char_;
  int block_size_;
  std::string location_;
  int64_t id_;

  bool exprs_prepared_;
  bool exprs_opened_;
  bool exprs_closed_;

  std::vector<ExprContext*> partition_key_value_ctxs_;

  THdfsFileFormat::type file_format_;

  ObjectPool* object_pool_;
};

class HdfsTableDescriptor : public TableDescriptor {
 public:
  HdfsTableDescriptor(const TTableDescriptor& tdesc, ObjectPool* pool);
  const std::string& hdfs_base_dir() const { return hdfs_base_dir_; }
  const std::string& null_partition_key_value() const {
    return null_partition_key_value_;
  }
  const std::string& null_column_value() const { return null_column_value_; }
  const std::string& avro_schema() const { return avro_schema_; }

  typedef std::map<int64_t, HdfsPartitionDescriptor*> PartitionIdToDescriptorMap;

  HdfsPartitionDescriptor* GetPartition(int64_t partition_id) const {
    PartitionIdToDescriptorMap::const_iterator it =
        partition_descriptors_.find(partition_id);
    if (it == partition_descriptors_.end()) return NULL;
    return it->second;
  }

  const PartitionIdToDescriptorMap& partition_descriptors() const {
    return partition_descriptors_;
  }

  virtual std::string DebugString() const;

 protected:
  std::string hdfs_base_dir_;
  std::string null_partition_key_value_;
  std::string null_column_value_;
  PartitionIdToDescriptorMap partition_descriptors_;
  std::string avro_schema_;
  ObjectPool* object_pool_;
};

class HBaseTableDescriptor : public TableDescriptor {
 public:
  HBaseTableDescriptor(const TTableDescriptor& tdesc);
  virtual std::string DebugString() const;
  const std::string table_name() const { return table_name_; }

  struct HBaseColumnDescriptor {
    std::string family;
    std::string qualifier;
    bool binary_encoded;

    HBaseColumnDescriptor(const std::string& col_family, const std::string& col_qualifier,
        bool col_binary_encoded)
      : family(col_family),
        qualifier(col_qualifier),
        binary_encoded(col_binary_encoded){
    }
  };
  const std::vector<HBaseColumnDescriptor>& cols() const { return cols_; }

 protected:
  std::string table_name_;

  std::vector<HBaseColumnDescriptor> cols_;
};

class DataSourceTableDescriptor : public TableDescriptor {
 public:
  DataSourceTableDescriptor(const TTableDescriptor& tdesc) : TableDescriptor(tdesc) { }
  virtual std::string DebugString() const;
};

class TupleDescriptor {
 public:
  int byte_size() const { return byte_size_; }
  int num_null_bytes() const { return num_null_bytes_; }
  const std::vector<SlotDescriptor*>& slots() const { return slots_; }
  const std::vector<SlotDescriptor*>& string_slots() const { return string_slots_; }
  const TableDescriptor* table_desc() const { return table_desc_; }

  TupleId id() const { return id_; }
  std::string DebugString() const;

  llvm::StructType* GenerateLlvmStruct(LlvmCodeGen* codegen);

 protected:
  friend class DescriptorTbl;

  const TupleId id_;
  TableDescriptor* table_desc_;
  const int byte_size_;
  const int num_null_bytes_;
  int num_materialized_slots_;
  std::vector<SlotDescriptor*> slots_;  // contains all slots
  std::vector<SlotDescriptor*> string_slots_;  // contains only materialized string slots
  llvm::StructType* llvm_struct_; // cache for the llvm struct type for this tuple desc

  TupleDescriptor(const TTupleDescriptor& tdesc);
  void AddSlot(SlotDescriptor* slot);
};

class DescriptorTbl {
 public:
  static Status Create(ObjectPool* pool, const TDescriptorTable& thrift_tbl,
                       DescriptorTbl** tbl);

  TableDescriptor* GetTableDescriptor(TableId id) const;
  TupleDescriptor* GetTupleDescriptor(TupleId id) const;
  SlotDescriptor* GetSlotDescriptor(SlotId id) const;

  void GetTupleDescs(std::vector<TupleDescriptor*>* descs) const;

  std::string DebugString() const;

 private:
  typedef std::tr1::unordered_map<TableId, TableDescriptor*> TableDescriptorMap;
  typedef std::tr1::unordered_map<TupleId, TupleDescriptor*> TupleDescriptorMap;
  typedef std::tr1::unordered_map<SlotId, SlotDescriptor*> SlotDescriptorMap;

  TableDescriptorMap tbl_desc_map_;
  TupleDescriptorMap tuple_desc_map_;
  SlotDescriptorMap slot_desc_map_;

  DescriptorTbl(): tbl_desc_map_(), tuple_desc_map_(), slot_desc_map_() {}
};

class RowDescriptor {
 public:
  RowDescriptor(const DescriptorTbl& desc_tbl, const std::vector<TTupleId>& row_tuples,
      const std::vector<bool>& nullable_tuples);

  RowDescriptor(const RowDescriptor& desc)
    : tuple_desc_map_(desc.tuple_desc_map_),
      tuple_idx_map_(desc.tuple_idx_map_) {
  }

  RowDescriptor(const RowDescriptor& lhs_row_desc, const RowDescriptor& rhs_row_desc);

  RowDescriptor(const std::vector<TupleDescriptor*>& tuple_descs,
      const std::vector<bool>& nullable_tuples);

  RowDescriptor(TupleDescriptor* tuple_desc, bool is_nullable);

  RowDescriptor() {}

  int GetRowSize() const;

  static const int INVALID_IDX = -1;

  int GetTupleIdx(TupleId id) const;

  bool TupleIsNullable(int tuple_idx) const;

  bool IsAnyTupleNullable() const;

  const std::vector<TupleDescriptor*>& tuple_descriptors() const {
    return tuple_desc_map_;
  }

  void ToThrift(std::vector<TTupleId>* row_tuple_ids);

  bool IsPrefixOf(const RowDescriptor& other_desc) const;

  bool Equals(const RowDescriptor& other_desc) const;

  std::string DebugString() const;

 private:
  void InitTupleIdxMap();

  std::vector<TupleDescriptor*> tuple_desc_map_;

  std::vector<bool> tuple_idx_nullable_map_;

  std::vector<int> tuple_idx_map_;
};

}

#endif