hdfs-parquet-scanner.h

Go to the documentation of this file.

// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef IMPALA_EXEC_HDFS_PARQUET_SCANNER_H
#define IMPALA_EXEC_HDFS_PARQUET_SCANNER_H

#include "exec/hdfs-scanner.h"
#include "exec/parquet-common.h"

namespace impala {

struct HdfsFileDesc;

//
//
class HdfsParquetScanner : public HdfsScanner {
 public:
  HdfsParquetScanner(HdfsScanNode* scan_node, RuntimeState* state);

  virtual ~HdfsParquetScanner();
  virtual Status Prepare(ScannerContext* context);
  virtual void Close();
  virtual Status ProcessSplit();

  static Status IssueInitialRanges(HdfsScanNode* scan_node,
                                   const std::vector<HdfsFileDesc*>& files);

  struct FileVersion {
    std::string application;

    struct {
      int major;
      int minor;
      int patch;
    } version;

    bool is_impala_internal;

    FileVersion() : is_impala_internal(false) { }

    FileVersion(const std::string& created_by);

    bool VersionLt(int major, int minor = 0, int patch = 0) const;

    bool VersionEq(int major, int minor, int patch) const;
  };

 private:
  struct SchemaNode {
    const parquet::SchemaElement* element;

    int col_idx;

    int max_def_level;

    std::vector<SchemaNode> children;

    SlotDescriptor* slot_desc;

    SchemaNode() : col_idx(-1), max_def_level(-1), slot_desc(NULL) { }
    std::string DebugString(int indent = 0) const;
  };

  static const int FOOTER_SIZE = 100 * 1024;

  class BaseColumnReader;
  friend class BaseColumnReader;

  template<typename T> class ColumnReader;
  template<typename T> friend class ColumnReader;
  class BoolColumnReader;
  friend class BoolColumnReader;

  std::vector<BaseColumnReader*> column_readers_;

  parquet::FileMetaData file_metadata_;

  FileVersion file_version_;

  SchemaNode schema_;

  const DiskIoMgr::ScanRange* metadata_range_;

  Status parse_status_;

  boost::scoped_ptr<MemPool> dictionary_pool_;

  ScopedTimer<MonotonicStopWatch> assemble_rows_timer_;

  RuntimeProfile::Counter* num_cols_counter_;

  Status AssembleRows(int row_group_idx);

  Status ProcessFooter(bool* eosr);

  Status CreateColumnReaders();

  BaseColumnReader* CreateReader(const SchemaNode& node);

  Status InitColumns(int row_group_idx);

  Status ValidateFileMetadata();

  Status ValidateColumn(const BaseColumnReader& col_reader, int row_group_idx);

  Status InitNewRange() { return Status::OK; };

  Status CreateSchemaTree(const std::vector<parquet::SchemaElement>& schema,
      SchemaNode* node) const;

  Status CreateSchemaTree(const std::vector<parquet::SchemaElement>& schema,
      int max_def_level, int* idx, int* col_idx, SchemaNode* node) const;
};

} // namespace impala

#endif