hdfs-rcfile-scanner.h

Go to the documentation of this file.

// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef IMPALA_EXEC_HDFS_RCFILE_SCANNER_H
#define IMPALA_EXEC_HDFS_RCFILE_SCANNER_H

//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//

#include "exec/base-sequence-scanner.h"

namespace impala {

struct HdfsFileDesc;
class HdfsScanNode;
class TupleDescriptor;
class Tuple;

class HdfsRCFileScanner : public BaseSequenceScanner {
 public:
  HdfsRCFileScanner(HdfsScanNode* scan_node, RuntimeState* state);
  virtual ~HdfsRCFileScanner();

  virtual Status Prepare(ScannerContext* context);

  void DebugString(int indentation_level, std::stringstream* out) const;

 private:
  static const char* const RCFILE_KEY_CLASS_NAME;

  static const char* const RCFILE_VALUE_CLASS_NAME;

  static const char* const RCFILE_METADATA_KEY_NUM_COLS;

  static const uint8_t RCFILE_VERSION_HEADER[4];

  virtual FileHeader* AllocateFileHeader();
  virtual Status ReadFileHeader();
  virtual Status InitNewRange();
  virtual Status ProcessRange();

  virtual THdfsFileFormat::type file_format() const {
    return THdfsFileFormat::RC_FILE;
  }

  Status ReadNumColumnsMetadata();

  Status ReadRowGroupHeader();

  Status ReadKeyBuffers();

  void GetCurrentKeyBuffer(int col_idx, bool skip_col_data, uint8_t** key_buf_ptr);

  Status ReadColumnBuffers();

  Status NextField(int col_idx);

  Status ReadRowGroup();

  void ResetRowGroup();

  Status NextRow();

  enum Version {
    SEQ6,     // Version for sequence file and pre hive-0.9 rc files
    RCF1      // The version post hive-0.9 which uses a new header
  };

  struct RcFileHeader : public BaseSequenceScanner::FileHeader {
    Version version;

    int num_cols;
  };

  struct ColumnInfo {
    bool materialize_column;

    int32_t buffer_len;
    int32_t uncompressed_buffer_len;

    int32_t key_buffer_len;
    uint8_t* key_buffer;

    int32_t key_buffer_pos;

    int32_t start_offset;

    int32_t buffer_pos;

    int32_t current_field_len;
    int32_t current_field_len_rep;
  };

  std::vector<ColumnInfo> columns_;

  std::vector<uint8_t> key_buffer_;

  int num_rows_;

  int row_pos_;

  int key_length_;

  int compressed_key_length_;

  bool reuse_row_group_buffer_;

  uint8_t* row_group_buffer_;

  int row_group_length_;

  int row_group_buffer_size_;
};

}

#endif