hdfs-text-scanner.h

Go to the documentation of this file.

// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef IMPALA_EXEC_HDFS_TEXT_SCANNER_H
#define IMPALA_EXEC_HDFS_TEXT_SCANNER_H

#include "exec/hdfs-scanner.h"
#include "runtime/string-buffer.h"

namespace impala {

class DelimitedTextParser;
class ScannerContext;
struct HdfsFileDesc;

class HdfsTextScanner : public HdfsScanner {
 public:
  HdfsTextScanner(HdfsScanNode* scan_node, RuntimeState* state);
  virtual ~HdfsTextScanner();

  virtual Status Prepare(ScannerContext* context);
  virtual Status ProcessSplit();
  virtual void Close();

  static Status IssueInitialRanges(HdfsScanNode* scan_node,
                                   const std::vector<HdfsFileDesc*>& files);

  static llvm::Function* Codegen(HdfsScanNode*,
                                 const std::vector<ExprContext*>& conjunct_ctxs);

  const static std::string LZO_INDEX_SUFFIX;

  static const char* LLVM_CLASS_NAME;

 protected:
  Status ResetScanner();

  char* byte_buffer_ptr_;

  char* byte_buffer_end_;

  int64_t byte_buffer_read_size_;

  bool only_parsing_header_;

 private:
  const static int NEXT_BLOCK_READ_SIZE = 1024; //bytes

  virtual Status InitNewRange();

  Status FindFirstTuple(bool* tuple_found);

  Status ProcessRange(int* num_tuples, bool past_scan_range);

  Status FinishScanRange();

  virtual Status FillByteBuffer(bool* eosr, int num_bytes = 0);

  Status FillByteBufferCompressedFile(bool* eosr);

  Status FillByteBufferGzip(bool* eosr);

  void CopyBoundaryField(FieldLocation* data, MemPool* pool);

  int WriteFields(MemPool*, TupleRow* tuple_row_mem, int num_fields, int num_tuples);

  int WritePartialTuple(FieldLocation*, int num_fields, bool copy_strings);

  virtual void LogRowParseError(int row_idx, std::stringstream*);

  boost::scoped_ptr<MemPool> boundary_pool_;

  StringBuffer boundary_row_;

  StringBuffer boundary_column_;

  int slot_idx_;

  boost::scoped_ptr<DelimitedTextParser> delimited_text_parser_;

  std::vector<FieldLocation> field_locations_;

  std::vector<char*> row_end_locations_;

  char* batch_start_ptr_;

  bool error_in_row_;

  Tuple* partial_tuple_;

  bool partial_tuple_empty_;

  RuntimeProfile::Counter* parse_delimiter_timer_;
};

}

#endif