delimited-text-parser.h

Go to the documentation of this file.

// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef IMPALA_EXEC_DELIMITED_TEXT_PARSER_H
#define IMPALA_EXEC_DELIMITED_TEXT_PARSER_H

#include "exec/hdfs-scanner.h"
#include "exec/hdfs-scan-node.h"
#include "util/sse-util.h"

namespace impala {

class DelimitedTextParser {
 public:

  //
  //
  //
  DelimitedTextParser(
      int num_cols, int num_partition_keys, const bool* is_materialized_col,
      char tuple_delim, char field_delim_ = '\0', char collection_item_delim = '^',
      char escape_char = '\0');

  void ParserReset();

  bool AtTupleStart() { return column_idx_ == num_partition_keys_; }

  char escape_char() const { return escape_char_; }

  Status ParseFieldLocations(int max_tuples, int64_t remaining_len,
      char** byte_buffer_ptr, char** row_end_locations,
      FieldLocation* field_locations,
      int* num_tuples, int* num_fields, char** next_column_start);

  template <bool process_escapes>
  void ParseSingleTuple(int64_t len, char* buffer, FieldLocation* field_locations, 
      int* num_fields);

  int FindFirstInstance(const char* buffer, int len);

  bool ReturnCurrentColumn() const {
    return column_idx_ < num_cols_ && is_materialized_col_[column_idx_];
  }

  template <bool process_escapes>
  void FillColumns(int len, char** last_column,
                   int* num_fields, impala::FieldLocation* field_locations);

  bool HasUnfinishedTuple() { return unfinished_tuple_; }

 private:
  void ParserInit(HdfsScanNode* scan_node);

  template <bool process_escapes>
  void AddColumn(int len, char** next_column_start, int* num_fields,
                 FieldLocation* field_locations);

  template <bool process_escapes>
  void ParseSse(int max_tuples, int64_t* remaining_len,
      char** byte_buffer_ptr, char** row_end_locations_,
      FieldLocation* field_locations,
      int* num_tuples, int* num_fields, char** next_column_start);

  __m128i xmm_tuple_search_;

  __m128i xmm_delim_search_;

  int num_delims_;

  __m128i xmm_escape_search_;

  char field_delim_;

  bool process_escapes_;

  char escape_char_;

  char collection_item_delim_;

  char tuple_delim_;

  bool current_column_has_escape_;

  bool last_char_is_escape_;

  int32_t last_row_delim_offset_;

  uint16_t low_mask_[16];
  uint16_t high_mask_[16];

  int num_cols_;

  int num_partition_keys_;

  const bool* is_materialized_col_;

  int column_idx_;

  bool unfinished_tuple_;
};

}// namespace impala
#endif// IMPALA_EXEC_DELIMITED_TEXT_PARSER_H