hdfs-table-writer.h

Go to the documentation of this file.

// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef IMPALA_EXEC_HDFS_TABLE_WRITER_H
#define IMPALA_EXEC_HDFS_TABLE_WRITER_H

#include <hdfs.h>
#include <boost/scoped_ptr.hpp>
#include <boost/unordered_map.hpp>

#include "runtime/descriptors.h"
#include "exec/hdfs-table-sink.h"
#include "util/hdfs-util.h"

namespace impala {

class HdfsTableWriter {
 public:
  HdfsTableWriter(HdfsTableSink* parent,
                  RuntimeState* state, OutputPartition* output_partition,
                  const HdfsPartitionDescriptor* partition_desc,
                  const HdfsTableDescriptor* table_desc,
                  const std::vector<ExprContext*>& output_expr_ctxs);

  virtual ~HdfsTableWriter() { }


  virtual Status Init() = 0;

  virtual Status InitNewFile() = 0;

  virtual Status AppendRowBatch(RowBatch* batch,
                                const std::vector<int32_t>& row_group_indices,
                                bool* new_file) = 0;

  virtual Status Finalize() = 0;

  virtual void Close() = 0;

  TInsertStats& stats() { return stats_; };

  virtual uint64_t default_block_size() const = 0;

  virtual std::string file_extension() const = 0;

 protected:
  static const int HDFS_FLUSH_WRITE_SIZE = 50 * 1024;

  Status Write(const char* data, int32_t len) {
    return Write(reinterpret_cast<const uint8_t*>(data), len);
  }
  Status Write(const uint8_t* data, int32_t len);

  template<typename T>
  Status Write(T v) {
    return Write(reinterpret_cast<uint8_t*>(&v), sizeof(T));
  }

  HdfsTableSink* parent_;

  RuntimeState* state_;

  OutputPartition* output_;

  const HdfsTableDescriptor* table_desc_;

  std::vector<ExprContext*> output_expr_ctxs_;

  TInsertStats stats_;

};
}
#endif