compress.h

Go to the documentation of this file.

// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef IMPALA_UTIL_COMPRESS_H
#define IMPALA_UTIL_COMPRESS_H

#include <zlib.h>

#include "util/codec.h"
#include "exec/hdfs-scanner.h"
#include "runtime/mem-pool.h"

namespace impala {


class GzipCompressor : public Codec {
 public:
  enum Format {
    ZLIB,
    DEFLATE,
    GZIP,
  };

  virtual ~GzipCompressor();
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);

  virtual std::string file_extension() const { return "gz"; }

 private:
  friend class Codec;
  GzipCompressor(Format format, MemPool* mem_pool = NULL, bool reuse_buffer = false);
  virtual Status Init();

  Format format_;

  z_stream stream_;

  const static int WINDOW_BITS = 15;    // Maximum window size
  const static int GZIP_CODEC = 16;     // Output Gzip.

  Status Compress(int64_t input_length, const uint8_t* input,
      int64_t* output_length, uint8_t* output);
};

class BzipCompressor : public Codec {
 public:
  virtual ~BzipCompressor() { }
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);
  virtual std::string file_extension() const { return "bz2"; }

 private:
  friend class Codec;
  BzipCompressor(MemPool* mem_pool, bool reuse_buffer);
  virtual Status Init() { return Status::OK; }
};

class SnappyBlockCompressor : public Codec {
 public:
  virtual ~SnappyBlockCompressor() { }
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);
  virtual std::string file_extension() const { return "snappy"; }

 private:
  friend class Codec;
  SnappyBlockCompressor(MemPool* mem_pool, bool reuse_buffer);
  virtual Status Init() { return Status::OK; }
};

class SnappyCompressor : public Codec {
 public:
  virtual ~SnappyCompressor() { }
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);
  virtual std::string file_extension() const { return "snappy"; }

  static uint32_t ComputeChecksum(int64_t input_len, const uint8_t* input);

 private:
  friend class Codec;
  SnappyCompressor(MemPool* mem_pool = NULL, bool reuse_buffer = false);
  virtual Status Init() { return Status::OK; }
};

class Lz4Compressor : public Codec {
 public:
  virtual ~Lz4Compressor() { }
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);
  virtual std::string file_extension() const { return "lz4"; }

 private:
  friend class Codec;
  Lz4Compressor(MemPool* mem_pool = NULL, bool reuse_buffer = false);
  virtual Status Init() { return Status::OK; }
};

}
#endif