decompress.h

Go to the documentation of this file.

// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef IMPALA_UTIL_DECOMPRESS_H
#define IMPALA_UTIL_DECOMPRESS_H

// We need zlib.h here to declare stream_ below.
#include <zlib.h>

#include "util/codec.h"
#include "exec/hdfs-scanner.h"
#include "runtime/mem-pool.h"

namespace impala {

class GzipDecompressor : public Codec {
 public:
  virtual ~GzipDecompressor();
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);
  virtual Status ProcessBlockStreaming(int64_t input_length, const uint8_t* input,
      int64_t* input_bytes_read, int64_t* output_length, uint8_t** output, bool* eos);
  virtual std::string file_extension() const { return "gz"; }

 private:
  friend class Codec;
  GzipDecompressor(
      MemPool* mem_pool = NULL, bool reuse_buffer = false, bool is_deflate = false);
  virtual Status Init();
  std::string DebugStreamState() const;

  bool is_deflate_;

  z_stream stream_;

  const static int WINDOW_BITS = 15;    // Maximum window size
  const static int DETECT_CODEC = 32;   // Determine if this is libz or gzip from header.
};

class BzipDecompressor : public Codec {
 public:
  virtual ~BzipDecompressor() { }
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated,
                              int64_t input_length, const uint8_t* input,
                              int64_t* output_length, uint8_t** output);
  virtual std::string file_extension() const { return "bz2"; }
 private:
  friend class Codec;
  BzipDecompressor(MemPool* mem_pool, bool reuse_buffer);

  virtual Status Init() { return Status::OK; }
};

class SnappyDecompressor : public Codec {
 public:
  static const uint TRAILING_CHECKSUM_LEN = 4;

  virtual ~SnappyDecompressor() { }
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);
  virtual std::string file_extension() const { return "snappy"; }

 private:
  friend class Codec;
  SnappyDecompressor(MemPool* mem_pool = NULL, bool reuse_buffer = false);
  virtual Status Init() { return Status::OK; }
};

class Lz4Decompressor : public Codec {
 public:
  virtual ~Lz4Decompressor() { }
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);
  virtual std::string file_extension() const { return "lz4"; }

 private:
  friend class Codec;
  Lz4Decompressor(MemPool* mem_pool = NULL, bool reuse_buffer = false);
  virtual Status Init() { return Status::OK; }
};

class SnappyBlockDecompressor : public Codec {
 public:
  virtual ~SnappyBlockDecompressor() { }
  virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t* input = NULL);
  virtual Status ProcessBlock(bool output_preallocated, int64_t input_length,
      const uint8_t* input, int64_t* output_length, uint8_t** output);
  virtual std::string file_extension() const { return "snappy"; }

 private:
  friend class Codec;
  SnappyBlockDecompressor(MemPool* mem_pool, bool reuse_buffer);
  virtual Status Init() { return Status::OK; }
};

}
#endif