timestamp-parse-util.h

Go to the documentation of this file.

// Copyright 2013 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef IMPALA_RUNTIME_TIMESTAMP_PARSE_UTIL_H
#define IMPALA_RUNTIME_TIMESTAMP_PARSE_UTIL_H

#include <boost/assign/list_of.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/foreach.hpp>
#include <boost/unordered_map.hpp>
#include "common/status.h"
#include "runtime/string-value.inline.h"
#include "util/string-parser.h"

namespace impala {

enum DateTimeFormatTokenType {
  UNKNOWN = 0,
  SEPARATOR,
  YEAR,
  MONTH_IN_YEAR,
  MONTH_IN_YEAR_SLT,
  DAY_IN_MONTH,
  HOUR_IN_DAY,
  MINUTE_IN_HOUR,
  SECOND_IN_MINUTE,
  FRACTION,
};

struct DateTimeFormatToken {
  DateTimeFormatTokenType type;
  int pos;
  int len;
  const char* val;

  DateTimeFormatToken(DateTimeFormatTokenType type, int pos, int len, const char* val)
    : type(type),
      pos(pos),
      len(len),
      val(val) {
  }
};

struct DateTimeFormatContext {
  const char* fmt;
  int fmt_len;
  int fmt_out_len;
  std::vector<DateTimeFormatToken> toks;
  bool has_date_toks;
  bool has_time_toks;

  DateTimeFormatContext() {
    Reset(NULL, 0);
  }

  DateTimeFormatContext(const char* fmt, int fmt_len) {
    Reset(fmt, fmt_len);
  }

  void Reset(const char* fmt, int fmt_len) {
    this->fmt = fmt;
    this->fmt_len = fmt_len;
    this->fmt_out_len = fmt_len;
    this->has_date_toks = false;
    this->has_time_toks = false;
    this->toks.clear();
  }
};

struct DateTimeParseResult {
  int year;
  int month;
  int day;
  int hour;
  int minute;
  int second;
  int32_t fraction;

  DateTimeParseResult()
    : year(0),
      month(0),
      day(0),
      hour(0),
      minute(0),
      second(0),
      fraction(0) {
  }
};

class TimestampParser {
 public:
  static void Init();

  static inline bool ParseFormatTokens(DateTimeFormatContext* dt_ctx) {
    DCHECK(dt_ctx != NULL);
    DCHECK(dt_ctx->fmt != NULL);
    DCHECK(dt_ctx->fmt_len > 0);
    DCHECK(dt_ctx->toks.size() == 0);
    const char* str_begin = dt_ctx->fmt;
    const char* str_end = str_begin + dt_ctx->fmt_len;
    const char* str = str_begin;
    // Parse the tokens from the format string
    while (str < str_end) {
      if (isdigit(*str)) return false;
      // Ignore T|Z|non aA-zZ chars but track them as separators (required for printing).
      if ((*str == 'T') || (*str == 'Z') || (!isalpha(*str))) {
        dt_ctx->toks.push_back(DateTimeFormatToken(SEPARATOR, str - str_begin, 1, str));
        ++str;
        continue;
      }
      DateTimeFormatTokenType tok_type = UNKNOWN;
      switch (*str) {
        case 'y': tok_type = YEAR; break;
        case 'M': tok_type = MONTH_IN_YEAR; break;
        case 'd': tok_type = DAY_IN_MONTH; break;
        case 'H': tok_type = HOUR_IN_DAY; break;
        case 'm': tok_type = MINUTE_IN_HOUR; break;
        case 's': tok_type = SECOND_IN_MINUTE; break;
        case 'S': tok_type = FRACTION; break;
        // Error on aA-zZ reserved characters that are not used yet.
        default: return false;
      }
      dt_ctx->has_date_toks |= tok_type < HOUR_IN_DAY;
      dt_ctx->has_time_toks |= tok_type >= HOUR_IN_DAY;
      // Get the token group length
      int tok_len = 1;
      char tok_chr = *str;
      const char* curr_tok_chr = str + 1;
      while (curr_tok_chr < str_end) {
        if (*curr_tok_chr != tok_chr) break;
        ++tok_len;
        ++curr_tok_chr;
      }
      if (tok_type == MONTH_IN_YEAR) {
        if (UNLIKELY(tok_len > 3)) return false;
        if (tok_len == 3) tok_type = MONTH_IN_YEAR_SLT;
      }
      // In an output scenario, fmt_out_len is used to determine the print buffer size.
      // If the format uses short token groups e.g. yyyy-MM-d, there must to be enough
      // room in the buffer for wider values e.g. 2013-12-16.
      if (tok_len == 1) ++dt_ctx->fmt_out_len;
      DateTimeFormatToken tok(tok_type, str - str_begin, tok_len, str);
      str += tok.len;
      dt_ctx->toks.push_back(tok);
    }
    return dt_ctx->has_date_toks || dt_ctx->has_time_toks;
  }

  static inline bool Parse(const char* str, int len, boost::gregorian::date* d,
      boost::posix_time::time_duration* t) {
    DCHECK(TimestampParser::initialized_);
    DCHECK(d != NULL);
    DCHECK(t != NULL);
    if (UNLIKELY(str == NULL || len <= 0)) {
      *d = boost::gregorian::date();
      *t = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
      return false;
    }
    // Remove leading white space.
    while (len > 0 && isspace(*str)) {
      ++str;
      --len;
    }
    // Strip the trailing blanks.
    while (len > 0 && isspace(str[len - 1])) --len;
    // Only process what we have to.
    if (len > DEFAULT_DATE_TIME_FMT_LEN) len = DEFAULT_DATE_TIME_FMT_LEN;
    // Determine the default formatting context that's required for parsing.
    DateTimeFormatContext* dt_ctx = NULL;
    if (LIKELY(len >= DEFAULT_TIME_FMT_LEN)) {
      // This string starts with a date component
      if (str[4] == '-') {
        switch (len) {
          case DEFAULT_DATE_FMT_LEN: {
            dt_ctx = &DEFAULT_DATE_CTX;
            break;
          }
          case DEFAULT_SHORT_DATE_TIME_FMT_LEN:  {
            switch (str[10]) {
              case ' ': dt_ctx = &DEFAULT_SHORT_DATE_TIME_CTX; break;
              case 'T': dt_ctx = &DEFAULT_SHORT_ISO_DATE_TIME_CTX; break;
            }
            break;
          }
          case DEFAULT_DATE_TIME_FMT_LEN: {
            switch (str[10]) {
              case ' ': dt_ctx = &DEFAULT_DATE_TIME_CTX[9]; break;
              case 'T': dt_ctx = &DEFAULT_ISO_DATE_TIME_CTX[9]; break;
            }
            break;
          }
          default: {
            // There is likely a fractional component that's below the expected 9 chars.
            // We will need to work out which default context to use that corresponds to
            // the fractional length in the string.
            if (LIKELY(len > DEFAULT_SHORT_DATE_TIME_FMT_LEN)) {
              switch (str[10]) {
                case ' ': {
                  dt_ctx =
                      &DEFAULT_DATE_TIME_CTX[len - DEFAULT_SHORT_DATE_TIME_FMT_LEN - 1];
                  break;
                }
                case 'T': {
                  dt_ctx = &DEFAULT_ISO_DATE_TIME_CTX
                      [len - DEFAULT_SHORT_DATE_TIME_FMT_LEN - 1];
                  break;
                }
              }
            }
            break;
          }
        }
      } else if (str[2] == ':') {
        if (len > DEFAULT_TIME_FRAC_FMT_LEN) len = DEFAULT_TIME_FRAC_FMT_LEN;
        if (len > DEFAULT_TIME_FMT_LEN && str[8] == '.') {
          dt_ctx = &DEFAULT_TIME_FRAC_CTX[len - DEFAULT_TIME_FMT_LEN - 1];
        } else {
          dt_ctx = &DEFAULT_TIME_CTX;
        }
      }
    }
    if (LIKELY(dt_ctx != NULL)) {
      return Parse(str, len, *dt_ctx, d, t);
    } else {
      *d = boost::gregorian::date();
      *t = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
      return false;
    }
  }

  static inline bool Parse(const char* str, int len, const DateTimeFormatContext& dt_ctx,
      boost::gregorian::date* d, boost::posix_time::time_duration* t) {
    DCHECK(TimestampParser::initialized_);
    DCHECK(dt_ctx.toks.size() > 0);
    DCHECK(d != NULL);
    DCHECK(t != NULL);
    DateTimeParseResult dt_result;
    if (UNLIKELY(str == NULL || len <= 0 ||
        !ParseDateTime(str, len, dt_ctx, &dt_result))) {
      *d = boost::gregorian::date();
      *t = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
      return false;
    }
    if (dt_ctx.has_date_toks) {
      try {
        *d = boost::gregorian::date(dt_result.year, dt_result.month, dt_result.day);
      } catch (boost::exception& e) {
        VLOG_ROW << "Invalid date: " << dt_result.year << "-" << dt_result.month << "-"
                 << dt_result.day;
        *d = boost::gregorian::date();
        *t = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
        return false;
      }
    } else {
      *d = boost::gregorian::date();
    }
    if (dt_ctx.has_time_toks) {
      *t = boost::posix_time::time_duration(dt_result.hour, dt_result.minute,
          dt_result.second, dt_result.fraction);
    } else {
      *t = boost::posix_time::time_duration(0, 0, 0, 0);
    }
    return true;
  }

  static inline int Format(const DateTimeFormatContext& dt_ctx,
      const boost::gregorian::date& d, const boost::posix_time::time_duration& t,
      int len, char* buff) {
    DCHECK(TimestampParser::initialized_);
    DCHECK(dt_ctx.toks.size() > 0);
    DCHECK(len > dt_ctx.fmt_out_len);
    DCHECK(buff != NULL);
    if (dt_ctx.has_date_toks && d.is_special()) return -1;
    if (dt_ctx.has_time_toks && t.is_special()) return -1;
    char* str = buff;
    BOOST_FOREACH(const DateTimeFormatToken& tok, dt_ctx.toks) {
      int32_t num_val = -1;
      const char* str_val = NULL;
      int str_val_len = 0;
      switch (tok.type) {
        case YEAR: {
          num_val = d.year();
          if (tok.len <= 3) num_val %= 100;
          break;
        }
        case MONTH_IN_YEAR: num_val = d.month().as_number(); break;
        case MONTH_IN_YEAR_SLT: {
          str_val = d.month().as_short_string();
          str_val_len = 3;
          break;
        }
        case DAY_IN_MONTH: num_val = d.day(); break;
        case HOUR_IN_DAY: num_val = t.hours(); break;
        case MINUTE_IN_HOUR: num_val = t.minutes(); break;
        case SECOND_IN_MINUTE: num_val = t.seconds(); break;
        case FRACTION: {
          num_val = t.fractional_seconds();
          if (num_val > 0) for (int j = tok.len; j < 9; ++j) num_val /= 10;
          break;
        }
        case SEPARATOR: {
          str_val = tok.val;
          str_val_len = tok.len;
          break;
        }
        default: DCHECK(false) << "Unknown date/time format token";
      }
      if (num_val > -1) {
        str += sprintf(str, "%0*d", tok.len, num_val);
      } else {
        memcpy(str, str_val, str_val_len);
        str += str_val_len;
      }
    }
    *str = '\0';
    return str - buff;
  }

 private:
  static inline bool ParseDateTime(const char* str, int str_len,
      const DateTimeFormatContext& dt_ctx, DateTimeParseResult* dt_result) {
    DCHECK(dt_ctx.fmt_len > 0);
    DCHECK(dt_ctx.toks.size() > 0);
    DCHECK(dt_result != NULL);
    if (str_len <= 0 || str_len < dt_ctx.fmt_len || str == NULL) return false;
    StringParser::ParseResult status;
    BOOST_FOREACH(const DateTimeFormatToken& tok, dt_ctx.toks) {
      const char* tok_val = str + tok.pos;
      if (tok.type == SEPARATOR) {
        if (UNLIKELY(*tok_val != *tok.val)) return false;
        continue;
      }
      switch (tok.type) {
        case YEAR: {
          dt_result->year = StringParser::StringToInt<int>(tok_val, tok.len, &status);
          if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
          if (UNLIKELY(dt_result->year < 1 || dt_result->year > 9999)) return false;
          if (tok.len < 4 && dt_result->year < 99) dt_result->year += 2000;
          break;
        }
        case MONTH_IN_YEAR: {
          dt_result->month = StringParser::StringToInt<int>(tok_val, tok.len, &status);
          if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
          if (UNLIKELY(dt_result->month < 1 || dt_result->month > 12)) return false;
          break;
        }
        case MONTH_IN_YEAR_SLT: {
          char raw_buff[tok.len];
          std::transform(tok_val, tok_val + tok.len, raw_buff, ::tolower);
          StringValue buff(raw_buff, tok.len);
          boost::unordered_map<StringValue, int>::const_iterator iter =
              REV_MONTH_INDEX.find(buff);
          if (UNLIKELY(iter == REV_MONTH_INDEX.end())) return false;
          dt_result->month = iter->second;
          break;
        }
        case DAY_IN_MONTH: {
          dt_result->day = StringParser::StringToInt<int>(tok_val, tok.len, &status);
          if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
          // TODO: Validate that the value of day is correct for the given month.
          if (UNLIKELY(dt_result->day < 1 || dt_result->day > 31)) return false;
          break;
        }
        case HOUR_IN_DAY: {
          dt_result->hour = StringParser::StringToInt<int>(tok_val, tok.len, &status);
          if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
          if (UNLIKELY(dt_result->hour < 0 || dt_result->hour > 23)) return false;
          break;
        }
        case MINUTE_IN_HOUR: {
          dt_result->minute = StringParser::StringToInt<int>(tok_val, tok.len, &status);
          if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
          if (UNLIKELY(dt_result->minute < 0 || dt_result->minute > 59)) return false;
          break;
        }
        case SECOND_IN_MINUTE: {
          dt_result->second = StringParser::StringToInt<int>(tok_val, tok.len, &status);
          if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
          if (UNLIKELY(dt_result->second < 0 || dt_result->second > 59)) return false;
          break;
        }
        case FRACTION: {
          dt_result->fraction =
              StringParser::StringToInt<int32_t>(tok_val, tok.len, &status);
          if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
          // A user may specify a time of 04:30:22.1238, the parser will return 1238 for
          // the fractional portion. This does not represent the intended value of
          // 123800000, therefore the number must be scaled up.
          for (int i = tok.len; i < 9; ++i) dt_result->fraction *= 10;
          break;
        }
        default: DCHECK(false) << "Unknown date/time format token";
      }
    }
    return true;
  }

  static const int DEFAULT_DATE_FMT_LEN = 10;
  static const int DEFAULT_TIME_FMT_LEN = 8;
  static const int DEFAULT_TIME_FRAC_FMT_LEN = 18;
  static const int DEFAULT_SHORT_DATE_TIME_FMT_LEN = 19;
  static const int DEFAULT_DATE_TIME_FMT_LEN = 29;

  static bool initialized_;

  static boost::unordered_map<StringValue, int> REV_MONTH_INDEX;

  static DateTimeFormatContext DEFAULT_SHORT_DATE_TIME_CTX;
  static DateTimeFormatContext DEFAULT_SHORT_ISO_DATE_TIME_CTX;
  static DateTimeFormatContext DEFAULT_DATE_CTX;
  static DateTimeFormatContext DEFAULT_TIME_CTX;
  static DateTimeFormatContext DEFAULT_DATE_TIME_CTX[10];
  static DateTimeFormatContext DEFAULT_ISO_DATE_TIME_CTX[10];
  static DateTimeFormatContext DEFAULT_TIME_FRAC_CTX[10];
};

}

#endif