TestFileParser.java

Go to the documentation of this file.

// Copyright (c) 2012 Cloudera, Inc. All rights reserved.

package com.cloudera.impala.testutil;

import static org.junit.Assert.fail;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.Scanner;

import org.apache.log4j.Logger;

import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

public class TestFileParser {
  private static final Logger LOG = Logger.getLogger(TestCase.class);

  public enum Section {
    QUERY,
    TYPES,
    COLLABELS,
    RESULTS,
    PLAN,
    DISTRIBUTEDPLAN,
    FILEERRORS,
    PARTITIONS,
    SETUP,
    ERRORS,
    SCANRANGELOCATIONS,
    LINEAGE;

    // Return header line for this section
    public String getHeader() {
      return "---- " + this.toString();
    }
  }

  public static class TestCase {

    private final EnumMap<Section, ArrayList<String>> expectedResultSections =
        Maps.newEnumMap(Section.class);

    // Line number in the test case file where this case started
    private final int startLineNum;

    public TestCase(int lineNum) {
      startLineNum = lineNum;
    }

    public int getStartingLineNum() {
      return startLineNum;
    }

    protected void addSection(Section section, ArrayList<String> contents) {
      expectedResultSections.put(section, contents);
    }

    public ArrayList<String> getSectionContents(Section section) {
      return getSectionContents(section, false);
    }

    public ArrayList<String> getSectionContents(Section section, boolean withComments) {
      return getSectionContents(section, withComments, null);
    }

    public ArrayList<String> getSectionContents(Section section, boolean withComments,
                                                String dbSuffix) {
      ArrayList<String> ret = expectedResultSections.get(section);
      if (ret == null) {
        return Lists.newArrayList();
      } else if (withComments && dbSuffix == null) {
        return ret;
      }

      ArrayList<String> retList = Lists.newArrayList();
      for (String s : ret) {
        if (!(s.startsWith("#") || s.startsWith("//"))) {
          if (dbSuffix != null) {
            retList.add(s.replaceAll("\\$DATABASE", dbSuffix));
          } else {
            retList.add(s);
          }
        } else if (withComments) {
          retList.add(s);
        }
      }

      return retList;
    }

    public String getSectionAsString(Section section, boolean withComments,
        String delimiter) {
      return getSectionAsString(section, withComments, delimiter, null);
    }

    public String getSectionAsString(Section section, boolean withComments,
                                     String delimiter, String dbSuffix) {
      List<String> sectionList = getSectionContents(section, withComments, dbSuffix);
      if (sectionList == null) {
        return null;
      }

      return Joiner.on(delimiter).join(sectionList);
    }

    public String getQuery() {
      return getSectionAsString(Section.QUERY, false, "\n");
    }

    public boolean isValid() {
      return !getQuery().isEmpty() && (!getSectionContents(Section.PLAN).isEmpty() ||
          !getSectionContents(Section.DISTRIBUTEDPLAN).isEmpty() ||
          !getSectionContents(Section.LINEAGE).isEmpty());
    }
  }

  private final List<TestCase> testCases = Lists.newArrayList();

  private int lineNum = 0;
  private final String fileName;
  private BufferedReader reader;
  private Scanner scanner;
  private boolean hasSetupSection = false;

  static private final ArrayList<Section> defaultSectionOrder =
    Lists.newArrayList(Section.QUERY, Section.TYPES, Section.RESULTS);

  public TestFileParser(String fileName) {
    this.fileName = fileName;
  }

  public List<TestCase> getTestCases() {
    return testCases;
  }

  public String getTestFileName() {
    return fileName;
  }

  public boolean hasSetupSection() {
    return hasSetupSection;
  }

  private void open(String table) {
    try {
      String testFileBaseDir =
          new File(System.getenv("IMPALA_HOME"), "testdata/workloads").getPath();
      String fullPath = new File(testFileBaseDir, fileName).getPath();
      reader = new BufferedReader(new FileReader(fullPath));
      scanner = new Scanner(reader);
    } catch (Exception e) {
      fail(e.getMessage());
    }
  }

  private TestCase parseOneTestCase() {
    Section currentSection = Section.QUERY;
    ArrayList<String> sectionContents = Lists.newArrayList();
    TestCase currentTestCase = new TestCase(lineNum);
    int sectionCount = 0;

    while (scanner.hasNextLine()) {
      String line = scanner.nextLine();
      ++lineNum;
      if (line.startsWith("====") && sectionCount > 0) {
        currentTestCase.addSection(currentSection, sectionContents);
        if(!currentTestCase.isValid()) {
          throw new IllegalStateException("Invalid test case" +
              " at line " + currentTestCase.startLineNum + " detected.");
        }
        return currentTestCase; // done with this test case
      }
      if (line.startsWith("----")) {
        sectionCount++;
        // New section
        currentTestCase.addSection(currentSection, sectionContents);
        boolean found = false;
        line = line.trim().toLowerCase();

        // Check for section header - a missing header probably means an old test file.
        if (!line.endsWith("----")) {
          for (Section s : Section.values()) {
            if (line.matches("----\\s+" + s.toString().toLowerCase() + "\\b.*")) {
              currentSection = s;
              if (s == Section.SETUP) {
                hasSetupSection = true;
              }
              found = true;
              break;
            }
          }
          if (!found) {
            throw new IllegalStateException("Unknown section name: " + line);
          }
        } else {
          // Backwards compatibility only - TODO remove once all test files have section
          // headers.
          if (sectionCount >= defaultSectionOrder.size()) {
            throw new IllegalStateException("Unexpected number of untitled sections: "
                + sectionCount);
          }
          currentSection = defaultSectionOrder.get(sectionCount);
          LOG.warn("No section header found. Guessing: " + currentSection);
        }

        sectionContents = Lists.newArrayList();
      } else {
        sectionContents.add(line);
      }
    }

    if(!currentTestCase.isValid()) {
      throw new IllegalStateException("Invalid test case" +
          " at line " + currentTestCase.startLineNum + " detected.");
    }

    return currentTestCase;
  }

  public void parseFile() {
    parseFile(null);
  }

  public void parseFile(String table) {
    try {
      open(table);
      testCases.clear();
      while (scanner.hasNextLine()) {
        testCases.add(parseOneTestCase());
      }
    } finally {
      close();
    }
  }

  private void close() {
    if (reader != null) {
      try {
        reader.close();
      } catch (IOException e) {
        fail(e.getMessage());
      }
    }

    if (scanner != null) {
      scanner.close();
    }
  }
}