org.eclipse.tracecompass.ctf.core.tests/src/org/eclipse/tracecompass/ctf/core/tests/ctftestsuite/CtfTestSuiteTests.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2014 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Alexandre Montplaisir - Initial API and implementation
  11  *******************************************************************************/
  12
  13 package org.eclipse.tracecompass.ctf.core.tests.ctftestsuite;
  14
  15 import static org.junit.Assert.assertNotNull;
  16 import static org.junit.Assert.fail;
  17
  18 import java.io.IOException;
  19 import java.nio.file.DirectoryStream;
  20 import java.nio.file.Files;
  21 import java.nio.file.Path;
  22 import java.nio.file.Paths;
  23 import java.util.LinkedList;
  24 import java.util.List;
  25
  26 import org.eclipse.tracecompass.ctf.core.CTFException;
  27 import org.eclipse.tracecompass.ctf.core.trace.CTFTrace;
  28 import org.eclipse.tracecompass.ctf.core.trace.CTFTraceReader;
  29 import org.junit.Rule;
  30 import org.junit.Test;
  31 import org.junit.rules.TestRule;
  32 import org.junit.rules.Timeout;
  33 import org.junit.runner.RunWith;
  34 import org.junit.runners.Parameterized;
  35 import org.junit.runners.Parameterized.Parameters;
  36
  37 /**
  38  * Parameterized test class running the CTF Test Suite
  39  *
  40  * (from https://github.com/efficios/ctf-testsuite).
  41  *
  42  * @author Alexandre Montplaisir
  43  */
  44 @RunWith(Parameterized.class)
  45 public class CtfTestSuiteTests {
  46
  47     /** Time-out tests after 10 seconds. */
  48     @Rule
  49     public TestRule globalTimeout = new Timeout(10000);
  50
  51     private static final Path BASE_PATH = Paths.get("traces", "ctf-testsuite", "tests", "1.8");
  52
  53     /**
  54      * Test we know are currently failing. Ignore them so we can at least run
  55      * the others.
  56      *
  57      * TODO Actually fix them!
  58      */
  59     private static final Path[] IGNORED_TESTS = {
  60             BASE_PATH.resolve(Paths.get("regression", "metadata", "pass", "sequence-typedef-length")),
  61             BASE_PATH.resolve(Paths.get("regression", "metadata", "pass", "array-of-struct")),
  62             BASE_PATH.resolve(Paths.get("regression", "stream", "pass", "integer-large-size")),
  63             BASE_PATH.resolve(Paths.get("regression", "metadata", "fail", "metadata-packetized-endianness-mismatch"))
  64     };
  65
  66     private final String fTracePath;
  67     private final boolean fExpectSuccess;
  68
  69     // ------------------------------------------------------------------------
  70     // Methods for the Parametrized runner
  71     // ------------------------------------------------------------------------
  72
  73     /**
  74      * Get the existing trace paths in the CTF-Testsuite git tree.
  75      *
  76      * @return The list of CTF traces (directories) to test
  77      */
  78     @Parameters(name = "{index}: {0}")
  79     public static Iterable<Object[]> getTracePaths() {
  80         final List<Object[]> dirs = new LinkedList<>();
  81
  82         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "metadata", "fail")), false);
  83         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "metadata", "pass")), true);
  84         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "stream", "fail")), false);
  85         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "stream", "pass")), true);
  86
  87         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "metadata", "fail")), false);
  88         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "metadata", "pass")), true);
  89         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "stream", "fail")), false);
  90         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "stream", "pass")), true);
  91
  92         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "metadata", "fail")), false);
  93         addDirsOneLevelDeepFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "metadata", "pass")), true);
  94         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "stream", "fail")), false);
  95         addDirsOneLevelDeepFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "stream", "pass")), true);
  96
  97         return dirs;
  98     }
  99
 100     private static void addDirsFrom(List<Object[]> dirs, Path path, boolean expectSuccess) {
 101         if (!Files.exists(path)) {
 102             /* Some planned directories may not exist yet in the test suite */
 103             return;
 104         }
 105         try (DirectoryStream<Path> ds = Files.newDirectoryStream(path, DIR_FILTER);) {
 106             for (Path p : ds) {
 107                 /* Add this test case to the list of tests to run */
 108                 Object array[] = new Object[] { p.toString(), expectSuccess };
 109                 dirs.add(array);
 110             }
 111         } catch (IOException e) {
 112             /* Something is wrong with the layout of the test suite? */
 113             e.printStackTrace();
 114         }
 115     }
 116
 117     /**
 118      * Some test traces are not in pass/trace1, pass/trace2, etc. but rather
 119      * pass/test1/trace1, pass/test1/trace2, etc.
 120      *
 121      * This methods adds the directories one level "down" instead of the very
 122      * next level.
 123      */
 124     private static void addDirsOneLevelDeepFrom(List<Object[]> dirs, Path path,
 125             boolean expectSuccess) {
 126         if (!Files.exists(path)) {
 127             return;
 128         }
 129         try (DirectoryStream<Path> ds = Files.newDirectoryStream(path, DIR_FILTER);) {
 130             for (Path p : ds) {
 131                 addDirsFrom(dirs, p, expectSuccess);
 132             }
 133         } catch (IOException e) {
 134             e.printStackTrace();
 135         }
 136     }
 137
 138     private static final DirectoryStream.Filter<Path> DIR_FILTER =
 139             new DirectoryStream.Filter<Path>() {
 140                 @Override
 141                 public boolean accept(Path entry) {
 142                     /* Only accept directories and non-blacklisted tests */
 143                     if (!Files.isDirectory(entry)) {
 144                         return false;
 145                     }
 146                     for (Path ignoredTestPath : IGNORED_TESTS) {
 147                         if (entry.equals(ignoredTestPath)) {
 148                             System.err.println("Skipping test " + entry.toString() + " as requested.");
 149                             return false;
 150                         }
 151                     }
 152                     return true;
 153                 }
 154             };
 155
 156     // ------------------------------------------------------------------------
 157     // Test constructor
 158     // ------------------------------------------------------------------------
 159
 160     /**
 161      * Constructor for the parametrized tests
 162      *
 163      * @param tracePath
 164      *            The complete path to the trace to test
 165      * @param expectSuccess
 166      *            Should this trace parse successfully, or not.
 167      */
 168     public CtfTestSuiteTests(String tracePath, boolean expectSuccess) {
 169         fTracePath = tracePath;
 170         fExpectSuccess = expectSuccess;
 171     }
 172
 173     // ------------------------------------------------------------------------
 174     // Test methods
 175     // ------------------------------------------------------------------------
 176
 177     /**
 178      * Test opening and reading the trace
 179      */
 180     @Test
 181     public void testTrace() {
 182         try {
 183             /* Instantiate the trace (which implies parsing the metadata) */
 184             CTFTrace trace = new CTFTrace(fTracePath);
 185             /* Read the trace until the end */
 186             try (CTFTraceReader reader = new CTFTraceReader(trace);) {
 187
 188                 reader.getCurrentEventDef();
 189                 while (reader.advance()) {
 190                     assertNotNull(reader.getCurrentEventDef());
 191                 }
 192
 193                 checkIfWeShoudlSucceed();
 194             }
 195         } catch (CTFException e) {
 196             checkIfWeShouldFail(e);
 197         } catch (OutOfMemoryError e) {
 198             checkIfWeShouldFail(e);
 199         }
 200     }
 201
 202     private void checkIfWeShoudlSucceed() {
 203         if (!fExpectSuccess) {
 204             fail("Trace was expected to fail parsing: " + fTracePath);
 205         }
 206     }
 207
 208     private void checkIfWeShouldFail(Throwable e) {
 209         if (fExpectSuccess) {
 210             fail("Trace was expected to succeed, but failed parsing: " +
 211                     fTracePath + " (" + e.getMessage() + ")");
 212         }
 213     }
 214 }