ctf/org.eclipse.tracecompass.ctf.core.tests/src/org/eclipse/tracecompass/ctf/core/tests/ctftestsuite/CtfTestSuiteTest.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2014 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Alexandre Montplaisir - Initial API and implementation
  11  *******************************************************************************/
  12
  13 package org.eclipse.tracecompass.ctf.core.tests.ctftestsuite;
  14
  15 import static org.junit.Assert.assertNotNull;
  16 import static org.junit.Assert.fail;
  17
  18 import java.io.IOException;
  19 import java.nio.file.DirectoryStream;
  20 import java.nio.file.Files;
  21 import java.nio.file.Path;
  22 import java.nio.file.Paths;
  23 import java.util.LinkedList;
  24 import java.util.List;
  25 import java.util.concurrent.TimeUnit;
  26
  27 import org.eclipse.tracecompass.ctf.core.CTFException;
  28 import org.eclipse.tracecompass.ctf.core.trace.CTFTrace;
  29 import org.eclipse.tracecompass.ctf.core.trace.CTFTraceReader;
  30 import org.junit.Rule;
  31 import org.junit.Test;
  32 import org.junit.rules.TestRule;
  33 import org.junit.rules.Timeout;
  34 import org.junit.runner.RunWith;
  35 import org.junit.runners.Parameterized;
  36 import org.junit.runners.Parameterized.Parameters;
  37
  38 /**
  39  * Parameterized test class running the CTF Test Suite
  40  *
  41  * (from https://github.com/efficios/ctf-testsuite).
  42  *
  43  * @author Alexandre Montplaisir
  44  */
  45 @RunWith(Parameterized.class)
  46 public class CtfTestSuiteTest {
  47
  48     /** Time-out tests after 10 seconds. */
  49     @Rule
  50     public TestRule globalTimeout = new Timeout(10, TimeUnit.SECONDS);
  51
  52     private static final Path BASE_PATH = Paths.get("traces", "ctf-testsuite", "tests", "1.8");
  53
  54     /**
  55      * Test we know are currently failing. Ignore them so we can at least run
  56      * the others.
  57      *
  58      * TODO Actually fix them!
  59      */
  60     private static final Path[] IGNORED_TESTS = {
  61             BASE_PATH.resolve(Paths.get("regression", "metadata", "pass", "sequence-typedef-length")),
  62             BASE_PATH.resolve(Paths.get("regression", "stream", "pass", "integer-large-size")),
  63     };
  64
  65     private final String fTracePath;
  66     private final boolean fExpectSuccess;
  67
  68     // ------------------------------------------------------------------------
  69     // Methods for the Parametrized runner
  70     // ------------------------------------------------------------------------
  71
  72     /**
  73      * Get the existing trace paths in the CTF-Testsuite git tree.
  74      *
  75      * @return The list of CTF traces (directories) to test
  76      */
  77     @Parameters(name = "{index}: {0}")
  78     public static Iterable<Object[]> getTracePaths() {
  79         final List<Object[]> dirs = new LinkedList<>();
  80
  81         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "metadata", "fail")), false);
  82         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "metadata", "pass")), true);
  83         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "stream", "fail")), false);
  84         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("fuzzing", "stream", "pass")), true);
  85
  86         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "metadata", "fail")), false);
  87         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "metadata", "pass")), true);
  88         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "stream", "fail")), false);
  89         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("regression", "stream", "pass")), true);
  90
  91         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "metadata", "fail")), false);
  92         addDirsOneLevelDeepFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "metadata", "pass")), true);
  93         addDirsFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "stream", "fail")), false);
  94         addDirsOneLevelDeepFrom(dirs, BASE_PATH.resolve(Paths.get("stress", "stream", "pass")), true);
  95
  96         return dirs;
  97     }
  98
  99     private static void addDirsFrom(List<Object[]> dirs, Path path, boolean expectSuccess) {
 100         if (!Files.exists(path)) {
 101             /* Some planned directories may not exist yet in the test suite */
 102             return;
 103         }
 104         try (DirectoryStream<Path> ds = Files.newDirectoryStream(path, DIR_FILTER);) {
 105             for (Path p : ds) {
 106                 /* Add this test case to the list of tests to run */
 107                 Object array[] = new Object[] { p.toString(), expectSuccess };
 108                 dirs.add(array);
 109             }
 110         } catch (IOException e) {
 111             /* Something is wrong with the layout of the test suite? */
 112             e.printStackTrace();
 113         }
 114     }
 115
 116     /**
 117      * Some test traces are not in pass/trace1, pass/trace2, etc. but rather
 118      * pass/test1/trace1, pass/test1/trace2, etc.
 119      *
 120      * This methods adds the directories one level "down" instead of the very
 121      * next level.
 122      */
 123     private static void addDirsOneLevelDeepFrom(List<Object[]> dirs, Path path,
 124             boolean expectSuccess) {
 125         if (!Files.exists(path)) {
 126             return;
 127         }
 128         try (DirectoryStream<Path> ds = Files.newDirectoryStream(path, DIR_FILTER);) {
 129             for (Path p : ds) {
 130                 addDirsFrom(dirs, p, expectSuccess);
 131             }
 132         } catch (IOException e) {
 133             e.printStackTrace();
 134         }
 135     }
 136
 137     private static final DirectoryStream.Filter<Path> DIR_FILTER =
 138             new DirectoryStream.Filter<Path>() {
 139                 @Override
 140                 public boolean accept(Path entry) {
 141                     /* Only accept directories and non-blacklisted tests */
 142                     if (!Files.isDirectory(entry)) {
 143                         return false;
 144                     }
 145                     for (Path ignoredTestPath : IGNORED_TESTS) {
 146                         if (entry.equals(ignoredTestPath)) {
 147                             System.err.println("Skipping test " + entry.toString() + " as requested.");
 148                             return false;
 149                         }
 150                     }
 151                     return true;
 152                 }
 153             };
 154
 155     // ------------------------------------------------------------------------
 156     // Test constructor
 157     // ------------------------------------------------------------------------
 158
 159     /**
 160      * Constructor for the parametrized tests
 161      *
 162      * @param tracePath
 163      *            The complete path to the trace to test
 164      * @param expectSuccess
 165      *            Should this trace parse successfully, or not.
 166      */
 167     public CtfTestSuiteTest(String tracePath, boolean expectSuccess) {
 168         fTracePath = tracePath;
 169         fExpectSuccess = expectSuccess;
 170     }
 171
 172     // ------------------------------------------------------------------------
 173     // Test methods
 174     // ------------------------------------------------------------------------
 175
 176     /**
 177      * Test opening and reading the trace
 178      */
 179     @Test
 180     public void testTrace() {
 181         try {
 182             /* Instantiate the trace (which implies parsing the metadata) */
 183             CTFTrace trace = new CTFTrace(fTracePath);
 184             /* Read the trace until the end */
 185             try (CTFTraceReader reader = new CTFTraceReader(trace);) {
 186
 187                 reader.getCurrentEventDef();
 188                 while (reader.advance()) {
 189                     assertNotNull(reader.getCurrentEventDef());
 190                 }
 191
 192                 checkIfWeShoudlSucceed();
 193             }
 194         } catch (CTFException e) {
 195             checkIfWeShouldFail(e);
 196         } catch (OutOfMemoryError e) {
 197             checkIfWeShouldFail(e);
 198         }
 199     }
 200
 201     private void checkIfWeShoudlSucceed() {
 202         if (!fExpectSuccess) {
 203             fail("Trace was expected to fail parsing: " + fTracePath);
 204         }
 205     }
 206
 207     private void checkIfWeShouldFail(Throwable e) {
 208         if (fExpectSuccess) {
 209             fail("Trace was expected to succeed, but failed parsing: " +
 210                     fTracePath + " (" + e.getMessage() + ")");
 211         }
 212     }
 213 }