releng/org.eclipse.tracecompass.alltests/src/org/eclipse/tracecompass/alltests/perf/PerfResultsToJSon.java

   1 /*******************************************************************************
   2  * Copyright (c) 2014 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Marc-Andre Laperle - Initial API and implementation
  11  *******************************************************************************/
  12
  13 package org.eclipse.tracecompass.alltests.perf;
  14
  15 import java.io.FileWriter;
  16 import java.io.IOException;
  17 import java.text.ParseException;
  18 import java.text.SimpleDateFormat;
  19 import java.util.ArrayList;
  20 import java.util.Date;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.regex.Matcher;
  24 import java.util.regex.Pattern;
  25
  26 import org.eclipse.jdt.annotation.NonNull;
  27 import org.eclipse.test.internal.performance.PerformanceTestPlugin;
  28 import org.eclipse.test.internal.performance.data.Dim;
  29 import org.eclipse.test.internal.performance.db.DB;
  30 import org.eclipse.test.internal.performance.db.Scenario;
  31 import org.eclipse.test.internal.performance.db.SummaryEntry;
  32 import org.eclipse.test.internal.performance.db.TimeSeries;
  33 import org.eclipse.test.internal.performance.db.Variations;
  34 import org.eclipse.tracecompass.alltests.Activator;
  35 import org.json.JSONArray;
  36 import org.json.JSONException;
  37 import org.json.JSONObject;
  38 import org.junit.Test;
  39
  40 /**
  41  * Convert results from the database to JSON suitable for display.
  42  *
  43  * Normal charts:
  44  *
  45  * Individual charts are generated into JSON files in the form chart#.json where
  46  * # is incremented for each new chart. A chart contains data points consisting
  47  * of X and Y values suitable for a line chart. Each point can also have
  48  * additional data, for example the commit id. This format is compatible with
  49  * nvd3. For example:
  50  *
  51  * <pre>
  52  * <code>
  53  * [{
  54  *   "key": "Experiment Benchmark:84 traces",
  55  *   "values": [{
  56  *       "label": {"commit": "fe3c142"},
  57  *       "x": 1405024320000,
  58  *       "y": 17592
  59  *   }]
  60  * }]
  61  * </code>
  62  * </pre>
  63  *
  64  * Normal charts metadata:
  65  *
  66  * Each chart has an entry in the metada.js file which organizes the charts per
  67  * component and contains additional information to augment the format expected
  68  * by nvd3. Each entry contains the combination of OS and JVM, the filename (in
  69  * JSON format), the title of the chart, the unit (seconds, etc) and the
  70  * dimension (CPU time, used heap, etc).
  71  *
  72  * <pre>
  73  *  <code>
  74  *  var MetaData = {
  75  *     "applicationComponents": {
  76  *         "Experiment benchmark": {
  77  *             "name": "Experiment benchmark",
  78  *             "tests": [
  79  *                 {
  80  *                     "dimension": "CPU Time",
  81  *                     "file": "chart12",
  82  *                     "jvm": "1.7",
  83  *                     "os": "linux",
  84  *                     "title": "Experiment Benchmark:84 traces",
  85  *                     "unit": "s"
  86  *                 },
  87  *                 {
  88  *                     "dimension": "CPU Time",
  89  *                     "file": "chart11",
  90  *                     "jvm": "1.7",
  91  *                     "os": "linux",
  92  *                     "title": "Experiment Benchmark:6 traces",
  93  *                     "unit": "s"
  94  *                 },
  95  * ...
  96  *  </code>
  97  * </pre>
  98  *
  99  * Overview charts:
 100  *
 101  * In addition to the normal charts, overview charts are generated. An overview
 102  * chart presents a summary of the scenarios ran for a given OS and JVM
 103  * combination. Only scenarios marked as "global" are added to the overview
 104  * because of space concerns. Overview charts are generated under the
 105  * chart_overview#.json name and look similar in structure to the normal charts
 106  * except that they contain more than one series.
 107  *
 108  * <pre>
 109  *   <code>
 110  * [
 111  *   {
 112  *       "key": "CTF Read & Seek Benchmark (500 seeks):tr",
 113  *       "values": [
 114  *           {
 115  *               "label": {"commit": "4d34345"},
 116  *               "x": 1405436820000,
 117  *               "y": 5382.5
 118  *           },
 119  *           ...
 120  *       ]
 121  *   },
 122  *   {
 123  *       "key": "CTF Read Benchmark:trace-kernel",
 124  *       "values": [
 125  *           {
 126  *               "label": {"commit": "4d34345"},
 127  *               "x": 1405436820000,
 128  *               "y": 1311.5
 129  *           },
 130  *           ...
 131  *       ]
 132  *   },
 133  *   ...
 134  *   </code>
 135  * </pre>
 136  *
 137  * Overview charts metadata:
 138  *
 139  * Overview charts also have similar metadata entries to normal charts except
 140  * they are not organized by component.
 141  *
 142  * <pre>
 143  *   <code>
 144  * var MetaData = {
 145  * ...
 146  *   "overviews": {
 147  *       "1": {
 148  *           "dimension": "",
 149  *           "file": "chart_overview0",
 150  *           "jvm": "1.7",
 151  *           "os": "linux",
 152  *           "title": "linux / 1.7",
 153  *           "unit": ""
 154  *       },
 155  *       "2": {
 156  *           "dimension": "",
 157  *           "file": "chart_overview1",
 158  *           "jvm": "1.7",
 159  *           "os": "windows",
 160  *           "title": "windows / 1.7",
 161  *           "unit": ""
 162  *       },
 163  *       ...
 164  *   </code>
 165  * </pre>
 166  *
 167  * Finally, since we want to be able to filter all the charts by OS/JVM
 168  * combination, there is a section in the metadata that lists all the
 169  * combinations:
 170  *
 171  * <pre>
 172  *   <code>
 173  *     "osjvm": {
 174  *       "1": {
 175  *           "description": "linux / 1.7",
 176  *           "jvm": "1.7",
 177  *           "os": "linux"
 178  *       },
 179  *       "2": {
 180  *           "description": "windows / 1.7",
 181  *           "jvm": "1.7",
 182  *           "os": "windows"
 183  *       },
 184  *       "3": {
 185  *           "description": "mac / 1.7",
 186  *           "jvm": "1.7",
 187  *           "os": "mac"
 188  *       }
 189  *   },
 190  *   </code>
 191  * </pre>
 192  *
 193  * All of this data is meant to be view on a website. Specifically, the source
 194  * code for our implementation is available on GitHub at
 195  * https://github.com/PSRCode/ITCFYWebsite
 196  *
 197  * It makes use of the NVD3 project to display the charts based on the data
 198  * generated by this class.
 199  */
 200 public class PerfResultsToJSon {
 201
 202     /*
 203      * Labels
 204      */
 205     private static final String APPLICATION_COMPONENTS_LABEL = "applicationComponents";
 206     private static final String BUILD_LABEL = "build";
 207     private static final String COMMIT_LABEL = "commit";
 208     private static final String CONFIG_LABEL = "config";
 209     private static final String DESCRIPTION_LABEL = "description";
 210     private static final String DIMENSION_LABEL = "dimension";
 211     private static final String FILE_LABEL = "file";
 212     private static final String HOST_LABEL = "host";
 213     private static final String JVM_LABEL = "jvm";
 214     private static final String KEY_LABEL = "key";
 215     private static final String LABEL_LABEL = "label";
 216     private static final String NAME_LABEL = "name";
 217     private static final String OS_LABEL = "os";
 218     private static final String OSJVM_LABEL = "osjvm";
 219     private static final String OVERVIEWS_LABEL = "overviews";
 220     private static final String TESTS_LABEL = "tests";
 221     private static final String TITLE_LABEL = "title";
 222     private static final String UNIT_LABEL = "unit";
 223     private static final String VALUES_LABEL = "values";
 224     private static final String X_LABEL = "x";
 225     private static final String Y_LABEL = "y";
 226
 227     private static final String BUILD_DATE_FORMAT = "yyyyMMdd-HHmm";
 228     private static final String OVERVIEW_CHART_FILE_NAME = "chart_overview";
 229     private static final String METADATA_FILE_NAME = "meta";
 230     private static final String METADATA_FILE_NAME_EXTENSION = ".js";
 231     private static final String CHART_FILE_NAME = "chart";
 232     private static final String CHART_FILE_NAME_EXTENSION = ".json";
 233     private static final String WILDCARD_PATTERN = "%";
 234     private static final @NonNull String COMPONENT_SEPARATOR = "#";
 235     private static final String META_DATA_JAVASCRIPT_START = "var MetaData = ";
 236
 237     private static Pattern BUILD_DATE_PATTERN = Pattern.compile("(\\w+-\\w+)(-\\w+)?");
 238     private static Pattern COMMIT_PATTERN = Pattern.compile(".*-.*-(.*)");
 239
 240     private JSONObject fApplicationComponents = new JSONObject();
 241     private JSONObject fOverviews = new JSONObject();
 242
 243     private int fNumChart = 0;
 244     private int fNumOverviewChart = 0;
 245
 246     /**
 247      * Convert results from the database to JSON suitable for display
 248      *
 249      * <pre>
 250      * For each variant (os/jvm combination)
 251      *    - For each summary entry (scenario)
 252      *      - Generate a chart
 253      *      - Add it to global summary (if needed)
 254      *      - Create the metadata for this test
 255      *    - Create an overview chart for this os/jvm
 256      * </pre>
 257      *
 258      * @throws JSONException
 259      *             JSON error
 260      * @throws IOException
 261      *             IO error
 262      */
 263     @Test
 264     public void parseResults() throws JSONException, IOException {
 265         Variations configVariations = PerformanceTestPlugin.getVariations();
 266         JSONObject osJvmVariants = createOsJvm();
 267
 268         Iterator<?> keysIt = osJvmVariants.keys();
 269         while (keysIt.hasNext()) {
 270             JSONArray overviewSummarySeries = new JSONArray();
 271
 272             JSONObject variant = osJvmVariants.getJSONObject((String) keysIt.next());
 273             String seriesKey = PerformanceTestPlugin.BUILD;
 274
 275             // Clone the variations from the environment because it might have
 276             // extra parameters like host=, etc.
 277             Variations buildVariations = (Variations) configVariations.clone();
 278             buildVariations.setProperty(JVM_LABEL, variant.getString(JVM_LABEL));
 279             buildVariations.setProperty(CONFIG_LABEL, variant.getString(OS_LABEL));
 280             buildVariations.setProperty(BUILD_LABEL, WILDCARD_PATTERN);
 281
 282             Scenario[] scenarios = DB.queryScenarios(buildVariations, WILDCARD_PATTERN, seriesKey, null);
 283             SummaryEntry[] summaryEntries = DB.querySummaries(buildVariations, WILDCARD_PATTERN);
 284             for (SummaryEntry entry : summaryEntries) {
 285                 Scenario scenario = getScenario(entry.scenarioName, scenarios);
 286                 JSONObject scenarioSeries = createScenarioChart(scenario, entry, buildVariations);
 287                 // Add to global summary
 288                 if (scenarioSeries != null && entry.isGlobal) {
 289                     overviewSummarySeries.put(scenarioSeries);
 290                 }
 291             }
 292
 293             JSONObject overviewMetadata = createOverviewChart(overviewSummarySeries, buildVariations);
 294             fOverviews.put(Integer.toString(fNumOverviewChart), overviewMetadata);
 295         }
 296
 297         // Create the matadata javascript file that includes OS/JVM combinations
 298         // (for filtering), application components and overviews (one of OS/JVM
 299         // combination)
 300         JSONObject rootMetadata = new JSONObject();
 301         rootMetadata.put(OSJVM_LABEL, osJvmVariants);
 302         rootMetadata.put(APPLICATION_COMPONENTS_LABEL, fApplicationComponents);
 303         rootMetadata.put(OVERVIEWS_LABEL, fOverviews);
 304         try (FileWriter fw1 = new FileWriter(METADATA_FILE_NAME + METADATA_FILE_NAME_EXTENSION)) {
 305             fw1.write(META_DATA_JAVASCRIPT_START + rootMetadata.toString(4));
 306         }
 307     }
 308
 309     /**
 310      * Create chart for a scenario instance and add it to the relevant metadatas
 311      *
 312      * @param scenario
 313      *            the scenario. For example,
 314      *            "CTF Read & Seek Benchmark (500 seeks)".
 315      * @param entry
 316      *            an entry from the summary. Only scenarios that are part of the
 317      *            summary are processed.
 318      * @param variations
 319      *            all variations to consider to create the scenario chart. For
 320      *            example build=%;jvm=1.7;config=linux will generate a chart for
 321      *            all builds on Linux / JVM 1.7
 322      *
 323      * @return
 324      * @throws JSONException
 325      *             JSON error
 326      * @throws IOException
 327      *             IO error
 328      */
 329     private JSONObject createScenarioChart(Scenario scenario, SummaryEntry entry, Variations variations) throws JSONException, IOException {
 330         if (scenario == null) {
 331             return null;
 332         }
 333         String[] split = entry.scenarioName.split(COMPONENT_SEPARATOR);
 334         if (split.length < 3) {
 335             Activator.logError("Invalid scenario name \"" + entry.scenarioName + "\", it must be in format: org.package.foo#component#test");
 336             return null;
 337         }
 338
 339         // Generate individual chart
 340         JSONArray rootScenario = new JSONArray();
 341         JSONObject series = createSerie(scenario, variations, entry.shortName, entry.dimension);
 342         rootScenario.put(series);
 343         int numChart = fNumChart++;
 344         try (FileWriter fw = new FileWriter(CHART_FILE_NAME + numChart + CHART_FILE_NAME_EXTENSION)) {
 345             fw.write(rootScenario.toString(4));
 346         }
 347
 348         // Create the metadata
 349         JSONObject testMetadata = new JSONObject();
 350         testMetadata.put(TITLE_LABEL, entry.shortName);
 351         testMetadata.put(FILE_LABEL, CHART_FILE_NAME + numChart);
 352         testMetadata.put(OS_LABEL, variations.getProperty(CONFIG_LABEL));
 353         testMetadata.put(JVM_LABEL, variations.getProperty(JVM_LABEL));
 354         testMetadata.put(DIMENSION_LABEL, entry.dimension.getName());
 355         testMetadata.put(UNIT_LABEL, entry.dimension.getUnit().getShortName());
 356
 357         // Add the scenario to the metadata, under the correct component
 358         String componentName = split[1];
 359         JSONObject componentObject = null;
 360         if (fApplicationComponents.has(componentName)) {
 361             componentObject = fApplicationComponents.getJSONObject(componentName);
 362         } else {
 363             componentObject = new JSONObject();
 364             componentObject.put(NAME_LABEL, componentName);
 365             componentObject.put(TESTS_LABEL, new JSONArray());
 366             fApplicationComponents.put(componentName, componentObject);
 367         }
 368         JSONArray tests = componentObject.getJSONArray(TESTS_LABEL);
 369         tests.put(testMetadata);
 370
 371         return series;
 372     }
 373
 374     /**
 375      * Create an overview chart for this OS / JVM combination. The chart is made
 376      * of multiple series (scenarios) that were marked as global.
 377      *
 378      * @param overviewSummarySeries
 379      *            an array of series to include in the chart (multiple
 380      *            scenarios)
 381      * @param variations
 382      *            the variations used to generate the series to be included in
 383      *            this overview chart. For example build=%;jvm=1.7;config=linux
 384      *            will generate an overview chart for Linux / JVM 1.7
 385      * @return the overview metadata JSON object
 386      * @throws JSONException
 387      *             JSON error
 388      * @throws IOException
 389      *             io error
 390      */
 391     private JSONObject createOverviewChart(JSONArray overviewSummarySeries, Variations variations) throws IOException, JSONException {
 392         int numOverviewChart = fNumOverviewChart++;
 393         try (FileWriter fw = new FileWriter(OVERVIEW_CHART_FILE_NAME + numOverviewChart + CHART_FILE_NAME_EXTENSION)) {
 394             fw.write(overviewSummarySeries.toString(4));
 395         }
 396
 397         String os = variations.getProperty(CONFIG_LABEL);
 398         String jvm = variations.getProperty(JVM_LABEL);
 399
 400         // Create the overview metadata
 401         JSONObject overviewMetadata = new JSONObject();
 402         overviewMetadata.put(TITLE_LABEL, os + " / " + jvm);
 403         overviewMetadata.put(FILE_LABEL, OVERVIEW_CHART_FILE_NAME + numOverviewChart);
 404         overviewMetadata.put(OS_LABEL, os);
 405         overviewMetadata.put(JVM_LABEL, jvm);
 406         overviewMetadata.put(DIMENSION_LABEL, "");
 407         overviewMetadata.put(UNIT_LABEL, "");
 408
 409         return overviewMetadata;
 410     }
 411
 412     private static Scenario getScenario(String scenarioName, Scenario[] scenarios) {
 413         for (int i = 0; i < scenarios.length; i++) {
 414             Scenario s = scenarios[i];
 415             if (s.getScenarioName().equals(scenarioName)) {
 416                 return s;
 417             }
 418
 419         }
 420         return null;
 421     }
 422
 423     /**
 424      * Get all combinations of OS / JVM. This will be used for filtering.
 425      *
 426      * @return the JSON object containing all the combinations
 427      * @throws JSONException
 428      *             JSON error
 429      */
 430     private static JSONObject createOsJvm() throws JSONException {
 431         JSONObject osjvm = new JSONObject();
 432         List<String> oses = getDistinctOses();
 433
 434         int osJvmIndex = 1;
 435         for (String os : oses) {
 436             String key = JVM_LABEL;
 437             Variations v = new Variations();
 438
 439             v.setProperty(BUILD_LABEL, WILDCARD_PATTERN);
 440             v.setProperty(HOST_LABEL, WILDCARD_PATTERN);
 441             v.setProperty(CONFIG_LABEL, os);
 442             v.setProperty(JVM_LABEL, WILDCARD_PATTERN);
 443
 444             List<String> jvms = new ArrayList<>();
 445             DB.queryDistinctValues(jvms, key, v, WILDCARD_PATTERN);
 446             for (String jvm : jvms) {
 447                 JSONObject osjvmItem = new JSONObject();
 448                 osjvmItem.put(OS_LABEL, os);
 449                 osjvmItem.put(JVM_LABEL, jvm);
 450                 osjvmItem.put(DESCRIPTION_LABEL, os + " / " + jvm);
 451                 osjvm.put(Integer.toString(osJvmIndex), osjvmItem);
 452                 osJvmIndex++;
 453             }
 454         }
 455
 456         return osjvm;
 457     }
 458
 459     /**
 460      * Get all the distinct OS values
 461      *
 462      * @return the distinct OS values
 463      */
 464     private static List<String> getDistinctOses() {
 465         List<String> configs = new ArrayList<>();
 466         String key = PerformanceTestPlugin.CONFIG;
 467         Variations v = new Variations();
 468         v.setProperty(WILDCARD_PATTERN, WILDCARD_PATTERN);
 469         DB.queryDistinctValues(configs, key, v, WILDCARD_PATTERN);
 470         return configs;
 471     }
 472
 473     /**
 474      * This main can be run from within Eclipse provided everything is on the
 475      * class path.
 476      *
 477      * @param args
 478      *            the arguments
 479      * @throws JSONException
 480      *             JSON error
 481      * @throws IOException
 482      *             io error
 483      */
 484     public static void main(String[] args) throws JSONException, IOException {
 485         new PerfResultsToJSon().parseResults();
 486     }
 487
 488     /**
 489      * Create a series of data points for a given scenario through variations
 490      *
 491      * @param scenario
 492      *            the scenario. For example,
 493      *            "CTF Read & Seek Benchmark (500 seeks)".
 494      * @param variations
 495      *            all variations to consider to create the series. For example
 496      *            build=%;jvm=1.7;config=linux will generate the series for all
 497      *            builds on Linux / JVM 1.7
 498      * @param shortName
 499      *            the short name of the scenario
 500      * @param dimension
 501      *            the dimension of interest (CPU time, used java heap, etc).
 502      * @return the generated JSON object representing a series of data points
 503      *         for this scenario
 504      * @throws JSONException
 505      */
 506     private static JSONObject createSerie(Scenario scenario, Variations variations, String shortName, Dim dimension) throws JSONException {
 507         JSONObject o = new JSONObject();
 508         o.putOpt(KEY_LABEL, shortName);
 509         o.putOpt(VALUES_LABEL, createDataPoints(scenario, variations, dimension));
 510         return o;
 511     }
 512
 513     /**
 514      * Create data points for a given scenario and variations.
 515      *
 516      * @param s
 517      *            the scenario. For example,
 518      *            "CTF Read & Seek Benchmark (500 seeks)".
 519      * @param variations
 520      *            all variations to consider to create the data points. For
 521      *            example build=%;jvm=1.7;config=linux will generate the data
 522      *            points for all builds on Linux / JVM 1.7
 523      * @param dimension
 524      *            the dimension of interest (CPU time, used java heap, etc).
 525      *
 526      * @return the generated JSON array of points
 527      * @throws JSONException
 528      *             JSON error
 529      */
 530     private static JSONArray createDataPoints(Scenario s, Variations variations, Dim dimension) throws JSONException {
 531         // Can be uncommented to see raw dump
 532         //s.dump(System.out, PerformanceTestPlugin.BUILD);
 533
 534         String[] builds = DB.querySeriesValues(s.getScenarioName(), variations, PerformanceTestPlugin.BUILD);
 535         Date[] dates = new Date[builds.length];
 536         String[] commits = new String[builds.length];
 537         for (int i = 0; i < builds.length; i++) {
 538             dates[i] = parseBuildDate(builds[i]);
 539             commits[i] = parseCommit(builds[i]);
 540         }
 541
 542         TimeSeries timeSeries = s.getTimeSeries(dimension);
 543         JSONArray dataPoints = new JSONArray();
 544         int length = timeSeries.getLength();
 545         for (int i = 0; i < length; i++) {
 546             JSONObject point = new JSONObject();
 547             if (dates[i] == null) {
 548                 continue;
 549             }
 550             point.put(X_LABEL, dates[i].getTime());
 551             double value = 0;
 552             if (timeSeries.getCount(i) > 0) {
 553                 value = timeSeries.getValue(i);
 554                 if (Double.isNaN(value)) {
 555                     value = 0;
 556                 }
 557             }
 558             point.put(Y_LABEL, value);
 559             dataPoints.put(point);
 560             point.put(LABEL_LABEL, createLabel(commits[i]));
 561         }
 562         return dataPoints;
 563     }
 564
 565     /**
 566      * Create a label JSONObject which is used to attach more information to a
 567      * data point.
 568      *
 569      * @param commit
 570      *            the commit id for this data point
 571      * @return the resulting JSON object
 572      * @throws JSONException
 573      *             JSON error
 574      */
 575     private static JSONObject createLabel(String commit) throws JSONException {
 576         /*
 577          * Here we could add more information about this specific data point
 578          * like the commit author, the commit message, etc.
 579          */
 580         JSONObject label = new JSONObject();
 581         if (commit != null && !commit.isEmpty()) {
 582             label.put(COMMIT_LABEL, commit);
 583         }
 584         return label;
 585     }
 586
 587     /**
 588      * Get the commit id out of the build= string
 589      *
 590      * @param build
 591      *            the build string
 592      * @return the parsed commit id
 593      */
 594     private static String parseCommit(String build) {
 595         Matcher matcher = COMMIT_PATTERN.matcher(build);
 596         if (matcher.matches()) {
 597             return matcher.group(1);
 598         }
 599         return null;
 600     }
 601
 602     /**
 603      * Get the Date out of the build= string
 604      *
 605      * @param build
 606      *            the build string
 607      * @return the parsed Date
 608      */
 609     private static Date parseBuildDate(String build) {
 610         Matcher matcher = BUILD_DATE_PATTERN.matcher(build);
 611         Date date = null;
 612         if (matcher.matches()) {
 613             String dateStr = matcher.group(1);
 614             SimpleDateFormat f = new SimpleDateFormat(BUILD_DATE_FORMAT);
 615             try {
 616                 date = dateStr.length() > BUILD_DATE_FORMAT.length() ?
 617                         f.parse(dateStr.substring(dateStr.length() - BUILD_DATE_FORMAT.length())) :
 618                         f.parse(dateStr);
 619             } catch (ParseException e) {
 620                 return null;
 621             }
 622         }
 623         return date;
 624     }
 625 }