1 /*******************************************************************************
2 * Copyright (c) 2014 Ericsson
4 * All rights reserved. This program and the accompanying materials are
5 * made available under the terms of the Eclipse Public License v1.0 which
6 * accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * Marc-Andre Laperle - Initial API and implementation
11 *******************************************************************************/
13 package org
.eclipse
.tracecompass
.alltests
.perf
;
15 import java
.io
.FileWriter
;
16 import java
.io
.IOException
;
17 import java
.text
.ParseException
;
18 import java
.text
.SimpleDateFormat
;
19 import java
.util
.ArrayList
;
20 import java
.util
.Date
;
21 import java
.util
.Iterator
;
22 import java
.util
.List
;
23 import java
.util
.regex
.Matcher
;
24 import java
.util
.regex
.Pattern
;
26 import org
.eclipse
.jdt
.annotation
.NonNull
;
27 import org
.eclipse
.test
.internal
.performance
.PerformanceTestPlugin
;
28 import org
.eclipse
.test
.internal
.performance
.data
.Dim
;
29 import org
.eclipse
.test
.internal
.performance
.db
.DB
;
30 import org
.eclipse
.test
.internal
.performance
.db
.Scenario
;
31 import org
.eclipse
.test
.internal
.performance
.db
.SummaryEntry
;
32 import org
.eclipse
.test
.internal
.performance
.db
.TimeSeries
;
33 import org
.eclipse
.test
.internal
.performance
.db
.Variations
;
34 import org
.eclipse
.tracecompass
.alltests
.Activator
;
35 import org
.json
.JSONArray
;
36 import org
.json
.JSONException
;
37 import org
.json
.JSONObject
;
38 import org
.junit
.Test
;
41 * Convert results from the database to JSON suitable for display.
45 * Individual charts are generated into JSON files in the form chart#.json where
46 * # is incremented for each new chart. A chart contains data points consisting
47 * of X and Y values suitable for a line chart. Each point can also have
48 * additional data, for example the commit id. This format is compatible with
54 * "key": "Experiment Benchmark:84 traces",
56 * "label": {"commit": "fe3c142"},
64 * Normal charts metadata:
66 * Each chart has an entry in the metada.js file which organizes the charts per
67 * component and contains additional information to augment the format expected
68 * by nvd3. Each entry contains the combination of OS and JVM, the filename (in
69 * JSON format), the title of the chart, the unit (seconds, etc) and the
70 * dimension (CPU time, used heap, etc).
75 * "applicationComponents": {
76 * "Experiment benchmark": {
77 * "name": "Experiment benchmark",
80 * "dimension": "CPU Time",
84 * "title": "Experiment Benchmark:84 traces",
88 * "dimension": "CPU Time",
92 * "title": "Experiment Benchmark:6 traces",
101 * In addition to the normal charts, overview charts are generated. An overview
102 * chart presents a summary of the scenarios ran for a given OS and JVM
103 * combination. Only scenarios marked as "global" are added to the overview
104 * because of space concerns. Overview charts are generated under the
105 * chart_overview#.json name and look similar in structure to the normal charts
106 * except that they contain more than one series.
112 * "key": "CTF Read & Seek Benchmark (500 seeks):tr",
115 * "label": {"commit": "4d34345"},
116 * "x": 1405436820000,
123 * "key": "CTF Read Benchmark:trace-kernel",
126 * "label": {"commit": "4d34345"},
127 * "x": 1405436820000,
137 * Overview charts metadata:
139 * Overview charts also have similar metadata entries to normal charts except
140 * they are not organized by component.
149 * "file": "chart_overview0",
152 * "title": "linux / 1.7",
157 * "file": "chart_overview1",
160 * "title": "windows / 1.7",
167 * Finally, since we want to be able to filter all the charts by OS/JVM
168 * combination, there is a section in the metadata that lists all the
175 * "description": "linux / 1.7",
180 * "description": "windows / 1.7",
185 * "description": "mac / 1.7",
193 * All of this data is meant to be view on a website. Specifically, the source
194 * code for our implementation is available on GitHub at
195 * https://github.com/PSRCode/ITCFYWebsite
197 * It makes use of the NVD3 project to display the charts based on the data
198 * generated by this class.
200 public class PerfResultsToJSon
{
205 private static final String APPLICATION_COMPONENTS_LABEL
= "applicationComponents";
206 private static final String BUILD_LABEL
= "build";
207 private static final String COMMIT_LABEL
= "commit";
208 private static final String CONFIG_LABEL
= "config";
209 private static final String DESCRIPTION_LABEL
= "description";
210 private static final String DIMENSION_LABEL
= "dimension";
211 private static final String FILE_LABEL
= "file";
212 private static final String HOST_LABEL
= "host";
213 private static final String JVM_LABEL
= "jvm";
214 private static final String KEY_LABEL
= "key";
215 private static final String LABEL_LABEL
= "label";
216 private static final String NAME_LABEL
= "name";
217 private static final String OS_LABEL
= "os";
218 private static final String OSJVM_LABEL
= "osjvm";
219 private static final String OVERVIEWS_LABEL
= "overviews";
220 private static final String TESTS_LABEL
= "tests";
221 private static final String TITLE_LABEL
= "title";
222 private static final String UNIT_LABEL
= "unit";
223 private static final String VALUES_LABEL
= "values";
224 private static final String X_LABEL
= "x";
225 private static final String Y_LABEL
= "y";
227 private static final String BUILD_DATE_FORMAT
= "yyyyMMdd-HHmm";
228 private static final String OVERVIEW_CHART_FILE_NAME
= "chart_overview";
229 private static final String METADATA_FILE_NAME
= "meta";
230 private static final String METADATA_FILE_NAME_EXTENSION
= ".js";
231 private static final String CHART_FILE_NAME
= "chart";
232 private static final String CHART_FILE_NAME_EXTENSION
= ".json";
233 private static final String WILDCARD_PATTERN
= "%";
234 private static final @NonNull String COMPONENT_SEPARATOR
= "#";
235 private static final String META_DATA_JAVASCRIPT_START
= "var MetaData = ";
237 private static Pattern BUILD_DATE_PATTERN
= Pattern
.compile("(\\w+-\\w+)(-\\w+)?");
238 private static Pattern COMMIT_PATTERN
= Pattern
.compile(".*-.*-(.*)");
240 private JSONObject fApplicationComponents
= new JSONObject();
241 private JSONObject fOverviews
= new JSONObject();
243 private int fNumChart
= 0;
244 private int fNumOverviewChart
= 0;
247 * Convert results from the database to JSON suitable for display
250 * For each variant (os/jvm combination)
251 * - For each summary entry (scenario)
253 * - Add it to global summary (if needed)
254 * - Create the metadata for this test
255 * - Create an overview chart for this os/jvm
258 * @throws JSONException
260 * @throws IOException
264 public void parseResults() throws JSONException
, IOException
{
265 Variations configVariations
= PerformanceTestPlugin
.getVariations();
266 JSONObject osJvmVariants
= createOsJvm();
268 Iterator
<?
> keysIt
= osJvmVariants
.keys();
269 while (keysIt
.hasNext()) {
270 JSONArray overviewSummarySeries
= new JSONArray();
272 JSONObject variant
= osJvmVariants
.getJSONObject((String
) keysIt
.next());
273 String seriesKey
= PerformanceTestPlugin
.BUILD
;
275 // Clone the variations from the environment because it might have
276 // extra parameters like host=, etc.
277 Variations buildVariations
= (Variations
) configVariations
.clone();
278 buildVariations
.setProperty(JVM_LABEL
, variant
.getString(JVM_LABEL
));
279 buildVariations
.setProperty(CONFIG_LABEL
, variant
.getString(OS_LABEL
));
280 buildVariations
.setProperty(BUILD_LABEL
, WILDCARD_PATTERN
);
282 Scenario
[] scenarios
= DB
.queryScenarios(buildVariations
, WILDCARD_PATTERN
, seriesKey
, null);
283 SummaryEntry
[] summaryEntries
= DB
.querySummaries(buildVariations
, WILDCARD_PATTERN
);
284 for (SummaryEntry entry
: summaryEntries
) {
285 Scenario scenario
= getScenario(entry
.scenarioName
, scenarios
);
286 JSONObject scenarioSeries
= createScenarioChart(scenario
, entry
, buildVariations
);
287 // Add to global summary
288 if (scenarioSeries
!= null && entry
.isGlobal
) {
289 overviewSummarySeries
.put(scenarioSeries
);
293 JSONObject overviewMetadata
= createOverviewChart(overviewSummarySeries
, buildVariations
);
294 fOverviews
.put(Integer
.toString(fNumOverviewChart
), overviewMetadata
);
297 // Create the matadata javascript file that includes OS/JVM combinations
298 // (for filtering), application components and overviews (one of OS/JVM
300 JSONObject rootMetadata
= new JSONObject();
301 rootMetadata
.put(OSJVM_LABEL
, osJvmVariants
);
302 rootMetadata
.put(APPLICATION_COMPONENTS_LABEL
, fApplicationComponents
);
303 rootMetadata
.put(OVERVIEWS_LABEL
, fOverviews
);
304 try (FileWriter fw1
= new FileWriter(METADATA_FILE_NAME
+ METADATA_FILE_NAME_EXTENSION
)) {
305 fw1
.write(META_DATA_JAVASCRIPT_START
+ rootMetadata
.toString(4));
310 * Create chart for a scenario instance and add it to the relevant metadatas
313 * the scenario. For example,
314 * "CTF Read & Seek Benchmark (500 seeks)".
316 * an entry from the summary. Only scenarios that are part of the
317 * summary are processed.
319 * all variations to consider to create the scenario chart. For
320 * example build=%;jvm=1.7;config=linux will generate a chart for
321 * all builds on Linux / JVM 1.7
324 * @throws JSONException
326 * @throws IOException
329 private JSONObject
createScenarioChart(Scenario scenario
, SummaryEntry entry
, Variations variations
) throws JSONException
, IOException
{
330 if (scenario
== null) {
333 String
[] split
= entry
.scenarioName
.split(COMPONENT_SEPARATOR
);
334 if (split
.length
< 3) {
335 Activator
.logError("Invalid scenario name \"" + entry
.scenarioName
+ "\", it must be in format: org.package.foo#component#test");
339 // Generate individual chart
340 JSONArray rootScenario
= new JSONArray();
341 JSONObject series
= createSerie(scenario
, variations
, entry
.shortName
, entry
.dimension
);
342 rootScenario
.put(series
);
343 int numChart
= fNumChart
++;
344 try (FileWriter fw
= new FileWriter(CHART_FILE_NAME
+ numChart
+ CHART_FILE_NAME_EXTENSION
)) {
345 fw
.write(rootScenario
.toString(4));
348 // Create the metadata
349 JSONObject testMetadata
= new JSONObject();
350 testMetadata
.put(TITLE_LABEL
, entry
.shortName
);
351 testMetadata
.put(FILE_LABEL
, CHART_FILE_NAME
+ numChart
);
352 testMetadata
.put(OS_LABEL
, variations
.getProperty(CONFIG_LABEL
));
353 testMetadata
.put(JVM_LABEL
, variations
.getProperty(JVM_LABEL
));
354 testMetadata
.put(DIMENSION_LABEL
, entry
.dimension
.getName());
355 testMetadata
.put(UNIT_LABEL
, entry
.dimension
.getUnit().getShortName());
357 // Add the scenario to the metadata, under the correct component
358 String componentName
= split
[1];
359 JSONObject componentObject
= null;
360 if (fApplicationComponents
.has(componentName
)) {
361 componentObject
= fApplicationComponents
.getJSONObject(componentName
);
363 componentObject
= new JSONObject();
364 componentObject
.put(NAME_LABEL
, componentName
);
365 componentObject
.put(TESTS_LABEL
, new JSONArray());
366 fApplicationComponents
.put(componentName
, componentObject
);
368 JSONArray tests
= componentObject
.getJSONArray(TESTS_LABEL
);
369 tests
.put(testMetadata
);
375 * Create an overview chart for this OS / JVM combination. The chart is made
376 * of multiple series (scenarios) that were marked as global.
378 * @param overviewSummarySeries
379 * an array of series to include in the chart (multiple
382 * the variations used to generate the series to be included in
383 * this overview chart. For example build=%;jvm=1.7;config=linux
384 * will generate an overview chart for Linux / JVM 1.7
385 * @return the overview metadata JSON object
386 * @throws JSONException
388 * @throws IOException
391 private JSONObject
createOverviewChart(JSONArray overviewSummarySeries
, Variations variations
) throws IOException
, JSONException
{
392 int numOverviewChart
= fNumOverviewChart
++;
393 try (FileWriter fw
= new FileWriter(OVERVIEW_CHART_FILE_NAME
+ numOverviewChart
+ CHART_FILE_NAME_EXTENSION
)) {
394 fw
.write(overviewSummarySeries
.toString(4));
397 String os
= variations
.getProperty(CONFIG_LABEL
);
398 String jvm
= variations
.getProperty(JVM_LABEL
);
400 // Create the overview metadata
401 JSONObject overviewMetadata
= new JSONObject();
402 overviewMetadata
.put(TITLE_LABEL
, os
+ " / " + jvm
);
403 overviewMetadata
.put(FILE_LABEL
, OVERVIEW_CHART_FILE_NAME
+ numOverviewChart
);
404 overviewMetadata
.put(OS_LABEL
, os
);
405 overviewMetadata
.put(JVM_LABEL
, jvm
);
406 overviewMetadata
.put(DIMENSION_LABEL
, "");
407 overviewMetadata
.put(UNIT_LABEL
, "");
409 return overviewMetadata
;
412 private static Scenario
getScenario(String scenarioName
, Scenario
[] scenarios
) {
413 for (int i
= 0; i
< scenarios
.length
; i
++) {
414 Scenario s
= scenarios
[i
];
415 if (s
.getScenarioName().equals(scenarioName
)) {
424 * Get all combinations of OS / JVM. This will be used for filtering.
426 * @return the JSON object containing all the combinations
427 * @throws JSONException
430 private static JSONObject
createOsJvm() throws JSONException
{
431 JSONObject osjvm
= new JSONObject();
432 List
<String
> oses
= getDistinctOses();
435 for (String os
: oses
) {
436 String key
= JVM_LABEL
;
437 Variations v
= new Variations();
439 v
.setProperty(BUILD_LABEL
, WILDCARD_PATTERN
);
440 v
.setProperty(HOST_LABEL
, WILDCARD_PATTERN
);
441 v
.setProperty(CONFIG_LABEL
, os
);
442 v
.setProperty(JVM_LABEL
, WILDCARD_PATTERN
);
444 List
<String
> jvms
= new ArrayList
<>();
445 DB
.queryDistinctValues(jvms
, key
, v
, WILDCARD_PATTERN
);
446 for (String jvm
: jvms
) {
447 JSONObject osjvmItem
= new JSONObject();
448 osjvmItem
.put(OS_LABEL
, os
);
449 osjvmItem
.put(JVM_LABEL
, jvm
);
450 osjvmItem
.put(DESCRIPTION_LABEL
, os
+ " / " + jvm
);
451 osjvm
.put(Integer
.toString(osJvmIndex
), osjvmItem
);
460 * Get all the distinct OS values
462 * @return the distinct OS values
464 private static List
<String
> getDistinctOses() {
465 List
<String
> configs
= new ArrayList
<>();
466 String key
= PerformanceTestPlugin
.CONFIG
;
467 Variations v
= new Variations();
468 v
.setProperty(WILDCARD_PATTERN
, WILDCARD_PATTERN
);
469 DB
.queryDistinctValues(configs
, key
, v
, WILDCARD_PATTERN
);
474 * This main can be run from within Eclipse provided everything is on the
479 * @throws JSONException
481 * @throws IOException
484 public static void main(String
[] args
) throws JSONException
, IOException
{
485 new PerfResultsToJSon().parseResults();
489 * Create a series of data points for a given scenario through variations
492 * the scenario. For example,
493 * "CTF Read & Seek Benchmark (500 seeks)".
495 * all variations to consider to create the series. For example
496 * build=%;jvm=1.7;config=linux will generate the series for all
497 * builds on Linux / JVM 1.7
499 * the short name of the scenario
501 * the dimension of interest (CPU time, used java heap, etc).
502 * @return the generated JSON object representing a series of data points
504 * @throws JSONException
506 private static JSONObject
createSerie(Scenario scenario
, Variations variations
, String shortName
, Dim dimension
) throws JSONException
{
507 JSONObject o
= new JSONObject();
508 o
.putOpt(KEY_LABEL
, shortName
);
509 o
.putOpt(VALUES_LABEL
, createDataPoints(scenario
, variations
, dimension
));
514 * Create data points for a given scenario and variations.
517 * the scenario. For example,
518 * "CTF Read & Seek Benchmark (500 seeks)".
520 * all variations to consider to create the data points. For
521 * example build=%;jvm=1.7;config=linux will generate the data
522 * points for all builds on Linux / JVM 1.7
524 * the dimension of interest (CPU time, used java heap, etc).
526 * @return the generated JSON array of points
527 * @throws JSONException
530 private static JSONArray
createDataPoints(Scenario s
, Variations variations
, Dim dimension
) throws JSONException
{
531 // Can be uncommented to see raw dump
532 //s.dump(System.out, PerformanceTestPlugin.BUILD);
534 String
[] builds
= DB
.querySeriesValues(s
.getScenarioName(), variations
, PerformanceTestPlugin
.BUILD
);
535 Date
[] dates
= new Date
[builds
.length
];
536 String
[] commits
= new String
[builds
.length
];
537 for (int i
= 0; i
< builds
.length
; i
++) {
538 dates
[i
] = parseBuildDate(builds
[i
]);
539 commits
[i
] = parseCommit(builds
[i
]);
542 TimeSeries timeSeries
= s
.getTimeSeries(dimension
);
543 JSONArray dataPoints
= new JSONArray();
544 int length
= timeSeries
.getLength();
545 for (int i
= 0; i
< length
; i
++) {
546 JSONObject point
= new JSONObject();
547 if (dates
[i
] == null) {
550 point
.put(X_LABEL
, dates
[i
].getTime());
552 if (timeSeries
.getCount(i
) > 0) {
553 value
= timeSeries
.getValue(i
);
554 if (Double
.isNaN(value
)) {
558 point
.put(Y_LABEL
, value
);
559 dataPoints
.put(point
);
560 point
.put(LABEL_LABEL
, createLabel(commits
[i
]));
566 * Create a label JSONObject which is used to attach more information to a
570 * the commit id for this data point
571 * @return the resulting JSON object
572 * @throws JSONException
575 private static JSONObject
createLabel(String commit
) throws JSONException
{
577 * Here we could add more information about this specific data point
578 * like the commit author, the commit message, etc.
580 JSONObject label
= new JSONObject();
581 if (commit
!= null && !commit
.isEmpty()) {
582 label
.put(COMMIT_LABEL
, commit
);
588 * Get the commit id out of the build= string
592 * @return the parsed commit id
594 private static String
parseCommit(String build
) {
595 Matcher matcher
= COMMIT_PATTERN
.matcher(build
);
596 if (matcher
.matches()) {
597 return matcher
.group(1);
603 * Get the Date out of the build= string
607 * @return the parsed Date
609 private static Date
parseBuildDate(String build
) {
610 Matcher matcher
= BUILD_DATE_PATTERN
.matcher(build
);
612 if (matcher
.matches()) {
613 String dateStr
= matcher
.group(1);
614 SimpleDateFormat f
= new SimpleDateFormat(BUILD_DATE_FORMAT
);
616 date
= dateStr
.length() > BUILD_DATE_FORMAT
.length() ?
617 f
.parse(dateStr
.substring(dateStr
.length() - BUILD_DATE_FORMAT
.length())) :
619 } catch (ParseException e
) {