1 /*******************************************************************************
2 * Copyright (c) 2014 Ericsson
4 * All rights reserved. This program and the accompanying materials are
5 * made available under the terms of the Eclipse Public License v1.0 which
6 * accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * Marc-Andre Laperle - Initial API and implementation
11 *******************************************************************************/
13 package org
.eclipse
.linuxtools
.lttng
.alltests
.perf
;
15 import java
.io
.FileWriter
;
16 import java
.io
.IOException
;
17 import java
.text
.ParseException
;
18 import java
.text
.SimpleDateFormat
;
19 import java
.util
.ArrayList
;
20 import java
.util
.Date
;
21 import java
.util
.Iterator
;
22 import java
.util
.List
;
23 import java
.util
.regex
.Matcher
;
24 import java
.util
.regex
.Pattern
;
26 import org
.eclipse
.linuxtools
.lttng
.alltests
.Activator
;
27 import org
.eclipse
.test
.internal
.performance
.PerformanceTestPlugin
;
28 import org
.eclipse
.test
.internal
.performance
.data
.Dim
;
29 import org
.eclipse
.test
.internal
.performance
.db
.DB
;
30 import org
.eclipse
.test
.internal
.performance
.db
.Scenario
;
31 import org
.eclipse
.test
.internal
.performance
.db
.SummaryEntry
;
32 import org
.eclipse
.test
.internal
.performance
.db
.TimeSeries
;
33 import org
.eclipse
.test
.internal
.performance
.db
.Variations
;
34 import org
.json
.JSONArray
;
35 import org
.json
.JSONException
;
36 import org
.json
.JSONObject
;
37 import org
.junit
.Test
;
40 * Convert results from the database to JSON suitable for display.
44 * Individual charts are generated into JSON files in the form chart#.json where
45 * # is incremented for each new chart. A chart contains data points consisting
46 * of X and Y values suitable for a line chart. Each point can also have
47 * additional data, for example the commit id. This format is compatible with
53 * "key": "Experiment Benchmark:84 traces",
55 * "label": {"commit": "fe3c142"},
63 * Normal charts metadata:
65 * Each chart has an entry in the metada.js file which organizes the charts per
66 * component and contains additional information to augment the format expected
67 * by nvd3. Each entry contains the combination of OS and JVM, the filename (in
68 * JSON format), the title of the chart, the unit (seconds, etc) and the
69 * dimension (CPU time, used heap, etc).
74 * "applicationComponents": {
75 * "Experiment benchmark": {
76 * "name": "Experiment benchmark",
79 * "dimension": "CPU Time",
83 * "title": "Experiment Benchmark:84 traces",
87 * "dimension": "CPU Time",
91 * "title": "Experiment Benchmark:6 traces",
100 * In addition to the normal charts, overview charts are generated. An overview
101 * chart presents a summary of the scenarios ran for a given OS and JVM
102 * combination. Only scenarios marked as "global" are added to the overview
103 * because of space concerns. Overview charts are generated under the
104 * chart_overview#.json name and look similar in structure to the normal charts
105 * except that they contain more than one series.
111 * "key": "CTF Read & Seek Benchmark (500 seeks):tr",
114 * "label": {"commit": "4d34345"},
115 * "x": 1405436820000,
122 * "key": "CTF Read Benchmark:trace-kernel",
125 * "label": {"commit": "4d34345"},
126 * "x": 1405436820000,
136 * Overview charts metadata:
138 * Overview charts also have similar metadata entries to normal charts except
139 * they are not organized by component.
148 * "file": "chart_overview0",
151 * "title": "linux / 1.7",
156 * "file": "chart_overview1",
159 * "title": "windows / 1.7",
166 * Finally, since we want to be able to filter all the charts by OS/JVM
167 * combination, there is a section in the metadata that lists all the
174 * "description": "linux / 1.7",
179 * "description": "windows / 1.7",
184 * "description": "mac / 1.7",
192 * All of this data is meant to be view on a website. Specifically, the source
193 * code for our implementation is available on GitHub at
194 * https://github.com/PSRCode/ITCFYWebsite
196 * It makes use of the NVD3 project to display the charts based on the data
197 * generated by this class.
199 public class PerfResultsToJSon
{
204 private static final String APPLICATION_COMPONENTS_LABEL
= "applicationComponents";
205 private static final String BUILD_LABEL
= "build";
206 private static final String COMMIT_LABEL
= "commit";
207 private static final String CONFIG_LABEL
= "config";
208 private static final String DESCRIPTION_LABEL
= "description";
209 private static final String DIMENSION_LABEL
= "dimension";
210 private static final String FILE_LABEL
= "file";
211 private static final String HOST_LABEL
= "host";
212 private static final String JVM_LABEL
= "jvm";
213 private static final String KEY_LABEL
= "key";
214 private static final String LABEL_LABEL
= "label";
215 private static final String NAME_LABEL
= "name";
216 private static final String OS_LABEL
= "os";
217 private static final String OSJVM_LABEL
= "osjvm";
218 private static final String OVERVIEWS_LABEL
= "overviews";
219 private static final String TESTS_LABEL
= "tests";
220 private static final String TITLE_LABEL
= "title";
221 private static final String UNIT_LABEL
= "unit";
222 private static final String VALUES_LABEL
= "values";
223 private static final String X_LABEL
= "x";
224 private static final String Y_LABEL
= "y";
226 private static final String BUILD_DATE_FORMAT
= "yyyyMMdd-HHmm";
227 private static final String OVERVIEW_CHART_FILE_NAME
= "chart_overview";
228 private static final String METADATA_FILE_NAME
= "meta";
229 private static final String METADATA_FILE_NAME_EXTENSION
= ".js";
230 private static final String CHART_FILE_NAME
= "chart";
231 private static final String CHART_FILE_NAME_EXTENSION
= ".json";
232 private static final String WILDCARD_PATTERN
= "%";
233 private static final String COMPONENT_SEPARATOR
= "#";
234 private static final String META_DATA_JAVASCRIPT_START
= "var MetaData = ";
236 private static Pattern BUILD_DATE_PATTERN
= Pattern
.compile("(\\w+-\\w+)(-\\w+)?");
237 private static Pattern COMMIT_PATTERN
= Pattern
.compile(".*-.*-(.*)");
239 private JSONObject fApplicationComponents
= new JSONObject();
240 private JSONObject fOverviews
= new JSONObject();
242 private int fNumChart
= 0;
243 private int fNumOverviewChart
= 0;
246 * Convert results from the database to JSON suitable for display
249 * For each variant (os/jvm combination)
250 * - For each summary entry (scenario)
252 * - Add it to global summary (if needed)
253 * - Create the metadata for this test
254 * - Create an overview chart for this os/jvm
257 * @throws JSONException
259 * @throws IOException
263 public void parseResults() throws JSONException
, IOException
{
264 Variations configVariations
= PerformanceTestPlugin
.getVariations();
265 JSONObject osJvmVariants
= createOsJvm();
267 Iterator
<?
> keysIt
= osJvmVariants
.keys();
268 while (keysIt
.hasNext()) {
269 JSONArray overviewSummarySeries
= new JSONArray();
271 JSONObject variant
= osJvmVariants
.getJSONObject((String
) keysIt
.next());
272 String seriesKey
= PerformanceTestPlugin
.BUILD
;
274 // Clone the variations from the environment because it might have
275 // extra parameters like host=, etc.
276 Variations buildVariations
= (Variations
) configVariations
.clone();
277 buildVariations
.setProperty(JVM_LABEL
, variant
.getString(JVM_LABEL
));
278 buildVariations
.setProperty(CONFIG_LABEL
, variant
.getString(OS_LABEL
));
279 buildVariations
.setProperty(BUILD_LABEL
, WILDCARD_PATTERN
);
281 Scenario
[] scenarios
= DB
.queryScenarios(buildVariations
, WILDCARD_PATTERN
, seriesKey
, null);
282 SummaryEntry
[] summaryEntries
= DB
.querySummaries(buildVariations
, WILDCARD_PATTERN
);
283 for (SummaryEntry entry
: summaryEntries
) {
284 Scenario scenario
= getScenario(entry
.scenarioName
, scenarios
);
285 JSONObject scenarioSeries
= createScenarioChart(scenario
, entry
, buildVariations
);
286 // Add to global summary
287 if (scenarioSeries
!= null && entry
.isGlobal
) {
288 overviewSummarySeries
.put(scenarioSeries
);
292 JSONObject overviewMetadata
= createOverviewChart(overviewSummarySeries
, buildVariations
);
293 fOverviews
.put(Integer
.toString(fNumOverviewChart
), overviewMetadata
);
296 // Create the matadata javascript file that includes OS/JVM combinations
297 // (for filtering), application components and overviews (one of OS/JVM
299 JSONObject rootMetadata
= new JSONObject();
300 rootMetadata
.put(OSJVM_LABEL
, osJvmVariants
);
301 rootMetadata
.put(APPLICATION_COMPONENTS_LABEL
, fApplicationComponents
);
302 rootMetadata
.put(OVERVIEWS_LABEL
, fOverviews
);
303 try (FileWriter fw1
= new FileWriter(METADATA_FILE_NAME
+ METADATA_FILE_NAME_EXTENSION
)) {
304 fw1
.write(META_DATA_JAVASCRIPT_START
+ rootMetadata
.toString(4));
309 * Create chart for a scenario instance and add it to the relevant metadatas
312 * the scenario. For example,
313 * "CTF Read & Seek Benchmark (500 seeks)".
315 * an entry from the summary. Only scenarios that are part of the
316 * summary are processed.
318 * all variations to consider to create the scenario chart. For
319 * example build=%;jvm=1.7;config=linux will generate a chart for
320 * all builds on Linux / JVM 1.7
323 * @throws JSONException
325 * @throws IOException
328 private JSONObject
createScenarioChart(Scenario scenario
, SummaryEntry entry
, Variations variations
) throws JSONException
, IOException
{
329 if (scenario
== null) {
332 String
[] split
= entry
.scenarioName
.split(COMPONENT_SEPARATOR
);
333 if (split
.length
< 3) {
334 Activator
.logError("Invalid scenario name \"" + entry
.scenarioName
+ "\", it must be in format: org.package.foo#component#test");
338 // Generate individual chart
339 JSONArray rootScenario
= new JSONArray();
340 JSONObject series
= createSerie(scenario
, variations
, entry
.shortName
, entry
.dimension
);
341 rootScenario
.put(series
);
342 int numChart
= fNumChart
++;
343 try (FileWriter fw
= new FileWriter(CHART_FILE_NAME
+ numChart
+ CHART_FILE_NAME_EXTENSION
)) {
344 fw
.write(rootScenario
.toString(4));
347 // Create the metadata
348 JSONObject testMetadata
= new JSONObject();
349 testMetadata
.put(TITLE_LABEL
, entry
.shortName
);
350 testMetadata
.put(FILE_LABEL
, CHART_FILE_NAME
+ numChart
);
351 testMetadata
.put(OS_LABEL
, variations
.getProperty(CONFIG_LABEL
));
352 testMetadata
.put(JVM_LABEL
, variations
.getProperty(JVM_LABEL
));
353 testMetadata
.put(DIMENSION_LABEL
, entry
.dimension
.getName());
354 testMetadata
.put(UNIT_LABEL
, entry
.dimension
.getUnit().getShortName());
356 // Add the scenario to the metadata, under the correct component
357 String componentName
= split
[1];
358 JSONObject componentObject
= null;
359 if (fApplicationComponents
.has(componentName
)) {
360 componentObject
= fApplicationComponents
.getJSONObject(componentName
);
362 componentObject
= new JSONObject();
363 componentObject
.put(NAME_LABEL
, componentName
);
364 componentObject
.put(TESTS_LABEL
, new JSONArray());
365 fApplicationComponents
.put(componentName
, componentObject
);
367 JSONArray tests
= componentObject
.getJSONArray(TESTS_LABEL
);
368 tests
.put(testMetadata
);
374 * Create an overview chart for this OS / JVM combination. The chart is made
375 * of multiple series (scenarios) that were marked as global.
377 * @param overviewSummarySeries
378 * an array of series to include in the chart (multiple
381 * the variations used to generate the series to be included in
382 * this overview chart. For example build=%;jvm=1.7;config=linux
383 * will generate an overview chart for Linux / JVM 1.7
384 * @return the overview metadata JSON object
385 * @throws JSONException
387 * @throws IOException
390 private JSONObject
createOverviewChart(JSONArray overviewSummarySeries
, Variations variations
) throws IOException
, JSONException
{
391 int numOverviewChart
= fNumOverviewChart
++;
392 try (FileWriter fw
= new FileWriter(OVERVIEW_CHART_FILE_NAME
+ numOverviewChart
+ CHART_FILE_NAME_EXTENSION
)) {
393 fw
.write(overviewSummarySeries
.toString(4));
396 String os
= variations
.getProperty(CONFIG_LABEL
);
397 String jvm
= variations
.getProperty(JVM_LABEL
);
399 // Create the overview metadata
400 JSONObject overviewMetadata
= new JSONObject();
401 overviewMetadata
.put(TITLE_LABEL
, os
+ " / " + jvm
);
402 overviewMetadata
.put(FILE_LABEL
, OVERVIEW_CHART_FILE_NAME
+ numOverviewChart
);
403 overviewMetadata
.put(OS_LABEL
, os
);
404 overviewMetadata
.put(JVM_LABEL
, jvm
);
405 overviewMetadata
.put(DIMENSION_LABEL
, "");
406 overviewMetadata
.put(UNIT_LABEL
, "");
408 return overviewMetadata
;
411 private static Scenario
getScenario(String scenarioName
, Scenario
[] scenarios
) {
412 for (int i
= 0; i
< scenarios
.length
; i
++) {
413 Scenario s
= scenarios
[i
];
414 if (s
.getScenarioName().equals(scenarioName
)) {
423 * Get all combinations of OS / JVM. This will be used for filtering.
425 * @return the JSON object containing all the combinations
426 * @throws JSONException
429 private static JSONObject
createOsJvm() throws JSONException
{
430 JSONObject osjvm
= new JSONObject();
431 List
<String
> oses
= getDistinctOses();
434 for (String os
: oses
) {
435 String key
= JVM_LABEL
;
436 Variations v
= new Variations();
438 v
.setProperty(BUILD_LABEL
, WILDCARD_PATTERN
);
439 v
.setProperty(HOST_LABEL
, WILDCARD_PATTERN
);
440 v
.setProperty(CONFIG_LABEL
, os
);
441 v
.setProperty(JVM_LABEL
, WILDCARD_PATTERN
);
443 List
<String
> jvms
= new ArrayList
<>();
444 DB
.queryDistinctValues(jvms
, key
, v
, WILDCARD_PATTERN
);
445 for (String jvm
: jvms
) {
446 JSONObject osjvmItem
= new JSONObject();
447 osjvmItem
.put(OS_LABEL
, os
);
448 osjvmItem
.put(JVM_LABEL
, jvm
);
449 osjvmItem
.put(DESCRIPTION_LABEL
, os
+ " / " + jvm
);
450 osjvm
.put(Integer
.toString(osJvmIndex
), osjvmItem
);
459 * Get all the distinct OS values
461 * @return the distinct OS values
463 private static List
<String
> getDistinctOses() {
464 List
<String
> configs
= new ArrayList
<>();
465 String key
= PerformanceTestPlugin
.CONFIG
;
466 Variations v
= new Variations();
467 v
.setProperty(WILDCARD_PATTERN
, WILDCARD_PATTERN
);
468 DB
.queryDistinctValues(configs
, key
, v
, WILDCARD_PATTERN
);
473 * This main can be run from within Eclipse provided everything is on the
478 * @throws JSONException
480 * @throws IOException
483 public static void main(String
[] args
) throws JSONException
, IOException
{
484 new PerfResultsToJSon().parseResults();
488 * Create a series of data points for a given scenario through variations
491 * the scenario. For example,
492 * "CTF Read & Seek Benchmark (500 seeks)".
494 * all variations to consider to create the series. For example
495 * build=%;jvm=1.7;config=linux will generate the series for all
496 * builds on Linux / JVM 1.7
498 * the short name of the scenario
500 * the dimension of interest (CPU time, used java heap, etc).
501 * @return the generated JSON object representing a series of data points
503 * @throws JSONException
505 private static JSONObject
createSerie(Scenario scenario
, Variations variations
, String shortName
, Dim dimension
) throws JSONException
{
506 JSONObject o
= new JSONObject();
507 o
.putOpt(KEY_LABEL
, shortName
);
508 o
.putOpt(VALUES_LABEL
, createDataPoints(scenario
, variations
, dimension
));
513 * Create data points for a given scenario and variations.
516 * the scenario. For example,
517 * "CTF Read & Seek Benchmark (500 seeks)".
519 * all variations to consider to create the data points. For
520 * example build=%;jvm=1.7;config=linux will generate the data
521 * points for all builds on Linux / JVM 1.7
523 * the dimension of interest (CPU time, used java heap, etc).
525 * @return the generated JSON array of points
526 * @throws JSONException
529 private static JSONArray
createDataPoints(Scenario s
, Variations variations
, Dim dimension
) throws JSONException
{
530 // Can be uncommented to see raw dump
531 //s.dump(System.out, PerformanceTestPlugin.BUILD);
533 String
[] builds
= DB
.querySeriesValues(s
.getScenarioName(), variations
, PerformanceTestPlugin
.BUILD
);
534 Date
[] dates
= new Date
[builds
.length
];
535 String
[] commits
= new String
[builds
.length
];
536 for (int i
= 0; i
< builds
.length
; i
++) {
537 dates
[i
] = parseBuildDate(builds
[i
]);
538 commits
[i
] = parseCommit(builds
[i
]);
541 TimeSeries timeSeries
= s
.getTimeSeries(dimension
);
542 JSONArray dataPoints
= new JSONArray();
543 int length
= timeSeries
.getLength();
544 for (int i
= 0; i
< length
; i
++) {
545 JSONObject point
= new JSONObject();
546 if (dates
[i
] == null) {
549 point
.put(X_LABEL
, dates
[i
].getTime());
551 if (timeSeries
.getCount(i
) > 0) {
552 value
= timeSeries
.getValue(i
);
553 if (Double
.isNaN(value
)) {
557 point
.put(Y_LABEL
, value
);
558 dataPoints
.put(point
);
559 point
.put(LABEL_LABEL
, createLabel(commits
[i
]));
565 * Create a label JSONObject which is used to attach more information to a
569 * the commit id for this data point
570 * @return the resulting JSON object
571 * @throws JSONException
574 private static JSONObject
createLabel(String commit
) throws JSONException
{
576 * Here we could add more information about this specific data point
577 * like the commit author, the commit message, etc.
579 JSONObject label
= new JSONObject();
580 if (commit
!= null && !commit
.isEmpty()) {
581 label
.put(COMMIT_LABEL
, commit
);
587 * Get the commit id out of the build= string
591 * @return the parsed commit id
593 private static String
parseCommit(String build
) {
594 Matcher matcher
= COMMIT_PATTERN
.matcher(build
);
595 if (matcher
.matches()) {
596 return matcher
.group(1);
602 * Get the Date out of the build= string
606 * @return the parsed Date
608 private static Date
parseBuildDate(String build
) {
609 Matcher matcher
= BUILD_DATE_PATTERN
.matcher(build
);
611 if (matcher
.matches()) {
612 String dateStr
= matcher
.group(1);
613 SimpleDateFormat f
= new SimpleDateFormat(BUILD_DATE_FORMAT
);
615 date
= dateStr
.length() > BUILD_DATE_FORMAT
.length() ?
616 f
.parse(dateStr
.substring(dateStr
.length() - BUILD_DATE_FORMAT
.length())) :
618 } catch (ParseException e
) {