--- /dev/null
+/*******************************************************************************
+ * Copyright (c) 2014 Ericsson
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v1.0 which
+ * accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Marc-Andre Laperle - Initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.linuxtools.lttng.alltests;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.eclipse.test.internal.performance.PerformanceTestPlugin;
+import org.eclipse.test.internal.performance.data.Dim;
+import org.eclipse.test.internal.performance.db.DB;
+import org.eclipse.test.internal.performance.db.Scenario;
+import org.eclipse.test.internal.performance.db.SummaryEntry;
+import org.eclipse.test.internal.performance.db.TimeSeries;
+import org.eclipse.test.internal.performance.db.Variations;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.Test;
+
+/**
+ * Convert results from the database to JSON suitable for display.
+ *
+ * Normal charts:
+ *
+ * Individual charts are generated into JSON files in the form chart#.json where
+ * # is incremented for each new chart. A chart contains data points consisting
+ * of X and Y values suitable for a line chart. Each point can also have
+ * additional data, for example the commit id. This format is compatible with
+ * nvd3. For example:
+ *
+ * <pre>
+ * <code>
+ * [{
+ * "key": "Experiment Benchmark:84 traces",
+ * "values": [{
+ * "label": {"commit": "fe3c142"},
+ * "x": 1405024320000,
+ * "y": 17592
+ * }]
+ * }]
+ * </code>
+ * </pre>
+ *
+ * Normal charts metadata:
+ *
+ * Each chart has an entry in the metada.js file which organizes the charts per
+ * component and contains additional information to augment the format expected
+ * by nvd3. Each entry contains the combination of OS and JVM, the filename (in
+ * JSON format), the title of the chart, the unit (seconds, etc) and the
+ * dimension (CPU time, used heap, etc).
+ *
+ * <pre>
+ * <code>
+ * var MetaData = {
+ * "applicationComponents": {
+ * "Experiment benchmark": {
+ * "name": "Experiment benchmark",
+ * "tests": [
+ * {
+ * "dimension": "CPU Time",
+ * "file": "chart12",
+ * "jvm": "1.7",
+ * "os": "linux",
+ * "title": "Experiment Benchmark:84 traces",
+ * "unit": "s"
+ * },
+ * {
+ * "dimension": "CPU Time",
+ * "file": "chart11",
+ * "jvm": "1.7",
+ * "os": "linux",
+ * "title": "Experiment Benchmark:6 traces",
+ * "unit": "s"
+ * },
+ * ...
+ * </code>
+ * </pre>
+ *
+ * Overview charts:
+ *
+ * In addition to the normal charts, overview charts are generated. An overview
+ * chart presents a summary of the scenarios ran for a given OS and JVM
+ * combination. Only scenarios marked as "global" are added to the overview
+ * because of space concerns. Overview charts are generated under the
+ * chart_overview#.json name and look similar in structure to the normal charts
+ * except that they contain more than one series.
+ *
+ * <pre>
+ * <code>
+ * [
+ * {
+ * "key": "CTF Read & Seek Benchmark (500 seeks):tr",
+ * "values": [
+ * {
+ * "label": {"commit": "4d34345"},
+ * "x": 1405436820000,
+ * "y": 5382.5
+ * },
+ * ...
+ * ]
+ * },
+ * {
+ * "key": "CTF Read Benchmark:trace-kernel",
+ * "values": [
+ * {
+ * "label": {"commit": "4d34345"},
+ * "x": 1405436820000,
+ * "y": 1311.5
+ * },
+ * ...
+ * ]
+ * },
+ * ...
+ * </code>
+ * </pre>
+ *
+ * Overview charts metadata:
+ *
+ * Overview charts also have similar metadata entries to normal charts except
+ * they are not organized by component.
+ *
+ * <pre>
+ * <code>
+ * var MetaData = {
+ * ...
+ * "overviews": {
+ * "1": {
+ * "dimension": "",
+ * "file": "chart_overview0",
+ * "jvm": "1.7",
+ * "os": "linux",
+ * "title": "linux / 1.7",
+ * "unit": ""
+ * },
+ * "2": {
+ * "dimension": "",
+ * "file": "chart_overview1",
+ * "jvm": "1.7",
+ * "os": "windows",
+ * "title": "windows / 1.7",
+ * "unit": ""
+ * },
+ * ...
+ * </code>
+ * </pre>
+ *
+ * Finally, since we want to be able to filter all the charts by OS/JVM
+ * combination, there is a section in the metadata that lists all the
+ * combinations:
+ *
+ * <pre>
+ * <code>
+ * "osjvm": {
+ * "1": {
+ * "description": "linux / 1.7",
+ * "jvm": "1.7",
+ * "os": "linux"
+ * },
+ * "2": {
+ * "description": "windows / 1.7",
+ * "jvm": "1.7",
+ * "os": "windows"
+ * },
+ * "3": {
+ * "description": "mac / 1.7",
+ * "jvm": "1.7",
+ * "os": "mac"
+ * }
+ * },
+ * </code>
+ * </pre>
+ *
+ * All of this data is meant to be view on a website. Specifically, the source
+ * code for our implementation is available on GitHub at
+ * https://github.com/PSRCode/ITCFYWebsite
+ *
+ * It makes use of the NVD3 project to display the charts based on the data
+ * generated by this class.
+ */
+public class PerfResultsToJSon {
+
+ /*
+ * Labels
+ */
+ private static final String APPLICATION_COMPONENTS_LABEL = "applicationComponents";
+ private static final String BUILD_LABEL = "build";
+ private static final String COMMIT_LABEL = "commit";
+ private static final String CONFIG_LABEL = "config";
+ private static final String DESCRIPTION_LABEL = "description";
+ private static final String DIMENSION_LABEL = "dimension";
+ private static final String FILE_LABEL = "file";
+ private static final String HOST_LABEL = "host";
+ private static final String JVM_LABEL = "jvm";
+ private static final String KEY_LABEL = "key";
+ private static final String LABEL_LABEL = "label";
+ private static final String NAME_LABEL = "name";
+ private static final String OS_LABEL = "os";
+ private static final String OSJVM_LABEL = "osjvm";
+ private static final String OVERVIEWS_LABEL = "overviews";
+ private static final String TESTS_LABEL = "tests";
+ private static final String TITLE_LABEL = "title";
+ private static final String UNIT_LABEL = "unit";
+ private static final String VALUES_LABEL = "values";
+ private static final String X_LABEL = "x";
+ private static final String Y_LABEL = "y";
+
+ private static final String BUILD_DATE_FORMAT = "yyyyMMdd-HHmm";
+ private static final String OVERVIEW_CHART_FILE_NAME = "chart_overview";
+ private static final String METADATA_FILE_NAME = "meta";
+ private static final String METADATA_FILE_NAME_EXTENSION = ".js";
+ private static final String CHART_FILE_NAME = "chart";
+ private static final String CHART_FILE_NAME_EXTENSION = ".json";
+ private static final String WILDCARD_PATTERN = "%";
+ private static final String COMPONENT_SEPARATOR = "#";
+ private static final String META_DATA_JAVASCRIPT_START = "var MetaData = ";
+
+ private static Pattern BUILD_DATE_PATTERN = Pattern.compile("(\\w+-\\w+)(-\\w+)?");
+ private static Pattern COMMIT_PATTERN = Pattern.compile(".*-.*-(.*)");
+
+ private JSONObject fApplicationComponents = new JSONObject();
+ private JSONObject fOverviews = new JSONObject();
+
+ private int fNumChart = 0;
+ private int fNumOverviewChart = 0;
+
+ /**
+ * Convert results from the database to JSON suitable for display
+ *
+ * <pre>
+ * For each variant (os/jvm combination)
+ * - For each summary entry (scenario)
+ * - Generate a chart
+ * - Add it to global summary (if needed)
+ * - Create the metadata for this test
+ * - Create an overview chart for this os/jvm
+ * </pre>
+ *
+ * @throws JSONException
+ * JSON error
+ * @throws IOException
+ * IO error
+ */
+ @Test
+ public void parseResults() throws JSONException, IOException {
+ Variations configVariations = PerformanceTestPlugin.getVariations();
+ JSONObject osJvmVariants = createOsJvm();
+
+ @SuppressWarnings("rawtypes")
+ Iterator keysIt = osJvmVariants.keys();
+ while (keysIt.hasNext()) {
+ JSONArray overviewSummarySeries = new JSONArray();
+
+ JSONObject variant = osJvmVariants.getJSONObject((String) keysIt.next());
+ String seriesKey = PerformanceTestPlugin.BUILD;
+
+ // Clone the variations from the environment because it might have
+ // extra parameters like host=, etc.
+ Variations buildVariations = (Variations) configVariations.clone();
+ buildVariations.setProperty(JVM_LABEL, variant.getString(JVM_LABEL));
+ buildVariations.setProperty(CONFIG_LABEL, variant.getString(OS_LABEL));
+ buildVariations.setProperty(BUILD_LABEL, WILDCARD_PATTERN);
+
+ Scenario[] scenarios = DB.queryScenarios(buildVariations, WILDCARD_PATTERN, seriesKey, null);
+ SummaryEntry[] summaryEntries = DB.querySummaries(buildVariations, WILDCARD_PATTERN);
+ for (SummaryEntry entry : summaryEntries) {
+ Scenario scenario = getScenario(entry.scenarioName, scenarios);
+ JSONObject scenarioSeries = createScenarioChart(scenario, entry, buildVariations);
+ // Add to global summary
+ if (scenarioSeries != null && entry.isGlobal) {
+ overviewSummarySeries.put(scenarioSeries);
+ }
+ }
+
+ JSONObject overviewMetadata = createOverviewChart(overviewSummarySeries, buildVariations);
+ fOverviews.put(Integer.toString(fNumOverviewChart), overviewMetadata);
+ }
+
+ // Create the matadata javascript file that includes OS/JVM combinations
+ // (for filtering), application components and overviews (one of OS/JVM
+ // combination)
+ JSONObject rootMetadata = new JSONObject();
+ rootMetadata.put(OSJVM_LABEL, osJvmVariants);
+ rootMetadata.put(APPLICATION_COMPONENTS_LABEL, fApplicationComponents);
+ rootMetadata.put(OVERVIEWS_LABEL, fOverviews);
+ try (FileWriter fw1 = new FileWriter(METADATA_FILE_NAME + METADATA_FILE_NAME_EXTENSION)) {
+ fw1.write(META_DATA_JAVASCRIPT_START + rootMetadata.toString(4));
+ }
+ }
+
+ /**
+ * Create chart for a scenario instance and add it to the relevant metadatas
+ *
+ * @param scenario
+ * the scenario. For example,
+ * "CTF Read & Seek Benchmark (500 seeks)".
+ * @param entry
+ * an entry from the summary. Only scenarios that are part of the
+ * summary are processed.
+ * @param variations
+ * all variations to consider to create the scenario chart. For
+ * example build=%;jvm=1.7;config=linux will generate a chart for
+ * all builds on Linux / JVM 1.7
+ *
+ * @return
+ * @throws JSONException
+ * JSON error
+ * @throws IOException
+ * IO error
+ */
+ private JSONObject createScenarioChart(Scenario scenario, SummaryEntry entry, Variations variations) throws JSONException, IOException {
+ if (scenario == null) {
+ return null;
+ }
+ String[] split = entry.scenarioName.split(COMPONENT_SEPARATOR);
+ if (split.length < 3) {
+ Activator.logError("Invalid scenario name \"" + entry.scenarioName + "\", it must be in format: org.package.foo#component#test");
+ return null;
+ }
+
+ // Generate individual chart
+ JSONArray rootScenario = new JSONArray();
+ JSONObject series = createSerie(scenario, variations, entry.shortName, entry.dimension);
+ rootScenario.put(series);
+ int numChart = fNumChart++;
+ try (FileWriter fw = new FileWriter(CHART_FILE_NAME + numChart + CHART_FILE_NAME_EXTENSION)) {
+ fw.write(rootScenario.toString(4));
+ }
+
+ // Create the metadata
+ JSONObject testMetadata = new JSONObject();
+ testMetadata.put(TITLE_LABEL, entry.shortName);
+ testMetadata.put(FILE_LABEL, CHART_FILE_NAME + numChart);
+ testMetadata.put(OS_LABEL, variations.getProperty(CONFIG_LABEL));
+ testMetadata.put(JVM_LABEL, variations.getProperty(JVM_LABEL));
+ testMetadata.put(DIMENSION_LABEL, entry.dimension.getName());
+ testMetadata.put(UNIT_LABEL, entry.dimension.getUnit().getShortName());
+
+ // Add the scenario to the metadata, under the correct component
+ String componentName = split[1];
+ JSONObject componentObject = null;
+ if (fApplicationComponents.has(componentName)) {
+ componentObject = fApplicationComponents.getJSONObject(componentName);
+ } else {
+ componentObject = new JSONObject();
+ componentObject.put(NAME_LABEL, componentName);
+ componentObject.put(TESTS_LABEL, new JSONArray());
+ fApplicationComponents.put(componentName, componentObject);
+ }
+ JSONArray tests = componentObject.getJSONArray(TESTS_LABEL);
+ tests.put(testMetadata);
+
+ return series;
+ }
+
+ /**
+ * Create an overview chart for this OS / JVM combination. The chart is made
+ * of multiple series (scenarios) that were marked as global.
+ *
+ * @param overviewSummarySeries
+ * an array of series to include in the chart (multiple
+ * scenarios)
+ * @param variations
+ * the variations used to generate the series to be included in
+ * this overview chart. For example build=%;jvm=1.7;config=linux
+ * will generate an overview chart for Linux / JVM 1.7
+ * @return the overview metadata JSON object
+ * @throws JSONException
+ * JSON error
+ * @throws IOException
+ * io error
+ */
+ private JSONObject createOverviewChart(JSONArray overviewSummarySeries, Variations variations) throws IOException, JSONException {
+ int numOverviewChart = fNumOverviewChart++;
+ try (FileWriter fw = new FileWriter(OVERVIEW_CHART_FILE_NAME + numOverviewChart + CHART_FILE_NAME_EXTENSION)) {
+ fw.write(overviewSummarySeries.toString(4));
+ }
+
+ String os = variations.getProperty(CONFIG_LABEL);
+ String jvm = variations.getProperty(JVM_LABEL);
+
+ // Create the overview metadata
+ JSONObject overviewMetadata = new JSONObject();
+ overviewMetadata.put(TITLE_LABEL, os + " / " + jvm);
+ overviewMetadata.put(FILE_LABEL, OVERVIEW_CHART_FILE_NAME + numOverviewChart);
+ overviewMetadata.put(OS_LABEL, os);
+ overviewMetadata.put(JVM_LABEL, jvm);
+ overviewMetadata.put(DIMENSION_LABEL, "");
+ overviewMetadata.put(UNIT_LABEL, "");
+
+ return overviewMetadata;
+ }
+
+ private static Scenario getScenario(String scenarioName, Scenario[] scenarios) {
+ for (int i = 0; i < scenarios.length; i++) {
+ Scenario s = scenarios[i];
+ if (s.getScenarioName().equals(scenarioName)) {
+ return s;
+ }
+
+ }
+ return null;
+ }
+
+ /**
+ * Get all combinations of OS / JVM. This will be used for filtering.
+ *
+ * @return the JSON object containing all the combinations
+ * @throws JSONException
+ * JSON error
+ */
+ private static JSONObject createOsJvm() throws JSONException {
+ JSONObject osjvm = new JSONObject();
+ List<String> oses = getDistinctOses();
+
+ int osJvmIndex = 1;
+ for (String os : oses) {
+ String key = JVM_LABEL;
+ Variations v = new Variations();
+
+ v.setProperty(BUILD_LABEL, WILDCARD_PATTERN);
+ v.setProperty(HOST_LABEL, WILDCARD_PATTERN);
+ v.setProperty(CONFIG_LABEL, os);
+ v.setProperty(JVM_LABEL, WILDCARD_PATTERN);
+
+ List<String> jvms = new ArrayList<>();
+ DB.queryDistinctValues(jvms, key, v, WILDCARD_PATTERN);
+ for (String jvm : jvms) {
+ JSONObject osjvmItem = new JSONObject();
+ osjvmItem.put(OS_LABEL, os);
+ osjvmItem.put(JVM_LABEL, jvm);
+ osjvmItem.put(DESCRIPTION_LABEL, os + " / " + jvm);
+ osjvm.put(Integer.toString(osJvmIndex), osjvmItem);
+ osJvmIndex++;
+ }
+ }
+
+ return osjvm;
+ }
+
+ /**
+ * Get all the distinct OS values
+ *
+ * @return the distinct OS values
+ */
+ private static List<String> getDistinctOses() {
+ List<String> configs = new ArrayList<>();
+ String key = PerformanceTestPlugin.CONFIG;
+ Variations v = new Variations();
+ v.setProperty(WILDCARD_PATTERN, WILDCARD_PATTERN);
+ DB.queryDistinctValues(configs, key, v, WILDCARD_PATTERN);
+ return configs;
+ }
+
+ /**
+ * This main can be run from within Eclipse provided everything is on the
+ * class path.
+ *
+ * @param args
+ * the arguments
+ * @throws JSONException
+ * JSON error
+ * @throws IOException
+ * io error
+ */
+ public static void main(String[] args) throws JSONException, IOException {
+ new PerfResultsToJSon().parseResults();
+ }
+
+ /**
+ * Create a series of data points for a given scenario through variations
+ *
+ * @param scenario
+ * the scenario. For example,
+ * "CTF Read & Seek Benchmark (500 seeks)".
+ * @param variations
+ * all variations to consider to create the series. For example
+ * build=%;jvm=1.7;config=linux will generate the series for all
+ * builds on Linux / JVM 1.7
+ * @param shortName
+ * the short name of the scenario
+ * @param dimension
+ * the dimension of interest (CPU time, used java heap, etc).
+ * @return the generated JSON object representing a series of data points
+ * for this scenario
+ * @throws JSONException
+ */
+ private static JSONObject createSerie(Scenario scenario, Variations variations, String shortName, Dim dimension) throws JSONException {
+ JSONObject o = new JSONObject();
+ o.putOpt(KEY_LABEL, shortName);
+ o.putOpt(VALUES_LABEL, createDataPoints(scenario, variations, dimension));
+ return o;
+ }
+
+ /**
+ * Create data points for a given scenario and variations.
+ *
+ * @param s
+ * the scenario. For example,
+ * "CTF Read & Seek Benchmark (500 seeks)".
+ * @param variations
+ * all variations to consider to create the data points. For
+ * example build=%;jvm=1.7;config=linux will generate the data
+ * points for all builds on Linux / JVM 1.7
+ * @param dimension
+ * the dimension of interest (CPU time, used java heap, etc).
+ *
+ * @return the generated JSON array of points
+ * @throws JSONException
+ * JSON error
+ */
+ private static JSONArray createDataPoints(Scenario s, Variations variations, Dim dimension) throws JSONException {
+ // Can be uncommented to see raw dump
+ //s.dump(System.out, PerformanceTestPlugin.BUILD);
+
+ String[] builds = DB.querySeriesValues(s.getScenarioName(), variations, PerformanceTestPlugin.BUILD);
+ Date[] dates = new Date[builds.length];
+ String[] commits = new String[builds.length];
+ for (int i = 0; i < builds.length; i++) {
+ dates[i] = parseBuildDate(builds[i]);
+ commits[i] = parseCommit(builds[i]);
+ }
+
+ TimeSeries timeSeries = s.getTimeSeries(dimension);
+ JSONArray dataPoints = new JSONArray();
+ int length = timeSeries.getLength();
+ for (int i = 0; i < length; i++) {
+ JSONObject point = new JSONObject();
+ if (dates[i] == null) {
+ continue;
+ }
+ point.put(X_LABEL, dates[i].getTime());
+ double value = 0;
+ if (timeSeries.getCount(i) > 0) {
+ value = timeSeries.getValue(i);
+ if (Double.isNaN(value)) {
+ value = 0;
+ }
+ }
+ point.put(Y_LABEL, value);
+ dataPoints.put(point);
+ point.put(LABEL_LABEL, createLabel(commits[i]));
+ }
+ return dataPoints;
+ }
+
+ /**
+ * Create a label JSONObject which is used to attach more information to a
+ * data point.
+ *
+ * @param commit
+ * the commit id for this data point
+ * @return the resulting JSON object
+ * @throws JSONException
+ * JSON error
+ */
+ private static JSONObject createLabel(String commit) throws JSONException {
+ /*
+ * Here we could add more information about this specific data point
+ * like the commit author, the commit message, etc.
+ */
+ JSONObject label = new JSONObject();
+ if (commit != null && !commit.isEmpty()) {
+ label.put(COMMIT_LABEL, commit);
+ }
+ return label;
+ }
+
+ /**
+ * Get the commit id out of the build= string
+ *
+ * @param build
+ * the build string
+ * @return the parsed commit id
+ */
+ private static String parseCommit(String build) {
+ Matcher matcher = COMMIT_PATTERN.matcher(build);
+ if (matcher.matches()) {
+ return matcher.group(1);
+ }
+ return null;
+ }
+
+ /**
+ * Get the Date out of the build= string
+ *
+ * @param build
+ * the build string
+ * @return the parsed Date
+ */
+ private static Date parseBuildDate(String build) {
+ Matcher matcher = BUILD_DATE_PATTERN.matcher(build);
+ Date date = null;
+ if (matcher.matches()) {
+ String dateStr = matcher.group(1);
+ SimpleDateFormat f = new SimpleDateFormat(BUILD_DATE_FORMAT);
+ try {
+ date = dateStr.length() > BUILD_DATE_FORMAT.length() ?
+ f.parse(dateStr.substring(dateStr.length() - BUILD_DATE_FORMAT.length())) :
+ f.parse(dateStr);
+ } catch (ParseException e) {
+ return null;
+ }
+ }
+ return date;
+ }
+}
projects / deliverable / tracecompass.git / commitdiff
