analysis/org.eclipse.tracecompass.analysis.lami.core/src/org/eclipse/tracecompass/internal/provisional/analysis/lami/core/module/LamiAnalysis.java

   1 /*******************************************************************************
   2  * Copyright (c) 2015, 2016 EfficiOS Inc., Alexandre Montplaisir
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *******************************************************************************/
   9
  10 package org.eclipse.tracecompass.internal.provisional.analysis.lami.core.module;
  11
  12 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNull;
  13 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNullContents;
  14 import static org.eclipse.tracecompass.common.core.NonNullUtils.nullToEmptyString;
  15
  16 import java.io.BufferedReader;
  17 import java.io.File;
  18 import java.io.IOException;
  19 import java.io.InputStreamReader;
  20 import java.nio.file.Files;
  21 import java.nio.file.Paths;
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.Collections;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.function.Predicate;
  28 import java.util.logging.Logger;
  29 import java.util.regex.Pattern;
  30 import java.util.stream.Collectors;
  31 import java.util.stream.Stream;
  32
  33 import org.eclipse.core.runtime.CoreException;
  34 import org.eclipse.core.runtime.IProgressMonitor;
  35 import org.eclipse.core.runtime.IStatus;
  36 import org.eclipse.core.runtime.MultiStatus;
  37 import org.eclipse.core.runtime.Status;
  38 import org.eclipse.jdt.annotation.NonNull;
  39 import org.eclipse.jdt.annotation.Nullable;
  40 import org.eclipse.tracecompass.common.core.log.TraceCompassLog;
  41 import org.eclipse.tracecompass.internal.analysis.lami.core.Activator;
  42 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.LamiStrings;
  43 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.ShellUtils;
  44 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiDurationAspect;
  45 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiEmptyAspect;
  46 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiGenericAspect;
  47 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQNameAspect;
  48 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQNumberAspect;
  49 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQTypeAspect;
  50 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiMixedAspect;
  51 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessNameAspect;
  52 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessPIDAspect;
  53 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessTIDAspect;
  54 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTableEntryAspect;
  55 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeBeginAspect;
  56 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeDurationAspect;
  57 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeEndAspect;
  58 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimestampAspect;
  59 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiData;
  60 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiData.DataType;
  61 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiTimeRange;
  62 import org.eclipse.tracecompass.tmf.core.analysis.ondemand.IOnDemandAnalysis;
  63 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
  64 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
  65 import org.json.JSONArray;
  66 import org.json.JSONException;
  67 import org.json.JSONObject;
  68
  69 import com.google.common.annotations.VisibleForTesting;
  70 import com.google.common.collect.ImmutableList;
  71 import com.google.common.collect.ImmutableMap;
  72 import com.google.common.collect.ImmutableMultimap;
  73 import com.google.common.collect.Multimap;
  74
  75 /**
  76  * Base class for analysis modules that call external scripts implementing the
  77  * LAMI protocol.
  78  *
  79  * @author Alexandre Montplaisir
  80  */
  81 public class LamiAnalysis implements IOnDemandAnalysis {
  82
  83     private static final Logger LOGGER = TraceCompassLog.getLogger(LamiAnalysis.class);
  84
  85     /**
  86      * Maximum major version of the LAMI protocol we support.
  87      *
  88      * Currently only 0.x/unversioned MI, outputted by lttng-analyses 0.4.x
  89      */
  90     private static final int MAX_SUPPORTED_MAJOR_VERSION = 0;
  91
  92     private static final String DOUBLE_QUOTES = "\""; //$NON-NLS-1$
  93
  94     /* Flags passed to the analysis scripts */
  95     private static final String METADATA_FLAG = "--metadata"; //$NON-NLS-1$
  96     private static final String PROGRESS_FLAG = "--output-progress"; //$NON-NLS-1$
  97     private static final String BEGIN_FLAG = "--begin"; //$NON-NLS-1$
  98     private static final String END_FLAG = "--end"; //$NON-NLS-1$
  99
 100     private final List<String> fScriptCommand;
 101
 102     /**
 103      * The LAMI analysis is considered initialized after we have read the
 104      * script's --metadata once. This will assign the fields below.
 105      */
 106     private boolean fInitialized = false;
 107
 108     private boolean fIsAvailable;
 109     private final String fName;
 110     private final boolean fIsUserDefined;
 111     private final Predicate<ITmfTrace> fAppliesTo;
 112
 113     /* Data defined by the analysis's metadata */
 114     private @Nullable String fAnalysisTitle;
 115     private @Nullable Map<String, LamiTableClass> fTableClasses;
 116     private boolean fUseProgressOutput;
 117
 118     /**
 119      * Constructor. To be called by implementing classes.
 120      *
 121      * @param name
 122      *            Name of this analysis
 123      * @param isUserDefined
 124      *            {@code true} if this is a user-defined analysis
 125      * @param appliesTo
 126      *            Predicate to use to check whether or not this analysis applies
 127      *            to a given trace
 128      * @param args
 129      *            Analysis arguments, including the executable name (first
 130      *            argument)
 131      */
 132     public LamiAnalysis(String name, boolean isUserDefined, Predicate<ITmfTrace> appliesTo,
 133             List<String> args) {
 134         fScriptCommand = ImmutableList.copyOf(args);
 135         fName = name;
 136         fIsUserDefined = isUserDefined;
 137         fAppliesTo = appliesTo;
 138     }
 139
 140     /**
 141      * Map of pre-defined charts, for every table class names.
 142      *
 143      * If a table class is not in this map then it means that table has no
 144      * predefined charts.
 145      *
 146      * @return The chart models, per table class names
 147      */
 148     protected Multimap<String, LamiChartModel> getPredefinedCharts() {
 149         return ImmutableMultimap.of();
 150     }
 151
 152     @Override
 153     public final boolean appliesTo(ITmfTrace trace) {
 154         return fAppliesTo.test(trace);
 155     }
 156
 157     @Override
 158     public boolean canExecute(ITmfTrace trace) {
 159         initialize();
 160         return fIsAvailable;
 161     }
 162
 163     private static boolean executableExists(String name) {
 164         if (name.contains(File.separator)) {
 165             // This seems like a path, not just an executable name
 166             return Files.isExecutable(Paths.get(name));
 167         }
 168
 169         // Check if this name is found in the PATH environment variable
 170         final String pathEnv = System.getenv("PATH"); //$NON-NLS-1$
 171         final String[] exeDirs = pathEnv.split(checkNotNull(Pattern.quote(File.pathSeparator)));
 172
 173         return Stream.of(exeDirs)
 174                 .map(Paths::get)
 175                 .anyMatch(path -> Files.isExecutable(path.resolve(name)));
 176     }
 177
 178     /**
 179      * Perform initialization of the LAMI script. This means verifying that it
 180      * is actually present on disk, and that it returns correct --metadata.
 181      */
 182     @VisibleForTesting
 183     protected synchronized void initialize() {
 184         if (fInitialized) {
 185             return;
 186         }
 187
 188         /* Do the analysis's initialization */
 189
 190         /* Check if the script's expected executable is on the PATH */
 191         final String executable = fScriptCommand.get(0);
 192         final boolean executableExists = executableExists(executable);
 193
 194         if (!executableExists) {
 195             /* Script is not found */
 196             fIsAvailable = false;
 197             fInitialized = true;
 198             return;
 199         }
 200
 201         fIsAvailable = checkMetadata();
 202         fInitialized = true;
 203     }
 204
 205     /**
 206      * Verify that this script returns valid metadata.
 207      *
 208      * This will populate all remaining non-final fields of this class.
 209      *
 210      * @return If the metadata is valid or not
 211      */
 212     @VisibleForTesting
 213     protected boolean checkMetadata() {
 214         /*
 215          * The initialize() phase of the analysis will be used to check the
 216          * script's metadata. Actual runs of the script will use the execute()
 217          * method below.
 218          */
 219         List<String> command = ImmutableList.<@NonNull String> builder()
 220                 .addAll(fScriptCommand).add(METADATA_FLAG).build();
 221
 222         LOGGER.info(() -> "[LamiAnalysis:RunningMetadataCommand] command=" + command.toString()); //$NON-NLS-1$
 223
 224         String output = getOutputFromCommand(command);
 225         if (output == null || output.isEmpty()) {
 226             return false;
 227         }
 228
 229         /*
 230          *
 231          * Metadata should look this this:
 232          *
 233          * {
 234          *   "version": [1, 5, 2, "dev"],
 235          *   "title": "I/O latency statistics",
 236          *   "authors": [
 237          *     "Julien Desfossez",
 238          *     "Antoine Busque"
 239          *   ],
 240          *   "description": "Provides statistics about the latency involved in various I/O operations.",
 241          *   "url": "https://github.com/lttng/lttng-analyses",
 242          *   "tags": [
 243          *     "io",
 244          *     "stats",
 245          *     "linux-kernel",
 246          *     "lttng-analyses"
 247          *   ],
 248          *   "table-classes": {
 249          *     "syscall-latency": {
 250          *       "title": "System calls latency statistics",
 251          *       "column-descriptions": [
 252          *         {"title": "System call", "type": "syscall"},
 253          *         {"title": "Count", "type": "int", "unit": "operations"},
 254          *         {"title": "Minimum duration", "type": "duration"},
 255          *         {"title": "Average duration", "type": "duration"},
 256          *         {"title": "Maximum duration", "type": "duration"},
 257          *         {"title": "Standard deviation", "type": "duration"}
 258          *       ]
 259          *     },
 260          *     "disk-latency": {
 261          *       "title": "Disk latency statistics",
 262          *       "column-descriptions": [
 263          *         {"title": "Disk name", "type": "disk"},
 264          *         {"title": "Count", "type": "int", "unit": "operations"},
 265          *         {"title": "Minimum duration", "type": "duration"},
 266          *         {"title": "Average duration", "type": "duration"},
 267          *         {"title": "Maximum duration", "type": "duration"},
 268          *         {"title": "Standard deviation", "type": "duration"}
 269          *       ]
 270          *     }
 271          *   }
 272          * }
 273          *
 274          */
 275
 276         try {
 277             JSONObject obj = new JSONObject(output);
 278             fAnalysisTitle = obj.getString(LamiStrings.TITLE);
 279
 280             /* Very early scripts may not contain the "mi-version" */
 281             JSONObject miVersion = obj.optJSONObject(LamiStrings.MI_VERSION);
 282             if (miVersion == null) {
 283                 /* Before version 0.1 */
 284                 fUseProgressOutput = false;
 285             } else {
 286                 int majorVersion = miVersion.getInt(LamiStrings.MAJOR);
 287                 if (majorVersion <= MAX_SUPPORTED_MAJOR_VERSION) {
 288                     fUseProgressOutput = true;
 289                 } else {
 290                     /* Unknown version, we do not support it */
 291                     return false;
 292                 }
 293             }
 294
 295             JSONObject tableClasses = obj.getJSONObject(LamiStrings.TABLE_CLASSES);
 296             @NonNull String[] tableClassNames = checkNotNullContents(JSONObject.getNames(tableClasses));
 297
 298             ImmutableMap.Builder<String, LamiTableClass> tablesBuilder = ImmutableMap.builder();
 299             for (String tableClassName : tableClassNames) {
 300                 JSONObject tableClass = tableClasses.getJSONObject(tableClassName);
 301
 302                 final String tableTitle = checkNotNull(tableClass.getString(LamiStrings.TITLE));
 303                 @NonNull JSONArray columnDescriptions = checkNotNull(tableClass.getJSONArray(LamiStrings.COLUMN_DESCRIPTIONS));
 304
 305                 List<LamiTableEntryAspect> aspects = getAspectsFromColumnDescriptions(columnDescriptions);
 306                 Collection<LamiChartModel> chartModels = getPredefinedCharts().get(tableClassName);
 307
 308                 tablesBuilder.put(tableClassName, new LamiTableClass(tableClassName, tableTitle, aspects, chartModels));
 309             }
 310
 311             try {
 312                 fTableClasses = tablesBuilder.build();
 313             } catch (IllegalArgumentException e) {
 314                 /*
 315                  * This is thrown if there are duplicate keys in the map
 316                  * builder.
 317                  */
 318                 throw new JSONException("Duplicate table class entry in " + fAnalysisTitle); //$NON-NLS-1$
 319             }
 320
 321         } catch (JSONException e) {
 322             /* Error in the parsing of the JSON, script is broken? */
 323             LOGGER.severe(() -> "[LamiAnalysis:ErrorParsingMetadata] msg=" + e.getMessage()); //$NON-NLS-1$
 324             Activator.instance().logError(e.getMessage());
 325             return false;
 326         }
 327         return true;
 328     }
 329
 330     private static List<LamiTableEntryAspect> getAspectsFromColumnDescriptions(JSONArray columnDescriptions) throws JSONException {
 331         ImmutableList.Builder<LamiTableEntryAspect> aspectsBuilder = new ImmutableList.Builder<>();
 332         for (int j = 0; j < columnDescriptions.length(); j++) {
 333             JSONObject column = columnDescriptions.getJSONObject(j);
 334             DataType columnDataType;
 335             String columnClass = column.optString(LamiStrings.CLASS, null);
 336
 337             if (columnClass == null) {
 338                 columnDataType = DataType.MIXED;
 339             } else {
 340                 columnDataType = getDataTypeFromString(columnClass);
 341             }
 342
 343             String columnTitle = column.optString(LamiStrings.TITLE, null);
 344
 345             if (columnTitle == null) {
 346                 columnTitle = String.format("%s #%d", columnDataType.getTitle(), j + 1); //$NON-NLS-1$
 347             }
 348
 349             final int colIndex = j;
 350             switch (columnDataType) {
 351             case TIME_RANGE:
 352                 /*
 353                  * We will add 3 aspects, to represent the start, end and
 354                  * duration of this time range.
 355                  */
 356                 aspectsBuilder.add(new LamiTimeRangeBeginAspect(columnTitle, colIndex));
 357                 aspectsBuilder.add(new LamiTimeRangeEndAspect(columnTitle, colIndex));
 358                 aspectsBuilder.add(new LamiTimeRangeDurationAspect(columnTitle, colIndex));
 359                 break;
 360
 361             case TIMESTAMP:
 362                 aspectsBuilder.add(new LamiTimestampAspect(columnTitle, colIndex));
 363                 break;
 364
 365             case PROCESS:
 366                 aspectsBuilder.add(new LamiProcessNameAspect(columnTitle, colIndex));
 367                 aspectsBuilder.add(new LamiProcessPIDAspect(columnTitle, colIndex));
 368                 aspectsBuilder.add(new LamiProcessTIDAspect(columnTitle, colIndex));
 369                 break;
 370
 371             case IRQ:
 372                 aspectsBuilder.add(new LamiIRQTypeAspect(columnTitle, colIndex));
 373                 aspectsBuilder.add(new LamiIRQNameAspect(columnTitle, colIndex));
 374                 aspectsBuilder.add(new LamiIRQNumberAspect(columnTitle, colIndex));
 375                 break;
 376
 377             case DURATION:
 378                 aspectsBuilder.add(new LamiDurationAspect(columnTitle, colIndex));
 379                 break;
 380
 381             case MIXED:
 382                 aspectsBuilder.add(new LamiMixedAspect(columnTitle, colIndex));
 383                 break;
 384
 385             // $CASES-OMITTED$
 386             default:
 387                 String units = column.optString(LamiStrings.UNIT, null);
 388
 389                 if (units == null) {
 390                     units = columnDataType.getUnits();
 391                 }
 392
 393                 /* We will add only one aspect representing the element */
 394                 LamiTableEntryAspect aspect = new LamiGenericAspect(columnTitle,
 395                         units, colIndex, columnDataType.isContinuous(), false);
 396                 aspectsBuilder.add(aspect);
 397                 break;
 398             }
 399         }
 400         /*
 401          * SWT quirk : we need an empty column at the end or else the last data
 402          * column will clamp to the right edge of the view if it is
 403          * right-aligned.
 404          */
 405         aspectsBuilder.add(LamiEmptyAspect.INSTANCE);
 406
 407         return aspectsBuilder.build();
 408     }
 409
 410     private static DataType getDataTypeFromString(String value) throws JSONException {
 411         try {
 412             return DataType.fromString(value);
 413         } catch (IllegalArgumentException e) {
 414             throw new JSONException("Unrecognized data type: " + value); //$NON-NLS-1$
 415         }
 416     }
 417
 418     /**
 419      * Get the title of this analysis, as read from the script's metadata.
 420      *
 421      * @return The analysis title. Should not be null after the initialization
 422      *         completed successfully.
 423      */
 424     public @Nullable String getAnalysisTitle() {
 425         return fAnalysisTitle;
 426     }
 427
 428     /**
 429      * Get the result table classes defined by this analysis, as read from the
 430      * script's metadata.
 431      *
 432      * @return The analysis' result table classes. Should not be null after the
 433      *         execution completed successfully.
 434      */
 435     public @Nullable Map<String, LamiTableClass> getTableClasses() {
 436         return fTableClasses;
 437     }
 438
 439     /**
 440      * Print the full command that will be run when calling {@link #execute},
 441      * with the exception of the 'extraParams' that will be passed to execute().
 442      *
 443      * This can be used to display the command in the UI before it is actually
 444      * run.
 445      *
 446      * @param trace
 447      *            The trace on which to run the analysis
 448      * @param range
 449      *            The time range to specify. Null will not specify a time range,
 450      *            which means the whole trace will be taken.
 451      * @return The command as a single, space-separated string
 452      */
 453     public String getFullCommandAsString(ITmfTrace trace, @Nullable TmfTimeRange range) {
 454         String tracePath = checkNotNull(trace.getPath());
 455
 456         ImmutableList.Builder<String> builder = getBaseCommand(range);
 457         /*
 458          * We can add double-quotes around the trace path, which could contain
 459          * spaces, so that the resulting command can be easily copy-pasted into
 460          * a shell.
 461          */
 462         builder.add(DOUBLE_QUOTES + tracePath + DOUBLE_QUOTES);
 463         List<String> list = builder.build();
 464         String ret = list.stream().collect(Collectors.joining(" ")); //$NON-NLS-1$
 465         return checkNotNull(ret);
 466     }
 467
 468     /**
 469      * Get the base part of the command that will be executed to run this
 470      * analysis, supplying the given time range. Base part meaning:
 471      *
 472      * <pre>
 473      * [script executable] [statically-defined parameters] [--begin/--end (if applicable)]
 474      * </pre>
 475      *
 476      * Note that it does not include the path to the trace, that is to be added
 477      * separately.
 478      *
 479      * @param range
 480      *            The time range that will be passed
 481      * @return The elements of the command
 482      */
 483     private ImmutableList.Builder<String> getBaseCommand(@Nullable TmfTimeRange range) {
 484         long begin = 0;
 485         long end = 0;
 486         if (range != null) {
 487             begin = range.getStartTime().getValue();
 488             end = range.getEndTime().getValue();
 489             if (range.getStartTime().compareTo(range.getEndTime()) > 0) {
 490                 begin = range.getEndTime().getValue();
 491                 end = range.getStartTime().getValue();
 492             }
 493         }
 494
 495         ImmutableList.Builder<String> builder = ImmutableList.builder();
 496         builder.addAll(fScriptCommand);
 497
 498         if (fUseProgressOutput) {
 499             builder.add(PROGRESS_FLAG);
 500         }
 501
 502         if (range != null) {
 503             builder.add(BEGIN_FLAG).add(String.valueOf(begin));
 504             builder.add(END_FLAG).add(String.valueOf(end));
 505         }
 506         return builder;
 507     }
 508
 509     @Override
 510     public List<LamiResultTable> execute(ITmfTrace trace, @Nullable TmfTimeRange timeRange,
 511             String extraParamsString, IProgressMonitor monitor) throws CoreException {
 512         /* Should have been called already, but in case it was not */
 513         initialize();
 514
 515         final @NonNull String tracePath = checkNotNull(trace.getPath());
 516         final @NonNull String trimmedExtraParamsString = checkNotNull(extraParamsString.trim());
 517         final List<String> extraParams = ShellUtils.commandStringToArgs(trimmedExtraParamsString);
 518
 519         ImmutableList.Builder<String> builder = getBaseCommand(timeRange);
 520
 521         builder.addAll(extraParams);
 522         builder.add(tracePath);
 523         List<String> command = builder.build();
 524
 525         LOGGER.info(() -> "[LamiAnalysis:RunningExecuteCommand] command=" + command.toString()); //$NON-NLS-1$
 526         String output = getResultsFromCommand(command, monitor);
 527
 528         if (output.isEmpty()) {
 529             IStatus status = new Status(IStatus.INFO, Activator.instance().getPluginId(), Messages.LamiAnalysis_NoResults);
 530             throw new CoreException(status);
 531         }
 532
 533         /*
 534          * {
 535          *   "results": [
 536          *     {
 537          *       "time-range": {
 538          *         "type": "time-range",
 539          *         "begin": 1444334398154194201,
 540          *         "end": 1444334425194487548
 541          *       },
 542          *       "class": "syscall-latency",
 543          *       "data": [
 544          *         [
 545          *           {"type": "syscall", "name": "open"},
 546          *           45,
 547          *           {"type": "duration", "value": 5562},
 548          *           {"type": "duration", "value": 13835},
 549          *           {"type": "duration", "value": 77683},
 550          *           {"type": "duration", "value": 15263}
 551          *         ],
 552          *         [
 553          *           {"type": "syscall", "name": "read"},
 554          *           109,
 555          *           {"type": "duration", "value": 316},
 556          *           {"type": "duration", "value": 5774},
 557          *           {"type": "duration", "value": 62569},
 558          *           {"type": "duration", "value": 9277}
 559          *         ]
 560          *       ]
 561          *     },
 562          *     {
 563          *       "time-range": {
 564          *         "type": "time-range",
 565          *         "begin": 1444334425194487549,
 566          *         "end": 1444334425254887190
 567          *       },
 568          *       "class": "syscall-latency",
 569          *       "data": [
 570          *         [
 571          *           {"type": "syscall", "name": "open"},
 572          *           45,
 573          *           {"type": "duration", "value": 1578},
 574          *           {"type": "duration", "value": 16648},
 575          *           {"type": "duration", "value": 15444},
 576          *           {"type": "duration", "value": 68540}
 577          *         ],
 578          *         [
 579          *           {"type": "syscall", "name": "read"},
 580          *           109,
 581          *           {"type": "duration", "value": 78},
 582          *           {"type": "duration", "value": 1948},
 583          *           {"type": "duration", "value": 11184},
 584          *           {"type": "duration", "value": 94670}
 585          *         ]
 586          *       ]
 587          *     }
 588          *   ]
 589          * }
 590          *
 591          */
 592
 593         ImmutableList.Builder<LamiResultTable> resultsBuilder = new ImmutableList.Builder<>();
 594
 595         try {
 596             JSONObject obj = new JSONObject(output);
 597             JSONArray results = obj.getJSONArray(LamiStrings.RESULTS);
 598
 599             if (results.length() == 0) {
 600                 /*
 601                  * No results were reported. This may be normal, but warn the
 602                  * user why a report won't be created.
 603                  */
 604                 IStatus status = new Status(IStatus.INFO, Activator.instance().getPluginId(), Messages.LamiAnalysis_NoResults);
 605                 throw new CoreException(status);
 606             }
 607
 608             for (int i = 0; i < results.length(); i++) {
 609                 JSONObject result = results.getJSONObject(i);
 610
 611                 /* Parse the time-range */
 612                 JSONObject trObject = checkNotNull(result.getJSONObject(LamiStrings.TIME_RANGE));
 613                 LamiData trData = LamiData.createFromObject(trObject);
 614                 if (!(trData instanceof LamiTimeRange)) {
 615                     throw new JSONException("Time range did not have expected class type."); //$NON-NLS-1$
 616                 }
 617                 LamiTimeRange tr = (LamiTimeRange) trData;
 618
 619                 /* Parse the table's class */
 620                 LamiTableClass tableClass;
 621                 JSONObject tableClassObject = result.optJSONObject(LamiStrings.CLASS);
 622                 if (tableClassObject == null) {
 623                     /*
 624                      * "class" is just a standard string, indicating we use a
 625                      * metadata-defined table class as-is
 626                      */
 627                     @NonNull String tableClassName = checkNotNull(result.getString(LamiStrings.CLASS));
 628                     tableClass = getTableClassFromName(tableClassName);
 629
 630                     // FIXME Rest will become more generic eventually in the LAMI format.
 631                 } else if (tableClassObject.has(LamiStrings.INHERIT)) {
 632                     /*
 633                      * Dynamic title: We reuse an existing table class but
 634                      * override the title.
 635                      */
 636                     String baseTableName = checkNotNull(tableClassObject.getString(LamiStrings.INHERIT));
 637                     LamiTableClass baseTableClass = getTableClassFromName(baseTableName);
 638                     String newTitle = checkNotNull(tableClassObject.getString(LamiStrings.TITLE));
 639
 640                     tableClass = new LamiTableClass(baseTableClass, newTitle);
 641                 } else {
 642                     /*
 643                      * Dynamic column descriptions: we implement a new table
 644                      * class entirely.
 645                      */
 646                     String title = checkNotNull(tableClassObject.getString(LamiStrings.TITLE));
 647                     JSONArray columnDescriptions = checkNotNull(tableClassObject.getJSONArray(LamiStrings.COLUMN_DESCRIPTIONS));
 648                     List<LamiTableEntryAspect> aspects = getAspectsFromColumnDescriptions(columnDescriptions);
 649
 650                     tableClass = new LamiTableClass(nullToEmptyString(Messages.LamiAnalysis_DefaultDynamicTableName), title, aspects, Collections.EMPTY_SET);
 651                 }
 652
 653                 /* Parse the "data", which is the array of rows */
 654                 JSONArray data = result.getJSONArray(LamiStrings.DATA);
 655                 ImmutableList.Builder<LamiTableEntry> dataBuilder = new ImmutableList.Builder<>();
 656
 657                 for (int j = 0; j < data.length(); j++) {
 658                     /* A row is an array of cells */
 659                     JSONArray row = data.getJSONArray(j);
 660                     ImmutableList.Builder<LamiData> rowBuilder = ImmutableList.builder();
 661
 662                     for (int k = 0; k < row.length(); k++) {
 663                         Object cellObject = checkNotNull(row.get(k));
 664                         LamiData cellValue = LamiData.createFromObject(cellObject);
 665                         rowBuilder.add(cellValue);
 666
 667                     }
 668                     dataBuilder.add(new LamiTableEntry(rowBuilder.build()));
 669                 }
 670                 resultsBuilder.add(new LamiResultTable(tr, tableClass, dataBuilder.build()));
 671             }
 672
 673         } catch (JSONException e) {
 674             LOGGER.severe(() -> "[LamiAnalysis:ErrorParsingExecutionOutput] msg=" + e.getMessage()); //$NON-NLS-1$
 675             IStatus status = new Status(IStatus.ERROR, Activator.instance().getPluginId(), e.getMessage(), e);
 676             throw new CoreException(status);
 677         }
 678
 679         return resultsBuilder.build();
 680     }
 681
 682     private LamiTableClass getTableClassFromName(String tableClassName) throws JSONException {
 683         Map<String, LamiTableClass> map = checkNotNull(fTableClasses);
 684         LamiTableClass tableClass = map.get(tableClassName);
 685         if (tableClass == null) {
 686             throw new JSONException("Table class " + tableClassName + //$NON-NLS-1$
 687                     " was not declared in the metadata"); //$NON-NLS-1$
 688         }
 689         return tableClass;
 690     }
 691
 692     /**
 693      * Get the output of an external command, used for getting the metadata.
 694      * Cannot be cancelled, and will not report errors, simply returns null if
 695      * the process ended abnormally.
 696      *
 697      * @param command
 698      *            The parameters of the command, passed to
 699      *            {@link ProcessBuilder}
 700      * @return The command output as a string
 701      */
 702     @VisibleForTesting
 703     protected @Nullable String getOutputFromCommand(List<String> command) {
 704         try {
 705             ProcessBuilder builder = new ProcessBuilder(command);
 706             builder.redirectErrorStream(true);
 707
 708             Process p = builder.start();
 709             try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));) {
 710                 int ret = p.waitFor();
 711                 String output = br.lines().collect(Collectors.joining());
 712
 713                 return (ret == 0 ? output : null);
 714             }
 715         } catch (IOException | InterruptedException e) {
 716             return null;
 717         }
 718     }
 719
 720     /**
 721      * Get the results of invoking the specified command.
 722      *
 723      * The result should start with '{"results":...', as specified by the LAMI
 724      * JSON protocol. The JSON itself may be split over multiple lines.
 725      *
 726      * @param command
 727      *            The command to run (program and its arguments)
 728      * @param monitor
 729      *            The progress monitor
 730      * @return The analysis results
 731      * @throws CoreException
 732      *             If the command ended abnormally, and normal results were not
 733      *             returned
 734      */
 735     @VisibleForTesting
 736     protected String getResultsFromCommand(List<String> command, IProgressMonitor monitor)
 737             throws CoreException {
 738
 739         final int scale = 1000;
 740         double workedSoFar = 0.0;
 741
 742         ProcessCanceller cancellerRunnable = null;
 743         Thread cancellerThread = null;
 744
 745         try {
 746             monitor.beginTask(Messages.LamiAnalysis_MainTaskName, scale);
 747
 748             ProcessBuilder builder = new ProcessBuilder(command);
 749             builder.redirectErrorStream(false);
 750
 751             Process p = checkNotNull(builder.start());
 752
 753             cancellerRunnable = new ProcessCanceller(p, monitor);
 754             cancellerThread = new Thread(cancellerRunnable);
 755             cancellerThread.start();
 756
 757             List<String> results = new ArrayList<>();
 758
 759             try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));) {
 760                 String line = in.readLine();
 761                 while (line != null && !line.matches("\\s*\\{.*")) { //$NON-NLS-1$
 762                     /*
 763                      * This is a line indicating progress, it has the form:
 764                      *
 765                      * 0.123 3000 of 5000 events processed
 766                      *
 767                      * The first part indicates the estimated fraction (out of
 768                      * 1.0) of work done. The second part is status text.
 769                      */
 770
 771                     // Trim the line first to make sure the first character is
 772                     // significant
 773                     line = line.trim();
 774
 775                     // Split at the first space
 776                     String[] elems = line.split(" ", 2); //$NON-NLS-1$
 777
 778                     if (elems[0].matches("\\d.*")) { //$NON-NLS-1$
 779                         // It looks like we have a progress indication
 780                         try {
 781                             // Try parsing the number
 782                             double cumulativeWork = Double.parseDouble(elems[0]) * scale;
 783                             double workedThisLoop = cumulativeWork - workedSoFar;
 784
 785                             // We're going backwards? Do not update the
 786                             // monitor's value
 787                             if (workedThisLoop > 0) {
 788                                 monitor.internalWorked(workedThisLoop);
 789                                 workedSoFar = cumulativeWork;
 790                             }
 791
 792                             // There is a message: update the monitor's task name
 793                             if (elems.length >= 2) {
 794                                 monitor.setTaskName(elems[1].trim());
 795                             }
 796                         } catch (NumberFormatException e) {
 797                             // Continue reading progress lines anyway
 798                         }
 799                     }
 800
 801                     line = in.readLine();
 802                 }
 803                 while (line != null) {
 804                     /*
 805                      * We have seen the first line containing a '{', this is our
 806                      * JSON output!
 807                      */
 808                     results.add(line);
 809                     line = in.readLine();
 810                 }
 811             }
 812             int ret = p.waitFor();
 813
 814             if (monitor.isCanceled()) {
 815                 /* We were interrupted by the canceller thread. */
 816                 IStatus status = new Status(IStatus.CANCEL, Activator.instance().getPluginId(), null);
 817                 throw new CoreException(status);
 818             }
 819
 820             if (ret != 0) {
 821                 /*
 822                  * Something went wrong running the external script. We will
 823                  * gather the stderr and report it to the user.
 824                  */
 825                 BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()));
 826                 List<String> stdErrOutput = br.lines().collect(Collectors.toList());
 827
 828                 MultiStatus status = new MultiStatus(Activator.instance().getPluginId(),
 829                         IStatus.ERROR, Messages.LamiAnalysis_ErrorDuringExecution, null);
 830                 for (String str : stdErrOutput) {
 831                     status.add(new Status(IStatus.ERROR, Activator.instance().getPluginId(), str));
 832                 }
 833                 if (stdErrOutput.isEmpty()) {
 834                     /*
 835                      * At least say "no output", so an error message actually
 836                      * shows up.
 837                      */
 838                     status.add(new Status(IStatus.ERROR, Activator.instance().getPluginId(), Messages.LamiAnalysis_ErrorNoOutput));
 839                 }
 840                 throw new CoreException(status);
 841             }
 842
 843             /* External script ended successfully, all is fine! */
 844             String resultsStr = results.stream().collect(Collectors.joining());
 845             return checkNotNull(resultsStr);
 846
 847         } catch (IOException | InterruptedException e) {
 848             IStatus status = new Status(IStatus.ERROR, Activator.instance().getPluginId(), Messages.LamiAnalysis_ExecutionInterrupted, e);
 849             throw new CoreException(status);
 850
 851         } finally {
 852             if (cancellerRunnable != null) {
 853                 cancellerRunnable.setFinished();
 854             }
 855             if (cancellerThread != null) {
 856                 try {
 857                     cancellerThread.join();
 858                 } catch (InterruptedException e) {
 859                 }
 860             }
 861
 862             monitor.done();
 863         }
 864     }
 865
 866     private static class ProcessCanceller implements Runnable {
 867
 868         private final Process fProcess;
 869         private final IProgressMonitor fMonitor;
 870
 871         private boolean fIsFinished = false;
 872
 873         public ProcessCanceller(Process process, IProgressMonitor monitor) {
 874             fProcess = process;
 875             fMonitor = monitor;
 876         }
 877
 878         public void setFinished() {
 879             fIsFinished = true;
 880         }
 881
 882         @Override
 883         public void run() {
 884             try {
 885                 while (!fIsFinished) {
 886                     Thread.sleep(500);
 887                     if (fMonitor.isCanceled()) {
 888                         fProcess.destroy();
 889                         return;
 890                     }
 891                 }
 892             } catch (InterruptedException e) {
 893             }
 894         }
 895
 896     }
 897
 898     @Override
 899     public @NonNull String getName() {
 900         return fName;
 901     }
 902
 903     @Override
 904     public boolean isUserDefined() {
 905         return fIsUserDefined;
 906     }
 907
 908 }