analysis/org.eclipse.tracecompass.analysis.lami.core/src/org/eclipse/tracecompass/internal/provisional/analysis/lami/core/module/LamiAnalysis.java

   1 /*******************************************************************************
   2  * Copyright (c) 2015, 2016 EfficiOS Inc., Alexandre Montplaisir
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *******************************************************************************/
   9
  10 package org.eclipse.tracecompass.internal.provisional.analysis.lami.core.module;
  11
  12 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNull;
  13 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNullContents;
  14 import static org.eclipse.tracecompass.common.core.NonNullUtils.nullToEmptyString;
  15
  16 import java.io.BufferedReader;
  17 import java.io.File;
  18 import java.io.IOException;
  19 import java.io.InputStreamReader;
  20 import java.nio.file.Files;
  21 import java.nio.file.Paths;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.function.Predicate;
  29 import java.util.logging.Logger;
  30 import java.util.regex.Pattern;
  31 import java.util.stream.Collectors;
  32 import java.util.stream.Stream;
  33
  34 import org.eclipse.core.runtime.CoreException;
  35 import org.eclipse.core.runtime.IProgressMonitor;
  36 import org.eclipse.core.runtime.IStatus;
  37 import org.eclipse.core.runtime.MultiStatus;
  38 import org.eclipse.core.runtime.Status;
  39 import org.eclipse.jdt.annotation.NonNull;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.eclipse.tracecompass.common.core.log.TraceCompassLog;
  42 import org.eclipse.tracecompass.internal.analysis.lami.core.Activator;
  43 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.LamiStrings;
  44 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiDurationAspect;
  45 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiEmptyAspect;
  46 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiGenericAspect;
  47 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQNameAspect;
  48 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQNumberAspect;
  49 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQTypeAspect;
  50 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiMixedAspect;
  51 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessNameAspect;
  52 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessPIDAspect;
  53 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessTIDAspect;
  54 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTableEntryAspect;
  55 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeBeginAspect;
  56 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeDurationAspect;
  57 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeEndAspect;
  58 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimestampAspect;
  59 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiData;
  60 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiData.DataType;
  61 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiTimeRange;
  62 import org.eclipse.tracecompass.tmf.core.analysis.ondemand.IOnDemandAnalysis;
  63 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
  64 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
  65 import org.json.JSONArray;
  66 import org.json.JSONException;
  67 import org.json.JSONObject;
  68
  69 import com.google.common.annotations.VisibleForTesting;
  70 import com.google.common.collect.ImmutableList;
  71 import com.google.common.collect.ImmutableMap;
  72 import com.google.common.collect.ImmutableMultimap;
  73 import com.google.common.collect.Multimap;
  74
  75 /**
  76  * Base class for analysis modules that call external scripts implementing the
  77  * LAMI protocol.
  78  *
  79  * @author Alexandre Montplaisir
  80  */
  81 public class LamiAnalysis implements IOnDemandAnalysis {
  82
  83     private static final Logger LOGGER = TraceCompassLog.getLogger(LamiAnalysis.class);
  84
  85     /**
  86      * Maximum major version of the LAMI protocol we support.
  87      *
  88      * Currently only 0.x/unversioned MI, outputted by lttng-analyses 0.4.x
  89      */
  90     private static final int MAX_SUPPORTED_MAJOR_VERSION = 0;
  91
  92     private static final String DOUBLE_QUOTES = "\""; //$NON-NLS-1$
  93
  94     /* Flags passed to the analysis scripts */
  95     private static final String METADATA_FLAG = "--metadata"; //$NON-NLS-1$
  96     private static final String PROGRESS_FLAG = "--output-progress"; //$NON-NLS-1$
  97     private static final String BEGIN_FLAG = "--begin"; //$NON-NLS-1$
  98     private static final String END_FLAG = "--end"; //$NON-NLS-1$
  99
 100     private final List<String> fScriptCommand;
 101
 102     /**
 103      * The LAMI analysis is considered initialized after we have read the
 104      * script's --metadata once. This will assign the fields below.
 105      */
 106     private boolean fInitialized = false;
 107
 108     private boolean fIsAvailable;
 109     private final String fName;
 110     private final boolean fIsUserDefined;
 111     private final Predicate<ITmfTrace> fAppliesTo;
 112
 113     /* Data defined by the analysis's metadata */
 114     private @Nullable String fAnalysisTitle;
 115     private @Nullable Map<String, LamiTableClass> fTableClasses;
 116     private boolean fUseProgressOutput;
 117
 118     /**
 119      * Constructor. To be called by implementing classes.
 120      *
 121      * @param name
 122      *            Name of this analysis
 123      * @param isUserDefined
 124      *            {@code true} if this is a user-defined analysis
 125      * @param appliesTo
 126      *            Predicate to use to check whether or not this analysis applies
 127      *            to a given trace
 128      * @param args
 129      *            Analysis arguments, including the executable name (first
 130      *            argument)
 131      */
 132     public LamiAnalysis(String name, boolean isUserDefined, Predicate<ITmfTrace> appliesTo,
 133             List<String> args) {
 134         fScriptCommand = ImmutableList.copyOf(args);
 135         fName = name;
 136         fIsUserDefined = isUserDefined;
 137         fAppliesTo = appliesTo;
 138     }
 139
 140     /**
 141      * Map of pre-defined charts, for every table class names.
 142      *
 143      * If a table class is not in this map then it means that table has no
 144      * predefined charts.
 145      *
 146      * @return The chart models, per table class names
 147      */
 148     protected Multimap<String, LamiChartModel> getPredefinedCharts() {
 149         return ImmutableMultimap.of();
 150     }
 151
 152     @Override
 153     public final boolean appliesTo(ITmfTrace trace) {
 154         return fAppliesTo.test(trace);
 155     }
 156
 157     @Override
 158     public boolean canExecute(ITmfTrace trace) {
 159         initialize();
 160         return fIsAvailable;
 161     }
 162
 163     /**
 164      * Perform initialization of the LAMI script. This means verifying that it
 165      * is actually present on disk, and that it returns correct --metadata.
 166      */
 167     @VisibleForTesting
 168     protected synchronized void initialize() {
 169         if (fInitialized) {
 170             return;
 171         }
 172
 173         /* Do the analysis's initialization */
 174
 175         /* Check if the script's expected executable is on the PATH */
 176         String executable = fScriptCommand.get(0);
 177         boolean exists = Stream.of(System.getenv("PATH").split(checkNotNull(Pattern.quote(File.pathSeparator)))) //$NON-NLS-1$
 178                 .map(Paths::get)
 179                 .anyMatch(path -> Files.exists(path.resolve(executable)));
 180         if (!exists) {
 181             /* Script is not found */
 182             fIsAvailable = false;
 183             fInitialized = true;
 184             return;
 185         }
 186
 187         fIsAvailable = checkMetadata();
 188         fInitialized = true;
 189     }
 190
 191     /**
 192      * Verify that this script returns valid metadata.
 193      *
 194      * This will populate all remaining non-final fields of this class.
 195      *
 196      * @return If the metadata is valid or not
 197      */
 198     @VisibleForTesting
 199     protected boolean checkMetadata() {
 200         /*
 201          * The initialize() phase of the analysis will be used to check the
 202          * script's metadata. Actual runs of the script will use the execute()
 203          * method below.
 204          */
 205         List<String> command = ImmutableList.<@NonNull String> builder()
 206                 .addAll(fScriptCommand).add(METADATA_FLAG).build();
 207
 208         LOGGER.info(() -> "[LamiAnalysis:RunningMetadataCommand] command=" + command.toString()); //$NON-NLS-1$
 209
 210         String output = getOutputFromCommand(command);
 211         if (output == null || output.isEmpty()) {
 212             return false;
 213         }
 214
 215         /*
 216          *
 217          * Metadata should look this this:
 218          *
 219          * {
 220          *   "version": [1, 5, 2, "dev"],
 221          *   "title": "I/O latency statistics",
 222          *   "authors": [
 223          *     "Julien Desfossez",
 224          *     "Antoine Busque"
 225          *   ],
 226          *   "description": "Provides statistics about the latency involved in various I/O operations.",
 227          *   "url": "https://github.com/lttng/lttng-analyses",
 228          *   "tags": [
 229          *     "io",
 230          *     "stats",
 231          *     "linux-kernel",
 232          *     "lttng-analyses"
 233          *   ],
 234          *   "table-classes": {
 235          *     "syscall-latency": {
 236          *       "title": "System calls latency statistics",
 237          *       "column-descriptions": [
 238          *         {"title": "System call", "type": "syscall"},
 239          *         {"title": "Count", "type": "int", "unit": "operations"},
 240          *         {"title": "Minimum duration", "type": "duration"},
 241          *         {"title": "Average duration", "type": "duration"},
 242          *         {"title": "Maximum duration", "type": "duration"},
 243          *         {"title": "Standard deviation", "type": "duration"}
 244          *       ]
 245          *     },
 246          *     "disk-latency": {
 247          *       "title": "Disk latency statistics",
 248          *       "column-descriptions": [
 249          *         {"title": "Disk name", "type": "disk"},
 250          *         {"title": "Count", "type": "int", "unit": "operations"},
 251          *         {"title": "Minimum duration", "type": "duration"},
 252          *         {"title": "Average duration", "type": "duration"},
 253          *         {"title": "Maximum duration", "type": "duration"},
 254          *         {"title": "Standard deviation", "type": "duration"}
 255          *       ]
 256          *     }
 257          *   }
 258          * }
 259          *
 260          */
 261
 262         try {
 263             JSONObject obj = new JSONObject(output);
 264             fAnalysisTitle = obj.getString(LamiStrings.TITLE);
 265
 266             /* Very early scripts may not contain the "mi-version" */
 267             JSONObject miVersion = obj.optJSONObject(LamiStrings.MI_VERSION);
 268             if (miVersion == null) {
 269                 /* Before version 0.1 */
 270                 fUseProgressOutput = false;
 271             } else {
 272                 int majorVersion = miVersion.getInt(LamiStrings.MAJOR);
 273                 if (majorVersion <= MAX_SUPPORTED_MAJOR_VERSION) {
 274                     fUseProgressOutput = true;
 275                 } else {
 276                     /* Unknown version, we do not support it */
 277                     return false;
 278                 }
 279             }
 280
 281             JSONObject tableClasses = obj.getJSONObject(LamiStrings.TABLE_CLASSES);
 282             @NonNull String[] tableClassNames = checkNotNullContents(JSONObject.getNames(tableClasses));
 283
 284             ImmutableMap.Builder<String, LamiTableClass> tablesBuilder = ImmutableMap.builder();
 285             for (String tableClassName : tableClassNames) {
 286                 JSONObject tableClass = tableClasses.getJSONObject(tableClassName);
 287
 288                 final String tableTitle = checkNotNull(tableClass.getString(LamiStrings.TITLE));
 289                 @NonNull JSONArray columnDescriptions = checkNotNull(tableClass.getJSONArray(LamiStrings.COLUMN_DESCRIPTIONS));
 290
 291                 List<LamiTableEntryAspect> aspects = getAspectsFromColumnDescriptions(columnDescriptions);
 292                 Collection<LamiChartModel> chartModels = getPredefinedCharts().get(tableClassName);
 293
 294                 tablesBuilder.put(tableClassName, new LamiTableClass(tableClassName, tableTitle, aspects, chartModels));
 295             }
 296
 297             try {
 298                 fTableClasses = tablesBuilder.build();
 299             } catch (IllegalArgumentException e) {
 300                 /*
 301                  * This is thrown if there are duplicate keys in the map
 302                  * builder.
 303                  */
 304                 throw new JSONException("Duplicate table class entry in " + fAnalysisTitle); //$NON-NLS-1$
 305             }
 306
 307         } catch (JSONException e) {
 308             /* Error in the parsing of the JSON, script is broken? */
 309             LOGGER.severe(() -> "[LamiAnalysis:ErrorParsingMetadata] msg=" + e.getMessage()); //$NON-NLS-1$
 310             Activator.instance().logError(e.getMessage());
 311             return false;
 312         }
 313         return true;
 314     }
 315
 316     private static List<LamiTableEntryAspect> getAspectsFromColumnDescriptions(JSONArray columnDescriptions) throws JSONException {
 317         ImmutableList.Builder<LamiTableEntryAspect> aspectsBuilder = new ImmutableList.Builder<>();
 318         for (int j = 0; j < columnDescriptions.length(); j++) {
 319             JSONObject column = columnDescriptions.getJSONObject(j);
 320             DataType columnDataType;
 321             String columnClass = column.optString(LamiStrings.CLASS, null);
 322
 323             if (columnClass == null) {
 324                 columnDataType = DataType.MIXED;
 325             } else {
 326                 columnDataType = getDataTypeFromString(columnClass);
 327             }
 328
 329             String columnTitle = column.optString(LamiStrings.TITLE, null);
 330
 331             if (columnTitle == null) {
 332                 columnTitle = String.format("%s #%d", columnDataType.getTitle(), j + 1); //$NON-NLS-1$
 333             }
 334
 335             final int colIndex = j;
 336             switch (columnDataType) {
 337             case TIME_RANGE:
 338                 /*
 339                  * We will add 3 aspects, to represent the start, end and
 340                  * duration of this time range.
 341                  */
 342                 aspectsBuilder.add(new LamiTimeRangeBeginAspect(columnTitle, colIndex));
 343                 aspectsBuilder.add(new LamiTimeRangeEndAspect(columnTitle, colIndex));
 344                 aspectsBuilder.add(new LamiTimeRangeDurationAspect(columnTitle, colIndex));
 345                 break;
 346
 347             case TIMESTAMP:
 348                 aspectsBuilder.add(new LamiTimestampAspect(columnTitle, colIndex));
 349                 break;
 350
 351             case PROCESS:
 352                 aspectsBuilder.add(new LamiProcessNameAspect(columnTitle, colIndex));
 353                 aspectsBuilder.add(new LamiProcessPIDAspect(columnTitle, colIndex));
 354                 aspectsBuilder.add(new LamiProcessTIDAspect(columnTitle, colIndex));
 355                 break;
 356
 357             case IRQ:
 358                 aspectsBuilder.add(new LamiIRQTypeAspect(columnTitle, colIndex));
 359                 aspectsBuilder.add(new LamiIRQNameAspect(columnTitle, colIndex));
 360                 aspectsBuilder.add(new LamiIRQNumberAspect(columnTitle, colIndex));
 361                 break;
 362
 363             case DURATION:
 364                 aspectsBuilder.add(new LamiDurationAspect(columnTitle, colIndex));
 365                 break;
 366
 367             case MIXED:
 368                 aspectsBuilder.add(new LamiMixedAspect(columnTitle, colIndex));
 369                 break;
 370
 371             // $CASES-OMITTED$
 372             default:
 373                 String units = column.optString(LamiStrings.UNIT, null);
 374
 375                 if (units == null) {
 376                     units = columnDataType.getUnits();
 377                 }
 378
 379                 /* We will add only one aspect representing the element */
 380                 LamiTableEntryAspect aspect = new LamiGenericAspect(columnTitle,
 381                         units, colIndex, columnDataType.isContinuous(), false);
 382                 aspectsBuilder.add(aspect);
 383                 break;
 384             }
 385         }
 386         /*
 387          * SWT quirk : we need an empty column at the end or else the last data
 388          * column will clamp to the right edge of the view if it is
 389          * right-aligned.
 390          */
 391         aspectsBuilder.add(LamiEmptyAspect.INSTANCE);
 392
 393         return aspectsBuilder.build();
 394     }
 395
 396     private static DataType getDataTypeFromString(String value) throws JSONException {
 397         try {
 398             return DataType.fromString(value);
 399         } catch (IllegalArgumentException e) {
 400             throw new JSONException("Unrecognized data type: " + value); //$NON-NLS-1$
 401         }
 402     }
 403
 404     /**
 405      * Get the title of this analysis, as read from the script's metadata.
 406      *
 407      * @return The analysis title. Should not be null after the initialization
 408      *         completed successfully.
 409      */
 410     public @Nullable String getAnalysisTitle() {
 411         return fAnalysisTitle;
 412     }
 413
 414     /**
 415      * Get the result table classes defined by this analysis, as read from the
 416      * script's metadata.
 417      *
 418      * @return The analysis' result table classes. Should not be null after the
 419      *         execution completed successfully.
 420      */
 421     public @Nullable Map<String, LamiTableClass> getTableClasses() {
 422         return fTableClasses;
 423     }
 424
 425     /**
 426      * Print the full command that will be run when calling {@link #execute},
 427      * with the exception of the 'extraParams' that will be passed to execute().
 428      *
 429      * This can be used to display the command in the UI before it is actually
 430      * run.
 431      *
 432      * @param trace
 433      *            The trace on which to run the analysis
 434      * @param range
 435      *            The time range to specify. Null will not specify a time range,
 436      *            which means the whole trace will be taken.
 437      * @return The command as a single, space-separated string
 438      */
 439     public String getFullCommandAsString(ITmfTrace trace, @Nullable TmfTimeRange range) {
 440         String tracePath = checkNotNull(trace.getPath());
 441
 442         ImmutableList.Builder<String> builder = getBaseCommand(range);
 443         /*
 444          * We can add double-quotes around the trace path, which could contain
 445          * spaces, so that the resulting command can be easily copy-pasted into
 446          * a shell.
 447          */
 448         builder.add(DOUBLE_QUOTES + tracePath + DOUBLE_QUOTES);
 449         List<String> list = builder.build();
 450         String ret = list.stream().collect(Collectors.joining(" ")); //$NON-NLS-1$
 451         return checkNotNull(ret);
 452     }
 453
 454     /**
 455      * Get the base part of the command that will be executed to run this
 456      * analysis, supplying the given time range. Base part meaning:
 457      *
 458      * <pre>
 459      * [script executable] [statically-defined parameters] [--begin/--end (if applicable)]
 460      * </pre>
 461      *
 462      * Note that it does not include the path to the trace, that is to be added
 463      * separately.
 464      *
 465      * @param range
 466      *            The time range that will be passed
 467      * @return The elements of the command
 468      */
 469     private ImmutableList.Builder<String> getBaseCommand(@Nullable TmfTimeRange range) {
 470         long begin = 0;
 471         long end = 0;
 472         if (range != null) {
 473             begin = range.getStartTime().getValue();
 474             end = range.getEndTime().getValue();
 475         }
 476
 477         ImmutableList.Builder<String> builder = ImmutableList.builder();
 478         builder.addAll(fScriptCommand);
 479
 480         if (fUseProgressOutput) {
 481             builder.add(PROGRESS_FLAG);
 482         }
 483
 484         if (range != null) {
 485             builder.add(BEGIN_FLAG).add(String.valueOf(begin));
 486             builder.add(END_FLAG).add(String.valueOf(end));
 487         }
 488         return builder;
 489     }
 490
 491     /**
 492      * Call the currently defined LAMI script with the given arguments.
 493      *
 494      * @param timeRange
 495      *            The time range. Null for the whole trace.
 496      * @param monitor
 497      *            The progress monitor used to report progress
 498      * @return The script's output, formatted into {@link LamiTableEntry}'s.
 499      * @throws CoreException
 500      *             If execution did not terminate normally
 501      */
 502     @Override
 503     public List<LamiResultTable> execute(ITmfTrace trace, @Nullable TmfTimeRange timeRange,
 504             String extraParams, IProgressMonitor monitor) throws CoreException {
 505         /* Should have been called already, but in case it was not */
 506         initialize();
 507
 508         final @NonNull String tracePath = checkNotNull(trace.getPath());
 509         final @NonNull String[] splitParams = extraParams.trim().split(" "); //$NON-NLS-1$
 510
 511         ImmutableList.Builder<String> builder = getBaseCommand(timeRange);
 512
 513         if (!extraParams.trim().equals("")) { //$NON-NLS-1$
 514             builder.addAll(Arrays.asList(splitParams));
 515         }
 516         builder.add(tracePath);
 517         List<String> command = builder.build();
 518
 519         LOGGER.info(() -> "[LamiAnalysis:RunningExecuteCommand] command=" + command.toString()); //$NON-NLS-1$
 520         String output = getResultsFromCommand(command, monitor);
 521
 522         if (output.isEmpty()) {
 523             IStatus status = new Status(IStatus.INFO, Activator.instance().getPluginId(), Messages.LamiAnalysis_NoResults);
 524             throw new CoreException(status);
 525         }
 526
 527         /*
 528          * {
 529          *   "results": [
 530          *     {
 531          *       "time-range": {
 532          *         "type": "time-range",
 533          *         "begin": 1444334398154194201,
 534          *         "end": 1444334425194487548
 535          *       },
 536          *       "class": "syscall-latency",
 537          *       "data": [
 538          *         [
 539          *           {"type": "syscall", "name": "open"},
 540          *           45,
 541          *           {"type": "duration", "value": 5562},
 542          *           {"type": "duration", "value": 13835},
 543          *           {"type": "duration", "value": 77683},
 544          *           {"type": "duration", "value": 15263}
 545          *         ],
 546          *         [
 547          *           {"type": "syscall", "name": "read"},
 548          *           109,
 549          *           {"type": "duration", "value": 316},
 550          *           {"type": "duration", "value": 5774},
 551          *           {"type": "duration", "value": 62569},
 552          *           {"type": "duration", "value": 9277}
 553          *         ]
 554          *       ]
 555          *     },
 556          *     {
 557          *       "time-range": {
 558          *         "type": "time-range",
 559          *         "begin": 1444334425194487549,
 560          *         "end": 1444334425254887190
 561          *       },
 562          *       "class": "syscall-latency",
 563          *       "data": [
 564          *         [
 565          *           {"type": "syscall", "name": "open"},
 566          *           45,
 567          *           {"type": "duration", "value": 1578},
 568          *           {"type": "duration", "value": 16648},
 569          *           {"type": "duration", "value": 15444},
 570          *           {"type": "duration", "value": 68540}
 571          *         ],
 572          *         [
 573          *           {"type": "syscall", "name": "read"},
 574          *           109,
 575          *           {"type": "duration", "value": 78},
 576          *           {"type": "duration", "value": 1948},
 577          *           {"type": "duration", "value": 11184},
 578          *           {"type": "duration", "value": 94670}
 579          *         ]
 580          *       ]
 581          *     }
 582          *   ]
 583          * }
 584          *
 585          */
 586
 587         ImmutableList.Builder<LamiResultTable> resultsBuilder = new ImmutableList.Builder<>();
 588
 589         try {
 590             JSONObject obj = new JSONObject(output);
 591             JSONArray results = obj.getJSONArray(LamiStrings.RESULTS);
 592
 593             if (results.length() == 0) {
 594                 /*
 595                  * No results were reported. This may be normal, but warn the
 596                  * user why a report won't be created.
 597                  */
 598                 IStatus status = new Status(IStatus.INFO, Activator.instance().getPluginId(), Messages.LamiAnalysis_NoResults);
 599                 throw new CoreException(status);
 600             }
 601
 602             for (int i = 0; i < results.length(); i++) {
 603                 JSONObject result = results.getJSONObject(i);
 604
 605                 /* Parse the time-range */
 606                 JSONObject trObject = result.getJSONObject(LamiStrings.TIME_RANGE);
 607                 long start = trObject.getLong(LamiStrings.BEGIN);
 608                 long end = trObject.getLong(LamiStrings.END);
 609                 LamiTimeRange tr = new LamiTimeRange(start, end);
 610
 611                 /* Parse the table's class */
 612                 LamiTableClass tableClass;
 613                 JSONObject tableClassObject = result.optJSONObject(LamiStrings.CLASS);
 614                 if (tableClassObject == null) {
 615                     /*
 616                      * "class" is just a standard string, indicating we use a
 617                      * metadata-defined table class as-is
 618                      */
 619                     @NonNull String tableClassName = checkNotNull(result.getString(LamiStrings.CLASS));
 620                     tableClass = getTableClassFromName(tableClassName);
 621
 622                     // FIXME Rest will become more generic eventually in the LAMI format.
 623                 } else if (tableClassObject.has(LamiStrings.INHERIT)) {
 624                     /*
 625                      * Dynamic title: We reuse an existing table class but
 626                      * override the title.
 627                      */
 628                     String baseTableName = checkNotNull(tableClassObject.getString(LamiStrings.INHERIT));
 629                     LamiTableClass baseTableClass = getTableClassFromName(baseTableName);
 630                     String newTitle = checkNotNull(tableClassObject.getString(LamiStrings.TITLE));
 631
 632                     tableClass = new LamiTableClass(baseTableClass, newTitle);
 633                 } else {
 634                     /*
 635                      * Dynamic column descriptions: we implement a new table
 636                      * class entirely.
 637                      */
 638                     String title = checkNotNull(tableClassObject.getString(LamiStrings.TITLE));
 639                     JSONArray columnDescriptions = checkNotNull(tableClassObject.getJSONArray(LamiStrings.COLUMN_DESCRIPTIONS));
 640                     List<LamiTableEntryAspect> aspects = getAspectsFromColumnDescriptions(columnDescriptions);
 641
 642                     tableClass = new LamiTableClass(nullToEmptyString(Messages.LamiAnalysis_DefaultDynamicTableName), title, aspects, Collections.EMPTY_SET);
 643                 }
 644
 645                 /* Parse the "data", which is the array of rows */
 646                 JSONArray data = result.getJSONArray(LamiStrings.DATA);
 647                 ImmutableList.Builder<LamiTableEntry> dataBuilder = new ImmutableList.Builder<>();
 648
 649                 for (int j = 0; j < data.length(); j++) {
 650                     /* A row is an array of cells */
 651                     JSONArray row = data.getJSONArray(j);
 652                     ImmutableList.Builder<LamiData> rowBuilder = ImmutableList.builder();
 653
 654                     for (int k = 0; k < row.length(); k++) {
 655                         Object cellObject = checkNotNull(row.get(k));
 656                         LamiData cellValue = LamiData.createFromObject(cellObject);
 657                         rowBuilder.add(cellValue);
 658
 659                     }
 660                     dataBuilder.add(new LamiTableEntry(rowBuilder.build()));
 661                 }
 662                 resultsBuilder.add(new LamiResultTable(tr, tableClass, dataBuilder.build()));
 663             }
 664
 665         } catch (JSONException e) {
 666             LOGGER.severe(() -> "[LamiAnalysis:ErrorParsingExecutionOutput] msg=" + e.getMessage()); //$NON-NLS-1$
 667             IStatus status = new Status(IStatus.ERROR, Activator.instance().getPluginId(), e.getMessage(), e);
 668             throw new CoreException(status);
 669         }
 670
 671         return resultsBuilder.build();
 672     }
 673
 674     private LamiTableClass getTableClassFromName(String tableClassName) throws JSONException {
 675         Map<String, LamiTableClass> map = checkNotNull(fTableClasses);
 676         LamiTableClass tableClass = map.get(tableClassName);
 677         if (tableClass == null) {
 678             throw new JSONException("Table class " + tableClassName + //$NON-NLS-1$
 679                     " was not declared in the metadata"); //$NON-NLS-1$
 680         }
 681         return tableClass;
 682     }
 683
 684     /**
 685      * Get the output of an external command, used for getting the metadata.
 686      * Cannot be cancelled, and will not report errors, simply returns null if
 687      * the process ended abnormally.
 688      *
 689      * @param command
 690      *            The parameters of the command, passed to
 691      *            {@link ProcessBuilder}
 692      * @return The command output as a string
 693      */
 694     @VisibleForTesting
 695     protected @Nullable String getOutputFromCommand(List<String> command) {
 696         try {
 697             ProcessBuilder builder = new ProcessBuilder(command);
 698             builder.redirectErrorStream(true);
 699
 700             Process p = builder.start();
 701             try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));) {
 702                 int ret = p.waitFor();
 703                 String output = br.lines().collect(Collectors.joining());
 704
 705                 return (ret == 0 ? output : null);
 706             }
 707         } catch (IOException | InterruptedException e) {
 708             return null;
 709         }
 710     }
 711
 712     /**
 713      * Get the results of invoking the specified command.
 714      *
 715      * The result should start with '{"results":...', as specified by the LAMI
 716      * JSON protocol. The JSON itself may be split over multiple lines.
 717      *
 718      * @param command
 719      *            The command to run (program and its arguments)
 720      * @param monitor
 721      *            The progress monitor
 722      * @return The analysis results
 723      * @throws CoreException
 724      *             If the command ended abnormally, and normal results were not
 725      *             returned
 726      */
 727     @VisibleForTesting
 728     protected String getResultsFromCommand(List<String> command, IProgressMonitor monitor)
 729             throws CoreException {
 730
 731         final int scale = 1000;
 732         double workedSoFar = 0.0;
 733
 734         ProcessCanceller cancellerRunnable = null;
 735         Thread cancellerThread = null;
 736
 737         try {
 738             monitor.beginTask(Messages.LamiAnalysis_MainTaskName, scale);
 739
 740             ProcessBuilder builder = new ProcessBuilder(command);
 741             builder.redirectErrorStream(false);
 742
 743             Process p = checkNotNull(builder.start());
 744
 745             cancellerRunnable = new ProcessCanceller(p, monitor);
 746             cancellerThread = new Thread(cancellerRunnable);
 747             cancellerThread.start();
 748
 749             List<String> results = new ArrayList<>();
 750
 751             try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));) {
 752                 String line = in.readLine();
 753                 while (line != null && !line.matches("\\s*\\{.*")) { //$NON-NLS-1$
 754                     /*
 755                      * This is a line indicating progress, it has the form:
 756                      *
 757                      * 0.123 3000 of 5000 events processed
 758                      *
 759                      * The first part indicates the estimated fraction (out of
 760                      * 1.0) of work done. The second part is status text.
 761                      */
 762
 763                     // Trim the line first to make sure the first character is
 764                     // significant
 765                     line = line.trim();
 766
 767                     // Split at the first space
 768                     String[] elems = line.split(" ", 2); //$NON-NLS-1$
 769
 770                     if (elems[0].matches("\\d.*")) { //$NON-NLS-1$
 771                         // It looks like we have a progress indication
 772                         try {
 773                             // Try parsing the number
 774                             double cumulativeWork = Double.parseDouble(elems[0]) * scale;
 775                             double workedThisLoop = cumulativeWork - workedSoFar;
 776
 777                             // We're going backwards? Do not update the
 778                             // monitor's value
 779                             if (workedThisLoop > 0) {
 780                                 monitor.internalWorked(workedThisLoop);
 781                                 workedSoFar = cumulativeWork;
 782                             }
 783
 784                             // There is a message: update the monitor's task name
 785                             if (elems.length >= 2) {
 786                                 monitor.setTaskName(elems[1].trim());
 787                             }
 788                         } catch (NumberFormatException e) {
 789                             // Continue reading progress lines anyway
 790                         }
 791                     }
 792
 793                     line = in.readLine();
 794                 }
 795                 while (line != null) {
 796                     /*
 797                      * We have seen the first line containing a '{', this is our
 798                      * JSON output!
 799                      */
 800                     results.add(line);
 801                     line = in.readLine();
 802                 }
 803             }
 804             int ret = p.waitFor();
 805
 806             if (monitor.isCanceled()) {
 807                 /* We were interrupted by the canceller thread. */
 808                 IStatus status = new Status(IStatus.CANCEL, Activator.instance().getPluginId(), null);
 809                 throw new CoreException(status);
 810             }
 811
 812             if (ret != 0) {
 813                 /*
 814                  * Something went wrong running the external script. We will
 815                  * gather the stderr and report it to the user.
 816                  */
 817                 BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()));
 818                 List<String> stdErrOutput = br.lines().collect(Collectors.toList());
 819
 820                 MultiStatus status = new MultiStatus(Activator.instance().getPluginId(),
 821                         IStatus.ERROR, Messages.LamiAnalysis_ErrorDuringExecution, null);
 822                 for (String str : stdErrOutput) {
 823                     status.add(new Status(IStatus.ERROR, Activator.instance().getPluginId(), str));
 824                 }
 825                 if (stdErrOutput.isEmpty()) {
 826                     /*
 827                      * At least say "no output", so an error message actually
 828                      * shows up.
 829                      */
 830                     status.add(new Status(IStatus.ERROR, Activator.instance().getPluginId(), Messages.LamiAnalysis_ErrorNoOutput));
 831                 }
 832                 throw new CoreException(status);
 833             }
 834
 835             /* External script ended successfully, all is fine! */
 836             String resultsStr = results.stream().collect(Collectors.joining());
 837             return checkNotNull(resultsStr);
 838
 839         } catch (IOException | InterruptedException e) {
 840             IStatus status = new Status(IStatus.ERROR, Activator.instance().getPluginId(), Messages.LamiAnalysis_ExecutionInterrupted, e);
 841             throw new CoreException(status);
 842
 843         } finally {
 844             if (cancellerRunnable != null) {
 845                 cancellerRunnable.setFinished();
 846             }
 847             if (cancellerThread != null) {
 848                 try {
 849                     cancellerThread.join();
 850                 } catch (InterruptedException e) {
 851                 }
 852             }
 853
 854             monitor.done();
 855         }
 856     }
 857
 858     private static class ProcessCanceller implements Runnable {
 859
 860         private final Process fProcess;
 861         private final IProgressMonitor fMonitor;
 862
 863         private boolean fIsFinished = false;
 864
 865         public ProcessCanceller(Process process, IProgressMonitor monitor) {
 866             fProcess = process;
 867             fMonitor = monitor;
 868         }
 869
 870         public void setFinished() {
 871             fIsFinished = true;
 872         }
 873
 874         @Override
 875         public void run() {
 876             try {
 877                 while (!fIsFinished) {
 878                     Thread.sleep(500);
 879                     if (fMonitor.isCanceled()) {
 880                         fProcess.destroy();
 881                         return;
 882                     }
 883                 }
 884             } catch (InterruptedException e) {
 885             }
 886         }
 887
 888     }
 889
 890     @Override
 891     public @NonNull String getName() {
 892         return fName;
 893     }
 894
 895     @Override
 896     public boolean isUserDefined() {
 897         return fIsUserDefined;
 898     }
 899
 900 }