analysis/org.eclipse.tracecompass.analysis.lami.core/src/org/eclipse/tracecompass/internal/provisional/analysis/lami/core/module/LamiAnalysis.java

   1 /*******************************************************************************
   2  * Copyright (c) 2015, 2016 EfficiOS Inc., Alexandre Montplaisir
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *******************************************************************************/
   9
  10 package org.eclipse.tracecompass.internal.provisional.analysis.lami.core.module;
  11
  12 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNull;
  13 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNullContents;
  14 import static org.eclipse.tracecompass.common.core.NonNullUtils.nullToEmptyString;
  15
  16 import java.io.BufferedReader;
  17 import java.io.File;
  18 import java.io.IOException;
  19 import java.io.InputStreamReader;
  20 import java.nio.file.Files;
  21 import java.nio.file.Paths;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.function.Predicate;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.regex.Pattern;
  30 import java.util.stream.Collectors;
  31 import java.util.stream.Stream;
  32
  33 import org.eclipse.core.runtime.IProgressMonitor;
  34 import org.eclipse.jdt.annotation.NonNull;
  35 import org.eclipse.jdt.annotation.Nullable;
  36 import org.eclipse.tracecompass.internal.analysis.lami.core.Activator;
  37 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.LamiStrings;
  38 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiGenericAspect;
  39 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQNameAspect;
  40 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQNumberAspect;
  41 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQTypeAspect;
  42 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiMixedAspect;
  43 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessNameAspect;
  44 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessPIDAspect;
  45 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessTIDAspect;
  46 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiDurationAspect;
  47 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiEmptyAspect;
  48 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTableEntryAspect;
  49 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeBeginAspect;
  50 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeDurationAspect;
  51 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeEndAspect;
  52 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimestampAspect;
  53 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiData;
  54 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiData.DataType;
  55 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiTimeRange;
  56 import org.eclipse.tracecompass.tmf.core.analysis.ondemand.IOnDemandAnalysis;
  57 import org.eclipse.tracecompass.tmf.core.analysis.ondemand.OnDemandAnalysisException;
  58 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
  59 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
  60 import org.json.JSONArray;
  61 import org.json.JSONException;
  62 import org.json.JSONObject;
  63
  64 import com.google.common.collect.ImmutableList;
  65 import com.google.common.collect.ImmutableMap;
  66 import com.google.common.collect.ImmutableMultimap;
  67 import com.google.common.collect.Multimap;
  68
  69 /**
  70  * Base class for analysis modules that call external scripts implementing the
  71  * LAMI protocol.
  72  *
  73  * @author Alexandre Montplaisir
  74  */
  75 public class LamiAnalysis implements IOnDemandAnalysis {
  76
  77     /** Maximum major version of the LAMI protocol we support */
  78     private static final int MAX_SUPPORTED_MAJOR_VERSION = 1;
  79
  80     private static final String DOUBLE_QUOTES = "\""; //$NON-NLS-1$
  81
  82     /* Flags passed to the analysis scripts */
  83     private static final String METADATA_FLAG = "--metadata"; //$NON-NLS-1$
  84     private static final String PROGRESS_FLAG = "--output-progress"; //$NON-NLS-1$
  85     private static final String BEGIN_FLAG = "--begin"; //$NON-NLS-1$
  86     private static final String END_FLAG = "--end"; //$NON-NLS-1$
  87
  88     /** Log message for commands being run */
  89     private static final String RUNNING_MESSAGE = "Running command:"; //$NON-NLS-1$
  90
  91     private final List<String> fScriptCommand;
  92
  93     /**
  94      * The LAMI analysis is considered initialized after we have read the
  95      * script's --metadata once. This will assign the fields below.
  96      */
  97     private boolean fInitialized = false;
  98
  99     private boolean fIsAvailable;
 100     private final String fName;
 101     private final boolean fIsUserDefined;
 102     private final Predicate<ITmfTrace> fAppliesTo;
 103
 104     /* Data defined by the analysis's metadata */
 105     private @Nullable String fAnalysisTitle;
 106     private @Nullable Map<String, LamiTableClass> fTableClasses;
 107     private boolean fUseProgressOutput;
 108
 109     /**
 110      * Constructor. To be called by implementing classes.
 111      *
 112      * @param name
 113      *            Name of this analysis
 114      * @param isUserDefined
 115      *            {@code true} if this is a user-defined analysis
 116      * @param appliesTo
 117      *            Predicate to use to check whether or not this analysis applies
 118      *            to a given trace
 119      * @param args
 120      *            Analysis arguments, including the executable name (first
 121      *            argument)
 122      */
 123     public LamiAnalysis(String name, boolean isUserDefined, Predicate<ITmfTrace> appliesTo,
 124             List<String> args) {
 125         fScriptCommand = ImmutableList.copyOf(args);
 126         fName = name;
 127         fIsUserDefined = isUserDefined;
 128         fAppliesTo = appliesTo;
 129     }
 130
 131     /**
 132      * Map of pre-defined charts, for every table class names.
 133      *
 134      * If a table class is not in this map then it means that table has no
 135      * predefined charts.
 136      *
 137      * @return The chart models, per table class names
 138      */
 139     protected Multimap<String, LamiChartModel> getPredefinedCharts() {
 140         return ImmutableMultimap.of();
 141     }
 142
 143     @Override
 144     public boolean appliesTo(ITmfTrace trace) {
 145         return fAppliesTo.test(trace);
 146     }
 147
 148     @Override
 149     public boolean canExecute(ITmfTrace trace) {
 150         initialize();
 151         return fIsAvailable;
 152     }
 153
 154     private synchronized void initialize() {
 155         if (fInitialized) {
 156             return;
 157         }
 158
 159         /* Do the analysis's initialization */
 160
 161         /* Check if the script's expected executable is on the PATH */
 162         String executable = fScriptCommand.get(0);
 163         boolean exists = Stream.of(System.getenv("PATH").split(checkNotNull(Pattern.quote(File.pathSeparator)))) //$NON-NLS-1$
 164                 .map(Paths::get)
 165                 .anyMatch(path -> Files.exists(path.resolve(executable)));
 166         if (!exists) {
 167             /* Script is not found */
 168             fIsAvailable = false;
 169             fInitialized = true;
 170             return;
 171         }
 172
 173         fIsAvailable = checkMetadata();
 174         fInitialized = true;
 175     }
 176
 177     private boolean checkMetadata() {
 178         /*
 179          * The initialize() phase of the analysis will be used to check the
 180          * script's metadata. Actual runs of the script will use the execute()
 181          * method below.
 182          */
 183         List<String> command = ImmutableList.<@NonNull String> builder()
 184                 .addAll(fScriptCommand).add(METADATA_FLAG).build();
 185
 186         Activator.instance().logInfo(RUNNING_MESSAGE + ' ' + command.toString());
 187
 188         String output = getOutputFromCommand(command);
 189         if (output == null || output.isEmpty()) {
 190             return false;
 191         }
 192
 193         /*
 194          *
 195          * Metadata should look this this:
 196          *
 197          * {
 198          *   "version": [1, 5, 2, "dev"],
 199          *   "title": "I/O latency statistics",
 200          *   "authors": [
 201          *     "Julien Desfossez",
 202          *     "Antoine Busque"
 203          *   ],
 204          *   "description": "Provides statistics about the latency involved in various I/O operations.",
 205          *   "url": "https://github.com/lttng/lttng-analyses",
 206          *   "tags": [
 207          *     "io",
 208          *     "stats",
 209          *     "linux-kernel",
 210          *     "lttng-analyses"
 211          *   ],
 212          *   "table-classes": {
 213          *     "syscall-latency": {
 214          *       "title": "System calls latency statistics",
 215          *       "column-descriptions": [
 216          *         {"title": "System call", "type": "syscall"},
 217          *         {"title": "Count", "type": "int", "unit": "operations"},
 218          *         {"title": "Minimum duration", "type": "duration"},
 219          *         {"title": "Average duration", "type": "duration"},
 220          *         {"title": "Maximum duration", "type": "duration"},
 221          *         {"title": "Standard deviation", "type": "duration"}
 222          *       ]
 223          *     },
 224          *     "disk-latency": {
 225          *       "title": "Disk latency statistics",
 226          *       "column-descriptions": [
 227          *         {"title": "Disk name", "type": "disk"},
 228          *         {"title": "Count", "type": "int", "unit": "operations"},
 229          *         {"title": "Minimum duration", "type": "duration"},
 230          *         {"title": "Average duration", "type": "duration"},
 231          *         {"title": "Maximum duration", "type": "duration"},
 232          *         {"title": "Standard deviation", "type": "duration"}
 233          *       ]
 234          *     }
 235          *   }
 236          * }
 237          *
 238          */
 239
 240         try {
 241             JSONObject obj = new JSONObject(output);
 242             fAnalysisTitle = obj.getString(LamiStrings.TITLE);
 243
 244             /* Very early scripts may not contain the "mi-version" */
 245             JSONObject miVersion = obj.optJSONObject(LamiStrings.MI_VERSION);
 246             if (miVersion == null) {
 247                 /* Before version 0.1 */
 248                 fUseProgressOutput = false;
 249             } else {
 250                 int majorVersion = miVersion.getInt(LamiStrings.MAJOR);
 251                 if (majorVersion <= MAX_SUPPORTED_MAJOR_VERSION) {
 252                     fUseProgressOutput = true;
 253                 } else {
 254                     /* Unknown version, we do not support it */
 255                     return false;
 256                 }
 257             }
 258
 259             JSONObject tableClasses = obj.getJSONObject(LamiStrings.TABLE_CLASSES);
 260             @NonNull String[] tableClassNames = checkNotNullContents(JSONObject.getNames(tableClasses));
 261
 262             ImmutableMap.Builder<String, LamiTableClass> tablesBuilder = ImmutableMap.builder();
 263             for (String tableClassName : tableClassNames) {
 264                 JSONObject tableClass = tableClasses.getJSONObject(tableClassName);
 265
 266                 final String tableTitle = checkNotNull(tableClass.getString(LamiStrings.TITLE));
 267                 @NonNull JSONArray columnDescriptions = checkNotNull(tableClass.getJSONArray(LamiStrings.COLUMN_DESCRIPTIONS));
 268
 269                 List<LamiTableEntryAspect> aspects = getAspectsFromColumnDescriptions(columnDescriptions);
 270                 Collection<LamiChartModel> chartModels = getPredefinedCharts().get(tableClassName);
 271
 272                 tablesBuilder.put(tableClassName, new LamiTableClass(tableClassName, tableTitle, aspects, chartModels));
 273             }
 274
 275             try {
 276                 fTableClasses = tablesBuilder.build();
 277             } catch (IllegalArgumentException e) {
 278                 /*
 279                  * This is thrown if there are duplicate keys in the map
 280                  * builder.
 281                  */
 282                 throw new JSONException("Duplicate table class entry in " + fAnalysisTitle); //$NON-NLS-1$
 283             }
 284
 285         } catch (JSONException e) {
 286             /* Error in the parsing of the JSON, script is broken? */
 287             Activator.instance().logError(e.getMessage());
 288             return false;
 289         }
 290         return true;
 291     }
 292
 293     private static List<LamiTableEntryAspect> getAspectsFromColumnDescriptions(JSONArray columnDescriptions) throws JSONException {
 294         ImmutableList.Builder<LamiTableEntryAspect> aspectsBuilder = new ImmutableList.Builder<>();
 295         for (int j = 0; j < columnDescriptions.length(); j++) {
 296             JSONObject column = columnDescriptions.getJSONObject(j);
 297             DataType columnDataType;
 298             String columnClass = column.optString(LamiStrings.CLASS, null);
 299
 300             if (columnClass == null) {
 301                 columnDataType = DataType.MIXED;
 302             } else {
 303                 columnDataType = getDataTypeFromString(columnClass);
 304             }
 305
 306             String columnTitle = column.optString(LamiStrings.TITLE, null);
 307
 308             if (columnTitle == null) {
 309                 columnTitle = String.format("%s #%d", columnDataType.getTitle(), j + 1); //$NON-NLS-1$
 310             }
 311
 312             final int colIndex = j;
 313             switch (columnDataType) {
 314             case TIME_RANGE:
 315                 /*
 316                  * We will add 3 aspects, to represent the start, end and
 317                  * duration of this time range.
 318                  */
 319                 aspectsBuilder.add(new LamiTimeRangeBeginAspect(columnTitle, colIndex));
 320                 aspectsBuilder.add(new LamiTimeRangeEndAspect(columnTitle, colIndex));
 321                 aspectsBuilder.add(new LamiTimeRangeDurationAspect(columnTitle, colIndex));
 322                 break;
 323
 324             case TIMESTAMP:
 325                 aspectsBuilder.add(new LamiTimestampAspect(columnTitle, colIndex));
 326                 break;
 327
 328             case PROCESS:
 329                 aspectsBuilder.add(new LamiProcessNameAspect(columnTitle, colIndex));
 330                 aspectsBuilder.add(new LamiProcessPIDAspect(columnTitle, colIndex));
 331                 aspectsBuilder.add(new LamiProcessTIDAspect(columnTitle, colIndex));
 332                 break;
 333
 334             case IRQ:
 335                 aspectsBuilder.add(new LamiIRQTypeAspect(columnTitle, colIndex));
 336                 aspectsBuilder.add(new LamiIRQNameAspect(columnTitle, colIndex));
 337                 aspectsBuilder.add(new LamiIRQNumberAspect(columnTitle, colIndex));
 338                 break;
 339
 340             case DURATION:
 341                 aspectsBuilder.add(new LamiDurationAspect(columnTitle, colIndex));
 342                 break;
 343
 344             case MIXED:
 345                 aspectsBuilder.add(new LamiMixedAspect(columnTitle, colIndex));
 346                 break;
 347
 348             // $CASES-OMITTED$
 349             default:
 350                 String units = column.optString(LamiStrings.UNIT, null);
 351
 352                 if (units == null) {
 353                     units = columnDataType.getUnits();
 354                 }
 355
 356                 /* We will add only one aspect representing the element */
 357                 LamiTableEntryAspect aspect = new LamiGenericAspect(columnTitle,
 358                         units, colIndex, columnDataType.isContinuous(), false);
 359                 aspectsBuilder.add(aspect);
 360                 break;
 361             }
 362         }
 363         /*
 364          * SWT quirk : we need an empty column at the end or else the last data
 365          * column will clamp to the right edge of the view if it is
 366          * right-aligned.
 367          */
 368         aspectsBuilder.add(LamiEmptyAspect.INSTANCE);
 369
 370         return aspectsBuilder.build();
 371     }
 372
 373     private static DataType getDataTypeFromString(String value) throws JSONException {
 374         try {
 375             return DataType.fromString(value);
 376         } catch (IllegalArgumentException e) {
 377             throw new JSONException("Unrecognized data type: " + value); //$NON-NLS-1$
 378         }
 379     }
 380
 381     /**
 382      * Get the title of this analysis, as read from the script's metadata.
 383      *
 384      * @return The analysis title. Should not be null after the initialization
 385      *         completed successfully.
 386      */
 387     public @Nullable String getAnalysisTitle() {
 388         return fAnalysisTitle;
 389     }
 390
 391     /**
 392      * Get the result table classes defined by this analysis, as read from the
 393      * script's metadata.
 394      *
 395      * @return The analysis' result table classes. Should not be null after the
 396      *         execution completed successfully.
 397      */
 398     public @Nullable Map<String, LamiTableClass> getTableClasses() {
 399         return fTableClasses;
 400     }
 401
 402     /**
 403      * Print the full command that will be run when calling {@link #execute},
 404      * with the exception of the 'extraParams' that will be passed to execute().
 405      *
 406      * This can be used to display the command in the UI before it is actually
 407      * run.
 408      *
 409      * @param trace
 410      *            The trace on which to run the analysis
 411      * @param range
 412      *            The time range to specify. Null will not specify a time range,
 413      *            which means the whole trace will be taken.
 414      * @return The command as a single, space-separated string
 415      */
 416     public String getFullCommandAsString(ITmfTrace trace, @Nullable TmfTimeRange range) {
 417         String tracePath = checkNotNull(trace.getPath());
 418
 419         ImmutableList.Builder<String> builder = getBaseCommand(range);
 420         /*
 421          * We can add double-quotes around the trace path, which could contain
 422          * spaces, so that the resulting command can be easily copy-pasted into
 423          * a shell.
 424          */
 425         builder.add(DOUBLE_QUOTES + tracePath + DOUBLE_QUOTES);
 426         List<String> list = builder.build();
 427         String ret = list.stream().collect(Collectors.joining(" ")); //$NON-NLS-1$
 428         return checkNotNull(ret);
 429     }
 430
 431     /**
 432      * Get the base part of the command that will be executed to run this
 433      * analysis, supplying the given time range. Base part meaning:
 434      *
 435      * <pre>
 436      * [script executable] [statically-defined parameters] [--begin/--end (if applicable)]
 437      * </pre>
 438      *
 439      * Note that it does not include the path to the trace, that is to be added
 440      * separately.
 441      *
 442      * @param range
 443      *            The time range that will be passed
 444      * @return The elements of the command
 445      */
 446     private ImmutableList.Builder<String> getBaseCommand(@Nullable TmfTimeRange range) {
 447         long begin = 0;
 448         long end = 0;
 449         if (range != null) {
 450             begin = range.getStartTime().getValue();
 451             end = range.getEndTime().getValue();
 452         }
 453
 454         ImmutableList.Builder<String> builder = ImmutableList.builder();
 455         builder.addAll(fScriptCommand);
 456
 457         if (fUseProgressOutput) {
 458             builder.add(PROGRESS_FLAG);
 459         }
 460
 461         if (range != null) {
 462             builder.add(BEGIN_FLAG).add(String.valueOf(begin));
 463             builder.add(END_FLAG).add(String.valueOf(end));
 464         }
 465         return builder;
 466     }
 467
 468     /**
 469      * Call the currently defined LAMI script with the given arguments.
 470      *
 471      * @param timeRange
 472      *            The time range. Null for the whole trace.
 473      * @param monitor
 474      *            The progress monitor used to report progress
 475      * @return The script's output, formatted into {@link LamiTableEntry}'s.
 476      * @throws OnDemandAnalysisException
 477      *             If execution did not terminate normally
 478      */
 479     @Override
 480     public List<LamiResultTable> execute(ITmfTrace trace, @Nullable TmfTimeRange timeRange,
 481             String extraParams, IProgressMonitor monitor) throws OnDemandAnalysisException {
 482         /* Should have been called already, but in case it was not */
 483         initialize();
 484
 485         final @NonNull String tracePath = checkNotNull(trace.getPath());
 486         final @NonNull String[] splitParams = extraParams.trim().split(" "); //$NON-NLS-1$
 487
 488         ImmutableList.Builder<String> builder = getBaseCommand(timeRange);
 489
 490         if (!extraParams.trim().equals("")) { //$NON-NLS-1$
 491             builder.addAll(Arrays.asList(splitParams));
 492         }
 493         builder.add(tracePath);
 494         List<String> command = builder.build();
 495
 496         Activator.instance().logInfo(RUNNING_MESSAGE + ' ' + command.toString());
 497         String output = getResultsFromCommand(command, monitor);
 498
 499         if (output.isEmpty()) {
 500             throw new OnDemandAnalysisException(Messages.LamiAnalysis_NoResults);
 501         }
 502
 503         /*
 504          * {
 505          *   "results": [
 506          *     {
 507          *       "time-range": {
 508          *         "type": "time-range",
 509          *         "begin": 1444334398154194201,
 510          *         "end": 1444334425194487548
 511          *       },
 512          *       "class": "syscall-latency",
 513          *       "data": [
 514          *         [
 515          *           {"type": "syscall", "name": "open"},
 516          *           45,
 517          *           {"type": "duration", "value": 5562},
 518          *           {"type": "duration", "value": 13835},
 519          *           {"type": "duration", "value": 77683},
 520          *           {"type": "duration", "value": 15263}
 521          *         ],
 522          *         [
 523          *           {"type": "syscall", "name": "read"},
 524          *           109,
 525          *           {"type": "duration", "value": 316},
 526          *           {"type": "duration", "value": 5774},
 527          *           {"type": "duration", "value": 62569},
 528          *           {"type": "duration", "value": 9277}
 529          *         ]
 530          *       ]
 531          *     },
 532          *     {
 533          *       "time-range": {
 534          *         "type": "time-range",
 535          *         "begin": 1444334425194487549,
 536          *         "end": 1444334425254887190
 537          *       },
 538          *       "class": "syscall-latency",
 539          *       "data": [
 540          *         [
 541          *           {"type": "syscall", "name": "open"},
 542          *           45,
 543          *           {"type": "duration", "value": 1578},
 544          *           {"type": "duration", "value": 16648},
 545          *           {"type": "duration", "value": 15444},
 546          *           {"type": "duration", "value": 68540}
 547          *         ],
 548          *         [
 549          *           {"type": "syscall", "name": "read"},
 550          *           109,
 551          *           {"type": "duration", "value": 78},
 552          *           {"type": "duration", "value": 1948},
 553          *           {"type": "duration", "value": 11184},
 554          *           {"type": "duration", "value": 94670}
 555          *         ]
 556          *       ]
 557          *     }
 558          *   ]
 559          * }
 560          *
 561          */
 562
 563         ImmutableList.Builder<LamiResultTable> resultsBuilder = new ImmutableList.Builder<>();
 564
 565         try {
 566             JSONObject obj = new JSONObject(output);
 567             JSONArray results = obj.getJSONArray(LamiStrings.RESULTS);
 568
 569             if (results.length() == 0) {
 570                 /*
 571                  * No results were reported. This may be normal, but warn the
 572                  * user why a report won't be created.
 573                  */
 574                 throw new OnDemandAnalysisException(Messages.LamiAnalysis_NoResults);
 575             }
 576
 577             for (int i = 0; i < results.length(); i++) {
 578                 JSONObject result = results.getJSONObject(i);
 579
 580                 /* Parse the time-range */
 581                 JSONObject trObject = result.getJSONObject(LamiStrings.TIME_RANGE);
 582                 long start = trObject.getLong(LamiStrings.BEGIN);
 583                 long end = trObject.getLong(LamiStrings.END);
 584                 LamiTimeRange tr = new LamiTimeRange(start, end);
 585
 586                 /* Parse the table's class */
 587                 LamiTableClass tableClass;
 588                 JSONObject tableClassObject = result.optJSONObject(LamiStrings.CLASS);
 589                 if (tableClassObject == null) {
 590                     /*
 591                      * "class" is just a standard string, indicating we use a
 592                      * metadata-defined table class as-is
 593                      */
 594                     @NonNull String tableClassName = checkNotNull(result.getString(LamiStrings.CLASS));
 595                     tableClass = getTableClassFromName(tableClassName);
 596
 597                     // FIXME Rest will become more generic eventually in the LAMI format.
 598                 } else if (tableClassObject.has(LamiStrings.INHERIT)) {
 599                     /*
 600                      * Dynamic title: We reuse an existing table class but
 601                      * override the title.
 602                      */
 603                     String baseTableName = checkNotNull(tableClassObject.getString(LamiStrings.INHERIT));
 604                     LamiTableClass baseTableClass = getTableClassFromName(baseTableName);
 605                     String newTitle = checkNotNull(tableClassObject.getString(LamiStrings.TITLE));
 606
 607                     tableClass = new LamiTableClass(baseTableClass, newTitle);
 608                 } else {
 609                     /*
 610                      * Dynamic column descriptions: we implement a new table
 611                      * class entirely.
 612                      */
 613                     String title = checkNotNull(tableClassObject.getString(LamiStrings.TITLE));
 614                     JSONArray columnDescriptions = checkNotNull(tableClassObject.getJSONArray(LamiStrings.COLUMN_DESCRIPTIONS));
 615                     List<LamiTableEntryAspect> aspects = getAspectsFromColumnDescriptions(columnDescriptions);
 616
 617                     tableClass = new LamiTableClass(nullToEmptyString(Messages.LamiAnalysis_DefaultDynamicTableName), title, aspects, Collections.EMPTY_SET);
 618                 }
 619
 620                 /* Parse the "data", which is the array of rows */
 621                 JSONArray data = result.getJSONArray(LamiStrings.DATA);
 622                 ImmutableList.Builder<LamiTableEntry> dataBuilder = new ImmutableList.Builder<>();
 623
 624                 for (int j = 0; j < data.length(); j++) {
 625                     /* A row is an array of cells */
 626                     JSONArray row = data.getJSONArray(j);
 627                     ImmutableList.Builder<LamiData> rowBuilder = ImmutableList.builder();
 628
 629                     for (int k = 0; k < row.length(); k++) {
 630                         Object cellObject = checkNotNull(row.get(k));
 631                         LamiData cellValue = LamiData.createFromObject(cellObject);
 632                         rowBuilder.add(cellValue);
 633
 634                     }
 635                     dataBuilder.add(new LamiTableEntry(rowBuilder.build()));
 636                 }
 637                 resultsBuilder.add(new LamiResultTable(tr, tableClass, dataBuilder.build()));
 638             }
 639
 640         } catch (JSONException e) {
 641             /* Error parsing the output */
 642             Activator.instance().logError(nullToEmptyString(e.getMessage()));
 643             return Collections.EMPTY_LIST;
 644         }
 645
 646         return resultsBuilder.build();
 647     }
 648
 649     private LamiTableClass getTableClassFromName(String tableClassName) throws JSONException {
 650         Map<String, LamiTableClass> map = checkNotNull(fTableClasses);
 651         LamiTableClass tableClass = map.get(tableClassName);
 652         if (tableClass == null) {
 653             throw new JSONException("Table class " + tableClassName + //$NON-NLS-1$
 654                     " was not declared in the metadata"); //$NON-NLS-1$
 655         }
 656         return tableClass;
 657     }
 658
 659     /**
 660      * Get the output of an external command, used for getting the metadata.
 661      * Cannot be cancelled, and will not report errors, simply returns null if
 662      * the process ended abnormally.
 663      */
 664     private static @Nullable String getOutputFromCommand(List<String> command) {
 665         try {
 666             ProcessBuilder builder = new ProcessBuilder(command);
 667             builder.redirectErrorStream(true);
 668
 669             Process p = builder.start();
 670             try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));) {
 671                 int ret = p.waitFor();
 672                 String output = br.lines().collect(Collectors.joining());
 673
 674                 return (ret == 0 ? output : null);
 675             }
 676         } catch (IOException | InterruptedException e) {
 677             return null;
 678         }
 679     }
 680
 681     /**
 682      * Get the results of invoking the specified command.
 683      *
 684      * The result should start with '{"results":...', as specified by the
 685      * LAMI JSON protocol. The JSON itself may be split over multiple lines.
 686      *
 687      * @param command
 688      *            The command to run (program and its arguments)
 689      * @return The analysis results
 690      */
 691     private static String getResultsFromCommand(List<String> command, IProgressMonitor monitor)
 692             throws OnDemandAnalysisException {
 693
 694         final int scale = 1000;
 695         double workedSoFar = 0.0;
 696
 697         ProcessCanceller cancellerRunnable = null;
 698         Thread cancellerThread = null;
 699
 700         try {
 701             monitor.beginTask(Messages.LamiAnalysis_MainTaskName, scale);
 702
 703             ProcessBuilder builder = new ProcessBuilder(command);
 704             builder.redirectErrorStream(false);
 705
 706             Process p = checkNotNull(builder.start());
 707
 708             cancellerRunnable = new ProcessCanceller(p, monitor);
 709             cancellerThread = new Thread(cancellerRunnable);
 710             cancellerThread.start();
 711
 712             List<String> results = new ArrayList<>();
 713
 714             try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));) {
 715                 String line = in.readLine();
 716                 while (line != null && !line.matches("\\s*\\{.*")) { //$NON-NLS-1$
 717                     /*
 718                      * This is a line indicating progress, it has the form:
 719                      *
 720                      * 0.123 3000 of 5000 events processed
 721                      *
 722                      * The first part indicates the estimated fraction (out of
 723                      * 1.0) of work done. The second part is status text.
 724                      */
 725
 726                     // Trim the line first to make sure the first character is
 727                     // significant
 728                     line = line.trim();
 729
 730                     // Split at the first space
 731                     String[] elems = line.split(" ", 2); //$NON-NLS-1$
 732
 733                     if (elems[0].matches("\\d.*")) { //$NON-NLS-1$
 734                         // It looks like we have a progress indication
 735                         try {
 736                             // Try parsing the number
 737                             double cumulativeWork = Double.parseDouble(elems[0]) * scale;
 738                             double workedThisLoop = cumulativeWork - workedSoFar;
 739
 740                             // We're going backwards? Do not update the
 741                             // monitor's value
 742                             if (workedThisLoop > 0) {
 743                                 monitor.internalWorked(workedThisLoop);
 744                                 workedSoFar = cumulativeWork;
 745                             }
 746
 747                             // There is a message: update the monitor's task name
 748                             if (elems.length >= 2) {
 749                                 monitor.setTaskName(elems[1].trim());
 750                             }
 751                         } catch (NumberFormatException e) {
 752                             // Continue reading progress lines anyway
 753                         }
 754                     }
 755
 756                     line = in.readLine();
 757                 }
 758                 while (line != null) {
 759                     /*
 760                      * We have seen the first line containing a '{', this is our
 761                      * JSON output!
 762                      */
 763                     results.add(line);
 764                     line = in.readLine();
 765                 }
 766             }
 767             int ret = p.waitFor();
 768
 769             if (monitor.isCanceled()) {
 770                 /* We were interrupted by the canceller thread. */
 771                 throw new OnDemandAnalysisException(null);
 772             }
 773
 774             if (ret != 0) {
 775                 /*
 776                  * Something went wrong running the external script. We will
 777                  * gather the stderr and report it to the user.
 778                  */
 779                 BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()));
 780                 String stdErrOutput = br.lines().collect(Collectors.joining("\n")); //$NON-NLS-1$
 781                 throw new OnDemandAnalysisException(stdErrOutput);
 782             }
 783
 784             /* External script ended successfully, all is fine! */
 785             String resultsStr = results.stream().collect(Collectors.joining());
 786             return checkNotNull(resultsStr);
 787
 788         } catch (IOException | InterruptedException e) {
 789             throw new OnDemandAnalysisException(Messages.LamiAnalysis_ExecutionInterrupted);
 790
 791         } finally {
 792             if (cancellerRunnable != null) {
 793                 cancellerRunnable.setFinished();
 794             }
 795             if (cancellerThread != null) {
 796                 try {
 797                     cancellerThread.join();
 798                 } catch (InterruptedException e) {
 799                 }
 800             }
 801
 802             monitor.done();
 803         }
 804     }
 805
 806     private static class ProcessCanceller implements Runnable {
 807
 808         private final Process fProcess;
 809         private final IProgressMonitor fMonitor;
 810
 811         private boolean fIsFinished = false;
 812
 813         public ProcessCanceller(Process process, IProgressMonitor monitor) {
 814             fProcess = process;
 815             fMonitor = monitor;
 816         }
 817
 818         public void setFinished() {
 819             fIsFinished = true;
 820         }
 821
 822         @Override
 823         public void run() {
 824             try {
 825                 while (!fIsFinished) {
 826                     Thread.sleep(500);
 827                     if (fMonitor.isCanceled()) {
 828                         fProcess.destroy();
 829                         return;
 830                     }
 831                 }
 832             } catch (InterruptedException e) {
 833             }
 834         }
 835
 836     }
 837
 838     @Override
 839     public @NonNull String getName() {
 840         return fName;
 841     }
 842
 843     @Override
 844     public boolean isUserDefined() {
 845         return fIsUserDefined;
 846     }
 847
 848 }