analysis/org.eclipse.tracecompass.analysis.lami.core/src/org/eclipse/tracecompass/internal/provisional/analysis/lami/core/module/LamiAnalysis.java

   1 /*******************************************************************************
   2  * Copyright (c) 2015, 2016 EfficiOS Inc., Alexandre Montplaisir
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *******************************************************************************/
   9
  10 package org.eclipse.tracecompass.internal.provisional.analysis.lami.core.module;
  11
  12 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNull;
  13 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNullContents;
  14 import static org.eclipse.tracecompass.common.core.NonNullUtils.nullToEmptyString;
  15
  16 import java.io.BufferedReader;
  17 import java.io.File;
  18 import java.io.IOException;
  19 import java.io.InputStreamReader;
  20 import java.nio.file.Files;
  21 import java.nio.file.Paths;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.function.Predicate;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.regex.Pattern;
  30 import java.util.stream.Collectors;
  31 import java.util.stream.Stream;
  32
  33 import org.eclipse.core.runtime.IProgressMonitor;
  34 import org.eclipse.jdt.annotation.NonNull;
  35 import org.eclipse.jdt.annotation.Nullable;
  36 import org.eclipse.tracecompass.internal.analysis.lami.core.Activator;
  37 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.LamiStrings;
  38 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiGenericAspect;
  39 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQNameAspect;
  40 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQNumberAspect;
  41 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiIRQTypeAspect;
  42 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiMixedAspect;
  43 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessNameAspect;
  44 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessPIDAspect;
  45 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiProcessTIDAspect;
  46 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiDurationAspect;
  47 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiEmptyAspect;
  48 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTableEntryAspect;
  49 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeBeginAspect;
  50 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeDurationAspect;
  51 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimeRangeEndAspect;
  52 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.aspect.LamiTimestampAspect;
  53 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiData;
  54 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiData.DataType;
  55 import org.eclipse.tracecompass.internal.provisional.analysis.lami.core.types.LamiTimeRange;
  56 import org.eclipse.tracecompass.tmf.core.analysis.ondemand.IOnDemandAnalysis;
  57 import org.eclipse.tracecompass.tmf.core.analysis.ondemand.OnDemandAnalysisException;
  58 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
  59 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
  60 import org.json.JSONArray;
  61 import org.json.JSONException;
  62 import org.json.JSONObject;
  63
  64 import com.google.common.annotations.VisibleForTesting;
  65 import com.google.common.collect.ImmutableList;
  66 import com.google.common.collect.ImmutableMap;
  67 import com.google.common.collect.ImmutableMultimap;
  68 import com.google.common.collect.Multimap;
  69
  70 /**
  71  * Base class for analysis modules that call external scripts implementing the
  72  * LAMI protocol.
  73  *
  74  * @author Alexandre Montplaisir
  75  */
  76 public class LamiAnalysis implements IOnDemandAnalysis {
  77
  78     /** Maximum major version of the LAMI protocol we support */
  79     private static final int MAX_SUPPORTED_MAJOR_VERSION = 1;
  80
  81     private static final String DOUBLE_QUOTES = "\""; //$NON-NLS-1$
  82
  83     /* Flags passed to the analysis scripts */
  84     private static final String METADATA_FLAG = "--metadata"; //$NON-NLS-1$
  85     private static final String PROGRESS_FLAG = "--output-progress"; //$NON-NLS-1$
  86     private static final String BEGIN_FLAG = "--begin"; //$NON-NLS-1$
  87     private static final String END_FLAG = "--end"; //$NON-NLS-1$
  88
  89     /** Log message for commands being run */
  90     private static final String RUNNING_MESSAGE = "Running command:"; //$NON-NLS-1$
  91
  92     private final List<String> fScriptCommand;
  93
  94     /**
  95      * The LAMI analysis is considered initialized after we have read the
  96      * script's --metadata once. This will assign the fields below.
  97      */
  98     private boolean fInitialized = false;
  99
 100     private boolean fIsAvailable;
 101     private final String fName;
 102     private final boolean fIsUserDefined;
 103     private final Predicate<ITmfTrace> fAppliesTo;
 104
 105     /* Data defined by the analysis's metadata */
 106     private @Nullable String fAnalysisTitle;
 107     private @Nullable Map<String, LamiTableClass> fTableClasses;
 108     private boolean fUseProgressOutput;
 109
 110     /**
 111      * Constructor. To be called by implementing classes.
 112      *
 113      * @param name
 114      *            Name of this analysis
 115      * @param isUserDefined
 116      *            {@code true} if this is a user-defined analysis
 117      * @param appliesTo
 118      *            Predicate to use to check whether or not this analysis applies
 119      *            to a given trace
 120      * @param args
 121      *            Analysis arguments, including the executable name (first
 122      *            argument)
 123      */
 124     public LamiAnalysis(String name, boolean isUserDefined, Predicate<ITmfTrace> appliesTo,
 125             List<String> args) {
 126         fScriptCommand = ImmutableList.copyOf(args);
 127         fName = name;
 128         fIsUserDefined = isUserDefined;
 129         fAppliesTo = appliesTo;
 130     }
 131
 132     /**
 133      * Map of pre-defined charts, for every table class names.
 134      *
 135      * If a table class is not in this map then it means that table has no
 136      * predefined charts.
 137      *
 138      * @return The chart models, per table class names
 139      */
 140     protected Multimap<String, LamiChartModel> getPredefinedCharts() {
 141         return ImmutableMultimap.of();
 142     }
 143
 144     @Override
 145     public final boolean appliesTo(ITmfTrace trace) {
 146         return fAppliesTo.test(trace);
 147     }
 148
 149     @Override
 150     public boolean canExecute(ITmfTrace trace) {
 151         initialize();
 152         return fIsAvailable;
 153     }
 154
 155     /**
 156      * Perform initialization of the LAMI script. This means verifying that it
 157      * is actually present on disk, and that it returns correct --metadata.
 158      */
 159     @VisibleForTesting
 160     protected synchronized void initialize() {
 161         if (fInitialized) {
 162             return;
 163         }
 164
 165         /* Do the analysis's initialization */
 166
 167         /* Check if the script's expected executable is on the PATH */
 168         String executable = fScriptCommand.get(0);
 169         boolean exists = Stream.of(System.getenv("PATH").split(checkNotNull(Pattern.quote(File.pathSeparator)))) //$NON-NLS-1$
 170                 .map(Paths::get)
 171                 .anyMatch(path -> Files.exists(path.resolve(executable)));
 172         if (!exists) {
 173             /* Script is not found */
 174             fIsAvailable = false;
 175             fInitialized = true;
 176             return;
 177         }
 178
 179         fIsAvailable = checkMetadata();
 180         fInitialized = true;
 181     }
 182
 183     /**
 184      * Verify that this script returns valid metadata.
 185      *
 186      * This will populate all remaining non-final fields of this class.
 187      *
 188      * @return If the metadata is valid or not
 189      */
 190     @VisibleForTesting
 191     protected boolean checkMetadata() {
 192         /*
 193          * The initialize() phase of the analysis will be used to check the
 194          * script's metadata. Actual runs of the script will use the execute()
 195          * method below.
 196          */
 197         List<String> command = ImmutableList.<@NonNull String> builder()
 198                 .addAll(fScriptCommand).add(METADATA_FLAG).build();
 199
 200         Activator.instance().logInfo(RUNNING_MESSAGE + ' ' + command.toString());
 201
 202         String output = getOutputFromCommand(command);
 203         if (output == null || output.isEmpty()) {
 204             return false;
 205         }
 206
 207         /*
 208          *
 209          * Metadata should look this this:
 210          *
 211          * {
 212          *   "version": [1, 5, 2, "dev"],
 213          *   "title": "I/O latency statistics",
 214          *   "authors": [
 215          *     "Julien Desfossez",
 216          *     "Antoine Busque"
 217          *   ],
 218          *   "description": "Provides statistics about the latency involved in various I/O operations.",
 219          *   "url": "https://github.com/lttng/lttng-analyses",
 220          *   "tags": [
 221          *     "io",
 222          *     "stats",
 223          *     "linux-kernel",
 224          *     "lttng-analyses"
 225          *   ],
 226          *   "table-classes": {
 227          *     "syscall-latency": {
 228          *       "title": "System calls latency statistics",
 229          *       "column-descriptions": [
 230          *         {"title": "System call", "type": "syscall"},
 231          *         {"title": "Count", "type": "int", "unit": "operations"},
 232          *         {"title": "Minimum duration", "type": "duration"},
 233          *         {"title": "Average duration", "type": "duration"},
 234          *         {"title": "Maximum duration", "type": "duration"},
 235          *         {"title": "Standard deviation", "type": "duration"}
 236          *       ]
 237          *     },
 238          *     "disk-latency": {
 239          *       "title": "Disk latency statistics",
 240          *       "column-descriptions": [
 241          *         {"title": "Disk name", "type": "disk"},
 242          *         {"title": "Count", "type": "int", "unit": "operations"},
 243          *         {"title": "Minimum duration", "type": "duration"},
 244          *         {"title": "Average duration", "type": "duration"},
 245          *         {"title": "Maximum duration", "type": "duration"},
 246          *         {"title": "Standard deviation", "type": "duration"}
 247          *       ]
 248          *     }
 249          *   }
 250          * }
 251          *
 252          */
 253
 254         try {
 255             JSONObject obj = new JSONObject(output);
 256             fAnalysisTitle = obj.getString(LamiStrings.TITLE);
 257
 258             /* Very early scripts may not contain the "mi-version" */
 259             JSONObject miVersion = obj.optJSONObject(LamiStrings.MI_VERSION);
 260             if (miVersion == null) {
 261                 /* Before version 0.1 */
 262                 fUseProgressOutput = false;
 263             } else {
 264                 int majorVersion = miVersion.getInt(LamiStrings.MAJOR);
 265                 if (majorVersion <= MAX_SUPPORTED_MAJOR_VERSION) {
 266                     fUseProgressOutput = true;
 267                 } else {
 268                     /* Unknown version, we do not support it */
 269                     return false;
 270                 }
 271             }
 272
 273             JSONObject tableClasses = obj.getJSONObject(LamiStrings.TABLE_CLASSES);
 274             @NonNull String[] tableClassNames = checkNotNullContents(JSONObject.getNames(tableClasses));
 275
 276             ImmutableMap.Builder<String, LamiTableClass> tablesBuilder = ImmutableMap.builder();
 277             for (String tableClassName : tableClassNames) {
 278                 JSONObject tableClass = tableClasses.getJSONObject(tableClassName);
 279
 280                 final String tableTitle = checkNotNull(tableClass.getString(LamiStrings.TITLE));
 281                 @NonNull JSONArray columnDescriptions = checkNotNull(tableClass.getJSONArray(LamiStrings.COLUMN_DESCRIPTIONS));
 282
 283                 List<LamiTableEntryAspect> aspects = getAspectsFromColumnDescriptions(columnDescriptions);
 284                 Collection<LamiChartModel> chartModels = getPredefinedCharts().get(tableClassName);
 285
 286                 tablesBuilder.put(tableClassName, new LamiTableClass(tableClassName, tableTitle, aspects, chartModels));
 287             }
 288
 289             try {
 290                 fTableClasses = tablesBuilder.build();
 291             } catch (IllegalArgumentException e) {
 292                 /*
 293                  * This is thrown if there are duplicate keys in the map
 294                  * builder.
 295                  */
 296                 throw new JSONException("Duplicate table class entry in " + fAnalysisTitle); //$NON-NLS-1$
 297             }
 298
 299         } catch (JSONException e) {
 300             /* Error in the parsing of the JSON, script is broken? */
 301             Activator.instance().logError(e.getMessage());
 302             return false;
 303         }
 304         return true;
 305     }
 306
 307     private static List<LamiTableEntryAspect> getAspectsFromColumnDescriptions(JSONArray columnDescriptions) throws JSONException {
 308         ImmutableList.Builder<LamiTableEntryAspect> aspectsBuilder = new ImmutableList.Builder<>();
 309         for (int j = 0; j < columnDescriptions.length(); j++) {
 310             JSONObject column = columnDescriptions.getJSONObject(j);
 311             DataType columnDataType;
 312             String columnClass = column.optString(LamiStrings.CLASS, null);
 313
 314             if (columnClass == null) {
 315                 columnDataType = DataType.MIXED;
 316             } else {
 317                 columnDataType = getDataTypeFromString(columnClass);
 318             }
 319
 320             String columnTitle = column.optString(LamiStrings.TITLE, null);
 321
 322             if (columnTitle == null) {
 323                 columnTitle = String.format("%s #%d", columnDataType.getTitle(), j + 1); //$NON-NLS-1$
 324             }
 325
 326             final int colIndex = j;
 327             switch (columnDataType) {
 328             case TIME_RANGE:
 329                 /*
 330                  * We will add 3 aspects, to represent the start, end and
 331                  * duration of this time range.
 332                  */
 333                 aspectsBuilder.add(new LamiTimeRangeBeginAspect(columnTitle, colIndex));
 334                 aspectsBuilder.add(new LamiTimeRangeEndAspect(columnTitle, colIndex));
 335                 aspectsBuilder.add(new LamiTimeRangeDurationAspect(columnTitle, colIndex));
 336                 break;
 337
 338             case TIMESTAMP:
 339                 aspectsBuilder.add(new LamiTimestampAspect(columnTitle, colIndex));
 340                 break;
 341
 342             case PROCESS:
 343                 aspectsBuilder.add(new LamiProcessNameAspect(columnTitle, colIndex));
 344                 aspectsBuilder.add(new LamiProcessPIDAspect(columnTitle, colIndex));
 345                 aspectsBuilder.add(new LamiProcessTIDAspect(columnTitle, colIndex));
 346                 break;
 347
 348             case IRQ:
 349                 aspectsBuilder.add(new LamiIRQTypeAspect(columnTitle, colIndex));
 350                 aspectsBuilder.add(new LamiIRQNameAspect(columnTitle, colIndex));
 351                 aspectsBuilder.add(new LamiIRQNumberAspect(columnTitle, colIndex));
 352                 break;
 353
 354             case DURATION:
 355                 aspectsBuilder.add(new LamiDurationAspect(columnTitle, colIndex));
 356                 break;
 357
 358             case MIXED:
 359                 aspectsBuilder.add(new LamiMixedAspect(columnTitle, colIndex));
 360                 break;
 361
 362             // $CASES-OMITTED$
 363             default:
 364                 String units = column.optString(LamiStrings.UNIT, null);
 365
 366                 if (units == null) {
 367                     units = columnDataType.getUnits();
 368                 }
 369
 370                 /* We will add only one aspect representing the element */
 371                 LamiTableEntryAspect aspect = new LamiGenericAspect(columnTitle,
 372                         units, colIndex, columnDataType.isContinuous(), false);
 373                 aspectsBuilder.add(aspect);
 374                 break;
 375             }
 376         }
 377         /*
 378          * SWT quirk : we need an empty column at the end or else the last data
 379          * column will clamp to the right edge of the view if it is
 380          * right-aligned.
 381          */
 382         aspectsBuilder.add(LamiEmptyAspect.INSTANCE);
 383
 384         return aspectsBuilder.build();
 385     }
 386
 387     private static DataType getDataTypeFromString(String value) throws JSONException {
 388         try {
 389             return DataType.fromString(value);
 390         } catch (IllegalArgumentException e) {
 391             throw new JSONException("Unrecognized data type: " + value); //$NON-NLS-1$
 392         }
 393     }
 394
 395     /**
 396      * Get the title of this analysis, as read from the script's metadata.
 397      *
 398      * @return The analysis title. Should not be null after the initialization
 399      *         completed successfully.
 400      */
 401     public @Nullable String getAnalysisTitle() {
 402         return fAnalysisTitle;
 403     }
 404
 405     /**
 406      * Get the result table classes defined by this analysis, as read from the
 407      * script's metadata.
 408      *
 409      * @return The analysis' result table classes. Should not be null after the
 410      *         execution completed successfully.
 411      */
 412     public @Nullable Map<String, LamiTableClass> getTableClasses() {
 413         return fTableClasses;
 414     }
 415
 416     /**
 417      * Print the full command that will be run when calling {@link #execute},
 418      * with the exception of the 'extraParams' that will be passed to execute().
 419      *
 420      * This can be used to display the command in the UI before it is actually
 421      * run.
 422      *
 423      * @param trace
 424      *            The trace on which to run the analysis
 425      * @param range
 426      *            The time range to specify. Null will not specify a time range,
 427      *            which means the whole trace will be taken.
 428      * @return The command as a single, space-separated string
 429      */
 430     public String getFullCommandAsString(ITmfTrace trace, @Nullable TmfTimeRange range) {
 431         String tracePath = checkNotNull(trace.getPath());
 432
 433         ImmutableList.Builder<String> builder = getBaseCommand(range);
 434         /*
 435          * We can add double-quotes around the trace path, which could contain
 436          * spaces, so that the resulting command can be easily copy-pasted into
 437          * a shell.
 438          */
 439         builder.add(DOUBLE_QUOTES + tracePath + DOUBLE_QUOTES);
 440         List<String> list = builder.build();
 441         String ret = list.stream().collect(Collectors.joining(" ")); //$NON-NLS-1$
 442         return checkNotNull(ret);
 443     }
 444
 445     /**
 446      * Get the base part of the command that will be executed to run this
 447      * analysis, supplying the given time range. Base part meaning:
 448      *
 449      * <pre>
 450      * [script executable] [statically-defined parameters] [--begin/--end (if applicable)]
 451      * </pre>
 452      *
 453      * Note that it does not include the path to the trace, that is to be added
 454      * separately.
 455      *
 456      * @param range
 457      *            The time range that will be passed
 458      * @return The elements of the command
 459      */
 460     private ImmutableList.Builder<String> getBaseCommand(@Nullable TmfTimeRange range) {
 461         long begin = 0;
 462         long end = 0;
 463         if (range != null) {
 464             begin = range.getStartTime().getValue();
 465             end = range.getEndTime().getValue();
 466         }
 467
 468         ImmutableList.Builder<String> builder = ImmutableList.builder();
 469         builder.addAll(fScriptCommand);
 470
 471         if (fUseProgressOutput) {
 472             builder.add(PROGRESS_FLAG);
 473         }
 474
 475         if (range != null) {
 476             builder.add(BEGIN_FLAG).add(String.valueOf(begin));
 477             builder.add(END_FLAG).add(String.valueOf(end));
 478         }
 479         return builder;
 480     }
 481
 482     /**
 483      * Call the currently defined LAMI script with the given arguments.
 484      *
 485      * @param timeRange
 486      *            The time range. Null for the whole trace.
 487      * @param monitor
 488      *            The progress monitor used to report progress
 489      * @return The script's output, formatted into {@link LamiTableEntry}'s.
 490      * @throws OnDemandAnalysisException
 491      *             If execution did not terminate normally
 492      */
 493     @Override
 494     public List<LamiResultTable> execute(ITmfTrace trace, @Nullable TmfTimeRange timeRange,
 495             String extraParams, IProgressMonitor monitor) throws OnDemandAnalysisException {
 496         /* Should have been called already, but in case it was not */
 497         initialize();
 498
 499         final @NonNull String tracePath = checkNotNull(trace.getPath());
 500         final @NonNull String[] splitParams = extraParams.trim().split(" "); //$NON-NLS-1$
 501
 502         ImmutableList.Builder<String> builder = getBaseCommand(timeRange);
 503
 504         if (!extraParams.trim().equals("")) { //$NON-NLS-1$
 505             builder.addAll(Arrays.asList(splitParams));
 506         }
 507         builder.add(tracePath);
 508         List<String> command = builder.build();
 509
 510         Activator.instance().logInfo(RUNNING_MESSAGE + ' ' + command.toString());
 511         String output = getResultsFromCommand(command, monitor);
 512
 513         if (output.isEmpty()) {
 514             throw new OnDemandAnalysisException(Messages.LamiAnalysis_NoResults);
 515         }
 516
 517         /*
 518          * {
 519          *   "results": [
 520          *     {
 521          *       "time-range": {
 522          *         "type": "time-range",
 523          *         "begin": 1444334398154194201,
 524          *         "end": 1444334425194487548
 525          *       },
 526          *       "class": "syscall-latency",
 527          *       "data": [
 528          *         [
 529          *           {"type": "syscall", "name": "open"},
 530          *           45,
 531          *           {"type": "duration", "value": 5562},
 532          *           {"type": "duration", "value": 13835},
 533          *           {"type": "duration", "value": 77683},
 534          *           {"type": "duration", "value": 15263}
 535          *         ],
 536          *         [
 537          *           {"type": "syscall", "name": "read"},
 538          *           109,
 539          *           {"type": "duration", "value": 316},
 540          *           {"type": "duration", "value": 5774},
 541          *           {"type": "duration", "value": 62569},
 542          *           {"type": "duration", "value": 9277}
 543          *         ]
 544          *       ]
 545          *     },
 546          *     {
 547          *       "time-range": {
 548          *         "type": "time-range",
 549          *         "begin": 1444334425194487549,
 550          *         "end": 1444334425254887190
 551          *       },
 552          *       "class": "syscall-latency",
 553          *       "data": [
 554          *         [
 555          *           {"type": "syscall", "name": "open"},
 556          *           45,
 557          *           {"type": "duration", "value": 1578},
 558          *           {"type": "duration", "value": 16648},
 559          *           {"type": "duration", "value": 15444},
 560          *           {"type": "duration", "value": 68540}
 561          *         ],
 562          *         [
 563          *           {"type": "syscall", "name": "read"},
 564          *           109,
 565          *           {"type": "duration", "value": 78},
 566          *           {"type": "duration", "value": 1948},
 567          *           {"type": "duration", "value": 11184},
 568          *           {"type": "duration", "value": 94670}
 569          *         ]
 570          *       ]
 571          *     }
 572          *   ]
 573          * }
 574          *
 575          */
 576
 577         ImmutableList.Builder<LamiResultTable> resultsBuilder = new ImmutableList.Builder<>();
 578
 579         try {
 580             JSONObject obj = new JSONObject(output);
 581             JSONArray results = obj.getJSONArray(LamiStrings.RESULTS);
 582
 583             if (results.length() == 0) {
 584                 /*
 585                  * No results were reported. This may be normal, but warn the
 586                  * user why a report won't be created.
 587                  */
 588                 throw new OnDemandAnalysisException(Messages.LamiAnalysis_NoResults);
 589             }
 590
 591             for (int i = 0; i < results.length(); i++) {
 592                 JSONObject result = results.getJSONObject(i);
 593
 594                 /* Parse the time-range */
 595                 JSONObject trObject = result.getJSONObject(LamiStrings.TIME_RANGE);
 596                 long start = trObject.getLong(LamiStrings.BEGIN);
 597                 long end = trObject.getLong(LamiStrings.END);
 598                 LamiTimeRange tr = new LamiTimeRange(start, end);
 599
 600                 /* Parse the table's class */
 601                 LamiTableClass tableClass;
 602                 JSONObject tableClassObject = result.optJSONObject(LamiStrings.CLASS);
 603                 if (tableClassObject == null) {
 604                     /*
 605                      * "class" is just a standard string, indicating we use a
 606                      * metadata-defined table class as-is
 607                      */
 608                     @NonNull String tableClassName = checkNotNull(result.getString(LamiStrings.CLASS));
 609                     tableClass = getTableClassFromName(tableClassName);
 610
 611                     // FIXME Rest will become more generic eventually in the LAMI format.
 612                 } else if (tableClassObject.has(LamiStrings.INHERIT)) {
 613                     /*
 614                      * Dynamic title: We reuse an existing table class but
 615                      * override the title.
 616                      */
 617                     String baseTableName = checkNotNull(tableClassObject.getString(LamiStrings.INHERIT));
 618                     LamiTableClass baseTableClass = getTableClassFromName(baseTableName);
 619                     String newTitle = checkNotNull(tableClassObject.getString(LamiStrings.TITLE));
 620
 621                     tableClass = new LamiTableClass(baseTableClass, newTitle);
 622                 } else {
 623                     /*
 624                      * Dynamic column descriptions: we implement a new table
 625                      * class entirely.
 626                      */
 627                     String title = checkNotNull(tableClassObject.getString(LamiStrings.TITLE));
 628                     JSONArray columnDescriptions = checkNotNull(tableClassObject.getJSONArray(LamiStrings.COLUMN_DESCRIPTIONS));
 629                     List<LamiTableEntryAspect> aspects = getAspectsFromColumnDescriptions(columnDescriptions);
 630
 631                     tableClass = new LamiTableClass(nullToEmptyString(Messages.LamiAnalysis_DefaultDynamicTableName), title, aspects, Collections.EMPTY_SET);
 632                 }
 633
 634                 /* Parse the "data", which is the array of rows */
 635                 JSONArray data = result.getJSONArray(LamiStrings.DATA);
 636                 ImmutableList.Builder<LamiTableEntry> dataBuilder = new ImmutableList.Builder<>();
 637
 638                 for (int j = 0; j < data.length(); j++) {
 639                     /* A row is an array of cells */
 640                     JSONArray row = data.getJSONArray(j);
 641                     ImmutableList.Builder<LamiData> rowBuilder = ImmutableList.builder();
 642
 643                     for (int k = 0; k < row.length(); k++) {
 644                         Object cellObject = checkNotNull(row.get(k));
 645                         LamiData cellValue = LamiData.createFromObject(cellObject);
 646                         rowBuilder.add(cellValue);
 647
 648                     }
 649                     dataBuilder.add(new LamiTableEntry(rowBuilder.build()));
 650                 }
 651                 resultsBuilder.add(new LamiResultTable(tr, tableClass, dataBuilder.build()));
 652             }
 653
 654         } catch (JSONException e) {
 655             throw new OnDemandAnalysisException(e.getMessage());
 656         }
 657
 658         return resultsBuilder.build();
 659     }
 660
 661     private LamiTableClass getTableClassFromName(String tableClassName) throws JSONException {
 662         Map<String, LamiTableClass> map = checkNotNull(fTableClasses);
 663         LamiTableClass tableClass = map.get(tableClassName);
 664         if (tableClass == null) {
 665             throw new JSONException("Table class " + tableClassName + //$NON-NLS-1$
 666                     " was not declared in the metadata"); //$NON-NLS-1$
 667         }
 668         return tableClass;
 669     }
 670
 671     /**
 672      * Get the output of an external command, used for getting the metadata.
 673      * Cannot be cancelled, and will not report errors, simply returns null if
 674      * the process ended abnormally.
 675      *
 676      * @param command
 677      *            The parameters of the command, passed to
 678      *            {@link ProcessBuilder}
 679      * @return The command output as a string
 680      */
 681     @VisibleForTesting
 682     protected @Nullable String getOutputFromCommand(List<String> command) {
 683         try {
 684             ProcessBuilder builder = new ProcessBuilder(command);
 685             builder.redirectErrorStream(true);
 686
 687             Process p = builder.start();
 688             try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));) {
 689                 int ret = p.waitFor();
 690                 String output = br.lines().collect(Collectors.joining());
 691
 692                 return (ret == 0 ? output : null);
 693             }
 694         } catch (IOException | InterruptedException e) {
 695             return null;
 696         }
 697     }
 698
 699     /**
 700      * Get the results of invoking the specified command.
 701      *
 702      * The result should start with '{"results":...', as specified by the LAMI
 703      * JSON protocol. The JSON itself may be split over multiple lines.
 704      *
 705      * @param command
 706      *            The command to run (program and its arguments)
 707      * @param monitor
 708      *            The progress monitor
 709      * @return The analysis results
 710      * @throws OnDemandAnalysisException
 711      *             If the command ended abnormally, and normal results were not
 712      *             returned
 713      */
 714     @VisibleForTesting
 715     protected String getResultsFromCommand(List<String> command, IProgressMonitor monitor)
 716             throws OnDemandAnalysisException {
 717
 718         final int scale = 1000;
 719         double workedSoFar = 0.0;
 720
 721         ProcessCanceller cancellerRunnable = null;
 722         Thread cancellerThread = null;
 723
 724         try {
 725             monitor.beginTask(Messages.LamiAnalysis_MainTaskName, scale);
 726
 727             ProcessBuilder builder = new ProcessBuilder(command);
 728             builder.redirectErrorStream(false);
 729
 730             Process p = checkNotNull(builder.start());
 731
 732             cancellerRunnable = new ProcessCanceller(p, monitor);
 733             cancellerThread = new Thread(cancellerRunnable);
 734             cancellerThread.start();
 735
 736             List<String> results = new ArrayList<>();
 737
 738             try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));) {
 739                 String line = in.readLine();
 740                 while (line != null && !line.matches("\\s*\\{.*")) { //$NON-NLS-1$
 741                     /*
 742                      * This is a line indicating progress, it has the form:
 743                      *
 744                      * 0.123 3000 of 5000 events processed
 745                      *
 746                      * The first part indicates the estimated fraction (out of
 747                      * 1.0) of work done. The second part is status text.
 748                      */
 749
 750                     // Trim the line first to make sure the first character is
 751                     // significant
 752                     line = line.trim();
 753
 754                     // Split at the first space
 755                     String[] elems = line.split(" ", 2); //$NON-NLS-1$
 756
 757                     if (elems[0].matches("\\d.*")) { //$NON-NLS-1$
 758                         // It looks like we have a progress indication
 759                         try {
 760                             // Try parsing the number
 761                             double cumulativeWork = Double.parseDouble(elems[0]) * scale;
 762                             double workedThisLoop = cumulativeWork - workedSoFar;
 763
 764                             // We're going backwards? Do not update the
 765                             // monitor's value
 766                             if (workedThisLoop > 0) {
 767                                 monitor.internalWorked(workedThisLoop);
 768                                 workedSoFar = cumulativeWork;
 769                             }
 770
 771                             // There is a message: update the monitor's task name
 772                             if (elems.length >= 2) {
 773                                 monitor.setTaskName(elems[1].trim());
 774                             }
 775                         } catch (NumberFormatException e) {
 776                             // Continue reading progress lines anyway
 777                         }
 778                     }
 779
 780                     line = in.readLine();
 781                 }
 782                 while (line != null) {
 783                     /*
 784                      * We have seen the first line containing a '{', this is our
 785                      * JSON output!
 786                      */
 787                     results.add(line);
 788                     line = in.readLine();
 789                 }
 790             }
 791             int ret = p.waitFor();
 792
 793             if (monitor.isCanceled()) {
 794                 /* We were interrupted by the canceller thread. */
 795                 throw new OnDemandAnalysisException(null);
 796             }
 797
 798             if (ret != 0) {
 799                 /*
 800                  * Something went wrong running the external script. We will
 801                  * gather the stderr and report it to the user.
 802                  */
 803                 BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()));
 804                 String stdErrOutput = br.lines().collect(Collectors.joining("\n")); //$NON-NLS-1$
 805                 throw new OnDemandAnalysisException(stdErrOutput);
 806             }
 807
 808             /* External script ended successfully, all is fine! */
 809             String resultsStr = results.stream().collect(Collectors.joining());
 810             return checkNotNull(resultsStr);
 811
 812         } catch (IOException | InterruptedException e) {
 813             throw new OnDemandAnalysisException(Messages.LamiAnalysis_ExecutionInterrupted);
 814
 815         } finally {
 816             if (cancellerRunnable != null) {
 817                 cancellerRunnable.setFinished();
 818             }
 819             if (cancellerThread != null) {
 820                 try {
 821                     cancellerThread.join();
 822                 } catch (InterruptedException e) {
 823                 }
 824             }
 825
 826             monitor.done();
 827         }
 828     }
 829
 830     private static class ProcessCanceller implements Runnable {
 831
 832         private final Process fProcess;
 833         private final IProgressMonitor fMonitor;
 834
 835         private boolean fIsFinished = false;
 836
 837         public ProcessCanceller(Process process, IProgressMonitor monitor) {
 838             fProcess = process;
 839             fMonitor = monitor;
 840         }
 841
 842         public void setFinished() {
 843             fIsFinished = true;
 844         }
 845
 846         @Override
 847         public void run() {
 848             try {
 849                 while (!fIsFinished) {
 850                     Thread.sleep(500);
 851                     if (fMonitor.isCanceled()) {
 852                         fProcess.destroy();
 853                         return;
 854                     }
 855                 }
 856             } catch (InterruptedException e) {
 857             }
 858         }
 859
 860     }
 861
 862     @Override
 863     public @NonNull String getName() {
 864         return fName;
 865     }
 866
 867     @Override
 868     public boolean isUserDefined() {
 869         return fIsUserDefined;
 870     }
 871
 872 }