tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/statesystem/TmfStateSystemAnalysisModule.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2015 École Polytechnique de Montréal
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Geneviève Bastien - Initial API and implementation
  11  *   Bernd Hufmann - Integrated history builder functionality
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.statesystem;
  15
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.util.Collections;
  19 import java.util.Map;
  20 import java.util.concurrent.CountDownLatch;
  21
  22 import org.apache.commons.io.FileUtils;
  23 import org.eclipse.core.runtime.IProgressMonitor;
  24 import org.eclipse.core.runtime.IStatus;
  25 import org.eclipse.core.runtime.NullProgressMonitor;
  26 import org.eclipse.jdt.annotation.NonNull;
  27 import org.eclipse.jdt.annotation.Nullable;
  28 import org.eclipse.tracecompass.common.core.NonNullUtils;
  29 import org.eclipse.tracecompass.internal.tmf.core.statesystem.backends.partial.PartialHistoryBackend;
  30 import org.eclipse.tracecompass.internal.tmf.core.statesystem.backends.partial.PartialStateSystem;
  31 import org.eclipse.tracecompass.statesystem.core.ITmfStateSystem;
  32 import org.eclipse.tracecompass.statesystem.core.ITmfStateSystemBuilder;
  33 import org.eclipse.tracecompass.statesystem.core.StateSystemFactory;
  34 import org.eclipse.tracecompass.statesystem.core.backend.IStateHistoryBackend;
  35 import org.eclipse.tracecompass.statesystem.core.backend.StateHistoryBackendFactory;
  36 import org.eclipse.tracecompass.tmf.core.analysis.TmfAbstractAnalysisModule;
  37 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  38 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  39 import org.eclipse.tracecompass.tmf.core.request.ITmfEventRequest;
  40 import org.eclipse.tracecompass.tmf.core.request.TmfEventRequest;
  41 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
  42 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
  43 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
  44 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
  45 import org.eclipse.tracecompass.tmf.core.trace.ITmfTraceCompleteness;
  46 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceManager;
  47 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  48 import org.eclipse.tracecompass.tmf.core.trace.experiment.TmfExperiment;
  49
  50 import com.google.common.annotations.VisibleForTesting;
  51
  52 /**
  53  * Abstract analysis module to generate a state system. It is a base class that
  54  * can be used as a shortcut by analysis who just need to build a single state
  55  * system with a state provider.
  56  *
  57  * Analysis implementing this class should only need to provide a state system
  58  * and optionally a backend (default to NULL) and, if required, a filename
  59  * (defaults to the analysis'ID)
  60  *
  61  * @author Geneviève Bastien
  62  */
  63 public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisModule
  64         implements ITmfAnalysisModuleWithStateSystems {
  65
  66     private static final String EXTENSION = ".ht"; //$NON-NLS-1$
  67
  68     private final CountDownLatch fInitialized = new CountDownLatch(1);
  69     private final Object fRequestSyncObj = new Object();
  70
  71     private @Nullable ITmfStateSystemBuilder fStateSystem;
  72     private @Nullable ITmfEventRequest fRequest;
  73     private @Nullable TmfTimeRange fTimeRange = null;
  74
  75     private int fNbRead = 0;
  76     private boolean fInitializationSucceeded;
  77
  78     private volatile @Nullable ITmfStateProvider fStateProvider;
  79
  80     /**
  81      * State system backend types
  82      *
  83      * @author Geneviève Bastien
  84      */
  85     protected enum StateSystemBackendType {
  86         /** Full history in file */
  87         FULL,
  88         /** In memory state system */
  89         INMEM,
  90         /** Null history */
  91         NULL,
  92         /** State system backed with partial history */
  93         PARTIAL
  94     }
  95
  96     /**
  97      * Retrieve a state system belonging to trace, by passing the ID of the
  98      * relevant analysis module.
  99      *
 100      * This will start the execution of the analysis module, and start the
 101      * construction of the state system, if needed.
 102      *
 103      * @param trace
 104      *            The trace for which you want the state system
 105      * @param moduleId
 106      *            The ID of the state system analysis module
 107      * @return The state system, or null if there was no match or the module was
 108      *         not initialized correctly
 109      */
 110     public static @Nullable ITmfStateSystem getStateSystem(ITmfTrace trace, String moduleId) {
 111         TmfStateSystemAnalysisModule module =
 112                 TmfTraceUtils.getAnalysisModuleOfClass(trace, TmfStateSystemAnalysisModule.class, moduleId);
 113         if (module != null) {
 114             ITmfStateSystem ss = module.getStateSystem();
 115             if (ss != null) {
 116                 return ss;
 117             }
 118             IStatus status = module.schedule();
 119             if (status.isOK()) {
 120                 return module.waitForInitialization() ? module.getStateSystem() : null;
 121             }
 122         }
 123         return null;
 124     }
 125
 126     /**
 127      * Get the state provider for this analysis module
 128      *
 129      * @return the state provider
 130      */
 131     protected abstract ITmfStateProvider createStateProvider();
 132
 133     /**
 134      * Get the state system backend type used by this module
 135      *
 136      * @return The {@link StateSystemBackendType}
 137      */
 138     protected StateSystemBackendType getBackendType() {
 139         /* Using full history by default, sub-classes can override */
 140         return StateSystemBackendType.FULL;
 141     }
 142
 143     /**
 144      * Get the supplementary file name where to save this state system. The
 145      * default is the ID of the analysis followed by the extension.
 146      *
 147      * @return The supplementary file name
 148      */
 149     protected String getSsFileName() {
 150         return getId() + EXTENSION;
 151     }
 152
 153     /**
 154      * Get the state system generated by this analysis, or null if it is not yet
 155      * created.
 156      *
 157      * @return The state system
 158      */
 159     @Nullable
 160     public ITmfStateSystem getStateSystem() {
 161         return fStateSystem;
 162     }
 163
 164     /**
 165      * @since 2.0
 166      */
 167     @Override
 168     public boolean waitForInitialization() {
 169         try {
 170             fInitialized.await();
 171         } catch (InterruptedException e) {
 172             return false;
 173         }
 174         return fInitializationSucceeded;
 175     }
 176
 177     /**
 178      * @since 2.0
 179      */
 180     @Override
 181     public boolean isQueryable(long ts) {
 182         /* Return true if there is no state provider available (the analysis is not being built) */
 183         ITmfStateProvider provider = fStateProvider;
 184         if (provider == null) {
 185             return true;
 186         }
 187         return ts <= provider.getLatestSafeTime();
 188     }
 189
 190     // ------------------------------------------------------------------------
 191     // TmfAbstractAnalysisModule
 192     // ------------------------------------------------------------------------
 193
 194     /**
 195      * Get the file where to save the results of the analysis
 196      *
 197      * @return The file to save the results in
 198      * @since 2.3
 199      */
 200     @VisibleForTesting
 201     protected @Nullable File getSsFile() {
 202         ITmfTrace trace = getTrace();
 203         if (trace == null) {
 204             return null;
 205         }
 206         String directory = TmfTraceManager.getSupplementaryFileDir(trace);
 207         File htFile = new File(directory + getSsFileName());
 208         return htFile;
 209     }
 210
 211     @Override
 212     protected boolean executeAnalysis(@Nullable final  IProgressMonitor monitor) {
 213         IProgressMonitor mon = (monitor == null ? new NullProgressMonitor() : monitor);
 214         final ITmfStateProvider provider = createStateProvider();
 215
 216         String id = getId();
 217
 218         /* FIXME: State systems should make use of the monitor, to be cancelled */
 219         try {
 220             /* Get the state system according to backend */
 221             StateSystemBackendType backend = getBackendType();
 222
 223             ITmfTrace trace = getTrace();
 224             if (trace == null) {
 225                 // Analysis was cancelled in the meantime
 226                 analysisReady(false);
 227                 return false;
 228             }
 229             switch (backend) {
 230             case FULL: {
 231                 File htFile = getSsFile();
 232                 if (htFile == null) {
 233                     return false;
 234                 }
 235                 createFullHistory(id, provider, htFile);
 236             }
 237                 break;
 238             case PARTIAL: {
 239                 File htFile = getSsFile();
 240                 if (htFile == null) {
 241                     return false;
 242                 }
 243                 createPartialHistory(id, provider, htFile);
 244             }
 245                 break;
 246             case INMEM:
 247                 createInMemoryHistory(id, provider);
 248                 break;
 249             case NULL:
 250                 createNullHistory(id, provider);
 251                 break;
 252             default:
 253                 break;
 254             }
 255         } catch (TmfTraceException e) {
 256             analysisReady(false);
 257             return false;
 258         }
 259         return !mon.isCanceled();
 260     }
 261
 262     /**
 263      * Make the module available and set whether the initialization succeeded or
 264      * not. If not, no state system is available and
 265      * {@link #waitForInitialization()} should return false.
 266      *
 267      * @param success
 268      *            True if the initialization succeeded, false otherwise
 269      */
 270     private void analysisReady(boolean succeeded) {
 271         fInitializationSucceeded = succeeded;
 272         fInitialized.countDown();
 273     }
 274
 275     @Override
 276     protected void canceling() {
 277         ITmfEventRequest req = fRequest;
 278         if ((req != null) && (!req.isCompleted())) {
 279             req.cancel();
 280         }
 281     }
 282
 283     @Override
 284     public void dispose() {
 285         super.dispose();
 286         if (fStateSystem != null) {
 287             fStateSystem.dispose();
 288         }
 289     }
 290
 291     // ------------------------------------------------------------------------
 292     // History creation methods
 293     // ------------------------------------------------------------------------
 294
 295     /*
 296      * Load the history file matching the target trace. If the file already
 297      * exists, it will be opened directly. If not, it will be created from
 298      * scratch.
 299      */
 300     private void createFullHistory(String id, ITmfStateProvider provider, File htFile) throws TmfTraceException {
 301
 302         /* If the target file already exists, do not rebuild it uselessly */
 303         // TODO for now we assume it's complete. Might be a good idea to check
 304         // at least if its range matches the trace's range.
 305
 306         if (htFile.exists()) {
 307            /* Load an existing history */
 308             final int version = provider.getVersion();
 309             try {
 310                 IStateHistoryBackend backend = StateHistoryBackendFactory.createHistoryTreeBackendExistingFile(
 311                         id, htFile, version);
 312                 fStateSystem = StateSystemFactory.newStateSystem(backend, false);
 313                 analysisReady(true);
 314                 return;
 315             } catch (IOException e) {
 316                 /*
 317                  * There was an error opening the existing file. Perhaps it was
 318                  * corrupted, perhaps it's an old version? We'll just
 319                  * fall-through and try to build a new one from scratch instead.
 320                  */
 321             }
 322         }
 323
 324         /* Size of the blocking queue to use when building a state history */
 325         final int QUEUE_SIZE = 10000;
 326
 327         try {
 328             IStateHistoryBackend backend = StateHistoryBackendFactory.createHistoryTreeBackendNewFile(
 329                     id, htFile, provider.getVersion(), provider.getStartTime(), QUEUE_SIZE);
 330             fStateSystem = StateSystemFactory.newStateSystem(backend);
 331             provider.assignTargetStateSystem(fStateSystem);
 332             build(provider);
 333         } catch (IOException e) {
 334             /*
 335              * If it fails here however, it means there was a problem writing to
 336              * the disk, so throw a real exception this time.
 337              */
 338             throw new TmfTraceException(e.toString(), e);
 339         }
 340     }
 341
 342     /*
 343      * Create a new state system backed with a partial history. A partial
 344      * history is similar to a "full" one (which you get with
 345      * {@link #newFullHistory}), except that the file on disk is much smaller,
 346      * but queries are a bit slower.
 347      *
 348      * Also note that single-queries are implemented using a full-query
 349      * underneath, (which are much slower), so this might not be a good fit for
 350      * a use case where you have to do lots of single queries.
 351      */
 352     private void createPartialHistory(String id, ITmfStateProvider provider, File htPartialFile)
 353             throws TmfTraceException {
 354         /*
 355          * The order of initializations is very tricky (but very important!)
 356          * here. We need to follow this pattern:
 357          * (1 is done before the call to this method)
 358          *
 359          * 1- Instantiate realStateProvider
 360          * 2- Instantiate realBackend
 361          * 3- Instantiate partialBackend, with prereqs:
 362          *  3a- Instantiate partialProvider, via realProvider.getNew()
 363          *  3b- Instantiate nullBackend (partialSS's backend)
 364          *  3c- Instantiate partialSS
 365          *  3d- partialProvider.assignSS(partialSS)
 366          * 4- Instantiate realSS
 367          * 5- partialSS.assignUpstream(realSS)
 368          * 6- realProvider.assignSS(realSS)
 369          * 7- Call HistoryBuilder(realProvider, realSS, partialBackend) to build the thing.
 370          */
 371
 372         /* Size of the blocking queue to use when building a state history */
 373         final int QUEUE_SIZE = 10000;
 374
 375         final long granularity = 50000;
 376
 377         /* 2 */
 378         IStateHistoryBackend realBackend = null;
 379         try {
 380             realBackend = StateHistoryBackendFactory.createHistoryTreeBackendNewFile(
 381                     id, htPartialFile, provider.getVersion(), provider.getStartTime(), QUEUE_SIZE);
 382         } catch (IOException e) {
 383             throw new TmfTraceException(e.toString(), e);
 384         }
 385
 386         /* 3a */
 387         ITmfStateProvider partialProvider = provider.getNewInstance();
 388
 389         /* 3b-3c, constructor automatically uses a NullBackend */
 390         PartialStateSystem pss = new PartialStateSystem();
 391
 392         /* 3d */
 393         partialProvider.assignTargetStateSystem(pss);
 394
 395         /* 3 */
 396         IStateHistoryBackend partialBackend = new PartialHistoryBackend(id + ".partial", partialProvider, pss, realBackend, granularity); //$NON-NLS-1$
 397
 398         /* 4 */
 399         @SuppressWarnings("restriction")
 400         org.eclipse.tracecompass.internal.statesystem.core.StateSystem realSS =
 401         (org.eclipse.tracecompass.internal.statesystem.core.StateSystem) StateSystemFactory.newStateSystem(partialBackend);
 402
 403         /* 5 */
 404         pss.assignUpstream(realSS);
 405
 406         /* 6 */
 407         provider.assignTargetStateSystem(realSS);
 408
 409         /* 7 */
 410         fStateSystem = realSS;
 411
 412         build(provider);
 413     }
 414
 415     /*
 416      * Create a new state system using a null history back-end. This means that
 417      * no history intervals will be saved anywhere, and as such only
 418      * {@link ITmfStateSystem#queryOngoingState} will be available.
 419      */
 420     private void createNullHistory(String id, ITmfStateProvider provider) {
 421         IStateHistoryBackend backend = StateHistoryBackendFactory.createNullBackend(id);
 422         fStateSystem = StateSystemFactory.newStateSystem(backend);
 423         provider.assignTargetStateSystem(fStateSystem);
 424         build(provider);
 425     }
 426
 427     /*
 428      * Create a new state system using in-memory interval storage. This should
 429      * only be done for very small state system, and will be naturally limited
 430      * to 2^31 intervals.
 431      */
 432     private void createInMemoryHistory(String id, ITmfStateProvider provider) {
 433         IStateHistoryBackend backend = StateHistoryBackendFactory.createInMemoryBackend(id, provider.getStartTime());
 434         fStateSystem = StateSystemFactory.newStateSystem(backend);
 435         provider.assignTargetStateSystem(fStateSystem);
 436         build(provider);
 437     }
 438
 439     private void disposeProvider(boolean deleteFiles) {
 440         ITmfStateProvider provider = fStateProvider;
 441         if (provider != null) {
 442             provider.dispose();
 443         }
 444         fStateProvider = null;
 445         if (deleteFiles && (fStateSystem != null)) {
 446             fStateSystem.removeFiles();
 447         }
 448     }
 449
 450     private void build(ITmfStateProvider provider) {
 451         if (fStateSystem == null) {
 452             throw new IllegalArgumentException();
 453         }
 454
 455         ITmfEventRequest request = fRequest;
 456         if ((request != null) && (!request.isCompleted())) {
 457             request.cancel();
 458         }
 459
 460         fTimeRange = TmfTimeRange.ETERNITY;
 461         final ITmfTrace trace = provider.getTrace();
 462         if (!isCompleteTrace(trace)) {
 463             fTimeRange = trace.getTimeRange();
 464         }
 465
 466         fStateProvider = provider;
 467         synchronized (fRequestSyncObj) {
 468             startRequest();
 469         }
 470
 471         /*
 472          * The state system object is now created, we can consider this module
 473          * "initialized" (components can retrieve it and start doing queries).
 474          */
 475         analysisReady(true);
 476
 477         /*
 478          * Block the executeAnalysis() construction is complete (so that the
 479          * progress monitor displays that it is running).
 480          */
 481         try {
 482             if (fRequest != null) {
 483                 fRequest.waitForCompletion();
 484             }
 485         } catch (InterruptedException e) {
 486              e.printStackTrace();
 487         }
 488     }
 489
 490     private class StateSystemEventRequest extends TmfEventRequest {
 491         private final ITmfStateProvider sci;
 492         private final ITmfTrace trace;
 493
 494         public StateSystemEventRequest(ITmfStateProvider sp, TmfTimeRange timeRange, int index) {
 495             super(ITmfEvent.class,
 496                     timeRange,
 497                     index,
 498                     ITmfEventRequest.ALL_DATA,
 499                     ITmfEventRequest.ExecutionType.BACKGROUND,
 500                     TmfStateSystemAnalysisModule.this.getDependencyLevel());
 501             this.sci = sp;
 502             trace = sci.getTrace();
 503
 504         }
 505
 506         @Override
 507         public void handleData(final ITmfEvent event) {
 508             super.handleData(event);
 509             if (event.getTrace() == trace) {
 510                 sci.processEvent(event);
 511             } else if (trace instanceof TmfExperiment) {
 512                 /*
 513                  * If the request is for an experiment, check if the event is
 514                  * from one of the child trace
 515                  */
 516                 for (ITmfTrace childTrace : ((TmfExperiment) trace).getTraces()) {
 517                     if (childTrace == event.getTrace()) {
 518                         sci.processEvent(event);
 519                     }
 520                 }
 521             }
 522         }
 523
 524         @Override
 525         public void handleSuccess() {
 526             super.handleSuccess();
 527             if (isCompleteTrace(trace)) {
 528                 disposeProvider(false);
 529             } else {
 530                 fNbRead += getNbRead();
 531                 synchronized (fRequestSyncObj) {
 532                     final TmfTimeRange timeRange = fTimeRange;
 533                     if (timeRange != null) {
 534                         if (getRange().getEndTime().getValue() < timeRange.getEndTime().getValue()) {
 535                             startRequest();
 536                         }
 537                     }
 538                 }
 539             }
 540         }
 541
 542         @Override
 543         public void handleCancel() {
 544             super.handleCancel();
 545             disposeProvider(true);
 546         }
 547
 548         @Override
 549         public void handleFailure() {
 550             super.handleFailure();
 551             disposeProvider(true);
 552         }
 553     }
 554
 555     // ------------------------------------------------------------------------
 556     // ITmfAnalysisModuleWithStateSystems
 557     // ------------------------------------------------------------------------
 558
 559     @Override
 560     @Nullable
 561     public ITmfStateSystem getStateSystem(String id) {
 562         if (id.equals(getId())) {
 563             return fStateSystem;
 564         }
 565         return null;
 566     }
 567
 568     @Override
 569     public @NonNull Iterable<@NonNull ITmfStateSystem> getStateSystems() {
 570         ITmfStateSystemBuilder stateSystem = fStateSystem;
 571         if (stateSystem == null) {
 572             return Collections.EMPTY_SET;
 573         }
 574         return Collections.singleton(stateSystem);
 575     }
 576
 577     /**
 578      * Signal handler for the TmfTraceRangeUpdatedSignal signal
 579      *
 580      * @param signal The incoming signal
 581      */
 582     @TmfSignalHandler
 583     public void traceRangeUpdated(final TmfTraceRangeUpdatedSignal signal) {
 584         fTimeRange = signal.getRange();
 585         ITmfStateProvider stateProvider = fStateProvider;
 586         synchronized (fRequestSyncObj) {
 587             if (signal.getTrace() == getTrace() && stateProvider != null && stateProvider.getAssignedStateSystem() != null) {
 588                 ITmfEventRequest request = fRequest;
 589                 if ((request == null) || request.isCompleted()) {
 590                     startRequest();
 591                 }
 592             }
 593         }
 594     }
 595
 596     private void startRequest() {
 597         ITmfStateProvider stateProvider = fStateProvider;
 598         TmfTimeRange timeRange = fTimeRange;
 599         if (stateProvider == null || timeRange == null) {
 600             return;
 601         }
 602         ITmfEventRequest request = new StateSystemEventRequest(stateProvider, timeRange, fNbRead);
 603         stateProvider.getTrace().sendRequest(request);
 604         fRequest = request;
 605     }
 606
 607     private static boolean isCompleteTrace(ITmfTrace trace) {
 608         return !(trace instanceof ITmfTraceCompleteness) || ((ITmfTraceCompleteness) trace).isComplete();
 609     }
 610
 611     // ------------------------------------------------------------------------
 612     // ITmfPropertiesProvider
 613     // ------------------------------------------------------------------------
 614
 615     /**
 616      * @since 2.0
 617      */
 618     @Override
 619     public @NonNull Map<@NonNull String, @NonNull String> getProperties() {
 620         Map<@NonNull String, @NonNull String> properties = super.getProperties();
 621
 622         StateSystemBackendType backend = getBackendType();
 623         properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesBackend), backend.name());
 624         switch (backend) {
 625         case FULL:
 626         case PARTIAL:
 627             File htFile = getSsFile();
 628             if (htFile != null) {
 629                 if (htFile.exists()) {
 630                     properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesFileSize), FileUtils.byteCountToDisplaySize(htFile.length()));
 631                 } else {
 632                     properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesFileSize), NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesAnalysisNotExecuted));
 633                 }
 634             }
 635             break;
 636         case INMEM:
 637         case NULL:
 638         default:
 639             break;
 640
 641         }
 642         return properties;
 643     }
 644 }