tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/statesystem/TmfStateSystemAnalysisModule.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2015 École Polytechnique de Montréal
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Geneviève Bastien - Initial API and implementation
  11  *   Bernd Hufmann - Integrated history builder functionality
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.statesystem;
  15
  16 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNull;
  17
  18 import java.io.File;
  19 import java.io.IOException;
  20 import java.util.Collections;
  21 import java.util.HashMap;
  22 import java.util.Map;
  23 import java.util.concurrent.CountDownLatch;
  24
  25 import org.apache.commons.io.FileUtils;
  26 import org.eclipse.core.runtime.IProgressMonitor;
  27 import org.eclipse.core.runtime.IStatus;
  28 import org.eclipse.core.runtime.NullProgressMonitor;
  29 import org.eclipse.jdt.annotation.NonNull;
  30 import org.eclipse.jdt.annotation.Nullable;
  31 import org.eclipse.tracecompass.common.core.NonNullUtils;
  32 import org.eclipse.tracecompass.internal.tmf.core.statesystem.backends.partial.PartialHistoryBackend;
  33 import org.eclipse.tracecompass.internal.tmf.core.statesystem.backends.partial.PartialStateSystem;
  34 import org.eclipse.tracecompass.statesystem.core.ITmfStateSystem;
  35 import org.eclipse.tracecompass.statesystem.core.ITmfStateSystemBuilder;
  36 import org.eclipse.tracecompass.statesystem.core.StateSystemFactory;
  37 import org.eclipse.tracecompass.statesystem.core.backend.IStateHistoryBackend;
  38 import org.eclipse.tracecompass.statesystem.core.backend.StateHistoryBackendFactory;
  39 import org.eclipse.tracecompass.tmf.core.analysis.TmfAbstractAnalysisModule;
  40 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  41 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  42 import org.eclipse.tracecompass.tmf.core.project.model.ITmfPropertiesProvider;
  43 import org.eclipse.tracecompass.tmf.core.request.ITmfEventRequest;
  44 import org.eclipse.tracecompass.tmf.core.request.TmfEventRequest;
  45 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
  46 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
  47 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
  48 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
  49 import org.eclipse.tracecompass.tmf.core.trace.ITmfTraceCompleteness;
  50 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceManager;
  51 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  52 import org.eclipse.tracecompass.tmf.core.trace.experiment.TmfExperiment;
  53
  54 /**
  55  * Abstract analysis module to generate a state system. It is a base class that
  56  * can be used as a shortcut by analysis who just need to build a single state
  57  * system with a state provider.
  58  *
  59  * Analysis implementing this class should only need to provide a state system
  60  * and optionally a backend (default to NULL) and, if required, a filename
  61  * (defaults to the analysis'ID)
  62  *
  63  * @author Geneviève Bastien
  64  */
  65 public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisModule
  66         implements ITmfAnalysisModuleWithStateSystems, ITmfPropertiesProvider {
  67
  68     private static final String EXTENSION = ".ht"; //$NON-NLS-1$
  69
  70     private final CountDownLatch fInitialized = new CountDownLatch(1);
  71     private final Object fRequestSyncObj = new Object();
  72
  73     @Nullable private ITmfStateSystemBuilder fStateSystem;
  74     @Nullable private ITmfStateProvider fStateProvider;
  75     @Nullable private IStateHistoryBackend fHtBackend;
  76     @Nullable private ITmfEventRequest fRequest;
  77     @Nullable private TmfTimeRange fTimeRange = null;
  78
  79     private int fNbRead = 0;
  80     private boolean fInitializationSucceeded;
  81
  82     /**
  83      * State system backend types
  84      *
  85      * @author Geneviève Bastien
  86      */
  87     protected enum StateSystemBackendType {
  88         /** Full history in file */
  89         FULL,
  90         /** In memory state system */
  91         INMEM,
  92         /** Null history */
  93         NULL,
  94         /** State system backed with partial history */
  95         PARTIAL
  96     }
  97
  98     /**
  99      * Retrieve a state system belonging to trace, by passing the ID of the
 100      * relevant analysis module.
 101      *
 102      * This will start the execution of the analysis module, and start the
 103      * construction of the state system, if needed.
 104      *
 105      * @param trace
 106      *            The trace for which you want the state system
 107      * @param moduleId
 108      *            The ID of the state system analysis module
 109      * @return The state system, or null if there was no match or the module was
 110      *         not initialized correctly
 111      */
 112     public static @Nullable ITmfStateSystem getStateSystem(ITmfTrace trace, String moduleId) {
 113         TmfStateSystemAnalysisModule module =
 114                 TmfTraceUtils.getAnalysisModuleOfClass(trace, TmfStateSystemAnalysisModule.class, moduleId);
 115         if (module != null) {
 116             ITmfStateSystem ss = module.getStateSystem();
 117             if (ss != null) {
 118                 return ss;
 119             }
 120             IStatus status = module.schedule();
 121             if (status.isOK()) {
 122                 return module.waitForInitialization() ? module.getStateSystem() : null;
 123             }
 124         }
 125         return null;
 126     }
 127
 128     /**
 129      * Get the state provider for this analysis module
 130      *
 131      * @return the state provider
 132      */
 133     protected abstract ITmfStateProvider createStateProvider();
 134
 135     /**
 136      * Get the state system backend type used by this module
 137      *
 138      * @return The {@link StateSystemBackendType}
 139      */
 140     protected StateSystemBackendType getBackendType() {
 141         /* Using full history by default, sub-classes can override */
 142         return StateSystemBackendType.FULL;
 143     }
 144
 145     /**
 146      * Get the supplementary file name where to save this state system. The
 147      * default is the ID of the analysis followed by the extension.
 148      *
 149      * @return The supplementary file name
 150      */
 151     protected String getSsFileName() {
 152         return getId() + EXTENSION;
 153     }
 154
 155     /**
 156      * Get the state system generated by this analysis, or null if it is not yet
 157      * created.
 158      *
 159      * @return The state system
 160      */
 161     @Nullable
 162     public ITmfStateSystem getStateSystem() {
 163         return fStateSystem;
 164     }
 165
 166     /**
 167      * @since 2.0
 168      */
 169     @Override
 170     public boolean waitForInitialization() {
 171         try {
 172             fInitialized.await();
 173         } catch (InterruptedException e) {
 174             return false;
 175         }
 176         return fInitializationSucceeded;
 177     }
 178
 179     // ------------------------------------------------------------------------
 180     // TmfAbstractAnalysisModule
 181     // ------------------------------------------------------------------------
 182
 183     private @Nullable File getSsFile() {
 184         ITmfTrace trace = getTrace();
 185         if (trace == null) {
 186             return null;
 187         }
 188         String directory = TmfTraceManager.getSupplementaryFileDir(trace);
 189         File htFile = new File(directory + getSsFileName());
 190         return htFile;
 191     }
 192
 193     @Override
 194     protected boolean executeAnalysis(@Nullable final  IProgressMonitor monitor) {
 195         IProgressMonitor mon = (monitor == null ? new NullProgressMonitor() : monitor);
 196         final ITmfStateProvider provider = createStateProvider();
 197
 198         String id = getId();
 199
 200         /* FIXME: State systems should make use of the monitor, to be cancelled */
 201         try {
 202             /* Get the state system according to backend */
 203             StateSystemBackendType backend = getBackendType();
 204
 205
 206             ITmfTrace trace = getTrace();
 207             if (trace == null) {
 208                 // Analysis was cancelled in the meantime
 209                 analysisReady(false);
 210                 return false;
 211             }
 212             switch (backend) {
 213             case FULL: {
 214                 File htFile = getSsFile();
 215                 if (htFile == null) {
 216                     return false;
 217                 }
 218                 createFullHistory(id, provider, htFile);
 219             }
 220                 break;
 221             case PARTIAL: {
 222                 File htFile = getSsFile();
 223                 if (htFile == null) {
 224                     return false;
 225                 }
 226                 createPartialHistory(id, provider, htFile);
 227             }
 228                 break;
 229             case INMEM:
 230                 createInMemoryHistory(id, provider);
 231                 break;
 232             case NULL:
 233                 createNullHistory(id, provider);
 234                 break;
 235             default:
 236                 break;
 237             }
 238         } catch (TmfTraceException e) {
 239             analysisReady(false);
 240             return false;
 241         }
 242         return !mon.isCanceled();
 243     }
 244
 245     /**
 246      * Make the module available and set whether the initialization succeeded or
 247      * not. If not, no state system is available and
 248      * {@link #waitForInitialization()} should return false.
 249      *
 250      * @param success
 251      *            True if the initialization succeeded, false otherwise
 252      */
 253     private void analysisReady(boolean succeeded) {
 254         fInitializationSucceeded = succeeded;
 255         fInitialized.countDown();
 256     }
 257
 258     @Override
 259     protected void canceling() {
 260         ITmfEventRequest req = fRequest;
 261         if ((req != null) && (!req.isCompleted())) {
 262             req.cancel();
 263         }
 264     }
 265
 266     @Override
 267     public void dispose() {
 268         super.dispose();
 269         if (fStateSystem != null) {
 270             fStateSystem.dispose();
 271         }
 272     }
 273
 274     // ------------------------------------------------------------------------
 275     // History creation methods
 276     // ------------------------------------------------------------------------
 277
 278     /*
 279      * Load the history file matching the target trace. If the file already
 280      * exists, it will be opened directly. If not, it will be created from
 281      * scratch.
 282      */
 283     private void createFullHistory(String id, ITmfStateProvider provider, File htFile) throws TmfTraceException {
 284
 285         /* If the target file already exists, do not rebuild it uselessly */
 286         // TODO for now we assume it's complete. Might be a good idea to check
 287         // at least if its range matches the trace's range.
 288
 289         if (htFile.exists()) {
 290            /* Load an existing history */
 291             final int version = provider.getVersion();
 292             try {
 293                 IStateHistoryBackend backend = StateHistoryBackendFactory.createHistoryTreeBackendExistingFile(
 294                         id, htFile, version);
 295                 fHtBackend = backend;
 296                 fStateSystem = StateSystemFactory.newStateSystem(backend, false);
 297                 analysisReady(true);
 298                 return;
 299             } catch (IOException e) {
 300                 /*
 301                  * There was an error opening the existing file. Perhaps it was
 302                  * corrupted, perhaps it's an old version? We'll just
 303                  * fall-through and try to build a new one from scratch instead.
 304                  */
 305             }
 306         }
 307
 308         /* Size of the blocking queue to use when building a state history */
 309         final int QUEUE_SIZE = 10000;
 310
 311         try {
 312             IStateHistoryBackend backend = StateHistoryBackendFactory.createHistoryTreeBackendNewFile(
 313                     id, htFile, provider.getVersion(), provider.getStartTime(), QUEUE_SIZE);
 314             fHtBackend = backend;
 315             fStateSystem = StateSystemFactory.newStateSystem(backend);
 316             provider.assignTargetStateSystem(fStateSystem);
 317             build(provider);
 318         } catch (IOException e) {
 319             /*
 320              * If it fails here however, it means there was a problem writing to
 321              * the disk, so throw a real exception this time.
 322              */
 323             throw new TmfTraceException(e.toString(), e);
 324         }
 325     }
 326
 327     /*
 328      * Create a new state system backed with a partial history. A partial
 329      * history is similar to a "full" one (which you get with
 330      * {@link #newFullHistory}), except that the file on disk is much smaller,
 331      * but queries are a bit slower.
 332      *
 333      * Also note that single-queries are implemented using a full-query
 334      * underneath, (which are much slower), so this might not be a good fit for
 335      * a use case where you have to do lots of single queries.
 336      */
 337     private void createPartialHistory(String id, ITmfStateProvider provider, File htPartialFile)
 338             throws TmfTraceException {
 339         /*
 340          * The order of initializations is very tricky (but very important!)
 341          * here. We need to follow this pattern:
 342          * (1 is done before the call to this method)
 343          *
 344          * 1- Instantiate realStateProvider
 345          * 2- Instantiate realBackend
 346          * 3- Instantiate partialBackend, with prereqs:
 347          *  3a- Instantiate partialProvider, via realProvider.getNew()
 348          *  3b- Instantiate nullBackend (partialSS's backend)
 349          *  3c- Instantiate partialSS
 350          *  3d- partialProvider.assignSS(partialSS)
 351          * 4- Instantiate realSS
 352          * 5- partialSS.assignUpstream(realSS)
 353          * 6- realProvider.assignSS(realSS)
 354          * 7- Call HistoryBuilder(realProvider, realSS, partialBackend) to build the thing.
 355          */
 356
 357         /* Size of the blocking queue to use when building a state history */
 358         final int QUEUE_SIZE = 10000;
 359
 360         final long granularity = 50000;
 361
 362         /* 2 */
 363         IStateHistoryBackend realBackend = null;
 364         try {
 365             realBackend = StateHistoryBackendFactory.createHistoryTreeBackendNewFile(
 366                     id, htPartialFile, provider.getVersion(), provider.getStartTime(), QUEUE_SIZE);
 367         } catch (IOException e) {
 368             throw new TmfTraceException(e.toString(), e);
 369         }
 370
 371         /* 3a */
 372         ITmfStateProvider partialProvider = provider.getNewInstance();
 373
 374         /* 3b-3c, constructor automatically uses a NullBackend */
 375         PartialStateSystem pss = new PartialStateSystem();
 376
 377         /* 3d */
 378         partialProvider.assignTargetStateSystem(pss);
 379
 380         /* 3 */
 381         IStateHistoryBackend partialBackend = new PartialHistoryBackend(id + ".partial", partialProvider, pss, realBackend, granularity); //$NON-NLS-1$
 382
 383         /* 4 */
 384         @SuppressWarnings("restriction")
 385         org.eclipse.tracecompass.internal.statesystem.core.StateSystem realSS =
 386         (org.eclipse.tracecompass.internal.statesystem.core.StateSystem) StateSystemFactory.newStateSystem(partialBackend);
 387
 388         /* 5 */
 389         pss.assignUpstream(realSS);
 390
 391         /* 6 */
 392         provider.assignTargetStateSystem(realSS);
 393
 394         /* 7 */
 395         fHtBackend = partialBackend;
 396         fStateSystem = realSS;
 397
 398         build(provider);
 399     }
 400
 401     /*
 402      * Create a new state system using a null history back-end. This means that
 403      * no history intervals will be saved anywhere, and as such only
 404      * {@link ITmfStateSystem#queryOngoingState} will be available.
 405      */
 406     private void createNullHistory(String id, ITmfStateProvider provider) {
 407         IStateHistoryBackend backend = StateHistoryBackendFactory.createNullBackend(id);
 408         fHtBackend = backend;
 409         fStateSystem = StateSystemFactory.newStateSystem(backend);
 410         provider.assignTargetStateSystem(fStateSystem);
 411         build(provider);
 412     }
 413
 414     /*
 415      * Create a new state system using in-memory interval storage. This should
 416      * only be done for very small state system, and will be naturally limited
 417      * to 2^31 intervals.
 418      */
 419     private void createInMemoryHistory(String id, ITmfStateProvider provider) {
 420         IStateHistoryBackend backend = StateHistoryBackendFactory.createInMemoryBackend(id, provider.getStartTime());
 421         fHtBackend = backend;
 422         fStateSystem = StateSystemFactory.newStateSystem(backend);
 423         provider.assignTargetStateSystem(fStateSystem);
 424         build(provider);
 425     }
 426
 427     private void disposeProvider(boolean deleteFiles) {
 428         ITmfStateProvider provider = fStateProvider;
 429         if (provider != null) {
 430             provider.dispose();
 431         }
 432         if (deleteFiles && (fHtBackend != null)) {
 433             fHtBackend.removeFiles();
 434         }
 435     }
 436
 437     private void build(ITmfStateProvider provider) {
 438         if ((fStateSystem == null) || (fHtBackend == null)) {
 439             throw new IllegalArgumentException();
 440         }
 441
 442         ITmfEventRequest request = fRequest;
 443         if ((request != null) && (!request.isCompleted())) {
 444             request.cancel();
 445         }
 446
 447         fTimeRange = TmfTimeRange.ETERNITY;
 448         final ITmfTrace trace = provider.getTrace();
 449         if (!isCompleteTrace(trace)) {
 450             fTimeRange = trace.getTimeRange();
 451         }
 452
 453         fStateProvider = provider;
 454         synchronized (fRequestSyncObj) {
 455             startRequest();
 456         }
 457
 458         /*
 459          * The state system object is now created, we can consider this module
 460          * "initialized" (components can retrieve it and start doing queries).
 461          */
 462         analysisReady(true);
 463
 464         /*
 465          * Block the executeAnalysis() construction is complete (so that the
 466          * progress monitor displays that it is running).
 467          */
 468         try {
 469             if (fRequest != null) {
 470                 fRequest.waitForCompletion();
 471             }
 472         } catch (InterruptedException e) {
 473              e.printStackTrace();
 474         }
 475     }
 476
 477     private class StateSystemEventRequest extends TmfEventRequest {
 478         private final ITmfStateProvider sci;
 479         private final ITmfTrace trace;
 480
 481         public StateSystemEventRequest(ITmfStateProvider sp, TmfTimeRange timeRange, int index) {
 482             super(ITmfEvent.class,
 483                     timeRange,
 484                     index,
 485                     ITmfEventRequest.ALL_DATA,
 486                     ITmfEventRequest.ExecutionType.BACKGROUND);
 487             this.sci = sp;
 488
 489             // sci.getTrace() will eventually return a @NonNull
 490             trace = checkNotNull(sci.getTrace());
 491
 492         }
 493
 494         @Override
 495         public void handleData(final ITmfEvent event) {
 496             super.handleData(event);
 497             if (event.getTrace() == trace) {
 498                 sci.processEvent(event);
 499             } else if (trace instanceof TmfExperiment) {
 500                 /*
 501                  * If the request is for an experiment, check if the event is
 502                  * from one of the child trace
 503                  */
 504                 for (ITmfTrace childTrace : ((TmfExperiment) trace).getTraces()) {
 505                     if (childTrace == event.getTrace()) {
 506                         sci.processEvent(event);
 507                     }
 508                 }
 509             }
 510         }
 511
 512         @Override
 513         public void handleSuccess() {
 514             super.handleSuccess();
 515             if (isCompleteTrace(trace)) {
 516                 disposeProvider(false);
 517             } else {
 518                 fNbRead += getNbRead();
 519                 synchronized (fRequestSyncObj) {
 520                     final TmfTimeRange timeRange = fTimeRange;
 521                     if (timeRange != null) {
 522                         if (getRange().getEndTime().getValue() < timeRange.getEndTime().getValue()) {
 523                             startRequest();
 524                         }
 525                     }
 526                 }
 527             }
 528         }
 529
 530         @Override
 531         public void handleCancel() {
 532             super.handleCancel();
 533             if (isCompleteTrace(trace)) {
 534                 disposeProvider(true);
 535             }
 536         }
 537
 538         @Override
 539         public void handleFailure() {
 540             super.handleFailure();
 541             disposeProvider(true);
 542         }
 543     }
 544
 545     // ------------------------------------------------------------------------
 546     // ITmfAnalysisModuleWithStateSystems
 547     // ------------------------------------------------------------------------
 548
 549     @Override
 550     @Nullable
 551     public ITmfStateSystem getStateSystem(String id) {
 552         if (id.equals(getId())) {
 553             return fStateSystem;
 554         }
 555         return null;
 556     }
 557
 558     @Override
 559     public @NonNull Iterable<@NonNull ITmfStateSystem> getStateSystems() {
 560         ITmfStateSystemBuilder stateSystem = fStateSystem;
 561         if (stateSystem == null) {
 562             return Collections.EMPTY_SET;
 563         }
 564         return Collections.singleton(stateSystem);
 565     }
 566
 567     /**
 568      * Signal handler for the TmfTraceRangeUpdatedSignal signal
 569      *
 570      * @param signal The incoming signal
 571      */
 572     @TmfSignalHandler
 573     public void traceRangeUpdated(final TmfTraceRangeUpdatedSignal signal) {
 574         fTimeRange = signal.getRange();
 575         ITmfStateProvider stateProvider = fStateProvider;
 576         synchronized (fRequestSyncObj) {
 577             if (signal.getTrace() == getTrace() && stateProvider != null && stateProvider.getAssignedStateSystem() != null) {
 578                 ITmfEventRequest request = fRequest;
 579                 if ((request == null) || request.isCompleted()) {
 580                     startRequest();
 581                 }
 582             }
 583         }
 584     }
 585
 586     private void startRequest() {
 587         ITmfStateProvider stateProvider = fStateProvider;
 588         TmfTimeRange timeRange = fTimeRange;
 589         if (stateProvider == null || timeRange == null) {
 590             return;
 591         }
 592         ITmfEventRequest request = new StateSystemEventRequest(stateProvider, timeRange, fNbRead);
 593         stateProvider.getTrace().sendRequest(request);
 594         fRequest = request;
 595     }
 596
 597     private static boolean isCompleteTrace(ITmfTrace trace) {
 598         return !(trace instanceof ITmfTraceCompleteness) || ((ITmfTraceCompleteness) trace).isComplete();
 599     }
 600
 601     // ------------------------------------------------------------------------
 602     // ITmfPropertiesProvider
 603     // ------------------------------------------------------------------------
 604
 605     /**
 606      * @since 2.0
 607      */
 608     @Override
 609     public @NonNull Map<@NonNull String, @NonNull String> getProperties() {
 610         Map<@NonNull String, @NonNull String> properties = new HashMap<>();
 611
 612         StateSystemBackendType backend = getBackendType();
 613         properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesBackend), NonNullUtils.checkNotNull(backend.name()));
 614         switch (backend) {
 615         case FULL:
 616         case PARTIAL:
 617             File htFile = getSsFile();
 618             if (htFile != null) {
 619                 if (htFile.exists()) {
 620                     properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesFileSize), FileUtils.byteCountToDisplaySize(htFile.length()));
 621                 } else {
 622                     properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesFileSize), NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesAnalysisNotExecuted));
 623                 }
 624             }
 625             break;
 626         case INMEM:
 627         case NULL:
 628         default:
 629             break;
 630
 631         }
 632         return properties;
 633     }
 634 }