tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/statesystem/TmfStateSystemAnalysisModule.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2015 École Polytechnique de Montréal
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Geneviève Bastien - Initial API and implementation
  11  *   Bernd Hufmann - Integrated history builder functionality
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.statesystem;
  15
  16 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNull;
  17
  18 import java.io.File;
  19 import java.io.IOException;
  20 import java.util.Collections;
  21 import java.util.HashMap;
  22 import java.util.Map;
  23 import java.util.concurrent.CountDownLatch;
  24
  25 import org.apache.commons.io.FileUtils;
  26 import org.eclipse.core.runtime.IProgressMonitor;
  27 import org.eclipse.core.runtime.IStatus;
  28 import org.eclipse.core.runtime.NullProgressMonitor;
  29 import org.eclipse.jdt.annotation.NonNull;
  30 import org.eclipse.jdt.annotation.Nullable;
  31 import org.eclipse.tracecompass.common.core.NonNullUtils;
  32 import org.eclipse.tracecompass.internal.tmf.core.statesystem.backends.partial.PartialHistoryBackend;
  33 import org.eclipse.tracecompass.internal.tmf.core.statesystem.backends.partial.PartialStateSystem;
  34 import org.eclipse.tracecompass.statesystem.core.ITmfStateSystem;
  35 import org.eclipse.tracecompass.statesystem.core.ITmfStateSystemBuilder;
  36 import org.eclipse.tracecompass.statesystem.core.StateSystemFactory;
  37 import org.eclipse.tracecompass.statesystem.core.backend.IStateHistoryBackend;
  38 import org.eclipse.tracecompass.statesystem.core.backend.StateHistoryBackendFactory;
  39 import org.eclipse.tracecompass.tmf.core.analysis.TmfAbstractAnalysisModule;
  40 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  41 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  42 import org.eclipse.tracecompass.tmf.core.project.model.ITmfPropertiesProvider;
  43 import org.eclipse.tracecompass.tmf.core.request.ITmfEventRequest;
  44 import org.eclipse.tracecompass.tmf.core.request.TmfEventRequest;
  45 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
  46 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
  47 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
  48 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
  49 import org.eclipse.tracecompass.tmf.core.trace.ITmfTraceCompleteness;
  50 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceManager;
  51 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  52 import org.eclipse.tracecompass.tmf.core.trace.experiment.TmfExperiment;
  53
  54 /**
  55  * Abstract analysis module to generate a state system. It is a base class that
  56  * can be used as a shortcut by analysis who just need to build a single state
  57  * system with a state provider.
  58  *
  59  * Analysis implementing this class should only need to provide a state system
  60  * and optionally a backend (default to NULL) and, if required, a filename
  61  * (defaults to the analysis'ID)
  62  *
  63  * @author Geneviève Bastien
  64  */
  65 public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisModule
  66         implements ITmfAnalysisModuleWithStateSystems, ITmfPropertiesProvider {
  67
  68     private static final String EXTENSION = ".ht"; //$NON-NLS-1$
  69
  70     private final CountDownLatch fInitialized = new CountDownLatch(1);
  71     private final Object fRequestSyncObj = new Object();
  72
  73     @Nullable private ITmfStateSystemBuilder fStateSystem;
  74     @Nullable private ITmfStateProvider fStateProvider;
  75     @Nullable private IStateHistoryBackend fHtBackend;
  76     @Nullable private ITmfEventRequest fRequest;
  77     @Nullable private TmfTimeRange fTimeRange = null;
  78
  79     private int fNbRead = 0;
  80
  81     /**
  82      * State system backend types
  83      *
  84      * @author Geneviève Bastien
  85      */
  86     protected enum StateSystemBackendType {
  87         /** Full history in file */
  88         FULL,
  89         /** In memory state system */
  90         INMEM,
  91         /** Null history */
  92         NULL,
  93         /** State system backed with partial history */
  94         PARTIAL
  95     }
  96
  97     /**
  98      * Retrieve a state system belonging to trace, by passing the ID of the
  99      * relevant analysis module.
 100      *
 101      * This will start the execution of the analysis module, and start the
 102      * construction of the state system, if needed.
 103      *
 104      * @param trace
 105      *            The trace for which you want the state system
 106      * @param moduleId
 107      *            The ID of the state system analysis module
 108      * @return The state system, or null if there was no match
 109      */
 110     public static @Nullable ITmfStateSystem getStateSystem(ITmfTrace trace, String moduleId) {
 111         TmfStateSystemAnalysisModule module =
 112                 TmfTraceUtils.getAnalysisModuleOfClass(trace, TmfStateSystemAnalysisModule.class, moduleId);
 113         if (module != null) {
 114             ITmfStateSystem ss = module.getStateSystem();
 115             if (ss != null) {
 116                 return ss;
 117             }
 118             IStatus status = module.schedule();
 119             if (status.isOK()) {
 120                 module.waitForInitialization();
 121                 return module.getStateSystem();
 122             }
 123         }
 124         return null;
 125     }
 126
 127     /**
 128      * Get the state provider for this analysis module
 129      *
 130      * @return the state provider
 131      */
 132     protected abstract ITmfStateProvider createStateProvider();
 133
 134     /**
 135      * Get the state system backend type used by this module
 136      *
 137      * @return The {@link StateSystemBackendType}
 138      */
 139     protected StateSystemBackendType getBackendType() {
 140         /* Using full history by default, sub-classes can override */
 141         return StateSystemBackendType.FULL;
 142     }
 143
 144     /**
 145      * Get the supplementary file name where to save this state system. The
 146      * default is the ID of the analysis followed by the extension.
 147      *
 148      * @return The supplementary file name
 149      */
 150     protected String getSsFileName() {
 151         return getId() + EXTENSION;
 152     }
 153
 154     /**
 155      * Get the state system generated by this analysis, or null if it is not yet
 156      * created.
 157      *
 158      * @return The state system
 159      */
 160     @Nullable
 161     public ITmfStateSystem getStateSystem() {
 162         return fStateSystem;
 163     }
 164
 165     /**
 166      * Block the calling thread until the analysis module has been initialized.
 167      * After this method returns, {@link #getStateSystem()} should not return
 168      * null anymore.
 169      */
 170     public void waitForInitialization() {
 171         try {
 172             fInitialized.await();
 173         } catch (InterruptedException e) {}
 174     }
 175
 176     // ------------------------------------------------------------------------
 177     // TmfAbstractAnalysisModule
 178     // ------------------------------------------------------------------------
 179
 180     private @Nullable File getSsFile() {
 181         ITmfTrace trace = getTrace();
 182         if (trace == null) {
 183             return null;
 184         }
 185         String directory = TmfTraceManager.getSupplementaryFileDir(trace);
 186         File htFile = new File(directory + getSsFileName());
 187         return htFile;
 188     }
 189
 190     @Override
 191     protected boolean executeAnalysis(@Nullable final  IProgressMonitor monitor) {
 192         IProgressMonitor mon = (monitor == null ? new NullProgressMonitor() : monitor);
 193         final ITmfStateProvider provider = createStateProvider();
 194
 195         String id = getId();
 196
 197         /* FIXME: State systems should make use of the monitor, to be cancelled */
 198         try {
 199             /* Get the state system according to backend */
 200             StateSystemBackendType backend = getBackendType();
 201
 202
 203             ITmfTrace trace = getTrace();
 204             if (trace == null) {
 205                 // Analysis was cancelled in the meantime
 206                 fInitialized.countDown();
 207                 return false;
 208             }
 209             switch (backend) {
 210             case FULL: {
 211                 File htFile = getSsFile();
 212                 if (htFile == null) {
 213                     return false;
 214                 }
 215                 createFullHistory(id, provider, htFile);
 216             }
 217                 break;
 218             case PARTIAL: {
 219                 File htFile = getSsFile();
 220                 if (htFile == null) {
 221                     return false;
 222                 }
 223                 createPartialHistory(id, provider, htFile);
 224             }
 225                 break;
 226             case INMEM:
 227                 createInMemoryHistory(id, provider);
 228                 break;
 229             case NULL:
 230                 createNullHistory(id, provider);
 231                 break;
 232             default:
 233                 break;
 234             }
 235         } catch (TmfTraceException e) {
 236             fInitialized.countDown();
 237             return false;
 238         }
 239         return !mon.isCanceled();
 240     }
 241
 242     @Override
 243     protected void canceling() {
 244         ITmfEventRequest req = fRequest;
 245         if ((req != null) && (!req.isCompleted())) {
 246             req.cancel();
 247         }
 248     }
 249
 250     @Override
 251     public void dispose() {
 252         super.dispose();
 253         if (fStateSystem != null) {
 254             fStateSystem.dispose();
 255         }
 256     }
 257
 258     // ------------------------------------------------------------------------
 259     // History creation methods
 260     // ------------------------------------------------------------------------
 261
 262     /*
 263      * Load the history file matching the target trace. If the file already
 264      * exists, it will be opened directly. If not, it will be created from
 265      * scratch.
 266      */
 267     private void createFullHistory(String id, ITmfStateProvider provider, File htFile) throws TmfTraceException {
 268
 269         /* If the target file already exists, do not rebuild it uselessly */
 270         // TODO for now we assume it's complete. Might be a good idea to check
 271         // at least if its range matches the trace's range.
 272
 273         if (htFile.exists()) {
 274            /* Load an existing history */
 275             final int version = provider.getVersion();
 276             try {
 277                 IStateHistoryBackend backend = StateHistoryBackendFactory.createHistoryTreeBackendExistingFile(
 278                         id, htFile, version);
 279                 fHtBackend = backend;
 280                 fStateSystem = StateSystemFactory.newStateSystem(backend, false);
 281                 fInitialized.countDown();
 282                 return;
 283             } catch (IOException e) {
 284                 /*
 285                  * There was an error opening the existing file. Perhaps it was
 286                  * corrupted, perhaps it's an old version? We'll just
 287                  * fall-through and try to build a new one from scratch instead.
 288                  */
 289             }
 290         }
 291
 292         /* Size of the blocking queue to use when building a state history */
 293         final int QUEUE_SIZE = 10000;
 294
 295         try {
 296             IStateHistoryBackend backend = StateHistoryBackendFactory.createHistoryTreeBackendNewFile(
 297                     id, htFile, provider.getVersion(), provider.getStartTime(), QUEUE_SIZE);
 298             fHtBackend = backend;
 299             fStateSystem = StateSystemFactory.newStateSystem(backend);
 300             provider.assignTargetStateSystem(fStateSystem);
 301             build(provider);
 302         } catch (IOException e) {
 303             /*
 304              * If it fails here however, it means there was a problem writing to
 305              * the disk, so throw a real exception this time.
 306              */
 307             throw new TmfTraceException(e.toString(), e);
 308         }
 309     }
 310
 311     /*
 312      * Create a new state system backed with a partial history. A partial
 313      * history is similar to a "full" one (which you get with
 314      * {@link #newFullHistory}), except that the file on disk is much smaller,
 315      * but queries are a bit slower.
 316      *
 317      * Also note that single-queries are implemented using a full-query
 318      * underneath, (which are much slower), so this might not be a good fit for
 319      * a use case where you have to do lots of single queries.
 320      */
 321     private void createPartialHistory(String id, ITmfStateProvider provider, File htPartialFile)
 322             throws TmfTraceException {
 323         /*
 324          * The order of initializations is very tricky (but very important!)
 325          * here. We need to follow this pattern:
 326          * (1 is done before the call to this method)
 327          *
 328          * 1- Instantiate realStateProvider
 329          * 2- Instantiate realBackend
 330          * 3- Instantiate partialBackend, with prereqs:
 331          *  3a- Instantiate partialProvider, via realProvider.getNew()
 332          *  3b- Instantiate nullBackend (partialSS's backend)
 333          *  3c- Instantiate partialSS
 334          *  3d- partialProvider.assignSS(partialSS)
 335          * 4- Instantiate realSS
 336          * 5- partialSS.assignUpstream(realSS)
 337          * 6- realProvider.assignSS(realSS)
 338          * 7- Call HistoryBuilder(realProvider, realSS, partialBackend) to build the thing.
 339          */
 340
 341         /* Size of the blocking queue to use when building a state history */
 342         final int QUEUE_SIZE = 10000;
 343
 344         final long granularity = 50000;
 345
 346         /* 2 */
 347         IStateHistoryBackend realBackend = null;
 348         try {
 349             realBackend = StateHistoryBackendFactory.createHistoryTreeBackendNewFile(
 350                     id, htPartialFile, provider.getVersion(), provider.getStartTime(), QUEUE_SIZE);
 351         } catch (IOException e) {
 352             throw new TmfTraceException(e.toString(), e);
 353         }
 354
 355         /* 3a */
 356         ITmfStateProvider partialProvider = provider.getNewInstance();
 357
 358         /* 3b-3c, constructor automatically uses a NullBackend */
 359         PartialStateSystem pss = new PartialStateSystem();
 360
 361         /* 3d */
 362         partialProvider.assignTargetStateSystem(pss);
 363
 364         /* 3 */
 365         IStateHistoryBackend partialBackend = new PartialHistoryBackend(id + ".partial", partialProvider, pss, realBackend, granularity); //$NON-NLS-1$
 366
 367         /* 4 */
 368         @SuppressWarnings("restriction")
 369         org.eclipse.tracecompass.internal.statesystem.core.StateSystem realSS =
 370         (org.eclipse.tracecompass.internal.statesystem.core.StateSystem) StateSystemFactory.newStateSystem(partialBackend);
 371
 372         /* 5 */
 373         pss.assignUpstream(realSS);
 374
 375         /* 6 */
 376         provider.assignTargetStateSystem(realSS);
 377
 378         /* 7 */
 379         fHtBackend = partialBackend;
 380         fStateSystem = realSS;
 381
 382         build(provider);
 383     }
 384
 385     /*
 386      * Create a new state system using a null history back-end. This means that
 387      * no history intervals will be saved anywhere, and as such only
 388      * {@link ITmfStateSystem#queryOngoingState} will be available.
 389      */
 390     private void createNullHistory(String id, ITmfStateProvider provider) {
 391         IStateHistoryBackend backend = StateHistoryBackendFactory.createNullBackend(id);
 392         fHtBackend = backend;
 393         fStateSystem = StateSystemFactory.newStateSystem(backend);
 394         provider.assignTargetStateSystem(fStateSystem);
 395         build(provider);
 396     }
 397
 398     /*
 399      * Create a new state system using in-memory interval storage. This should
 400      * only be done for very small state system, and will be naturally limited
 401      * to 2^31 intervals.
 402      */
 403     private void createInMemoryHistory(String id, ITmfStateProvider provider) {
 404         IStateHistoryBackend backend = StateHistoryBackendFactory.createInMemoryBackend(id, provider.getStartTime());
 405         fHtBackend = backend;
 406         fStateSystem = StateSystemFactory.newStateSystem(backend);
 407         provider.assignTargetStateSystem(fStateSystem);
 408         build(provider);
 409     }
 410
 411     private void disposeProvider(boolean deleteFiles) {
 412         ITmfStateProvider provider = fStateProvider;
 413         if (provider != null) {
 414             provider.dispose();
 415         }
 416         if (deleteFiles && (fHtBackend != null)) {
 417             fHtBackend.removeFiles();
 418         }
 419     }
 420
 421     private void build(ITmfStateProvider provider) {
 422         if ((fStateSystem == null) || (fHtBackend == null)) {
 423             throw new IllegalArgumentException();
 424         }
 425
 426         ITmfEventRequest request = fRequest;
 427         if ((request != null) && (!request.isCompleted())) {
 428             request.cancel();
 429         }
 430
 431         fTimeRange = TmfTimeRange.ETERNITY;
 432         final ITmfTrace trace = provider.getTrace();
 433         if (!isCompleteTrace(trace)) {
 434             fTimeRange = trace.getTimeRange();
 435         }
 436
 437         fStateProvider = provider;
 438         synchronized (fRequestSyncObj) {
 439             startRequest();
 440         }
 441
 442         /*
 443          * The state system object is now created, we can consider this module
 444          * "initialized" (components can retrieve it and start doing queries).
 445          */
 446         fInitialized.countDown();
 447
 448         /*
 449          * Block the executeAnalysis() construction is complete (so that the
 450          * progress monitor displays that it is running).
 451          */
 452         try {
 453             if (fRequest != null) {
 454                 fRequest.waitForCompletion();
 455             }
 456         } catch (InterruptedException e) {
 457              e.printStackTrace();
 458         }
 459     }
 460
 461     private class StateSystemEventRequest extends TmfEventRequest {
 462         private final ITmfStateProvider sci;
 463         private final ITmfTrace trace;
 464
 465         public StateSystemEventRequest(ITmfStateProvider sp, TmfTimeRange timeRange, int index) {
 466             super(ITmfEvent.class,
 467                     timeRange,
 468                     index,
 469                     ITmfEventRequest.ALL_DATA,
 470                     ITmfEventRequest.ExecutionType.BACKGROUND);
 471             this.sci = sp;
 472
 473             // sci.getTrace() will eventually return a @NonNull
 474             trace = checkNotNull(sci.getTrace());
 475
 476         }
 477
 478         @Override
 479         public void handleData(final ITmfEvent event) {
 480             super.handleData(event);
 481             if (event.getTrace() == trace) {
 482                 sci.processEvent(event);
 483             } else if (trace instanceof TmfExperiment) {
 484                 /*
 485                  * If the request is for an experiment, check if the event is
 486                  * from one of the child trace
 487                  */
 488                 for (ITmfTrace childTrace : ((TmfExperiment) trace).getTraces()) {
 489                     if (childTrace == event.getTrace()) {
 490                         sci.processEvent(event);
 491                     }
 492                 }
 493             }
 494         }
 495
 496         @Override
 497         public void handleSuccess() {
 498             super.handleSuccess();
 499             if (isCompleteTrace(trace)) {
 500                 disposeProvider(false);
 501             } else {
 502                 fNbRead += getNbRead();
 503                 synchronized (fRequestSyncObj) {
 504                     final TmfTimeRange timeRange = fTimeRange;
 505                     if (timeRange != null) {
 506                         if (getRange().getEndTime().getValue() < timeRange.getEndTime().getValue()) {
 507                             startRequest();
 508                         }
 509                     }
 510                 }
 511             }
 512         }
 513
 514         @Override
 515         public void handleCancel() {
 516             super.handleCancel();
 517             if (isCompleteTrace(trace)) {
 518                 disposeProvider(true);
 519             }
 520         }
 521
 522         @Override
 523         public void handleFailure() {
 524             super.handleFailure();
 525             disposeProvider(true);
 526         }
 527     }
 528
 529     // ------------------------------------------------------------------------
 530     // ITmfAnalysisModuleWithStateSystems
 531     // ------------------------------------------------------------------------
 532
 533     @Override
 534     @Nullable
 535     public ITmfStateSystem getStateSystem(String id) {
 536         if (id.equals(getId())) {
 537             return fStateSystem;
 538         }
 539         return null;
 540     }
 541
 542     @Override
 543     public Iterable<ITmfStateSystem> getStateSystems() {
 544         ITmfStateSystemBuilder stateSystem = fStateSystem;
 545         if (stateSystem == null) {
 546             return Collections.EMPTY_SET;
 547         }
 548         return Collections.singleton(stateSystem);
 549     }
 550
 551     /**
 552      * Signal handler for the TmfTraceRangeUpdatedSignal signal
 553      *
 554      * @param signal The incoming signal
 555      */
 556     @TmfSignalHandler
 557     public void traceRangeUpdated(final TmfTraceRangeUpdatedSignal signal) {
 558         fTimeRange = signal.getRange();
 559         ITmfStateProvider stateProvider = fStateProvider;
 560         synchronized (fRequestSyncObj) {
 561             if (signal.getTrace() == getTrace() && stateProvider != null && stateProvider.getAssignedStateSystem() != null) {
 562                 ITmfEventRequest request = fRequest;
 563                 if ((request == null) || request.isCompleted()) {
 564                     startRequest();
 565                 }
 566             }
 567         }
 568     }
 569
 570     private void startRequest() {
 571         ITmfStateProvider stateProvider = fStateProvider;
 572         TmfTimeRange timeRange = fTimeRange;
 573         if (stateProvider == null || timeRange == null) {
 574             return;
 575         }
 576         ITmfEventRequest request = new StateSystemEventRequest(stateProvider, timeRange, fNbRead);
 577         stateProvider.getTrace().sendRequest(request);
 578         fRequest = request;
 579     }
 580
 581     private static boolean isCompleteTrace(ITmfTrace trace) {
 582         return !(trace instanceof ITmfTraceCompleteness) || ((ITmfTraceCompleteness) trace).isComplete();
 583     }
 584
 585     // ------------------------------------------------------------------------
 586     // ITmfPropertiesProvider
 587     // ------------------------------------------------------------------------
 588
 589     /**
 590      * @since 2.0
 591      */
 592     @Override
 593     public @NonNull Map<@NonNull String, @NonNull String> getProperties() {
 594         Map<@NonNull String, @NonNull String> properties = new HashMap<>();
 595
 596         StateSystemBackendType backend = getBackendType();
 597         properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesBackend), NonNullUtils.checkNotNull(backend.name()));
 598         switch (backend) {
 599         case FULL:
 600         case PARTIAL:
 601             File htFile = getSsFile();
 602             if (htFile != null) {
 603                 if (htFile.exists()) {
 604                     properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesFileSize), FileUtils.byteCountToDisplaySize(htFile.length()));
 605                 } else {
 606                     properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesFileSize), NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesAnalysisNotExecuted));
 607                 }
 608             }
 609             break;
 610         case INMEM:
 611         case NULL:
 612         default:
 613             break;
 614
 615         }
 616         return properties;
 617     }
 618 }