tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/statesystem/TmfStateSystemAnalysisModule.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2015 École Polytechnique de Montréal
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Geneviève Bastien - Initial API and implementation
  11  *   Bernd Hufmann - Integrated history builder functionality
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.statesystem;
  15
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.util.Collections;
  19 import java.util.Map;
  20 import java.util.concurrent.CountDownLatch;
  21
  22 import org.apache.commons.io.FileUtils;
  23 import org.eclipse.core.runtime.IProgressMonitor;
  24 import org.eclipse.core.runtime.IStatus;
  25 import org.eclipse.core.runtime.NullProgressMonitor;
  26 import org.eclipse.jdt.annotation.NonNull;
  27 import org.eclipse.jdt.annotation.Nullable;
  28 import org.eclipse.tracecompass.common.core.NonNullUtils;
  29 import org.eclipse.tracecompass.internal.tmf.core.statesystem.backends.partial.PartialHistoryBackend;
  30 import org.eclipse.tracecompass.internal.tmf.core.statesystem.backends.partial.PartialStateSystem;
  31 import org.eclipse.tracecompass.statesystem.core.ITmfStateSystem;
  32 import org.eclipse.tracecompass.statesystem.core.ITmfStateSystemBuilder;
  33 import org.eclipse.tracecompass.statesystem.core.StateSystemFactory;
  34 import org.eclipse.tracecompass.statesystem.core.backend.IStateHistoryBackend;
  35 import org.eclipse.tracecompass.statesystem.core.backend.StateHistoryBackendFactory;
  36 import org.eclipse.tracecompass.tmf.core.analysis.TmfAbstractAnalysisModule;
  37 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  38 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  39 import org.eclipse.tracecompass.tmf.core.request.ITmfEventRequest;
  40 import org.eclipse.tracecompass.tmf.core.request.TmfEventRequest;
  41 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
  42 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
  43 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
  44 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
  45 import org.eclipse.tracecompass.tmf.core.trace.ITmfTraceCompleteness;
  46 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceManager;
  47 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  48 import org.eclipse.tracecompass.tmf.core.trace.experiment.TmfExperiment;
  49
  50 /**
  51  * Abstract analysis module to generate a state system. It is a base class that
  52  * can be used as a shortcut by analysis who just need to build a single state
  53  * system with a state provider.
  54  *
  55  * Analysis implementing this class should only need to provide a state system
  56  * and optionally a backend (default to NULL) and, if required, a filename
  57  * (defaults to the analysis'ID)
  58  *
  59  * @author Geneviève Bastien
  60  */
  61 public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisModule
  62         implements ITmfAnalysisModuleWithStateSystems {
  63
  64     private static final String EXTENSION = ".ht"; //$NON-NLS-1$
  65
  66     private final CountDownLatch fInitialized = new CountDownLatch(1);
  67     private final Object fRequestSyncObj = new Object();
  68
  69     private @Nullable ITmfStateSystemBuilder fStateSystem;
  70     private @Nullable ITmfEventRequest fRequest;
  71     private @Nullable TmfTimeRange fTimeRange = null;
  72
  73     private int fNbRead = 0;
  74     private boolean fInitializationSucceeded;
  75
  76     private volatile @Nullable ITmfStateProvider fStateProvider;
  77
  78     /**
  79      * State system backend types
  80      *
  81      * @author Geneviève Bastien
  82      */
  83     protected enum StateSystemBackendType {
  84         /** Full history in file */
  85         FULL,
  86         /** In memory state system */
  87         INMEM,
  88         /** Null history */
  89         NULL,
  90         /** State system backed with partial history */
  91         PARTIAL
  92     }
  93
  94     /**
  95      * Retrieve a state system belonging to trace, by passing the ID of the
  96      * relevant analysis module.
  97      *
  98      * This will start the execution of the analysis module, and start the
  99      * construction of the state system, if needed.
 100      *
 101      * @param trace
 102      *            The trace for which you want the state system
 103      * @param moduleId
 104      *            The ID of the state system analysis module
 105      * @return The state system, or null if there was no match or the module was
 106      *         not initialized correctly
 107      */
 108     public static @Nullable ITmfStateSystem getStateSystem(ITmfTrace trace, String moduleId) {
 109         TmfStateSystemAnalysisModule module =
 110                 TmfTraceUtils.getAnalysisModuleOfClass(trace, TmfStateSystemAnalysisModule.class, moduleId);
 111         if (module != null) {
 112             ITmfStateSystem ss = module.getStateSystem();
 113             if (ss != null) {
 114                 return ss;
 115             }
 116             IStatus status = module.schedule();
 117             if (status.isOK()) {
 118                 return module.waitForInitialization() ? module.getStateSystem() : null;
 119             }
 120         }
 121         return null;
 122     }
 123
 124     /**
 125      * Get the state provider for this analysis module
 126      *
 127      * @return the state provider
 128      */
 129     protected abstract ITmfStateProvider createStateProvider();
 130
 131     /**
 132      * Get the state system backend type used by this module
 133      *
 134      * @return The {@link StateSystemBackendType}
 135      */
 136     protected StateSystemBackendType getBackendType() {
 137         /* Using full history by default, sub-classes can override */
 138         return StateSystemBackendType.FULL;
 139     }
 140
 141     /**
 142      * Get the supplementary file name where to save this state system. The
 143      * default is the ID of the analysis followed by the extension.
 144      *
 145      * @return The supplementary file name
 146      */
 147     protected String getSsFileName() {
 148         return getId() + EXTENSION;
 149     }
 150
 151     /**
 152      * Get the state system generated by this analysis, or null if it is not yet
 153      * created.
 154      *
 155      * @return The state system
 156      */
 157     @Nullable
 158     public ITmfStateSystem getStateSystem() {
 159         return fStateSystem;
 160     }
 161
 162     /**
 163      * @since 2.0
 164      */
 165     @Override
 166     public boolean waitForInitialization() {
 167         try {
 168             fInitialized.await();
 169         } catch (InterruptedException e) {
 170             return false;
 171         }
 172         return fInitializationSucceeded;
 173     }
 174
 175     /**
 176      * @since 2.0
 177      */
 178     @Override
 179     public boolean isQueryable(long ts) {
 180         /* Return true if there is no state provider available (the analysis is not being built) */
 181         ITmfStateProvider provider = fStateProvider;
 182         if (provider == null) {
 183             return true;
 184         }
 185         return ts <= provider.getLatestSafeTime();
 186     }
 187
 188     // ------------------------------------------------------------------------
 189     // TmfAbstractAnalysisModule
 190     // ------------------------------------------------------------------------
 191
 192     private @Nullable File getSsFile() {
 193         ITmfTrace trace = getTrace();
 194         if (trace == null) {
 195             return null;
 196         }
 197         String directory = TmfTraceManager.getSupplementaryFileDir(trace);
 198         File htFile = new File(directory + getSsFileName());
 199         return htFile;
 200     }
 201
 202     @Override
 203     protected boolean executeAnalysis(@Nullable final  IProgressMonitor monitor) {
 204         IProgressMonitor mon = (monitor == null ? new NullProgressMonitor() : monitor);
 205         final ITmfStateProvider provider = createStateProvider();
 206
 207         String id = getId();
 208
 209         /* FIXME: State systems should make use of the monitor, to be cancelled */
 210         try {
 211             /* Get the state system according to backend */
 212             StateSystemBackendType backend = getBackendType();
 213
 214             ITmfTrace trace = getTrace();
 215             if (trace == null) {
 216                 // Analysis was cancelled in the meantime
 217                 analysisReady(false);
 218                 return false;
 219             }
 220             switch (backend) {
 221             case FULL: {
 222                 File htFile = getSsFile();
 223                 if (htFile == null) {
 224                     return false;
 225                 }
 226                 createFullHistory(id, provider, htFile);
 227             }
 228                 break;
 229             case PARTIAL: {
 230                 File htFile = getSsFile();
 231                 if (htFile == null) {
 232                     return false;
 233                 }
 234                 createPartialHistory(id, provider, htFile);
 235             }
 236                 break;
 237             case INMEM:
 238                 createInMemoryHistory(id, provider);
 239                 break;
 240             case NULL:
 241                 createNullHistory(id, provider);
 242                 break;
 243             default:
 244                 break;
 245             }
 246         } catch (TmfTraceException e) {
 247             analysisReady(false);
 248             return false;
 249         }
 250         return !mon.isCanceled();
 251     }
 252
 253     /**
 254      * Make the module available and set whether the initialization succeeded or
 255      * not. If not, no state system is available and
 256      * {@link #waitForInitialization()} should return false.
 257      *
 258      * @param success
 259      *            True if the initialization succeeded, false otherwise
 260      */
 261     private void analysisReady(boolean succeeded) {
 262         fInitializationSucceeded = succeeded;
 263         fInitialized.countDown();
 264     }
 265
 266     @Override
 267     protected void canceling() {
 268         ITmfEventRequest req = fRequest;
 269         if ((req != null) && (!req.isCompleted())) {
 270             req.cancel();
 271         }
 272     }
 273
 274     @Override
 275     public void dispose() {
 276         super.dispose();
 277         if (fStateSystem != null) {
 278             fStateSystem.dispose();
 279         }
 280     }
 281
 282     // ------------------------------------------------------------------------
 283     // History creation methods
 284     // ------------------------------------------------------------------------
 285
 286     /*
 287      * Load the history file matching the target trace. If the file already
 288      * exists, it will be opened directly. If not, it will be created from
 289      * scratch.
 290      */
 291     private void createFullHistory(String id, ITmfStateProvider provider, File htFile) throws TmfTraceException {
 292
 293         /* If the target file already exists, do not rebuild it uselessly */
 294         // TODO for now we assume it's complete. Might be a good idea to check
 295         // at least if its range matches the trace's range.
 296
 297         if (htFile.exists()) {
 298            /* Load an existing history */
 299             final int version = provider.getVersion();
 300             try {
 301                 IStateHistoryBackend backend = StateHistoryBackendFactory.createHistoryTreeBackendExistingFile(
 302                         id, htFile, version);
 303                 fStateSystem = StateSystemFactory.newStateSystem(backend, false);
 304                 analysisReady(true);
 305                 return;
 306             } catch (IOException e) {
 307                 /*
 308                  * There was an error opening the existing file. Perhaps it was
 309                  * corrupted, perhaps it's an old version? We'll just
 310                  * fall-through and try to build a new one from scratch instead.
 311                  */
 312             }
 313         }
 314
 315         /* Size of the blocking queue to use when building a state history */
 316         final int QUEUE_SIZE = 10000;
 317
 318         try {
 319             IStateHistoryBackend backend = StateHistoryBackendFactory.createHistoryTreeBackendNewFile(
 320                     id, htFile, provider.getVersion(), provider.getStartTime(), QUEUE_SIZE);
 321             fStateSystem = StateSystemFactory.newStateSystem(backend);
 322             provider.assignTargetStateSystem(fStateSystem);
 323             build(provider);
 324         } catch (IOException e) {
 325             /*
 326              * If it fails here however, it means there was a problem writing to
 327              * the disk, so throw a real exception this time.
 328              */
 329             throw new TmfTraceException(e.toString(), e);
 330         }
 331     }
 332
 333     /*
 334      * Create a new state system backed with a partial history. A partial
 335      * history is similar to a "full" one (which you get with
 336      * {@link #newFullHistory}), except that the file on disk is much smaller,
 337      * but queries are a bit slower.
 338      *
 339      * Also note that single-queries are implemented using a full-query
 340      * underneath, (which are much slower), so this might not be a good fit for
 341      * a use case where you have to do lots of single queries.
 342      */
 343     private void createPartialHistory(String id, ITmfStateProvider provider, File htPartialFile)
 344             throws TmfTraceException {
 345         /*
 346          * The order of initializations is very tricky (but very important!)
 347          * here. We need to follow this pattern:
 348          * (1 is done before the call to this method)
 349          *
 350          * 1- Instantiate realStateProvider
 351          * 2- Instantiate realBackend
 352          * 3- Instantiate partialBackend, with prereqs:
 353          *  3a- Instantiate partialProvider, via realProvider.getNew()
 354          *  3b- Instantiate nullBackend (partialSS's backend)
 355          *  3c- Instantiate partialSS
 356          *  3d- partialProvider.assignSS(partialSS)
 357          * 4- Instantiate realSS
 358          * 5- partialSS.assignUpstream(realSS)
 359          * 6- realProvider.assignSS(realSS)
 360          * 7- Call HistoryBuilder(realProvider, realSS, partialBackend) to build the thing.
 361          */
 362
 363         /* Size of the blocking queue to use when building a state history */
 364         final int QUEUE_SIZE = 10000;
 365
 366         final long granularity = 50000;
 367
 368         /* 2 */
 369         IStateHistoryBackend realBackend = null;
 370         try {
 371             realBackend = StateHistoryBackendFactory.createHistoryTreeBackendNewFile(
 372                     id, htPartialFile, provider.getVersion(), provider.getStartTime(), QUEUE_SIZE);
 373         } catch (IOException e) {
 374             throw new TmfTraceException(e.toString(), e);
 375         }
 376
 377         /* 3a */
 378         ITmfStateProvider partialProvider = provider.getNewInstance();
 379
 380         /* 3b-3c, constructor automatically uses a NullBackend */
 381         PartialStateSystem pss = new PartialStateSystem();
 382
 383         /* 3d */
 384         partialProvider.assignTargetStateSystem(pss);
 385
 386         /* 3 */
 387         IStateHistoryBackend partialBackend = new PartialHistoryBackend(id + ".partial", partialProvider, pss, realBackend, granularity); //$NON-NLS-1$
 388
 389         /* 4 */
 390         @SuppressWarnings("restriction")
 391         org.eclipse.tracecompass.internal.statesystem.core.StateSystem realSS =
 392         (org.eclipse.tracecompass.internal.statesystem.core.StateSystem) StateSystemFactory.newStateSystem(partialBackend);
 393
 394         /* 5 */
 395         pss.assignUpstream(realSS);
 396
 397         /* 6 */
 398         provider.assignTargetStateSystem(realSS);
 399
 400         /* 7 */
 401         fStateSystem = realSS;
 402
 403         build(provider);
 404     }
 405
 406     /*
 407      * Create a new state system using a null history back-end. This means that
 408      * no history intervals will be saved anywhere, and as such only
 409      * {@link ITmfStateSystem#queryOngoingState} will be available.
 410      */
 411     private void createNullHistory(String id, ITmfStateProvider provider) {
 412         IStateHistoryBackend backend = StateHistoryBackendFactory.createNullBackend(id);
 413         fStateSystem = StateSystemFactory.newStateSystem(backend);
 414         provider.assignTargetStateSystem(fStateSystem);
 415         build(provider);
 416     }
 417
 418     /*
 419      * Create a new state system using in-memory interval storage. This should
 420      * only be done for very small state system, and will be naturally limited
 421      * to 2^31 intervals.
 422      */
 423     private void createInMemoryHistory(String id, ITmfStateProvider provider) {
 424         IStateHistoryBackend backend = StateHistoryBackendFactory.createInMemoryBackend(id, provider.getStartTime());
 425         fStateSystem = StateSystemFactory.newStateSystem(backend);
 426         provider.assignTargetStateSystem(fStateSystem);
 427         build(provider);
 428     }
 429
 430     private void disposeProvider(boolean deleteFiles) {
 431         ITmfStateProvider provider = fStateProvider;
 432         if (provider != null) {
 433             provider.dispose();
 434         }
 435         fStateProvider = null;
 436         if (deleteFiles && (fStateSystem != null)) {
 437             fStateSystem.removeFiles();
 438         }
 439     }
 440
 441     private void build(ITmfStateProvider provider) {
 442         if (fStateSystem == null) {
 443             throw new IllegalArgumentException();
 444         }
 445
 446         ITmfEventRequest request = fRequest;
 447         if ((request != null) && (!request.isCompleted())) {
 448             request.cancel();
 449         }
 450
 451         fTimeRange = TmfTimeRange.ETERNITY;
 452         final ITmfTrace trace = provider.getTrace();
 453         if (!isCompleteTrace(trace)) {
 454             fTimeRange = trace.getTimeRange();
 455         }
 456
 457         fStateProvider = provider;
 458         synchronized (fRequestSyncObj) {
 459             startRequest();
 460         }
 461
 462         /*
 463          * The state system object is now created, we can consider this module
 464          * "initialized" (components can retrieve it and start doing queries).
 465          */
 466         analysisReady(true);
 467
 468         /*
 469          * Block the executeAnalysis() construction is complete (so that the
 470          * progress monitor displays that it is running).
 471          */
 472         try {
 473             if (fRequest != null) {
 474                 fRequest.waitForCompletion();
 475             }
 476         } catch (InterruptedException e) {
 477              e.printStackTrace();
 478         }
 479     }
 480
 481     private class StateSystemEventRequest extends TmfEventRequest {
 482         private final ITmfStateProvider sci;
 483         private final ITmfTrace trace;
 484
 485         public StateSystemEventRequest(ITmfStateProvider sp, TmfTimeRange timeRange, int index) {
 486             super(ITmfEvent.class,
 487                     timeRange,
 488                     index,
 489                     ITmfEventRequest.ALL_DATA,
 490                     ITmfEventRequest.ExecutionType.BACKGROUND,
 491                     TmfStateSystemAnalysisModule.this.getDependencyLevel());
 492             this.sci = sp;
 493             trace = sci.getTrace();
 494
 495         }
 496
 497         @Override
 498         public void handleData(final ITmfEvent event) {
 499             super.handleData(event);
 500             if (event.getTrace() == trace) {
 501                 sci.processEvent(event);
 502             } else if (trace instanceof TmfExperiment) {
 503                 /*
 504                  * If the request is for an experiment, check if the event is
 505                  * from one of the child trace
 506                  */
 507                 for (ITmfTrace childTrace : ((TmfExperiment) trace).getTraces()) {
 508                     if (childTrace == event.getTrace()) {
 509                         sci.processEvent(event);
 510                     }
 511                 }
 512             }
 513         }
 514
 515         @Override
 516         public void handleSuccess() {
 517             super.handleSuccess();
 518             if (isCompleteTrace(trace)) {
 519                 disposeProvider(false);
 520             } else {
 521                 fNbRead += getNbRead();
 522                 synchronized (fRequestSyncObj) {
 523                     final TmfTimeRange timeRange = fTimeRange;
 524                     if (timeRange != null) {
 525                         if (getRange().getEndTime().getValue() < timeRange.getEndTime().getValue()) {
 526                             startRequest();
 527                         }
 528                     }
 529                 }
 530             }
 531         }
 532
 533         @Override
 534         public void handleCancel() {
 535             super.handleCancel();
 536             disposeProvider(true);
 537         }
 538
 539         @Override
 540         public void handleFailure() {
 541             super.handleFailure();
 542             disposeProvider(true);
 543         }
 544     }
 545
 546     // ------------------------------------------------------------------------
 547     // ITmfAnalysisModuleWithStateSystems
 548     // ------------------------------------------------------------------------
 549
 550     @Override
 551     @Nullable
 552     public ITmfStateSystem getStateSystem(String id) {
 553         if (id.equals(getId())) {
 554             return fStateSystem;
 555         }
 556         return null;
 557     }
 558
 559     @Override
 560     public @NonNull Iterable<@NonNull ITmfStateSystem> getStateSystems() {
 561         ITmfStateSystemBuilder stateSystem = fStateSystem;
 562         if (stateSystem == null) {
 563             return Collections.EMPTY_SET;
 564         }
 565         return Collections.singleton(stateSystem);
 566     }
 567
 568     /**
 569      * Signal handler for the TmfTraceRangeUpdatedSignal signal
 570      *
 571      * @param signal The incoming signal
 572      */
 573     @TmfSignalHandler
 574     public void traceRangeUpdated(final TmfTraceRangeUpdatedSignal signal) {
 575         fTimeRange = signal.getRange();
 576         ITmfStateProvider stateProvider = fStateProvider;
 577         synchronized (fRequestSyncObj) {
 578             if (signal.getTrace() == getTrace() && stateProvider != null && stateProvider.getAssignedStateSystem() != null) {
 579                 ITmfEventRequest request = fRequest;
 580                 if ((request == null) || request.isCompleted()) {
 581                     startRequest();
 582                 }
 583             }
 584         }
 585     }
 586
 587     private void startRequest() {
 588         ITmfStateProvider stateProvider = fStateProvider;
 589         TmfTimeRange timeRange = fTimeRange;
 590         if (stateProvider == null || timeRange == null) {
 591             return;
 592         }
 593         ITmfEventRequest request = new StateSystemEventRequest(stateProvider, timeRange, fNbRead);
 594         stateProvider.getTrace().sendRequest(request);
 595         fRequest = request;
 596     }
 597
 598     private static boolean isCompleteTrace(ITmfTrace trace) {
 599         return !(trace instanceof ITmfTraceCompleteness) || ((ITmfTraceCompleteness) trace).isComplete();
 600     }
 601
 602     // ------------------------------------------------------------------------
 603     // ITmfPropertiesProvider
 604     // ------------------------------------------------------------------------
 605
 606     /**
 607      * @since 2.0
 608      */
 609     @Override
 610     public @NonNull Map<@NonNull String, @NonNull String> getProperties() {
 611         Map<@NonNull String, @NonNull String> properties = super.getProperties();
 612
 613         StateSystemBackendType backend = getBackendType();
 614         properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesBackend), backend.name());
 615         switch (backend) {
 616         case FULL:
 617         case PARTIAL:
 618             File htFile = getSsFile();
 619             if (htFile != null) {
 620                 if (htFile.exists()) {
 621                     properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesFileSize), FileUtils.byteCountToDisplaySize(htFile.length()));
 622                 } else {
 623                     properties.put(NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesFileSize), NonNullUtils.checkNotNull(Messages.TmfStateSystemAnalysisModule_PropertiesAnalysisNotExecuted));
 624                 }
 625             }
 626             break;
 627         case INMEM:
 628         case NULL:
 629         default:
 630             break;
 631
 632         }
 633         return properties;
 634     }
 635 }