From baa96b1ddfbe8f1ebd02a9a67107cefedf628648 Mon Sep 17 00:00:00 2001 From: Bernd Hufmann Date: Thu, 5 Dec 2013 10:57:22 -0500 Subject: [PATCH] tmf: Merge HistoryBuilder into the state system analysis module The state system analysis module acts as kind of a state system factory. There is no need to have a separate TmfStateSystemFactory and HistoryBuilder class anymore. Merging the history builder functionality into the state system analysis module also allows for better handling of requests to analyze the trace. Now it's possible to cancel the analysis job. Later on, when all state systems (e.g. statistic state systems) are built using the state system analysis module, the TmfStateSystemFactory and HistoryBuilder can be removed. Change-Id: Ibb52b20b6fdb16b665ad318cda04cf92d19ccf72 Signed-off-by: Bernd Hufmann Signed-off-by: Alexandre Montplaisir Reviewed-on: https://git.eclipse.org/r/19441 Tested-by: Hudson CI Reviewed-by: Bernd Hufmann --- .../stubs/analysis/TestStateSystemModule.java | 5 +- .../analysis/TmfAbstractAnalysisModule.java | 2 + .../TmfStateSystemAnalysisModule.java | 301 ++++++++++++++++-- 3 files changed, 284 insertions(+), 24 deletions(-) diff --git a/org.eclipse.linuxtools.tmf.core.tests/stubs/org/eclipse/linuxtools/tmf/tests/stubs/analysis/TestStateSystemModule.java b/org.eclipse.linuxtools.tmf.core.tests/stubs/org/eclipse/linuxtools/tmf/tests/stubs/analysis/TestStateSystemModule.java index 63c9e12668..1d0ba12fd0 100644 --- a/org.eclipse.linuxtools.tmf.core.tests/stubs/org/eclipse/linuxtools/tmf/tests/stubs/analysis/TestStateSystemModule.java +++ b/org.eclipse.linuxtools.tmf.core.tests/stubs/org/eclipse/linuxtools/tmf/tests/stubs/analysis/TestStateSystemModule.java @@ -12,7 +12,7 @@ package org.eclipse.linuxtools.tmf.tests.stubs.analysis; -import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; import org.eclipse.linuxtools.tmf.core.statesystem.ITmfStateProvider; import org.eclipse.linuxtools.tmf.core.statesystem.TmfStateSystemAnalysisModule; @@ -21,10 +21,11 @@ import org.eclipse.linuxtools.tmf.core.statesystem.TmfStateSystemAnalysisModule; * * @author Geneviève Bastien */ +@NonNullByDefault public class TestStateSystemModule extends TmfStateSystemAnalysisModule { @Override - protected @NonNull ITmfStateProvider createStateProvider() { + protected ITmfStateProvider createStateProvider() { return new TestStateSystemProvider(getTrace()); } diff --git a/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/analysis/TmfAbstractAnalysisModule.java b/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/analysis/TmfAbstractAnalysisModule.java index 116f268825..82fe1be282 100644 --- a/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/analysis/TmfAbstractAnalysisModule.java +++ b/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/analysis/TmfAbstractAnalysisModule.java @@ -275,6 +275,8 @@ public abstract class TmfAbstractAnalysisModule extends TmfComponent implements if (!fAnalysisCancelled) { return Status.OK_STATUS; } + // Reset analysis so that it can be executed again. + resetAnalysis(); return Status.CANCEL_STATUS; } diff --git a/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/statesystem/TmfStateSystemAnalysisModule.java b/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/statesystem/TmfStateSystemAnalysisModule.java index a5692ac4fd..f1cdb21399 100644 --- a/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/statesystem/TmfStateSystemAnalysisModule.java +++ b/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/statesystem/TmfStateSystemAnalysisModule.java @@ -8,17 +8,35 @@ * * Contributors: * Geneviève Bastien - Initial API and implementation + * Bernd Hufmann - Integrated history builder functionality *******************************************************************************/ package org.eclipse.linuxtools.tmf.core.statesystem; import java.io.File; +import java.io.IOException; import java.util.Collections; import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.NullProgressMonitor; import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.linuxtools.internal.tmf.core.statesystem.StateSystem; +import org.eclipse.linuxtools.internal.tmf.core.statesystem.backends.IStateHistoryBackend; +import org.eclipse.linuxtools.internal.tmf.core.statesystem.backends.InMemoryBackend; +import org.eclipse.linuxtools.internal.tmf.core.statesystem.backends.NullBackend; +import org.eclipse.linuxtools.internal.tmf.core.statesystem.backends.historytree.HistoryTreeBackend; +import org.eclipse.linuxtools.internal.tmf.core.statesystem.backends.historytree.ThreadedHistoryTreeBackend; +import org.eclipse.linuxtools.internal.tmf.core.statesystem.backends.partial.PartialHistoryBackend; +import org.eclipse.linuxtools.internal.tmf.core.statesystem.backends.partial.PartialStateSystem; import org.eclipse.linuxtools.tmf.core.analysis.TmfAbstractAnalysisModule; +import org.eclipse.linuxtools.tmf.core.event.ITmfEvent; import org.eclipse.linuxtools.tmf.core.exceptions.TmfTraceException; +import org.eclipse.linuxtools.tmf.core.request.ITmfEventRequest; +import org.eclipse.linuxtools.tmf.core.request.TmfEventRequest; +import org.eclipse.linuxtools.tmf.core.timestamp.TmfTimeRange; +import org.eclipse.linuxtools.tmf.core.trace.ITmfTrace; import org.eclipse.linuxtools.tmf.core.trace.TmfTraceManager; /** @@ -33,12 +51,16 @@ import org.eclipse.linuxtools.tmf.core.trace.TmfTraceManager; * @author Geneviève Bastien * @since 3.0 */ +@NonNullByDefault public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisModule implements ITmfAnalysisModuleWithStateSystems { private static final String EXTENSION = ".ht"; //$NON-NLS-1$ - private ITmfStateSystem fStateSystem = null; + @Nullable private ITmfStateSystemBuilder fStateSystem; + @Nullable private ITmfStateProvider fStateProvider; + @Nullable private IStateHistoryBackend fHtBackend; + @Nullable private ITmfEventRequest fRequest; /** * State system backend types @@ -61,7 +83,6 @@ public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisMo * * @return the state provider */ - @NonNull protected abstract ITmfStateProvider createStateProvider(); /** @@ -82,40 +103,47 @@ public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisMo } /** - * Get the state system generated by this analysis + * Get the state system generated by this analysis, or null if it is not yet + * created. * * @return The state system */ + @Nullable public ITmfStateSystem getStateSystem() { return fStateSystem; } - @Override - protected boolean executeAnalysis(final IProgressMonitor monitor) { + // ------------------------------------------------------------------------ + // TmfAbstractAnalysisModule + // ------------------------------------------------------------------------ - final ITmfStateProvider htInput = createStateProvider(); + @Override + protected boolean executeAnalysis(@Nullable final IProgressMonitor monitor) { + IProgressMonitor mon = (monitor == null ? new NullProgressMonitor() : monitor); + final ITmfStateProvider provider = createStateProvider(); /* FIXME: State systems should make use of the monitor, to be cancelled */ try { /* Get the state system according to backend */ StateSystemBackendType backend = getBackendType(); String directory; + File htFile; switch (backend) { case FULL: directory = TmfTraceManager.getSupplementaryFileDir(getTrace()); - final File htFile = new File(directory + getSsFileName()); - fStateSystem = TmfStateSystemFactory.newFullHistory(htFile, htInput, true); + htFile = new File(directory + getSsFileName()); + createFullHistory(provider, htFile); break; case PARTIAL: directory = TmfTraceManager.getSupplementaryFileDir(getTrace()); - final File htPartialFile = new File(directory + getSsFileName()); - fStateSystem = TmfStateSystemFactory.newPartialHistory(htPartialFile, htInput, true); + htFile = new File(directory + getSsFileName()); + createPartialHistory(provider, htFile); break; case INMEM: - fStateSystem = TmfStateSystemFactory.newInMemHistory(htInput, true); + createInMemoryHistory(provider); break; case NULL: - fStateSystem = TmfStateSystemFactory.newNullHistory(htInput); + createNullHistory(provider); break; default: break; @@ -123,17 +151,244 @@ public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisMo } catch (TmfTraceException e) { return false; } - return true; + return !mon.isCanceled(); } @Override protected void canceling() { + ITmfEventRequest req = fRequest; + if ((req != null) && (!req.isCompleted())) { + req.cancel(); + } + } + + // ------------------------------------------------------------------------ + // History creation methods + // ------------------------------------------------------------------------ + + /* + * Load the history file matching the target trace. If the file already + * exists, it will be opened directly. If not, it will be created from + * scratch. + */ + private void createFullHistory(ITmfStateProvider provider, File htFile) throws TmfTraceException { + + /* If the target file already exists, do not rebuild it uselessly */ + // TODO for now we assume it's complete. Might be a good idea to check + // at least if its range matches the trace's range. + + if (htFile.exists()) { + /* Load an existing history */ + final int version = provider.getVersion(); + try { + fHtBackend = new HistoryTreeBackend(htFile, version); + fStateSystem = new StateSystem(fHtBackend, false); + return; + } catch (IOException e) { + /* + * There was an error opening the existing file. Perhaps it was + * corrupted, perhaps it's an old version? We'll just + * fall-through and try to build a new one from scratch instead. + */ + } + } + + /* Size of the blocking queue to use when building a state history */ + final int QUEUE_SIZE = 10000; + + try { + fHtBackend = new ThreadedHistoryTreeBackend(htFile, + provider.getStartTime(), provider.getVersion(), QUEUE_SIZE); + fStateSystem = new StateSystem(fHtBackend); + provider.assignTargetStateSystem(fStateSystem); + build(provider); + } catch (IOException e) { + /* + * If it fails here however, it means there was a problem writing to + * the disk, so throw a real exception this time. + */ + throw new TmfTraceException(e.toString(), e); + } + } + + /* + * Create a new state system backed with a partial history. A partial + * history is similar to a "full" one (which you get with + * {@link #newFullHistory}), except that the file on disk is much smaller, + * but queries are a bit slower. + * + * Also note that single-queries are implemented using a full-query + * underneath, (which are much slower), so this might not be a good fit for + * a use case where you have to do lots of single queries. + */ + private void createPartialHistory(ITmfStateProvider provider, File htPartialFile) + throws TmfTraceException { + /* + * The order of initializations is very tricky (but very important!) + * here. We need to follow this pattern: + * (1 is done before the call to this method) + * + * 1- Instantiate realStateProvider + * 2- Instantiate realBackend + * 3- Instantiate partialBackend, with prereqs: + * 3a- Instantiate partialProvider, via realProvider.getNew() + * 3b- Instantiate nullBackend (partialSS's backend) + * 3c- Instantiate partialSS + * 3d- partialProvider.assignSS(partialSS) + * 4- Instantiate realSS + * 5- partialSS.assignUpstream(realSS) + * 6- realProvider.assignSS(realSS) + * 7- Call HistoryBuilder(realProvider, realSS, partialBackend) to build the thing. + */ + + /* Size of the blocking queue to use when building a state history */ + final int QUEUE_SIZE = 10000; + + final long granularity = 50000; + + /* 2 */ + IStateHistoryBackend realBackend = null; + try { + realBackend = new ThreadedHistoryTreeBackend(htPartialFile, + provider.getStartTime(), provider.getVersion(), QUEUE_SIZE); + } catch (IOException e) { + throw new TmfTraceException(e.toString(), e); + } + + /* 3a */ + ITmfStateProvider partialProvider = provider.getNewInstance(); + + /* 3b-3c, constructor automatically uses a NullBackend */ + PartialStateSystem pss = new PartialStateSystem(); + + /* 3d */ + partialProvider.assignTargetStateSystem(pss); + + /* 3 */ + IStateHistoryBackend partialBackend = + new PartialHistoryBackend(partialProvider, pss, realBackend, granularity); + + /* 4 */ + StateSystem realSS = new StateSystem(partialBackend); + + /* 5 */ + pss.assignUpstream(realSS); + + /* 6 */ + provider.assignTargetStateSystem(realSS); + + /* 7 */ + fHtBackend = partialBackend; + fStateSystem = realSS; + + build(provider); + } + + /* + * Create a new state system using a null history back-end. This means that + * no history intervals will be saved anywhere, and as such only + * {@link ITmfStateSystem#queryOngoingState} will be available. + */ + private void createNullHistory(ITmfStateProvider provider) { + fHtBackend = new NullBackend(); + fStateSystem = new StateSystem(fHtBackend); + provider.assignTargetStateSystem(fStateSystem); + build(provider); + } + + /* + * Create a new state system using in-memory interval storage. This should + * only be done for very small state system, and will be naturally limited + * to 2^31 intervals. + */ + private void createInMemoryHistory(ITmfStateProvider provider) { + fHtBackend = new InMemoryBackend(provider.getStartTime()); + fStateSystem = new StateSystem(fHtBackend); + provider.assignTargetStateSystem(fStateSystem); + build(provider); + } + + private void dispose(boolean deleteFiles) { + ITmfStateProvider provider = fStateProvider; + if (provider != null) { + provider.dispose(); + } + if (deleteFiles && (fHtBackend != null)) { + fHtBackend.removeFiles(); + } + } + + private void build(ITmfStateProvider provider) { + if ((fStateSystem == null) || (fHtBackend == null)) { + throw new IllegalArgumentException(); + } + + ITmfEventRequest request = fRequest; + if ((request != null) && (!request.isCompleted())) { + request.cancel(); + } + + request = new StateSystemEventRequest(provider); + provider.getTrace().sendRequest(request); + /* - * FIXME: I guess that will do to cancel the state system building, but - * it may be preferable to just tell the state system and he will handle - * himself how to cancel its work + * Only now that we've actually started the build, we'll update the + * class fields, so that they become visible for other callers. */ - fStateSystem.dispose(); + fStateProvider = provider; + fRequest = request; + + try { + fRequest.waitForCompletion(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + private class StateSystemEventRequest extends TmfEventRequest { + private final ITmfStateProvider sci; + private final ITmfTrace trace; + + public StateSystemEventRequest(ITmfStateProvider sp) { + super(sp.getExpectedEventType(), + TmfTimeRange.ETERNITY, + 0, + ITmfEventRequest.ALL_DATA, + ITmfEventRequest.ExecutionType.BACKGROUND); + this.sci = sp; + + // sci.getTrace() will eventually return a @NonNull + @SuppressWarnings("null") + @NonNull ITmfTrace tr = sci.getTrace(); + + this.trace = tr; + } + + @Override + public void handleData(final @Nullable ITmfEvent event) { + super.handleData(event); + if (event != null && event.getTrace() == trace) { + sci.processEvent(event); + } + } + + @Override + public void handleSuccess() { + super.handleSuccess(); + dispose(false); + } + + @Override + public void handleCancel() { + super.handleCancel(); + dispose(true); + } + + @Override + public void handleFailure() { + super.handleFailure(); + dispose(true); + } } // ------------------------------------------------------------------------ @@ -141,7 +396,8 @@ public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisMo // ------------------------------------------------------------------------ @Override - public ITmfStateSystem getStateSystem(@NonNull String id) { + @Nullable + public ITmfStateSystem getStateSystem(String id) { if (id.equals(getId())) { return fStateSystem; } @@ -149,14 +405,15 @@ public abstract class TmfStateSystemAnalysisModule extends TmfAbstractAnalysisMo } @Override - public String getStateSystemId(@NonNull ITmfStateSystem ss) { + @Nullable + public String getStateSystemId(ITmfStateSystem ss) { return getId(); } - @SuppressWarnings("null") @Override - @NonNull public Iterable getStateSystems() { - return Collections.singleton(fStateSystem); + @SuppressWarnings("null") + @NonNull Iterable ret = Collections.singleton((ITmfStateSystem) fStateSystem); + return ret; } } -- 2.34.1