TMF: Synchronize an experiment instead of collection of traces
authorGeneviève Bastien <gbastien+lttng@versatic.net>
Wed, 4 Jun 2014 13:55:08 +0000 (09:55 -0400)
committerGenevieve Bastien <gbastien+lttng@versatic.net>
Fri, 14 Nov 2014 20:56:36 +0000 (15:56 -0500)
That will make it compatible with streaming since one full trace does not
have to be read before the others. It will also accelerate the time necessary
to get results, as matches will be found earlier. The overhead of reading the
experiment instead of the traces decreases performance though.

Change-Id: I7c94c21e63781b2cae0faead915be8959170a992
Signed-off-by: Geneviève Bastien <gbastien+lttng@versatic.net>
Reviewed-on: https://git.eclipse.org/r/35350
Tested-by: Hudson CI
Reviewed-by: Alexandre Montplaisir <alexmonthy@voxpopuli.im>
org.eclipse.tracecompass.lttng2.kernel.core.tests/perf/org/eclipse/tracecompass/lttng2/kernel/core/tests/perf/event/matching/TraceSynchronizationBenchmark.java
org.eclipse.tracecompass.lttng2.kernel.core.tests/src/org/eclipse/tracecompass/lttng2/kernel/core/tests/event/matchandsync/MatchAndSyncTest.java
org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatches.java
org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfEventMatching.java
org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/event/matching/TmfNetworkEventMatching.java
org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/experiment/TmfExperiment.java

index 61e6bb7f5a6a53cb06e525a5db8b1d8e205665e9..2e124b1fa5b970ad1d2803a98068f413afbe6808 100644 (file)
@@ -15,7 +15,7 @@ package org.eclipse.tracecompass.lttng2.kernel.core.tests.perf.event.matching;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assume.assumeTrue;
 
-import java.util.Arrays;
+import java.util.Collections;
 
 import org.eclipse.test.performance.Dimension;
 import org.eclipse.test.performance.Performance;
@@ -26,6 +26,8 @@ import org.eclipse.tracecompass.tmf.core.event.matching.TmfEventMatching;
 import org.eclipse.tracecompass.tmf.core.synchronization.SynchronizationAlgorithm;
 import org.eclipse.tracecompass.tmf.core.synchronization.SynchronizationManager;
 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
+import org.eclipse.tracecompass.tmf.core.trace.experiment.TmfExperiment;
+import org.eclipse.tracecompass.tmf.ctf.core.event.CtfTmfEvent;
 import org.eclipse.tracecompass.tmf.ctf.core.tests.shared.CtfTmfTestTrace;
 import org.eclipse.tracecompass.tmf.ctf.core.trace.CtfTmfTrace;
 import org.junit.BeforeClass;
@@ -62,7 +64,8 @@ public class TraceSynchronizationBenchmark {
         try (CtfTmfTrace trace1 = CtfTmfTestTrace.SYNC_SRC.getTrace();
                 CtfTmfTrace trace2 = CtfTmfTestTrace.SYNC_DEST.getTrace();) {
             ITmfTrace[] traces = { trace1, trace2 };
-            runCpuTest(traces, "Match TCP events", 40);
+            TmfExperiment experiment = new TmfExperiment(CtfTmfEvent.class, "Test experiment", traces, TmfExperiment.DEFAULT_INDEX_PAGE_SIZE, null);
+            runCpuTest(experiment, "Match TCP events", 40);
         }
     }
 
@@ -78,19 +81,20 @@ public class TraceSynchronizationBenchmark {
                 CtfTmfTrace trace2 = CtfTmfTestTrace.DJANGO_DB.getTrace();
                 CtfTmfTrace trace3 = CtfTmfTestTrace.DJANGO_HTTPD.getTrace();) {
             ITmfTrace[] traces = { trace1, trace2, trace3 };
-            runCpuTest(traces, "Django traces", 10);
-            runMemoryTest(traces, "Django traces", 10);
+            TmfExperiment experiment = new TmfExperiment(CtfTmfEvent.class, "Test experiment", traces, TmfExperiment.DEFAULT_INDEX_PAGE_SIZE, null);
+            runCpuTest(experiment, "Django traces", 10);
+            runMemoryTest(experiment, "Django traces", 10);
         }
     }
 
-    private static void runCpuTest(ITmfTrace[] testTraces, String testName, int loop_count) {
+    private static void runCpuTest(TmfExperiment experiment, String testName, int loop_count) {
         Performance perf = Performance.getDefault();
         PerformanceMeter pm = perf.createPerformanceMeter(TEST_ID + testName + TIME);
         perf.tagAsSummary(pm, TEST_SUMMARY + ':' + testName + TIME, Dimension.CPU_TIME);
 
         for (int i = 0; i < loop_count; i++) {
             pm.start();
-            SynchronizationManager.synchronizeTraces(null, Arrays.asList(testTraces), true);
+            SynchronizationManager.synchronizeTraces(null, Collections.<ITmfTrace> singleton(experiment), true);
             pm.stop();
         }
         pm.commit();
@@ -98,7 +102,7 @@ public class TraceSynchronizationBenchmark {
     }
 
     /* Benchmark memory used by the algorithm */
-    private static void runMemoryTest(ITmfTrace[] testTraces, String testName, int loop_count) {
+    private static void runMemoryTest(TmfExperiment experiment, String testName, int loop_count) {
         Performance perf = Performance.getDefault();
         PerformanceMeter pm = perf.createPerformanceMeter(TEST_ID + testName + MEMORY);
         perf.tagAsSummary(pm, TEST_SUMMARY + ':' + testName + MEMORY, Dimension.USED_JAVA_HEAP);
@@ -107,7 +111,7 @@ public class TraceSynchronizationBenchmark {
 
             System.gc();
             pm.start();
-            SynchronizationAlgorithm algo = SynchronizationManager.synchronizeTraces(null, Arrays.asList(testTraces), true);
+            SynchronizationAlgorithm algo = SynchronizationManager.synchronizeTraces(null, Collections.<ITmfTrace> singleton(experiment), true);
             assertNotNull(algo);
 
             System.gc();
index c6bdc822ac791f292d798d49bb15d7dcd66e77b5..6caf09cbf662c82eba404cc2b956c88b0cf8e59a 100644 (file)
@@ -14,13 +14,19 @@ package org.eclipse.tracecompass.lttng2.kernel.core.tests.event.matchandsync;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 
 import org.eclipse.tracecompass.lttng2.kernel.core.event.matching.TcpEventMatching;
 import org.eclipse.tracecompass.lttng2.kernel.core.event.matching.TcpLttngEventMatching;
+import org.eclipse.tracecompass.tmf.core.event.matching.IMatchProcessingUnit;
 import org.eclipse.tracecompass.tmf.core.event.matching.TmfEventMatching;
 import org.eclipse.tracecompass.tmf.core.event.matching.TmfNetworkEventMatching;
 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
@@ -41,7 +47,6 @@ public class MatchAndSyncTest {
      */
     @Test
     public void testMatching() {
-        final String cr = System.getProperty("line.separator");
         assumeTrue(CtfTmfTestTrace.SYNC_SRC.exists());
         assumeTrue(CtfTmfTestTrace.SYNC_DEST.exists());
         try (CtfTmfTrace trace1 = CtfTmfTestTrace.SYNC_SRC.getTrace();
@@ -57,14 +62,35 @@ public class MatchAndSyncTest {
             TmfNetworkEventMatching twoTraceMatch = new TmfNetworkEventMatching(tracearr);
             assertTrue(twoTraceMatch.matchEvents());
 
-            String stats = twoTraceMatch.toString();
-            assertEquals("TmfEventMatches [ Number of matches found: 46 ]" +
-                    "Trace 0:" + cr +
-                    "  3 unmatched incoming events" + cr +
-                    "  2 unmatched outgoing events" + cr +
-                    "Trace 1:" + cr +
-                    "  2 unmatched incoming events" + cr +
-                    "  1 unmatched outgoing events" + cr, stats);
+            /* Set method and fields accessible to make sure the counts are ok */
+            try {
+                /* Verify number of matches */
+                Method method = TmfEventMatching.class.getDeclaredMethod("getProcessingUnit");
+                method.setAccessible(true);
+                IMatchProcessingUnit procUnit = (IMatchProcessingUnit) method.invoke(twoTraceMatch);
+                assertEquals(46, procUnit.countMatches());
+
+                /* Verify unmatched incoming */
+                Field fieldIn = twoTraceMatch.getClass().getDeclaredField("fUnmatchedIn");
+                fieldIn.setAccessible(true);
+                Map<?, ?> unmatched = (Map<?, ?> ) fieldIn.get(twoTraceMatch);
+                Map<?, ?> unmatchedTrace = (Map<?, ?>) unmatched.get(trace1);
+                assertEquals(3, unmatchedTrace.size());
+                unmatchedTrace = (Map<?, ?>) unmatched.get(trace2);
+                assertEquals(2, unmatchedTrace.size());
+
+                /* Verify unmatched outgoing */
+                Field fieldOut = twoTraceMatch.getClass().getDeclaredField("fUnmatchedOut");
+                fieldOut.setAccessible(true);
+                unmatched = (Map<?, ?> ) fieldOut.get(twoTraceMatch);
+                unmatchedTrace = (Map<?, ?>) unmatched.get(trace1);
+                assertEquals(2, unmatchedTrace.size());
+                unmatchedTrace = (Map<?, ?>) unmatched.get(trace2);
+                assertEquals(1, unmatchedTrace.size());
+
+            } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchFieldException e) {
+                fail(e.getMessage());
+            }
 
         }
     }
index c512ee26bb6f04fd80be77f4b63469e294192845..b32ba7e1ec10bb8c9574ea03aa2cc7732cdfe01c 100644 (file)
@@ -38,7 +38,6 @@ public class TmfEventMatches implements IMatchProcessingUnit {
     /**
      * IMatchProcessingUnit overrides
      */
-
     @Override
     public void init(Collection<ITmfTrace> fTraces) {
         fMatchCount = 0;
index 7b7f1837b85377bdfd018f5e8e65e9f18615cc45..4f992f1789516cb64d4c8454f6779645a1c46c01 100644 (file)
 
 package org.eclipse.tracecompass.tmf.core.event.matching;
 
-import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.eclipse.tracecompass.internal.tmf.core.Activator;
 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
@@ -24,6 +23,8 @@ import org.eclipse.tracecompass.tmf.core.request.ITmfEventRequest;
 import org.eclipse.tracecompass.tmf.core.request.TmfEventRequest;
 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimeRange;
 import org.eclipse.tracecompass.tmf.core.trace.ITmfTrace;
+import org.eclipse.tracecompass.tmf.core.trace.TmfTraceManager;
+import org.eclipse.tracecompass.tmf.core.trace.experiment.TmfExperiment;
 
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
@@ -50,6 +51,8 @@ public abstract class TmfEventMatching implements ITmfEventMatching {
         NETWORK
     }
 
+    private static final Multimap<MatchingType, ITmfMatchEventDefinition> MATCH_DEFINITIONS = HashMultimap.create();
+
     /**
      * The array of traces to match
      */
@@ -60,8 +63,6 @@ public abstract class TmfEventMatching implements ITmfEventMatching {
      */
     private final IMatchProcessingUnit fMatches;
 
-    private static final Map<MatchingType, List<ITmfMatchEventDefinition>> fMatchDefinitions = new HashMap<>();
-
     private final Multimap<ITmfTrace, ITmfMatchEventDefinition> fMatchMap = HashMultimap.create();
 
     /**
@@ -76,17 +77,35 @@ public abstract class TmfEventMatching implements ITmfEventMatching {
         if (tmfEventMatches == null) {
             throw new IllegalArgumentException();
         }
-        fTraces = traces;
+        fTraces = new HashSet<>(traces);
         fMatches = tmfEventMatches;
     }
 
     /**
-     * Returns the traces to process
+     * Returns the traces to synchronize. These are the traces that were
+     * specified in the constructor, they may contain either traces or
+     * experiment.
      *
-     * @return The traces
+     * @return The traces to synchronize
      */
-    protected Collection<? extends ITmfTrace> getTraces() {
-        return fTraces;
+    protected Collection<ITmfTrace> getTraces() {
+        return new HashSet<>(fTraces);
+    }
+
+    /**
+     * Returns the individual traces to process. If some of the traces specified
+     * to synchronize in the constructor were experiments, only the traces
+     * contained in this experiment will be returned. No {@link TmfExperiment}
+     * are returned by this method.
+     *
+     * @return The individual traces to synchronize, no experiments
+     */
+    protected Collection<ITmfTrace> getIndividualTraces() {
+        Set<ITmfTrace> traces = new HashSet<>();
+        for (ITmfTrace trace : fTraces) {
+            traces.addAll(Arrays.asList(TmfTraceManager.getTraceSet(trace)));
+        }
+        return traces;
     }
 
     /**
@@ -116,11 +135,11 @@ public abstract class TmfEventMatching implements ITmfEventMatching {
      */
     protected void initMatching() {
         fMatches.init(fTraces);
-        List<ITmfMatchEventDefinition> deflist = fMatchDefinitions.get(getMatchingType());
+        Collection<ITmfMatchEventDefinition> deflist = MATCH_DEFINITIONS.get(getMatchingType());
         if (deflist == null) {
             return;
         }
-        for (ITmfTrace trace : fTraces) {
+        for (ITmfTrace trace : getIndividualTraces()) {
             for (ITmfMatchEventDefinition def : deflist) {
                 if (def.canMatchTrace(trace)) {
                     fMatchMap.put(trace, def);
@@ -227,10 +246,7 @@ public abstract class TmfEventMatching implements ITmfEventMatching {
      */
     public static void registerMatchObject(ITmfMatchEventDefinition match) {
         for (MatchingType type : match.getApplicableMatchingTypes()) {
-            if (!fMatchDefinitions.containsKey(type)) {
-                fMatchDefinitions.put(type, new ArrayList<ITmfMatchEventDefinition>());
-            }
-            fMatchDefinitions.get(type).add(match);
+            MATCH_DEFINITIONS.put(type, match);
         }
     }
 
index bb57160ab68ca7b222aaefb6603f96971df48f87..2cd83857115f87692a1df6989f9f5ea45f966b07 100644 (file)
@@ -84,7 +84,7 @@ public class TmfNetworkEventMatching extends TmfEventMatching {
         // Initialize the matching infrastructure (unmatched event lists)
         fUnmatchedIn.clear();
         fUnmatchedOut.clear();
-        for (ITmfTrace trace : getTraces()) {
+        for (ITmfTrace trace : getIndividualTraces()) {
             fUnmatchedIn.put(trace, new HashMap<List<Object>, ITmfEvent>());
             fUnmatchedOut.put(trace, new HashMap<List<Object>, ITmfEvent>());
         }
@@ -187,8 +187,8 @@ public class TmfNetworkEventMatching extends TmfEventMatching {
              * events as value for the unmatched table. Not necessary right now
              * though
              */
-            if (!unmatchedTbl.get(trace).containsKey(eventKey)) {
-                unmatchedTbl.get(trace).put(eventKey, event);
+            if (!unmatchedTbl.get(event.getTrace()).containsKey(eventKey)) {
+                unmatchedTbl.get(event.getTrace()).put(eventKey, event);
             }
         }
 
@@ -206,7 +206,7 @@ public class TmfNetworkEventMatching extends TmfEventMatching {
         StringBuilder b = new StringBuilder();
         b.append(getProcessingUnit());
         int i = 0;
-        for (ITmfTrace trace : getTraces()) {
+        for (ITmfTrace trace : getIndividualTraces()) {
             b.append("Trace " + i++ + ":" + cr +
                     "  " + countEvents(fUnmatchedIn.get(trace)) + " unmatched incoming events" + cr +
                     "  " + countEvents(fUnmatchedOut.get(trace)) + " unmatched outgoing events" + cr);
index 29bbe6d76e3a1f87da3f0a2d058863ff7cdec8b3..f6de77f1e0eb47add5e9b8e208cbdfe169a1ded0 100644 (file)
@@ -19,7 +19,9 @@ package org.eclipse.tracecompass.tmf.core.trace.experiment;
 
 import java.io.File;
 import java.nio.ByteBuffer;
-import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.IResource;
@@ -107,6 +109,12 @@ public class TmfExperiment extends TmfTrace implements ITmfEventParser, ITmfPers
      */
     private boolean fInitialized = false;
 
+    /**
+     * Lock for synchronization methods. These methods cannot be 'synchronized'
+     * since it makes it impossible to use an event request on the experiment
+     * during synchronization (the request thread would block)
+     */
+    private final Lock fSyncLock = new ReentrantLock();
 
     // ------------------------------------------------------------------------
     // Construction
@@ -520,7 +528,7 @@ public class TmfExperiment extends TmfTrace implements ITmfEventParser, ITmfPers
      * @return The synchronization object
      * @since 3.0
      */
-    public synchronized SynchronizationAlgorithm synchronizeTraces() {
+    public SynchronizationAlgorithm synchronizeTraces() {
         return synchronizeTraces(false);
     }
 
@@ -533,25 +541,30 @@ public class TmfExperiment extends TmfTrace implements ITmfEventParser, ITmfPers
      * @return The synchronization object
      * @since 3.0
      */
-    public synchronized SynchronizationAlgorithm synchronizeTraces(boolean doSync) {
+    public SynchronizationAlgorithm synchronizeTraces(boolean doSync) {
+        fSyncLock.lock();
 
-        String syncDirectory = getSynchronizationFolder(true);
+        try {
+            String syncDirectory = getSynchronizationFolder(true);
 
-        final File syncFile = (syncDirectory != null) ? new File(syncDirectory + File.separator + SYNCHRONIZATION_FILE_NAME) : null;
+            final File syncFile = (syncDirectory != null) ? new File(syncDirectory + File.separator + SYNCHRONIZATION_FILE_NAME) : null;
 
-        final SynchronizationAlgorithm syncAlgo = SynchronizationManager.synchronizeTraces(syncFile, Arrays.asList(fTraces), doSync);
+            final SynchronizationAlgorithm syncAlgo = SynchronizationManager.synchronizeTraces(syncFile, Collections.<ITmfTrace> singleton(this), doSync);
 
-        final TmfTraceSynchronizedSignal signal = new TmfTraceSynchronizedSignal(this, syncAlgo);
+            final TmfTraceSynchronizedSignal signal = new TmfTraceSynchronizedSignal(this, syncAlgo);
 
-        /* Broadcast in separate thread to prevent deadlock */
-        new Thread() {
-            @Override
-            public void run() {
-                broadcast(signal);
-            }
-        }.start();
+            /* Broadcast in separate thread to prevent deadlock */
+            new Thread() {
+                @Override
+                public void run() {
+                    broadcast(signal);
+                }
+            }.start();
 
-        return syncAlgo;
+            return syncAlgo;
+        } finally {
+            fSyncLock.unlock();
+        }
     }
 
     @Override
This page took 0.047272 seconds and 5 git commands to generate.