os.linux: Use HistoryTreeSegmentStore for the system calls
authorGeneviève Bastien <gbastien+lttng@versatic.net>
Thu, 19 Jan 2017 16:50:31 +0000 (11:50 -0500)
committerGenevieve Bastien <gbastien+lttng@versatic.net>
Tue, 23 May 2017 13:09:24 +0000 (09:09 -0400)
Change-Id: I1632a5fa35afade438c92f874b0e41d490dde5a6
Signed-off-by: Geneviève Bastien <gbastien+lttng@versatic.net>
Reviewed-on: https://git.eclipse.org/r/89143
Reviewed-by: Matthew Khouzam <matthew.khouzam@ericsson.com>
Tested-by: Matthew Khouzam <matthew.khouzam@ericsson.com>
Reviewed-by: Hudson CI
analysis/org.eclipse.tracecompass.analysis.os.linux.core/META-INF/MANIFEST.MF
analysis/org.eclipse.tracecompass.analysis.os.linux.core/src/org/eclipse/tracecompass/internal/analysis/os/linux/core/latency/SystemCall.java
analysis/org.eclipse.tracecompass.analysis.os.linux.core/src/org/eclipse/tracecompass/internal/analysis/os/linux/core/latency/SystemCallLatencyAnalysis.java
analysis/org.eclipse.tracecompass.analysis.timing.core/src/org/eclipse/tracecompass/analysis/timing/core/segmentstore/AbstractSegmentStoreAnalysisModule.java
analysis/org.eclipse.tracecompass.analysis.timing.core/src/org/eclipse/tracecompass/analysis/timing/core/segmentstore/statistics/AbstractSegmentStatisticsAnalysis.java
common/org.eclipse.tracecompass.common.core/annotations/java/nio/file/Paths.eea [new file with mode: 0644]
statesystem/org.eclipse.tracecompass.segmentstore.core/src/org/eclipse/tracecompass/internal/segmentstore/core/segmentHistoryTree/HistoryTreeSegmentStore.java
statesystem/org.eclipse.tracecompass.segmentstore.core/src/org/eclipse/tracecompass/segmentstore/core/SegmentStoreFactory.java
tmf/org.eclipse.tracecompass.tmf.analysis.xml.core/src/org/eclipse/tracecompass/internal/tmf/analysis/xml/core/pattern/stateprovider/XmlPatternSegmentStoreModule.java

index a8cd95d03bf520c3df6e7ea606fadf12b7fc1967..7d518622aedb4bfbb78a93d6cc34e7b824209b2d 100644 (file)
@@ -15,7 +15,8 @@ Require-Bundle: org.eclipse.core.runtime,
  org.eclipse.tracecompass.datastore.core,
  org.eclipse.tracecompass.segmentstore.core,
  org.eclipse.tracecompass.analysis.timing.core,
- org.eclipse.tracecompass.statesystem.core
+ org.eclipse.tracecompass.statesystem.core,
+ org.eclipse.tracecompass.datastore.core
 Import-Package: com.google.common.base,
  com.google.common.collect,
  com.google.common.hash,
index fd1a3176be0845602d231987c900959779a604ee..06f2c799e4ff8748ebe02e2eb0fcc1ce046f1bc1 100644 (file)
@@ -9,11 +9,10 @@
 
 package org.eclipse.tracecompass.internal.analysis.os.linux.core.latency;
 
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
 import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.tracecompass.datastore.core.interval.IHTIntervalReader;
+import org.eclipse.tracecompass.datastore.core.serialization.ISafeByteBufferWriter;
+import org.eclipse.tracecompass.datastore.core.serialization.SafeByteBufferFactory;
 import org.eclipse.tracecompass.segmentstore.core.ISegment;
 import org.eclipse.tracecompass.segmentstore.core.segment.interfaces.INamedSegment;
 
@@ -27,6 +26,11 @@ public final class SystemCall implements INamedSegment {
 
     private static final long serialVersionUID = 1554494342105208730L;
 
+    /**
+     * The reader for this segment class
+     */
+    public static final IHTIntervalReader<ISegment> READER = buffer -> new SystemCall(buffer.getLong(), buffer.getLong(), buffer.getString());
+
     /**
      * The subset of information that is available from the syscall entry event.
      */
@@ -49,9 +53,9 @@ public final class SystemCall implements INamedSegment {
         }
     }
 
-    private long fStartTime;
-    private long fEndTime;
-    private String fName;
+    private final long fStartTime;
+    private final long fEndTime;
+    private final String fName;
 
     /**
      * @param info
@@ -67,16 +71,10 @@ public final class SystemCall implements INamedSegment {
         fEndTime = endTime;
     }
 
-    private void writeObject(ObjectOutputStream out) throws IOException {
-        out.writeLong(fStartTime);
-        out.writeLong(fEndTime);
-        out.writeUTF(fName);
-    }
-
-    private void readObject(ObjectInputStream in) throws IOException {
-        fStartTime = in.readLong();
-        fEndTime = in.readLong();
-        fName = in.readUTF().intern();
+    private SystemCall(long startTime, long endTime, String name) {
+        fStartTime = startTime;
+        fEndTime = endTime;
+        fName = name;
     }
 
     @Override
@@ -99,6 +97,18 @@ public final class SystemCall implements INamedSegment {
         return fName;
     }
 
+    @Override
+    public int getSizeOnDisk() {
+        return 2 * Long.BYTES + SafeByteBufferFactory.getStringSizeInBuffer(fName);
+    }
+
+    @Override
+    public void writeSegment(@NonNull ISafeByteBufferWriter buffer) {
+        buffer.putLong(fStartTime);
+        buffer.putLong(fEndTime);
+        buffer.putString(fName);
+    }
+
     @Override
     public int compareTo(@NonNull ISegment o) {
         int ret = INamedSegment.super.compareTo(o);
@@ -115,4 +125,5 @@ public final class SystemCall implements INamedSegment {
                 "; Duration = " + getLength() + //$NON-NLS-1$
                 "; Name = " + getName(); //$NON-NLS-1$
     }
+
 }
index f7cb1c17bbdb051c1f145a77f904595edbcc234a..c591f72a4e1b02908e1efc413953122e95592dfc 100644 (file)
@@ -28,8 +28,10 @@ import org.eclipse.tracecompass.analysis.os.linux.core.tid.TidAnalysisModule;
 import org.eclipse.tracecompass.analysis.os.linux.core.trace.IKernelAnalysisEventLayout;
 import org.eclipse.tracecompass.analysis.os.linux.core.trace.IKernelTrace;
 import org.eclipse.tracecompass.analysis.timing.core.segmentstore.AbstractSegmentStoreAnalysisEventBasedModule;
+import org.eclipse.tracecompass.datastore.core.interval.IHTIntervalReader;
 import org.eclipse.tracecompass.segmentstore.core.ISegment;
 import org.eclipse.tracecompass.segmentstore.core.ISegmentStore;
+import org.eclipse.tracecompass.segmentstore.core.SegmentStoreFactory.SegmentStoreType;
 import org.eclipse.tracecompass.tmf.core.analysis.IAnalysisModule;
 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
 import org.eclipse.tracecompass.tmf.core.segment.ISegmentAspect;
@@ -49,8 +51,6 @@ public class SystemCallLatencyAnalysis extends AbstractSegmentStoreAnalysisEvent
      */
     public static final String ID = "org.eclipse.tracecompass.analysis.os.linux.latency.syscall"; //$NON-NLS-1$
 
-    private static final String DATA_FILENAME = "latency-analysis.dat"; //$NON-NLS-1$
-
     private static final Collection<ISegmentAspect> BASE_ASPECTS =
             ImmutableList.of(SyscallNameAspect.INSTANCE);
 
@@ -78,15 +78,21 @@ public class SystemCallLatencyAnalysis extends AbstractSegmentStoreAnalysisEvent
     }
 
     @Override
-    public @NonNull String getDataFileName() {
-        return DATA_FILENAME;
+    protected @NonNull SegmentStoreType getSegmentStoreType() {
+        return SegmentStoreType.OnDisk;
     }
 
     @Override
-    public AbstractSegmentStoreAnalysisRequest createAnalysisRequest(ISegmentStore<ISegment> syscalls) {
+    protected  AbstractSegmentStoreAnalysisRequest createAnalysisRequest(ISegmentStore<ISegment> syscalls) {
         return new SyscallLatencyAnalysisRequest(syscalls);
     }
 
+    @Override
+    protected @NonNull IHTIntervalReader<ISegment> getSegmentReader() {
+        return SystemCall.READER;
+    }
+
+    @Deprecated
     @Override
     protected Object[] readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException {
         return checkNotNull((Object[]) ois.readObject());
@@ -159,7 +165,7 @@ public class SystemCallLatencyAnalysis extends AbstractSegmentStoreAnalysisEvent
                 }
 
                 long endTime = event.getTimestamp().toNanos();
-                ISegment syscall = new SystemCall(info, endTime);
+                SystemCall syscall = new SystemCall(info, endTime);
                 getSegmentStore().add(syscall);
             }
         }
index 0e20dd60f6ad93f989e4bcf0086fc38e36f39752..678ff3c5e094090086d96f02f9be6e627bc1f09e 100644 (file)
@@ -12,21 +12,26 @@ import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNull;
 
 import java.io.IOException;
 import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.ListenerList;
+import org.eclipse.jdt.annotation.NonNull;
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tracecompass.common.core.NonNullUtils;
+import org.eclipse.tracecompass.datastore.core.interval.IHTIntervalReader;
+import org.eclipse.tracecompass.internal.analysis.timing.core.Activator;
 import org.eclipse.tracecompass.segmentstore.core.ISegment;
 import org.eclipse.tracecompass.segmentstore.core.ISegmentStore;
 import org.eclipse.tracecompass.segmentstore.core.SegmentStoreFactory;
+import org.eclipse.tracecompass.segmentstore.core.SegmentStoreFactory.SegmentStoreType;
 import org.eclipse.tracecompass.tmf.core.analysis.TmfAbstractAnalysisModule;
 import org.eclipse.tracecompass.tmf.core.exceptions.TmfAnalysisException;
 import org.eclipse.tracecompass.tmf.core.segment.ISegmentAspect;
@@ -44,6 +49,7 @@ import org.eclipse.tracecompass.tmf.core.trace.TmfTraceManager;
  */
 public abstract class AbstractSegmentStoreAnalysisModule extends TmfAbstractAnalysisModule implements ISegmentStoreProvider {
 
+    private static final String EXTENSION = ".ss"; //$NON-NLS-1$
     private final ListenerList fListeners = new ListenerList(ListenerList.IDENTITY);
 
     private @Nullable ISegmentStore<ISegment> fSegmentStore;
@@ -81,10 +87,10 @@ public abstract class AbstractSegmentStoreAnalysisModule extends TmfAbstractAnal
     /**
      * Returns the file name for storing segment store
      *
-     * @return segment store fine name, or null if you don't want a file
+     * @return segment store file name
      */
-    protected @Nullable String getDataFileName() {
-        return null;
+    protected String getDataFileName() {
+        return getId() + EXTENSION;
     }
 
     /**
@@ -97,7 +103,10 @@ public abstract class AbstractSegmentStoreAnalysisModule extends TmfAbstractAnal
      *             - Class of a serialized object cannot be found.
      * @throws IOException
      *             - Any of the usual Input/Output related exceptions.
+     * @deprecated The segment store analysis modules are either on disk or all
+     *             in memory, no in between anymore
      */
+    @Deprecated
     protected abstract Object[] readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException;
 
     /**
@@ -119,6 +128,28 @@ public abstract class AbstractSegmentStoreAnalysisModule extends TmfAbstractAnal
      */
     protected abstract boolean buildAnalysisSegments(ISegmentStore<ISegment> segmentStore, IProgressMonitor monitor) throws TmfAnalysisException;
 
+    /**
+     * Get the reader for the segments on disk. If the segment store is not on
+     * disk, this method can return null.
+     *
+     * @return The segment reader
+     * @since 3.0
+     */
+    protected IHTIntervalReader<ISegment> getSegmentReader() {
+        throw new UnsupportedOperationException("getSegmentReader: This method should be overriden in classes that saves the segment store on disk"); //$NON-NLS-1$
+    }
+
+    /**
+     * Get the type of segment store to build. By default it is
+     * {@link SegmentStoreType#Fast}
+     *
+     * @return The type of segment store to build
+     * @since 3.0
+     */
+    protected SegmentStoreType getSegmentStoreType() {
+        return SegmentStoreType.Fast;
+    }
+
     @Override
     public @Nullable ISegmentStore<ISegment> getSegmentStore() {
         return fSegmentStore;
@@ -135,60 +166,83 @@ public abstract class AbstractSegmentStoreAnalysisModule extends TmfAbstractAnal
 
     @Override
     protected boolean executeAnalysis(IProgressMonitor monitor) throws TmfAnalysisException {
+        SegmentStoreType type = getSegmentStoreType();
+        ISegmentStore<ISegment> store = null;
+        switch (type) {
+        case Distinct:
+            // Fall-through
+        case Fast:
+            // Fall-through
+        case Stable:
+            store = buildInMemorySegmentStore(type, monitor);
+            break;
+        case OnDisk:
+            final @Nullable String dataFileName = getDataFileName();
+            store = buildOnDiskSegmentStore(dataFileName, monitor);
+            break;
+        default:
+            Activator.getInstance().logError("Unknown segment store type: " + type); //$NON-NLS-1$
+            break;
+        }
+
+        if (store == null) {
+            return false;
+        }
+
+        fSegmentStore = store;
+        sendUpdate(store);
+        return true;
+    }
+
+    private @Nullable ISegmentStore<@NonNull ISegment> buildOnDiskSegmentStore(@Nullable String dataFileName, IProgressMonitor monitor) throws TmfAnalysisException {
         ITmfTrace trace = checkNotNull(getTrace());
 
-        final @Nullable String dataFileName = getDataFileName();
-        if (dataFileName != null) {
-            /* See if the data file already exists on disk */
-            String dir = TmfTraceManager.getSupplementaryFileDir(trace);
-            final Path file = Paths.get(dir, dataFileName);
-
-            if (Files.exists(file)) {
-                /* Attempt to read the existing file */
-                try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(file))) {
-                    Object[] segmentArray = readObject(ois);
-                    ISegmentStore<ISegment> store = SegmentStoreFactory.createSegmentStore(NonNullUtils.checkNotNullContents(segmentArray));
-                    fSegmentStore = store;
-                    sendUpdate(store);
-                    return true;
-                } catch (IOException | ClassNotFoundException | ClassCastException e) {
-                    /*
-                     * We did not manage to read the file successfully, we will
-                     * just fall-through to rebuild a new one.
-                     */
-                    try {
-                        Files.delete(file);
-                    } catch (IOException e1) {
-                    }
-                }
+        String fileName = dataFileName;
+        if (fileName == null) {
+            fileName = getId() + ".ss"; //$NON-NLS-1$
+        }
+        /* See if the data file already exists on disk */
+        String dir = TmfTraceManager.getSupplementaryFileDir(trace);
+        final Path file = Paths.get(dir, fileName);
+
+        boolean built = false;
+        ISegmentStore<ISegment> segmentStore;
+        try {
+            // Compare the file creation time to determine if this analysis is
+            // built from scratch or not
+            FileTime origCreationTime = (Files.exists(file) ? NonNullUtils.checkNotNull(Files.readAttributes(file, BasicFileAttributes.class)).creationTime() : FileTime.fromMillis(0));
+            segmentStore = SegmentStoreFactory.createOnDiskSegmentStore(file, getSegmentReader());
+            FileTime creationTime = NonNullUtils.checkNotNull(Files.readAttributes(file, BasicFileAttributes.class)).creationTime();
+            built = origCreationTime.equals(creationTime);
+        } catch (IOException e) {
+            try {
+                Files.deleteIfExists(file);
+            } catch (IOException e1) {
+                // Ignore
             }
+            Activator.getInstance().logError("Error creating segment store", e); //$NON-NLS-1$
+            return null;
         }
 
-        ISegmentStore<ISegment> segmentStore = SegmentStoreFactory.createSegmentStore();
+        if (built) {
+            return segmentStore;
+        }
         boolean completed = buildAnalysisSegments(segmentStore, monitor);
         if (!completed) {
-            return false;
-        }
-        fSegmentStore = segmentStore;
-
-        if (dataFileName != null) {
-            String dir = TmfTraceManager.getSupplementaryFileDir(trace);
-            final Path file = Paths.get(dir, dataFileName);
-
-            /* Serialize the collections to disk for future usage */
-            try (ObjectOutputStream oos = new ObjectOutputStream(Files.newOutputStream(file))) {
-                oos.writeObject(segmentStore.toArray());
-            } catch (IOException e) {
-                /*
-                 * Didn't work, oh well. We will just re-read the trace next
-                 * time
-                 */
-            }
+            return null;
         }
 
-        sendUpdate(segmentStore);
+        return segmentStore;
+    }
 
-        return true;
+    private @Nullable ISegmentStore<@NonNull ISegment> buildInMemorySegmentStore(SegmentStoreType type, IProgressMonitor monitor) throws TmfAnalysisException {
+        ISegmentStore<ISegment> segmentStore = SegmentStoreFactory.createSegmentStore(type);
+        boolean completed = buildAnalysisSegments(segmentStore, monitor);
+        if (!completed) {
+            return null;
+        }
+
+        return segmentStore;
     }
 
     /**
index 29524bf60a28e318057df0fff7ce2106c26c52ac..e43066a5ecae4856244f765776c99ed10ea54395 100644 (file)
@@ -159,10 +159,16 @@ public abstract class AbstractSegmentStatisticsAnalysis extends TmfAbstractAnaly
         if (segmentStoreProviderModule instanceof IAnalysisModule) {
             ((IAnalysisModule) segmentStoreProviderModule).waitForCompletion();
         }
-
+        long t0 = start;
+        long t1 = end;
+        if (end < start) {
+            t0 = end;
+            t1 = start;
+        }
         ISegmentStore<@NonNull ISegment> segmentStore = segmentStoreProviderModule.getSegmentStore();
-        return segmentStore != null ? start != TmfTimeRange.ETERNITY.getStartTime().toNanos() || end != TmfTimeRange.ETERNITY.getEndTime().toNanos() ? (Iterable<@NonNull ISegment>) segmentStore.getIntersectingElements(start, end) : segmentStore
-                : Collections.EMPTY_LIST;
+        return segmentStore != null ? t0 != TmfTimeRange.ETERNITY.getStartTime().toNanos() || t1 != TmfTimeRange.ETERNITY.getEndTime().toNanos() ?
+                (Iterable<@NonNull ISegment>) segmentStore.getIntersectingElements(t0, t1) : segmentStore
+                : Collections.emptyList();
     }
 
     private static @Nullable IStatistics<ISegment> calculateTotalManual(Iterable<@NonNull ISegment> segments, IProgressMonitor monitor) {
diff --git a/common/org.eclipse.tracecompass.common.core/annotations/java/nio/file/Paths.eea b/common/org.eclipse.tracecompass.common.core/annotations/java/nio/file/Paths.eea
new file mode 100644 (file)
index 0000000..3d39c44
--- /dev/null
@@ -0,0 +1,4 @@
+class java/nio/file/Paths
+get
+ (Ljava/lang/String;[Ljava/lang/String;)Ljava/nio/file/Path;
+ (Ljava/lang/String;[Ljava/lang/String;)L1java/nio/file/Path;
index 428f23238ba71bf86b1d0492f90f63d6a4065cfe..68e849f181e994bf37134d9823e641f25d9b9a01 100644 (file)
@@ -145,6 +145,15 @@ public class HistoryTreeSegmentStore<E extends ISegment> implements ISegmentStor
         fFinishedBuilding = true;
     }
 
+    @Override
+    public void close(boolean deleteFiles) {
+        if (deleteFiles) {
+            removeFiles();
+        } else {
+            finishedBuilding(getEndTime());
+        }
+    }
+
     /**
      * delete the SHT files from disk
      */
index 5a2d1ea7fe199457fc7845684d1082176bbf7392..c95b55229e1320831b1a39e16b9070b8463de778 100644 (file)
@@ -57,7 +57,17 @@ public final class SegmentStoreFactory<E> {
         /**
          * Segment Store should contain no duplicate segments
          */
-        Distinct
+        Distinct,
+        /**
+         * Segment store that doesn't have to reside entirely in memory, ideal
+         * for very large stores, the performance are not as high as the other
+         * segment stores. These kind of stores should be created using the
+         * {@link SegmentStoreFactory#createOnDiskSegmentStore(Path, IHTIntervalReader)}
+         * factory method
+         *
+         * @since 2.0
+         */
+        OnDisk
     }
 
     private SegmentStoreFactory() {
index 7bc836732b47b419d907916c2822f3d126ba186b..0007dba18717cc36cc042cf5696f3837f1f4093f 100644 (file)
@@ -54,6 +54,7 @@ public class XmlPatternSegmentStoreModule extends AbstractSegmentStoreAnalysisMo
         fParent = parent;
     }
 
+    @Deprecated
     @Override
     protected Object @NonNull [] readObject(@NonNull ObjectInputStream ois) throws ClassNotFoundException, IOException {
         return checkNotNull((Object[]) ois.readObject());
@@ -84,7 +85,7 @@ public class XmlPatternSegmentStoreModule extends AbstractSegmentStoreAnalysisMo
     }
 
     @Override
-    protected @Nullable String getDataFileName() {
+    protected String getDataFileName() {
         return getId() + XmlPatternAnalysis.SEGMENT_STORE_EXTENSION;
     }
 
This page took 0.033216 seconds and 5 git commands to generate.