org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/trace/CTFStreamInput.java

   1 /*******************************************************************************
   2  * Copyright (c) 2011, 2014 Ericsson, Ecole Polytechnique de Montreal and others
   3  *
   4  * All rights reserved. This program and the accompanying materials are made
   5  * available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors: Matthew Khouzam - Initial API and implementation
  10  * Contributors: Simon Marchi - Initial API and implementation
  11  *******************************************************************************/
  12
  13 package org.eclipse.linuxtools.ctf.core.trace;
  14
  15 import java.io.File;
  16 import java.io.IOException;
  17 import java.nio.ByteBuffer;
  18 import java.nio.channels.FileChannel;
  19 import java.nio.channels.FileChannel.MapMode;
  20 import java.nio.file.StandardOpenOption;
  21 import java.util.UUID;
  22
  23 import org.eclipse.jdt.annotation.NonNull;
  24 import org.eclipse.linuxtools.ctf.core.event.io.BitBuffer;
  25 import org.eclipse.linuxtools.ctf.core.event.scope.IDefinitionScope;
  26 import org.eclipse.linuxtools.ctf.core.event.scope.LexicalScope;
  27 import org.eclipse.linuxtools.ctf.core.event.types.Definition;
  28 import org.eclipse.linuxtools.ctf.core.event.types.EnumDefinition;
  29 import org.eclipse.linuxtools.ctf.core.event.types.FloatDefinition;
  30 import org.eclipse.linuxtools.ctf.core.event.types.IDefinition;
  31 import org.eclipse.linuxtools.ctf.core.event.types.IntegerDefinition;
  32 import org.eclipse.linuxtools.ctf.core.event.types.StringDefinition;
  33 import org.eclipse.linuxtools.ctf.core.event.types.StructDeclaration;
  34 import org.eclipse.linuxtools.ctf.core.event.types.StructDefinition;
  35 import org.eclipse.linuxtools.internal.ctf.core.SafeMappedByteBuffer;
  36 import org.eclipse.linuxtools.internal.ctf.core.event.types.ArrayDefinition;
  37 import org.eclipse.linuxtools.internal.ctf.core.trace.StreamInputPacketIndex;
  38 import org.eclipse.linuxtools.internal.ctf.core.trace.StreamInputPacketIndexEntry;
  39
  40 /**
  41  * <b><u>StreamInput</u></b>
  42  * <p>
  43  * Represents a trace file that belongs to a certain stream.
  44  *
  45  * @since 3.0
  46  */
  47 // TODO: remove AutoCloseable
  48 public class CTFStreamInput implements IDefinitionScope, AutoCloseable {
  49
  50     // ------------------------------------------------------------------------
  51     // Attributes
  52     // ------------------------------------------------------------------------
  53
  54     /**
  55      * The associated Stream
  56      */
  57     private final CTFStream fStream;
  58
  59     /**
  60      * Information on the file (used for debugging)
  61      */
  62     @NonNull
  63     private final File fFile;
  64
  65     /**
  66      * The packet index of this input
  67      */
  68     private final StreamInputPacketIndex fIndex;
  69
  70     private long fTimestampEnd;
  71
  72     /**
  73      * Definition of trace packet header
  74      */
  75     private StructDeclaration fTracePacketHeaderDecl = null;
  76
  77     /**
  78      * Definition of trace stream packet context
  79      */
  80     private StructDeclaration fStreamPacketContextDecl = null;
  81
  82     /**
  83      * Total number of lost events in this stream
  84      */
  85     private long fLostSoFar = 0;
  86
  87     // ------------------------------------------------------------------------
  88     // Constructors
  89     // ------------------------------------------------------------------------
  90
  91     /**
  92      * Constructs a StreamInput.
  93      *
  94      * @param stream
  95      *            The stream to which this StreamInput belongs to.
  96      * @param file
  97      *            Information about the trace file (for debugging purposes).
  98      */
  99     public CTFStreamInput(CTFStream stream, @NonNull File file) {
 100         fStream = stream;
 101         fFile = file;
 102         fIndex = new StreamInputPacketIndex();
 103     }
 104
 105     @Override
 106     public void close() throws IOException {
 107     }
 108
 109     // ------------------------------------------------------------------------
 110     // Getters/Setters/Predicates
 111     // ------------------------------------------------------------------------
 112
 113     /**
 114      * Gets the stream the streamInput wrapper is wrapping
 115      *
 116      * @return the stream the streamInput wrapper is wrapping
 117      */
 118     public CTFStream getStream() {
 119         return fStream;
 120     }
 121
 122     /**
 123      * The common streamInput Index
 124      *
 125      * @return the stream input Index
 126      */
 127     StreamInputPacketIndex getIndex() {
 128         return fIndex;
 129     }
 130
 131     /**
 132      * Gets the filename of the streamInput file.
 133      *
 134      * @return the filename of the streaminput file.
 135      */
 136     public String getFilename() {
 137         return fFile.getName();
 138     }
 139
 140     /**
 141      * Gets the last read timestamp of a stream. (this is not necessarily the
 142      * last time in the stream.)
 143      *
 144      * @return the last read timestamp
 145      */
 146     public long getTimestampEnd() {
 147         return fTimestampEnd;
 148     }
 149
 150     /**
 151      * Sets the last read timestamp of a stream. (this is not necessarily the
 152      * last time in the stream.)
 153      *
 154      * @param timestampEnd
 155      *            the last read timestamp
 156      */
 157     public void setTimestampEnd(long timestampEnd) {
 158         fTimestampEnd = timestampEnd;
 159     }
 160
 161     /**
 162      * Useless for streaminputs
 163      */
 164     @Override
 165     public LexicalScope getScopePath() {
 166         return LexicalScope.STREAM;
 167     }
 168
 169     // ------------------------------------------------------------------------
 170     // Operations
 171     // ------------------------------------------------------------------------
 172
 173     @Override
 174     public Definition lookupDefinition(String lookupPath) {
 175         /* TODO: lookup in different dynamic scopes is not supported yet. */
 176         return null;
 177     }
 178
 179     /**
 180      * Create the index for this trace file.
 181      */
 182     public void setupIndex() {
 183
 184         /*
 185          * The BitBuffer to extract data from the StreamInput
 186          */
 187         BitBuffer bitBuffer = new BitBuffer();
 188         bitBuffer.setByteOrder(getStream().getTrace().getByteOrder());
 189
 190         /*
 191          * Create the definitions we need to read the packet headers + contexts
 192          */
 193         if (getStream().getTrace().getPacketHeader() != null) {
 194             fTracePacketHeaderDecl = getStream().getTrace().getPacketHeader();
 195         }
 196
 197         if (getStream().getPacketContextDecl() != null) {
 198             fStreamPacketContextDecl = getStream().getPacketContextDecl();
 199         }
 200
 201     }
 202
 203     /**
 204      * Adds the next packet header index entry to the index of a stream input.
 205      *
 206      * <strong>This method is slow and can corrupt data if not used
 207      * properly</strong>
 208      *
 209      * @return true if there are more packets to add
 210      * @throws CTFReaderException
 211      *             If there was a problem reading the packed header
 212      */
 213     public boolean addPacketHeaderIndex() throws CTFReaderException {
 214         long currentPos = 0L;
 215         if (!fIndex.getEntries().isEmpty()) {
 216             StreamInputPacketIndexEntry pos = fIndex.getEntries().lastElement();
 217             currentPos = computeNextOffset(pos);
 218         }
 219         long fileSize = getStreamSize();
 220         if (currentPos < fileSize) {
 221
 222             StreamInputPacketIndexEntry packetIndex = new StreamInputPacketIndexEntry(
 223                     currentPos);
 224             createPacketIndexEntry(fileSize, currentPos, packetIndex);
 225             fIndex.addEntry(packetIndex);
 226             return true;
 227         }
 228         return false;
 229     }
 230
 231     private long getStreamSize() {
 232         return fFile.length();
 233     }
 234
 235     private long createPacketIndexEntry(long fileSizeBytes, long packetOffsetBytes, StreamInputPacketIndexEntry packetIndex)
 236             throws CTFReaderException {
 237
 238         long pos = readPacketHeader(fileSizeBytes, packetOffsetBytes, packetIndex);
 239
 240         /* Basic validation */
 241         if (packetIndex.getContentSizeBits() > packetIndex.getPacketSizeBits()) {
 242             throw new CTFReaderException("Content size > packet size"); //$NON-NLS-1$
 243         }
 244
 245         if (packetIndex.getPacketSizeBits() > ((fileSizeBytes - packetIndex
 246                 .getOffsetBytes()) * 8)) {
 247             throw new CTFReaderException("Not enough data remaining in the file for the size of this packet"); //$NON-NLS-1$
 248         }
 249
 250         /*
 251          * Offset in the file, in bits
 252          */
 253         packetIndex.setDataOffsetBits(pos);
 254
 255         /*
 256          * Update the counting packet offset
 257          */
 258         return computeNextOffset(packetIndex);
 259     }
 260
 261     /**
 262      * @param packetIndex
 263      * @return
 264      */
 265     private static long computeNextOffset(
 266             StreamInputPacketIndexEntry packetIndex) {
 267         return packetIndex.getOffsetBytes()
 268                 + ((packetIndex.getPacketSizeBits() + 7) / 8);
 269     }
 270
 271     private long readPacketHeader(long fileSizeBytes,
 272             long packetOffsetBytes, StreamInputPacketIndexEntry packetIndex) throws CTFReaderException {
 273         long position = -1;
 274         /*
 275          * Initial size, it should map at least the packet header + context
 276          * size.
 277          *
 278          * TODO: use a less arbitrary size.
 279          */
 280         long mapSize = 4096;
 281         /*
 282          * If there is less data remaining than what we want to map, reduce the
 283          * map size.
 284          */
 285         if ((fileSizeBytes - packetIndex.getOffsetBytes()) < mapSize) {
 286             mapSize = fileSizeBytes - packetIndex.getOffsetBytes();
 287         }
 288
 289         /*
 290          * Map the packet.
 291          */
 292         try (FileChannel fc = FileChannel.open(fFile.toPath(), StandardOpenOption.READ)) {
 293             ByteBuffer map = SafeMappedByteBuffer.map(fc, MapMode.READ_ONLY, packetOffsetBytes, mapSize);
 294             if (map == null) {
 295                 throw new CTFReaderException("Failed to allocate mapped byte buffer"); //$NON-NLS-1$
 296             }
 297             /*
 298              * create a packet bit buffer to read the packet header
 299              */
 300             BitBuffer bitBuffer = new BitBuffer(map);
 301             bitBuffer.setByteOrder(getStream().getTrace().getByteOrder());
 302             /*
 303              * Read the trace packet header if it exists.
 304              */
 305             if (fTracePacketHeaderDecl != null) {
 306                 parseTracePacketHeader(fTracePacketHeaderDecl, bitBuffer);
 307             }
 308
 309             /*
 310              * Read the stream packet context if it exists.
 311              */
 312             if (fStreamPacketContextDecl != null) {
 313                 parsePacketContext(fileSizeBytes, fStreamPacketContextDecl,
 314                         bitBuffer, packetIndex);
 315             } else {
 316                 setPacketContextNull(fileSizeBytes, packetIndex);
 317             }
 318
 319             position = bitBuffer.position();
 320         } catch (IOException e) {
 321             throw new CTFReaderException(e);
 322         }
 323         return position;
 324     }
 325
 326     private void parseTracePacketHeader(StructDeclaration tracePacketHeaderDecl,
 327             @NonNull BitBuffer bitBuffer) throws CTFReaderException {
 328         StructDefinition tracePacketHeaderDef = tracePacketHeaderDecl.createDefinition(fStream.getTrace(), LexicalScope.TRACE_PACKET_HEADER, bitBuffer);
 329
 330         /*
 331          * Check the CTF magic number
 332          */
 333         IntegerDefinition magicDef = (IntegerDefinition) tracePacketHeaderDef
 334                 .lookupDefinition("magic"); //$NON-NLS-1$
 335         if (magicDef != null) {
 336             int magic = (int) magicDef.getValue();
 337             if (magic != Utils.CTF_MAGIC) {
 338                 throw new CTFReaderException(
 339                         "CTF magic mismatch " + Integer.toHexString(magic) + " vs " + Integer.toHexString(Utils.CTF_MAGIC)); //$NON-NLS-1$//$NON-NLS-2$
 340             }
 341         }
 342
 343         /*
 344          * Check the trace UUID
 345          */
 346         ArrayDefinition uuidDef =
 347                 (ArrayDefinition) tracePacketHeaderDef.lookupDefinition("uuid"); //$NON-NLS-1$
 348         if (uuidDef != null) {
 349             UUID uuid = Utils.getUUIDfromDefinition(uuidDef);
 350
 351             if (!getStream().getTrace().getUUID().equals(uuid)) {
 352                 throw new CTFReaderException("UUID mismatch"); //$NON-NLS-1$
 353             }
 354         }
 355
 356         /*
 357          * Check that the stream id did not change
 358          */
 359         IntegerDefinition streamIDDef = (IntegerDefinition) tracePacketHeaderDef
 360                 .lookupDefinition("stream_id"); //$NON-NLS-1$
 361         if (streamIDDef != null) {
 362             long streamID = streamIDDef.getValue();
 363
 364             if (streamID != getStream().getId()) {
 365                 throw new CTFReaderException("Stream ID changing within a StreamInput"); //$NON-NLS-1$
 366             }
 367         }
 368     }
 369
 370     /**
 371      * Gets the wrapped file
 372      *
 373      * @return the file
 374      */
 375     @NonNull
 376     File getFile() {
 377         return fFile;
 378     }
 379
 380     private static void setPacketContextNull(long fileSizeBytes,
 381             StreamInputPacketIndexEntry packetIndex) {
 382         /*
 383          * If there is no packet context, infer the content and packet size from
 384          * the file size (assume that there is only one packet and no padding)
 385          */
 386         packetIndex.setContentSizeBits(fileSizeBytes * 8);
 387         packetIndex.setPacketSizeBits(fileSizeBytes * 8);
 388     }
 389
 390     private void parsePacketContext(long fileSizeBytes,
 391             StructDeclaration streamPacketContextDecl, @NonNull BitBuffer bitBuffer,
 392             StreamInputPacketIndexEntry packetIndex) throws CTFReaderException {
 393         StructDefinition streamPacketContextDef = streamPacketContextDecl.createDefinition(this, LexicalScope.STREAM_PACKET_CONTEXT, bitBuffer);
 394
 395         for (String field : streamPacketContextDef.getDeclaration()
 396                 .getFieldsList()) {
 397             IDefinition id = streamPacketContextDef.lookupDefinition(field);
 398             if (id instanceof IntegerDefinition) {
 399                 packetIndex.addAttribute(field,
 400                         ((IntegerDefinition) id).getValue());
 401             } else if (id instanceof FloatDefinition) {
 402                 packetIndex.addAttribute(field,
 403                         ((FloatDefinition) id).getValue());
 404             } else if (id instanceof EnumDefinition) {
 405                 packetIndex.addAttribute(field,
 406                         ((EnumDefinition) id).getValue());
 407             } else if (id instanceof StringDefinition) {
 408                 packetIndex.addAttribute(field,
 409                         ((StringDefinition) id).getValue());
 410             }
 411         }
 412
 413         Long contentSize = (Long) packetIndex.lookupAttribute("content_size"); //$NON-NLS-1$
 414         Long packetSize = (Long) packetIndex.lookupAttribute("packet_size"); //$NON-NLS-1$
 415         Long tsBegin = (Long) packetIndex.lookupAttribute("timestamp_begin"); //$NON-NLS-1$
 416         Long tsEnd = (Long) packetIndex.lookupAttribute("timestamp_end"); //$NON-NLS-1$
 417         String device = (String) packetIndex.lookupAttribute("device"); //$NON-NLS-1$
 418         // LTTng Specific
 419         Long cpuId = (Long) packetIndex.lookupAttribute("cpu_id"); //$NON-NLS-1$
 420         Long lostEvents = (Long) packetIndex.lookupAttribute("events_discarded"); //$NON-NLS-1$
 421
 422         /* Read the content size in bits */
 423         if (contentSize != null) {
 424             packetIndex.setContentSizeBits(contentSize.intValue());
 425         } else if (packetSize != null) {
 426             packetIndex.setContentSizeBits(packetSize.longValue());
 427         } else {
 428             packetIndex.setContentSizeBits((int) (fileSizeBytes * 8));
 429         }
 430
 431         /* Read the packet size in bits */
 432         if (packetSize != null) {
 433             packetIndex.setPacketSizeBits(packetSize.intValue());
 434         } else if (packetIndex.getContentSizeBits() != 0) {
 435             packetIndex.setPacketSizeBits(packetIndex.getContentSizeBits());
 436         } else {
 437             packetIndex.setPacketSizeBits((int) (fileSizeBytes * 8));
 438         }
 439
 440         /* Read the begin timestamp */
 441         if (tsBegin != null) {
 442             packetIndex.setTimestampBegin(tsBegin.longValue());
 443         }
 444
 445         /* Read the end timestamp */
 446         if (tsEnd != null) {
 447             if (tsEnd == -1) {
 448                 tsEnd = Long.MAX_VALUE;
 449             }
 450             packetIndex.setTimestampEnd(tsEnd.longValue());
 451             setTimestampEnd(packetIndex.getTimestampEnd());
 452         }
 453
 454         if (device != null) {
 455             packetIndex.setTarget(device);
 456         }
 457
 458         if (cpuId != null) {
 459             packetIndex.setTarget("CPU" + cpuId.toString()); //$NON-NLS-1$
 460         }
 461
 462         if (lostEvents != null) {
 463             packetIndex.setLostEvents(lostEvents - fLostSoFar);
 464             fLostSoFar = lostEvents;
 465         }
 466     }
 467
 468     @Override
 469     public int hashCode() {
 470         final int prime = 31;
 471         int result = 1;
 472         result = (prime * result) + fFile.hashCode();
 473         return result;
 474     }
 475
 476     @Override
 477     public boolean equals(Object obj) {
 478         if (this == obj) {
 479             return true;
 480         }
 481         if (obj == null) {
 482             return false;
 483         }
 484         if (!(obj instanceof CTFStreamInput)) {
 485             return false;
 486         }
 487         CTFStreamInput other = (CTFStreamInput) obj;
 488         if (!fFile.equals(other.fFile)) {
 489             return false;
 490         }
 491         return true;
 492     }
 493
 494 }