org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/trace/Metadata.java

   1 /*******************************************************************************
   2  * Copyright (c) 2011, 2014 Ericsson, Ecole Polytechnique de Montreal and others
   3  *
   4  * All rights reserved. This program and the accompanying materials are made
   5  * available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     Matthew Khouzam - Initial API and implementation
  11  *     Simon Marchi - Initial API and implementation
  12  *     Matthew Khouzam - Update for live trace reading support
  13  *******************************************************************************/
  14
  15 package org.eclipse.linuxtools.ctf.core.trace;
  16
  17 import java.io.FileInputStream;
  18 import java.io.FileNotFoundException;
  19 import java.io.FileReader;
  20 import java.io.IOException;
  21 import java.io.Reader;
  22 import java.io.StringReader;
  23 import java.nio.ByteBuffer;
  24 import java.nio.ByteOrder;
  25 import java.nio.channels.FileChannel;
  26 import java.util.UUID;
  27
  28 import org.antlr.runtime.ANTLRReaderStream;
  29 import org.antlr.runtime.CommonTokenStream;
  30 import org.antlr.runtime.RecognitionException;
  31 import org.antlr.runtime.tree.CommonTree;
  32 import org.antlr.runtime.tree.RewriteCardinalityException;
  33 import org.eclipse.linuxtools.ctf.parser.CTFLexer;
  34 import org.eclipse.linuxtools.ctf.parser.CTFParser;
  35 import org.eclipse.linuxtools.ctf.parser.CTFParser.parse_return;
  36 import org.eclipse.linuxtools.internal.ctf.core.event.metadata.IOStructGen;
  37 import org.eclipse.linuxtools.internal.ctf.core.event.metadata.exceptions.CtfAntlrException;
  38 import org.eclipse.linuxtools.internal.ctf.core.event.metadata.exceptions.ParseException;
  39
  40 /**
  41  * The CTF trace metadata TSDL file
  42  *
  43  * @version 1.0
  44  * @author Matthew Khouzam
  45  * @author Simon Marchi
  46  */
  47 public class Metadata {
  48
  49     // ------------------------------------------------------------------------
  50     // Constants
  51     // ------------------------------------------------------------------------
  52
  53     /**
  54      * Name of the metadata file in the trace directory
  55      */
  56     private static final String METADATA_FILENAME = "metadata"; //$NON-NLS-1$
  57
  58     /**
  59      * Size of the metadata packet header, in bytes, computed by hand.
  60      */
  61     private static final int METADATA_PACKET_HEADER_SIZE = 37;
  62
  63     // ------------------------------------------------------------------------
  64     // Attributes
  65     // ------------------------------------------------------------------------
  66
  67     /**
  68      * Byte order as detected when reading the TSDL magic number.
  69      */
  70     private ByteOrder detectedByteOrder = null;
  71
  72     /**
  73      * The trace file to which belongs this metadata file.
  74      */
  75     private final CTFTrace trace;
  76
  77     private IOStructGen fTreeParser;
  78
  79     // ------------------------------------------------------------------------
  80     // Constructors
  81     // ------------------------------------------------------------------------
  82
  83     /**
  84      * Constructs a Metadata object.
  85      *
  86      * @param trace
  87      *            The trace to which belongs this metadata file.
  88      */
  89     public Metadata(CTFTrace trace) {
  90         this.trace = trace;
  91     }
  92
  93     /**
  94      * For network streaming
  95      *
  96      * @since 3.0
  97      */
  98     public Metadata() {
  99         trace = new CTFTrace();
 100     }
 101
 102     // ------------------------------------------------------------------------
 103     // Getters/Setters/Predicates
 104     // ------------------------------------------------------------------------
 105
 106     /**
 107      * Returns the ByteOrder that was detected while parsing the metadata.
 108      *
 109      * @return The byte order.
 110      */
 111     public ByteOrder getDetectedByteOrder() {
 112         return detectedByteOrder;
 113     }
 114
 115     /**
 116      * Gets the parent trace
 117      *
 118      * @return the parent trace
 119      * @since 3.0
 120      */
 121     public CTFTrace getTrace() {
 122         return trace;
 123     }
 124
 125     // ------------------------------------------------------------------------
 126     // Operations
 127     // ------------------------------------------------------------------------
 128
 129     /**
 130      * Parse the metadata file.
 131      *
 132      * @throws CTFReaderException
 133      *             If there was a problem parsing the metadata
 134      * @since 3.0
 135      */
 136     public void parseFile() throws CTFReaderException {
 137
 138         /*
 139          * Reader. It will contain a StringReader if we are using packet-based
 140          * metadata and it will contain a FileReader if we have text-based
 141          * metadata.
 142          */
 143
 144         try (FileInputStream fis = new FileInputStream(getMetadataPath());
 145                 FileChannel metadataFileChannel = fis.getChannel();
 146                 /* Check if metadata is packet-based, if not it is text based */
 147                 Reader metadataTextInput =
 148                         (isPacketBased(metadataFileChannel) ?
 149                                 readBinaryMetaData(metadataFileChannel) :
 150                                 new FileReader(getMetadataPath()));) {
 151
 152             readMetaDataText(metadataTextInput);
 153
 154         } catch (FileNotFoundException e) {
 155             throw new CTFReaderException("Cannot find metadata file!"); //$NON-NLS-1$
 156         } catch (IOException | ParseException e) {
 157             throw new CTFReaderException(e);
 158         } catch (RecognitionException | RewriteCardinalityException e) {
 159             throw new CtfAntlrException(e);
 160         }
 161     }
 162
 163     private Reader readBinaryMetaData(FileChannel metadataFileChannel) throws CTFReaderException {
 164         /* Create StringBuffer to receive metadata text */
 165         StringBuffer metadataText = new StringBuffer();
 166
 167         /*
 168          * Read metadata packet one by one, appending the text to the
 169          * StringBuffer
 170          */
 171         MetadataPacketHeader packetHeader = readMetadataPacket(
 172                 metadataFileChannel, metadataText);
 173         while (packetHeader != null) {
 174             packetHeader = readMetadataPacket(metadataFileChannel,
 175                     metadataText);
 176         }
 177
 178         /* Wrap the metadata string with a StringReader */
 179         return new StringReader(metadataText.toString());
 180     }
 181
 182     /**
 183      * Read the metadata from a formatted TSDL string
 184      *
 185      * @param data
 186      *            the data to read
 187      * @throws CTFReaderException
 188      *             this exception wraps a ParseException, IOException or
 189      *             CtfAntlrException, three exceptions that can be obtained from
 190      *             parsing a TSDL file
 191      * @since 3.0
 192      */
 193     public void parseText(String data) throws CTFReaderException {
 194         Reader metadataTextInput = new StringReader(data);
 195         try {
 196             readMetaDataText(metadataTextInput);
 197         } catch (IOException | ParseException e) {
 198             throw new CTFReaderException(e);
 199         } catch (RecognitionException | RewriteCardinalityException e) {
 200             throw new CtfAntlrException(e);
 201         }
 202
 203     }
 204
 205     private void readMetaDataText(Reader metadataTextInput) throws IOException, RecognitionException, ParseException {
 206         CommonTree tree = createAST(metadataTextInput);
 207
 208         /* Generate IO structures (declarations) */
 209         fTreeParser = new IOStructGen(tree, trace);
 210         fTreeParser.generate();
 211     }
 212
 213     /**
 214      * Read a metadata fragment from a formatted TSDL string
 215      *
 216      * @param dataFragment
 217      *            the data to read
 218      * @throws CTFReaderException
 219      *             this exception wraps a ParseException, IOException or
 220      *             CtfAntlrException, three exceptions that can be obtained from
 221      *             parsing a TSDL file
 222      * @since 3.0
 223      */
 224     public void parseTextFragment(String dataFragment) throws CTFReaderException {
 225         Reader metadataTextInput = new StringReader(dataFragment);
 226         try {
 227             readMetaDataTextFragment(metadataTextInput);
 228         } catch (IOException | ParseException e) {
 229             throw new CTFReaderException(e);
 230         } catch (RecognitionException | RewriteCardinalityException e) {
 231             throw new CtfAntlrException(e);
 232         }
 233     }
 234
 235     private void readMetaDataTextFragment(Reader metadataTextInput) throws IOException, RecognitionException, ParseException {
 236         CommonTree tree = createAST(metadataTextInput);
 237         fTreeParser.setTree(tree);
 238         fTreeParser.generateFragment();
 239     }
 240
 241     private static CommonTree createAST(Reader metadataTextInput) throws IOException,
 242             RecognitionException {
 243         /* Create an ANTLR reader */
 244         ANTLRReaderStream antlrStream;
 245         antlrStream = new ANTLRReaderStream(metadataTextInput);
 246
 247         /* Parse the metadata text and get the AST */
 248         CTFLexer ctfLexer = new CTFLexer(antlrStream);
 249         CommonTokenStream tokens = new CommonTokenStream(ctfLexer);
 250         CTFParser ctfParser = new CTFParser(tokens, false);
 251
 252         parse_return pr = ctfParser.parse();
 253         return pr.getTree();
 254     }
 255
 256     /**
 257      * Determines whether the metadata file is packet-based by looking at the
 258      * TSDL magic number. If it is packet-based, it also gives information about
 259      * the endianness of the trace using the detectedByteOrder attribute.
 260      *
 261      * @param metadataFileChannel
 262      *            FileChannel of the metadata file.
 263      * @return True if the metadata is packet-based.
 264      * @throws CTFReaderException
 265      */
 266     private boolean isPacketBased(FileChannel metadataFileChannel)
 267             throws CTFReaderException {
 268         /*
 269          * Create a ByteBuffer to read the TSDL magic number (default is
 270          * big-endian)
 271          */
 272         ByteBuffer magicByteBuffer = ByteBuffer.allocate(Utils.TSDL_MAGIC_LEN);
 273
 274         /* Read without changing file position */
 275         try {
 276             metadataFileChannel.read(magicByteBuffer, 0);
 277         } catch (IOException e) {
 278             throw new CTFReaderException("Unable to read metadata file channel.", e); //$NON-NLS-1$
 279         }
 280
 281         /* Get the first int from the file */
 282         int magic = magicByteBuffer.getInt(0);
 283
 284         /* Check if it matches */
 285         if (Utils.TSDL_MAGIC == magic) {
 286             detectedByteOrder = ByteOrder.BIG_ENDIAN;
 287             return true;
 288         }
 289
 290         /* Try the same thing, but with little-endian */
 291         magicByteBuffer.order(ByteOrder.LITTLE_ENDIAN);
 292         magic = magicByteBuffer.getInt(0);
 293
 294         if (Utils.TSDL_MAGIC == magic) {
 295             detectedByteOrder = ByteOrder.LITTLE_ENDIAN;
 296             return true;
 297         }
 298
 299         return false;
 300     }
 301
 302     private String getMetadataPath() {
 303         /* Path of metadata file = trace directory path + metadata filename */
 304         if (trace.getTraceDirectory() == null) {
 305             return new String();
 306         }
 307         return trace.getTraceDirectory().getPath()
 308                 + Utils.SEPARATOR + METADATA_FILENAME;
 309     }
 310
 311     /**
 312      * Reads a metadata packet from the given metadata FileChannel, do some
 313      * basic validation and append the text to the StringBuffer.
 314      *
 315      * @param metadataFileChannel
 316      *            Metadata FileChannel
 317      * @param metadataText
 318      *            StringBuffer to which the metadata text will be appended.
 319      * @return A structure describing the header of the metadata packet, or null
 320      *         if the end of the file is reached.
 321      * @throws CTFReaderException
 322      */
 323     private MetadataPacketHeader readMetadataPacket(
 324             FileChannel metadataFileChannel, StringBuffer metadataText)
 325             throws CTFReaderException {
 326         /* Allocate a ByteBuffer for the header */
 327         ByteBuffer headerByteBuffer = ByteBuffer.allocate(METADATA_PACKET_HEADER_SIZE);
 328
 329         /* Read the header */
 330         try {
 331             int nbBytesRead = metadataFileChannel.read(headerByteBuffer);
 332
 333             /* Return null if EOF */
 334             if (nbBytesRead < 0) {
 335                 return null;
 336             }
 337
 338             if (nbBytesRead != METADATA_PACKET_HEADER_SIZE) {
 339                 throw new CTFReaderException("Error reading the metadata header."); //$NON-NLS-1$
 340             }
 341
 342         } catch (IOException e) {
 343             throw new CTFReaderException("Error reading the metadata header.", e); //$NON-NLS-1$
 344         }
 345
 346         /* Set ByteBuffer's position to 0 */
 347         headerByteBuffer.position(0);
 348
 349         /* Use byte order that was detected with the magic number */
 350         headerByteBuffer.order(detectedByteOrder);
 351
 352         MetadataPacketHeader header = new MetadataPacketHeader(headerByteBuffer);
 353
 354         /* Check TSDL magic number */
 355         if (!header.isMagicValid()) {
 356             throw new CTFReaderException("TSDL magic number does not match"); //$NON-NLS-1$
 357         }
 358
 359         /* Check UUID */
 360         if (!trace.uuidIsSet()) {
 361             trace.setUUID(header.getUuid());
 362         } else if (!trace.getUUID().equals(header.getUuid())) {
 363             throw new CTFReaderException("UUID mismatch"); //$NON-NLS-1$
 364         }
 365
 366         /* Extract the text from the packet */
 367         int payloadSize = ((header.getContentSize() / 8) - METADATA_PACKET_HEADER_SIZE);
 368         if (payloadSize < 0) {
 369             throw new CTFReaderException("Invalid metadata packet payload size."); //$NON-NLS-1$
 370         }
 371         int skipSize = (header.getPacketSize() - header.getContentSize()) / 8;
 372
 373         /* Read the payload + the padding in a ByteBuffer */
 374         ByteBuffer payloadByteBuffer = ByteBuffer.allocateDirect(payloadSize
 375                 + skipSize);
 376         try {
 377             metadataFileChannel.read(payloadByteBuffer);
 378         } catch (IOException e) {
 379             throw new CTFReaderException("Error reading metadata packet payload.", e); //$NON-NLS-1$
 380         }
 381         payloadByteBuffer.rewind();
 382
 383         /* Read only the payload from the ByteBuffer into a byte array */
 384         byte payloadByteArray[] = new byte[payloadByteBuffer.remaining()];
 385         payloadByteBuffer.get(payloadByteArray, 0, payloadSize);
 386
 387         /* Convert the byte array to a String */
 388         String str = new String(payloadByteArray, 0, payloadSize);
 389
 390         /* Append it to the existing metadata */
 391         metadataText.append(str);
 392
 393         return header;
 394     }
 395
 396     private static class MetadataPacketHeader {
 397
 398         private final int fMagic;
 399         private final UUID fUuid;
 400         private final int fChecksum;
 401         private final int fContentSize;
 402         private final int fPacketSize;
 403         private final byte fCompressionScheme;
 404         private final byte fEncryptionScheme;
 405         private final byte fChecksumScheme;
 406         private final byte fCtfMajorVersion;
 407         private final byte fCtfMinorVersion;
 408
 409         public MetadataPacketHeader(ByteBuffer headerByteBuffer) {
 410             /* Read from the ByteBuffer */
 411             fMagic = headerByteBuffer.getInt();
 412             byte[] uuidBytes = new byte[16];
 413             headerByteBuffer.get(uuidBytes);
 414             fUuid = Utils.makeUUID(uuidBytes);
 415             fChecksum = headerByteBuffer.getInt();
 416             fContentSize = headerByteBuffer.getInt();
 417             fPacketSize = headerByteBuffer.getInt();
 418             fCompressionScheme = headerByteBuffer.get();
 419             fEncryptionScheme = headerByteBuffer.get();
 420             fChecksumScheme = headerByteBuffer.get();
 421             fCtfMajorVersion = headerByteBuffer.get();
 422             fCtfMinorVersion = headerByteBuffer.get();
 423         }
 424
 425         public boolean isMagicValid() {
 426             return fMagic == Utils.TSDL_MAGIC;
 427         }
 428
 429         public UUID getUuid() {
 430             return fUuid;
 431         }
 432
 433         public int getContentSize() {
 434             return fContentSize;
 435         }
 436
 437         public int getPacketSize() {
 438             return fPacketSize;
 439         }
 440
 441         @Override
 442         public String toString() {
 443             /* Only for debugging, shouldn't be externalized */
 444             /* Therefore it cannot be covered by test cases */
 445             return "MetadataPacketHeader [magic=0x" //$NON-NLS-1$
 446                     + Integer.toHexString(fMagic) + ", uuid=" //$NON-NLS-1$
 447                     + fUuid.toString() + ", checksum=" + fChecksum //$NON-NLS-1$
 448                     + ", contentSize=" + fContentSize + ", packetSize=" //$NON-NLS-1$ //$NON-NLS-2$
 449                     + fPacketSize + ", compressionScheme=" + fCompressionScheme //$NON-NLS-1$
 450                     + ", encryptionScheme=" + fEncryptionScheme //$NON-NLS-1$
 451                     + ", checksumScheme=" + fChecksumScheme //$NON-NLS-1$
 452                     + ", ctfMajorVersion=" + fCtfMajorVersion //$NON-NLS-1$
 453                     + ", ctfMinorVersion=" + fCtfMinorVersion + ']'; //$NON-NLS-1$
 454         }
 455
 456     }
 457 }