ctf/org.eclipse.tracecompass.ctf.core/src/org/eclipse/tracecompass/ctf/core/trace/Metadata.java

   1 /*******************************************************************************
   2  * Copyright (c) 2011, 2014 Ericsson, Ecole Polytechnique de Montreal and others
   3  *
   4  * All rights reserved. This program and the accompanying materials are made
   5  * available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     Matthew Khouzam - Initial API and implementation
  11  *     Simon Marchi - Initial API and implementation
  12  *     Matthew Khouzam - Update for live trace reading support
  13  *     Bernd Hufmann - Add method to copy metadata file
  14  *******************************************************************************/
  15
  16 package org.eclipse.tracecompass.ctf.core.trace;
  17
  18 import java.io.File;
  19 import java.io.FileInputStream;
  20 import java.io.FileNotFoundException;
  21 import java.io.FileReader;
  22 import java.io.IOException;
  23 import java.io.Reader;
  24 import java.io.StringReader;
  25 import java.nio.ByteBuffer;
  26 import java.nio.ByteOrder;
  27 import java.nio.channels.FileChannel;
  28 import java.nio.charset.Charset;
  29 import java.nio.file.FileSystems;
  30 import java.nio.file.Files;
  31 import java.nio.file.Path;
  32 import java.nio.file.StandardOpenOption;
  33 import java.util.UUID;
  34
  35 import org.antlr.runtime.ANTLRReaderStream;
  36 import org.antlr.runtime.CommonTokenStream;
  37 import org.antlr.runtime.RecognitionException;
  38 import org.antlr.runtime.tree.CommonTree;
  39 import org.antlr.runtime.tree.RewriteCardinalityException;
  40 import org.eclipse.tracecompass.common.core.NonNullUtils;
  41 import org.eclipse.tracecompass.ctf.core.CTFException;
  42 import org.eclipse.tracecompass.ctf.parser.CTFLexer;
  43 import org.eclipse.tracecompass.ctf.parser.CTFParser;
  44 import org.eclipse.tracecompass.ctf.parser.CTFParser.parse_return;
  45 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.CtfAntlrException;
  46 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.IOStructGen;
  47 import org.eclipse.tracecompass.internal.ctf.core.event.metadata.ParseException;
  48 import org.eclipse.tracecompass.internal.ctf.core.trace.Utils;
  49
  50 /**
  51  * The CTF trace metadata TSDL file
  52  *
  53  * @version 1.0
  54  * @author Matthew Khouzam
  55  * @author Simon Marchi
  56  */
  57 public class Metadata {
  58
  59     // ------------------------------------------------------------------------
  60     // Constants
  61     // ------------------------------------------------------------------------
  62
  63     private static final Charset ASCII_CHARSET = Charset.forName("ASCII"); //$NON-NLS-1$
  64
  65     private static final String TEXT_ONLY_METADATA_HEADER_PREFIX = "/* CTF"; //$NON-NLS-1$
  66
  67     private static final int PREVALIDATION_SIZE = 8;
  68
  69     private static final int BITS_PER_BYTE = Byte.SIZE;
  70
  71     /**
  72      * Name of the metadata file in the trace directory
  73      */
  74     private static final String METADATA_FILENAME = "metadata"; //$NON-NLS-1$
  75
  76     /**
  77      * Size of the metadata packet header, in bytes, computed by hand.
  78      */
  79     private static final int METADATA_PACKET_HEADER_SIZE = 37;
  80
  81     // ------------------------------------------------------------------------
  82     // Attributes
  83     // ------------------------------------------------------------------------
  84
  85     /**
  86      * Byte order as detected when reading the TSDL magic number.
  87      */
  88     private ByteOrder fDetectedByteOrder = null;
  89
  90     /**
  91      * The trace file to which belongs this metadata file.
  92      */
  93     private final CTFTrace fTrace;
  94
  95     private IOStructGen fTreeParser;
  96
  97     // ------------------------------------------------------------------------
  98     // Constructors
  99     // ------------------------------------------------------------------------
 100
 101     /**
 102      * Constructs a Metadata object.
 103      *
 104      * @param trace
 105      *            The trace to which belongs this metadata file.
 106      */
 107     public Metadata(CTFTrace trace) {
 108         fTrace = trace;
 109     }
 110
 111     /**
 112      * For network streaming
 113      */
 114     public Metadata() {
 115         fTrace = new CTFTrace();
 116     }
 117
 118     // ------------------------------------------------------------------------
 119     // Getters/Setters/Predicates
 120     // ------------------------------------------------------------------------
 121
 122     /**
 123      * Returns the ByteOrder that was detected while parsing the metadata.
 124      *
 125      * @return The byte order.
 126      */
 127     public ByteOrder getDetectedByteOrder() {
 128         return fDetectedByteOrder;
 129     }
 130
 131     /**
 132      * Gets the parent trace
 133      *
 134      * @return the parent trace
 135      */
 136     public CTFTrace getTrace() {
 137         return fTrace;
 138     }
 139
 140     // ------------------------------------------------------------------------
 141     // Operations
 142     // ------------------------------------------------------------------------
 143
 144     /**
 145      * Parse the metadata file.
 146      *
 147      * @throws CTFException
 148      *             If there was a problem parsing the metadata
 149      */
 150     public void parseFile() throws CTFException {
 151
 152         /*
 153          * Reader. It will contain a StringReader if we are using packet-based
 154          * metadata and it will contain a FileReader if we have text-based
 155          * metadata.
 156          */
 157
 158         try (FileInputStream fis = new FileInputStream(getMetadataPath());
 159                 FileChannel metadataFileChannel = fis.getChannel();
 160                 /* Check if metadata is packet-based, if not it is text based */
 161                 Reader metadataTextInput = (isPacketBased(metadataFileChannel) ? readBinaryMetaData(metadataFileChannel) : new FileReader(fis.getFD()));) {
 162
 163             readMetaDataText(metadataTextInput);
 164
 165         } catch (FileNotFoundException e) {
 166             throw new CTFException("Cannot find metadata file!", e); //$NON-NLS-1$
 167         } catch (IOException | ParseException e) {
 168             throw new CTFException(e);
 169         } catch (RecognitionException | RewriteCardinalityException e) {
 170             throw new CtfAntlrException(e);
 171         }
 172     }
 173
 174     private Reader readBinaryMetaData(FileChannel metadataFileChannel) throws CTFException {
 175         /* Create StringBuffer to receive metadata text */
 176         StringBuffer metadataText = new StringBuffer();
 177
 178         /*
 179          * Read metadata packet one by one, appending the text to the
 180          * StringBuffer
 181          */
 182         MetadataPacketHeader packetHeader = readMetadataPacket(
 183                 metadataFileChannel, metadataText);
 184         while (packetHeader != null) {
 185             packetHeader = readMetadataPacket(metadataFileChannel,
 186                     metadataText);
 187         }
 188
 189         /* Wrap the metadata string with a StringReader */
 190         return new StringReader(metadataText.toString());
 191     }
 192
 193     /**
 194      * Executes a weak validation of the metadata. It checks if a file with name
 195      * metadata exists and if one of the following conditions are met:
 196      * <ul>
 197      * <li>For text-only metadata, the file starts with "/* CTF" (without the
 198      * quotes)</li>
 199      * <li>For packet-based metadata, the file starts with correct magic number
 200      * </li>
 201      * </ul>
 202      *
 203      * @param path
 204      *            path to CTF trace directory
 205      * @return <code>true</code> if pre-validation is ok else <code>false</code>
 206      * @throws CTFException
 207      *             file channel cannot be created
 208      * @since 1.0
 209      */
 210     public static boolean preValidate(String path) throws CTFException {
 211         String metadataPath = path + Utils.SEPARATOR + METADATA_FILENAME;
 212         File metadataFile = new File(metadataPath);
 213         if (metadataFile.exists() && metadataFile.length() > PREVALIDATION_SIZE) {
 214             try (FileChannel fc = FileChannel.open(metadataFile.toPath(), StandardOpenOption.READ)) {
 215                 ByteBuffer bb = ByteBuffer.allocate(PREVALIDATION_SIZE);
 216                 bb.clear();
 217                 fc.read(bb);
 218                 bb.flip();
 219                 if (bb.getInt(0) == Utils.TSDL_MAGIC) {
 220                     return true;
 221                 }
 222                 bb.order(ByteOrder.LITTLE_ENDIAN);
 223                 if (bb.getInt(0) == Utils.TSDL_MAGIC) {
 224                     return true;
 225                 }
 226                 bb.position(0);
 227                 byte bytes[] = new byte[PREVALIDATION_SIZE];
 228                 bb.get(bytes);
 229                 String text = new String(bytes, ASCII_CHARSET);
 230                 return text.startsWith(TEXT_ONLY_METADATA_HEADER_PREFIX);
 231             } catch (IOException e) {
 232                 throw new CTFException(e.getMessage(), e);
 233             }
 234         }
 235         return false;
 236     }
 237
 238     /**
 239      * Read the metadata from a formatted TSDL string
 240      *
 241      * @param data
 242      *            the data to read
 243      * @throws CTFException
 244      *             this exception wraps a ParseException, IOException or
 245      *             CtfAntlrException, three exceptions that can be obtained from
 246      *             parsing a TSDL file
 247      */
 248     public void parseText(String data) throws CTFException {
 249         Reader metadataTextInput = new StringReader(data);
 250         try {
 251             readMetaDataText(metadataTextInput);
 252         } catch (IOException | ParseException e) {
 253             throw new CTFException(e);
 254         } catch (RecognitionException | RewriteCardinalityException e) {
 255             throw new CtfAntlrException(e);
 256         }
 257
 258     }
 259
 260     private void readMetaDataText(Reader metadataTextInput) throws IOException, RecognitionException, ParseException {
 261         CommonTree tree = createAST(metadataTextInput);
 262
 263         /* Generate IO structures (declarations) */
 264         fTreeParser = new IOStructGen(tree, NonNullUtils.checkNotNull(fTrace));
 265         fTreeParser.generate();
 266         /* store locally in case of concurrent modification */
 267         ByteOrder detectedByteOrder = getDetectedByteOrder();
 268         if (detectedByteOrder != null && fTrace.getByteOrder() != detectedByteOrder) {
 269             throw new ParseException("Metadata byte order and trace byte order inconsistent."); //$NON-NLS-1$
 270         }
 271     }
 272
 273     /**
 274      * Read a metadata fragment from a formatted TSDL string
 275      *
 276      * @param dataFragment
 277      *            the data to read
 278      * @throws CTFException
 279      *             this exception wraps a ParseException, IOException or
 280      *             CtfAntlrException, three exceptions that can be obtained from
 281      *             parsing a TSDL file
 282      */
 283     public void parseTextFragment(String dataFragment) throws CTFException {
 284         Reader metadataTextInput = new StringReader(dataFragment);
 285         try {
 286             readMetaDataTextFragment(metadataTextInput);
 287         } catch (IOException | ParseException e) {
 288             throw new CTFException(e);
 289         } catch (RecognitionException | RewriteCardinalityException e) {
 290             throw new CtfAntlrException(e);
 291         }
 292     }
 293
 294     private void readMetaDataTextFragment(Reader metadataTextInput) throws IOException, RecognitionException, ParseException {
 295         CommonTree tree = createAST(metadataTextInput);
 296         fTreeParser.setTree(tree);
 297         fTreeParser.generateFragment();
 298     }
 299
 300     private static CommonTree createAST(Reader metadataTextInput) throws IOException,
 301             RecognitionException {
 302         /* Create an ANTLR reader */
 303         ANTLRReaderStream antlrStream;
 304         antlrStream = new ANTLRReaderStream(metadataTextInput);
 305
 306         /* Parse the metadata text and get the AST */
 307         CTFLexer ctfLexer = new CTFLexer(antlrStream);
 308         CommonTokenStream tokens = new CommonTokenStream(ctfLexer);
 309         CTFParser ctfParser = new CTFParser(tokens, false);
 310
 311         parse_return pr = ctfParser.parse();
 312         return pr.getTree();
 313     }
 314
 315     /**
 316      * Determines whether the metadata file is packet-based by looking at the
 317      * TSDL magic number. If it is packet-based, it also gives information about
 318      * the endianness of the trace using the detectedByteOrder attribute.
 319      *
 320      * @param metadataFileChannel
 321      *            FileChannel of the metadata file.
 322      * @return True if the metadata is packet-based.
 323      * @throws CTFException
 324      */
 325     private boolean isPacketBased(FileChannel metadataFileChannel)
 326             throws CTFException {
 327         /*
 328          * Create a ByteBuffer to read the TSDL magic number (default is
 329          * big-endian)
 330          */
 331         ByteBuffer magicByteBuffer = ByteBuffer.allocate(Utils.TSDL_MAGIC_LEN);
 332
 333         /* Read without changing file position */
 334         try {
 335             metadataFileChannel.read(magicByteBuffer, 0);
 336         } catch (IOException e) {
 337             throw new CTFException("Unable to read metadata file channel.", e); //$NON-NLS-1$
 338         }
 339
 340         /* Get the first int from the file */
 341         int magic = magicByteBuffer.getInt(0);
 342
 343         /* Check if it matches */
 344         if (Utils.TSDL_MAGIC == magic) {
 345             fDetectedByteOrder = ByteOrder.BIG_ENDIAN;
 346             return true;
 347         }
 348
 349         /* Try the same thing, but with little-endian */
 350         magicByteBuffer.order(ByteOrder.LITTLE_ENDIAN);
 351         magic = magicByteBuffer.getInt(0);
 352
 353         if (Utils.TSDL_MAGIC == magic) {
 354             fDetectedByteOrder = ByteOrder.LITTLE_ENDIAN;
 355             return true;
 356         }
 357
 358         return false;
 359     }
 360
 361     private String getMetadataPath() {
 362         /* Path of metadata file = trace directory path + metadata filename */
 363         if (fTrace.getTraceDirectory() == null) {
 364             return ""; //$NON-NLS-1$
 365         }
 366         return fTrace.getTraceDirectory().getPath()
 367                 + Utils.SEPARATOR + METADATA_FILENAME;
 368     }
 369
 370     /**
 371      * Reads a metadata packet from the given metadata FileChannel, do some
 372      * basic validation and append the text to the StringBuffer.
 373      *
 374      * @param metadataFileChannel
 375      *            Metadata FileChannel
 376      * @param metadataText
 377      *            StringBuffer to which the metadata text will be appended.
 378      * @return A structure describing the header of the metadata packet, or null
 379      *         if the end of the file is reached.
 380      * @throws CTFException
 381      */
 382     private MetadataPacketHeader readMetadataPacket(
 383             FileChannel metadataFileChannel, StringBuffer metadataText)
 384                     throws CTFException {
 385         /* Allocate a ByteBuffer for the header */
 386         ByteBuffer headerByteBuffer = ByteBuffer.allocate(METADATA_PACKET_HEADER_SIZE);
 387
 388         /* Read the header */
 389         try {
 390             int nbBytesRead = metadataFileChannel.read(headerByteBuffer);
 391
 392             /* Return null if EOF */
 393             if (nbBytesRead < 0) {
 394                 return null;
 395             }
 396
 397             if (nbBytesRead != METADATA_PACKET_HEADER_SIZE) {
 398                 throw new CTFException("Error reading the metadata header."); //$NON-NLS-1$
 399             }
 400
 401         } catch (IOException e) {
 402             throw new CTFException("Error reading the metadata header.", e); //$NON-NLS-1$
 403         }
 404
 405         /* Set ByteBuffer's position to 0 */
 406         headerByteBuffer.position(0);
 407
 408         /* Use byte order that was detected with the magic number */
 409         headerByteBuffer.order(fDetectedByteOrder);
 410
 411         MetadataPacketHeader header = new MetadataPacketHeader(headerByteBuffer);
 412
 413         /* Check TSDL magic number */
 414         if (!header.isMagicValid()) {
 415             throw new CTFException("TSDL magic number does not match"); //$NON-NLS-1$
 416         }
 417
 418         /* Check UUID */
 419         if (!fTrace.uuidIsSet()) {
 420             fTrace.setUUID(header.getUuid());
 421         } else if (!fTrace.getUUID().equals(header.getUuid())) {
 422             throw new CTFException("UUID mismatch"); //$NON-NLS-1$
 423         }
 424
 425         /* Extract the text from the packet */
 426         int payloadSize = ((header.getContentSize() / BITS_PER_BYTE) - METADATA_PACKET_HEADER_SIZE);
 427         if (payloadSize < 0) {
 428             throw new CTFException("Invalid metadata packet payload size."); //$NON-NLS-1$
 429         }
 430         int skipSize = (header.getPacketSize() - header.getContentSize()) / BITS_PER_BYTE;
 431
 432         /* Read the payload + the padding in a ByteBuffer */
 433         ByteBuffer payloadByteBuffer = ByteBuffer.allocateDirect(payloadSize
 434                 + skipSize);
 435         try {
 436             metadataFileChannel.read(payloadByteBuffer);
 437         } catch (IOException e) {
 438             throw new CTFException("Error reading metadata packet payload.", e); //$NON-NLS-1$
 439         }
 440         payloadByteBuffer.rewind();
 441
 442         /* Read only the payload from the ByteBuffer into a byte array */
 443         byte payloadByteArray[] = new byte[payloadByteBuffer.remaining()];
 444         payloadByteBuffer.get(payloadByteArray, 0, payloadSize);
 445
 446         /* Convert the byte array to a String */
 447         String str = new String(payloadByteArray, 0, payloadSize, ASCII_CHARSET);
 448
 449         /* Append it to the existing metadata */
 450         metadataText.append(str);
 451
 452         return header;
 453     }
 454
 455     private static class MetadataPacketHeader {
 456
 457         private static final int UUID_SIZE = 16;
 458         private final int fMagic;
 459         private final UUID fUuid;
 460         private final int fChecksum;
 461         private final int fContentSize;
 462         private final int fPacketSize;
 463         private final byte fCompressionScheme;
 464         private final byte fEncryptionScheme;
 465         private final byte fChecksumScheme;
 466         private final byte fCtfMajorVersion;
 467         private final byte fCtfMinorVersion;
 468
 469         public MetadataPacketHeader(ByteBuffer headerByteBuffer) {
 470             /* Read from the ByteBuffer */
 471             fMagic = headerByteBuffer.getInt();
 472             byte[] uuidBytes = new byte[UUID_SIZE];
 473             headerByteBuffer.get(uuidBytes);
 474             fUuid = Utils.makeUUID(uuidBytes);
 475             fChecksum = headerByteBuffer.getInt();
 476             fContentSize = headerByteBuffer.getInt();
 477             fPacketSize = headerByteBuffer.getInt();
 478             fCompressionScheme = headerByteBuffer.get();
 479             fEncryptionScheme = headerByteBuffer.get();
 480             fChecksumScheme = headerByteBuffer.get();
 481             fCtfMajorVersion = headerByteBuffer.get();
 482             fCtfMinorVersion = headerByteBuffer.get();
 483         }
 484
 485         public boolean isMagicValid() {
 486             return fMagic == Utils.TSDL_MAGIC;
 487         }
 488
 489         public UUID getUuid() {
 490             return fUuid;
 491         }
 492
 493         public int getContentSize() {
 494             return fContentSize;
 495         }
 496
 497         public int getPacketSize() {
 498             return fPacketSize;
 499         }
 500
 501         @Override
 502         public String toString() {
 503             /* Only for debugging, shouldn't be externalized */
 504             /* Therefore it cannot be covered by test cases */
 505             return "MetadataPacketHeader [magic=0x" //$NON-NLS-1$
 506                     + Integer.toHexString(fMagic) + ", uuid=" //$NON-NLS-1$
 507                     + fUuid.toString() + ", checksum=" + fChecksum //$NON-NLS-1$
 508                     + ", contentSize=" + fContentSize + ", packetSize=" //$NON-NLS-1$ //$NON-NLS-2$
 509                     + fPacketSize + ", compressionScheme=" + fCompressionScheme //$NON-NLS-1$
 510                     + ", encryptionScheme=" + fEncryptionScheme //$NON-NLS-1$
 511                     + ", checksumScheme=" + fChecksumScheme //$NON-NLS-1$
 512                     + ", ctfMajorVersion=" + fCtfMajorVersion //$NON-NLS-1$
 513                     + ", ctfMinorVersion=" + fCtfMinorVersion + ']'; //$NON-NLS-1$
 514         }
 515
 516     }
 517
 518     /**
 519      * Copies the metadata file to a destination directory.
 520      *
 521      * @param path
 522      *            the destination directory
 523      * @return the path to the target file
 524      * @throws IOException
 525      *             if an error occurred
 526      *
 527      * @since 1.0
 528      */
 529     public Path copyTo(final File path) throws IOException {
 530         Path source = FileSystems.getDefault().getPath(fTrace.getTraceDirectory().getAbsolutePath(), METADATA_FILENAME);
 531         Path destPath = FileSystems.getDefault().getPath(path.getAbsolutePath());
 532         return Files.copy(source, destPath.resolve(source.getFileName()));
 533     }
 534 }