btf/org.eclipse.tracecompass.btf.core/src/org/eclipse/tracecompass/btf/core/trace/BtfTrace.java

   1 /*******************************************************************************
   2  * Copyright (c) 2014, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Matthew Khouzam - Initial API and implementation
  11  *   Patrick Tasse - Fix parsing of instance numbers
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.btf.core.trace;
  15
  16 import java.io.File;
  17 import java.io.FileNotFoundException;
  18 import java.io.IOException;
  19 import java.io.RandomAccessFile;
  20 import java.nio.ByteBuffer;
  21 import java.text.ParseException;
  22 import java.text.SimpleDateFormat;
  23 import java.util.Date;
  24 import java.util.HashMap;
  25 import java.util.Map;
  26 import java.util.TreeMap;
  27
  28 import org.eclipse.core.resources.IProject;
  29 import org.eclipse.core.resources.IResource;
  30 import org.eclipse.core.runtime.IStatus;
  31 import org.eclipse.core.runtime.Status;
  32 import org.eclipse.jdt.annotation.NonNull;
  33 import org.eclipse.tracecompass.btf.core.Activator;
  34 import org.eclipse.tracecompass.btf.core.event.BtfEvent;
  35 import org.eclipse.tracecompass.btf.core.event.BtfEventType;
  36 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  37 import org.eclipse.tracecompass.tmf.core.event.ITmfEventField;
  38 import org.eclipse.tracecompass.tmf.core.event.aspect.ITmfEventAspect;
  39 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  40 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
  41 import org.eclipse.tracecompass.tmf.core.parsers.custom.CustomTxtTraceContext;
  42 import org.eclipse.tracecompass.tmf.core.project.model.ITmfPropertiesProvider;
  43 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
  44 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
  45 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
  46 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
  47 import org.eclipse.tracecompass.tmf.core.trace.TmfContext;
  48 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
  49 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  50 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
  51 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
  52 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
  53 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
  54 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
  55 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
  56 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
  57 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  58
  59 import com.google.common.collect.ImmutableMap;
  60
  61 /**
  62  * BTF reader. Reads Best Trace Format traces.
  63  *
  64  * @author Matthew Khouzam
  65  */
  66 public class BtfTrace extends TmfTrace implements ITmfPersistentlyIndexable, ITmfPropertiesProvider {
  67
  68     private static final int MAX_FIELDS = 7;
  69
  70     private static final long MICROSECONDS_IN_A_SECOND = 1000000L;
  71
  72     private static final String VERSION = "#version"; //$NON-NLS-1$
  73     private static final String CREATOR = "#creator"; //$NON-NLS-1$
  74     private static final String CREATIONDATE = "#creationDate"; //$NON-NLS-1$
  75     private static final String INPUTFILE = "#inputFile"; //$NON-NLS-1$
  76     private static final String TIMESCALE = "#timeScale"; //$NON-NLS-1$
  77     private static final String ENTITYTYPE = "#entityType"; //$NON-NLS-1$
  78     private static final String ENTITYTABLE = "#entityTable"; //$NON-NLS-1$
  79     private static final String ENTITYTYPETABLE = "#entityTypeTable"; //$NON-NLS-1$
  80
  81     // lower-case helpers
  82     private static final String lCREATIONDATE = "#creationdate"; //$NON-NLS-1$
  83     private static final String lINPUTFILE = "#inputfile"; //$NON-NLS-1$
  84     private static final String lTIMESCALE = "#timescale"; //$NON-NLS-1$
  85     private static final String lENTITYTYPE = "#entitytype"; //$NON-NLS-1$
  86     private static final String lENTITYTABLE = "#entitytable"; //$NON-NLS-1$
  87     private static final String lENTITYTYPETABLE = "#entitytypetable"; //$NON-NLS-1$
  88
  89     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
  90
  91     private static final int CACHE_SIZE = 256;
  92     private static final int MAX_CONFIDENCE = 100;
  93     private static final int MAX_LINES = 100;
  94
  95     private static int fCheckpointSize = -1;
  96
  97     private final @NonNull Map<String, String> fProperties = new HashMap<>();
  98
  99     private final @NonNull Map<Integer, String> fEntityTable = new TreeMap<>();
 100     private final @NonNull Map<BtfEventType, String> fEntityTypeTable = new HashMap<>();
 101     private final @NonNull Map<Integer, BtfEventType> fEntityTypes = new TreeMap<>();
 102
 103     private String fVersion;
 104     private String fCreator;
 105     private String fCreationDate;
 106     private String fInputFile;
 107     // default unit is ns
 108     private BtfTimestampFormat fTsFormat = BtfTimestampFormat.NS;
 109
 110     private File fFile;
 111     private RandomAccessFile fFileInput;
 112     private long fDataOffset;
 113     private long fTsOffset = 0;
 114
 115     /**
 116      * Default constructor
 117      */
 118     public BtfTrace() {
 119         super();
 120         setCacheSize(CACHE_SIZE);
 121         fProperties.put(TIMESCALE, fTsFormat.toString());
 122     }
 123
 124     private void parseHeader(RandomAccessFile input) throws IOException {
 125         String line = input.readLine();
 126         long pos = 0;
 127         while (line != null && line.startsWith("#")) { //$NON-NLS-1$
 128             String[] tokens = line.split(" ", 2); //$NON-NLS-1$
 129             /*
 130              * please note that the examples we were given and the spec are NOT
 131              * consistent, so we are ignoring the case to avoid issues
 132              */
 133             switch (tokens[0].toLowerCase()) {
 134             case VERSION:
 135                 fVersion = tokens[1];
 136                 fProperties.put(VERSION, fVersion);
 137                 break;
 138             case CREATOR:
 139                 fCreator = tokens[1];
 140                 fProperties.put(CREATOR, fCreator);
 141                 break;
 142             case lCREATIONDATE:
 143                 fCreationDate = tokens[1];
 144                 fProperties.put(CREATIONDATE, fCreationDate);
 145
 146                 try {
 147                     // DateFormats are inherently unsafe for multithreaded use so we can't make this a field. Just in case.
 148                     final SimpleDateFormat ISO8601DATEFORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX"); //$NON-NLS-1$
 149                     Date dateTime = ISO8601DATEFORMAT.parse(fCreationDate);
 150                     fTsOffset = dateTime.getTime() * MICROSECONDS_IN_A_SECOND;
 151                 } catch (ParseException e) {
 152                     Activator.logWarning("Creation date error: " + e.getMessage()); //$NON-NLS-1$
 153                 }
 154                 break;
 155             case lINPUTFILE:
 156                 fInputFile = tokens[1];
 157                 fProperties.put(INPUTFILE, fInputFile);
 158                 break;
 159             case lTIMESCALE:
 160                 fTsFormat = BtfTimestampFormat.parse(tokens[1]);
 161                 fProperties.put(TIMESCALE, fTsFormat.toString());
 162                 break;
 163             case lENTITYTYPE:
 164                 pos = fFileInput.getFilePointer();
 165                 line = fFileInput.readLine();
 166                 while (line.startsWith("#-")) { //$NON-NLS-1$
 167                     String tempLine = line.substring(1);
 168                     String[] elements = tempLine.split(" ", 2); //$NON-NLS-1$
 169                     fEntityTypes.put(Integer.parseInt(elements[0]), BtfEventTypeFactory.parse(elements[1]));
 170                     pos = fFileInput.getFilePointer();
 171                     line = fFileInput.readLine();
 172                 }
 173                 fFileInput.seek(pos);
 174                 fProperties.put(ENTITYTYPE, fEntityTypes.toString());
 175                 break;
 176             case lENTITYTABLE:
 177                 pos = fFileInput.getFilePointer();
 178                 line = fFileInput.readLine();
 179                 while (line.startsWith("#-")) { //$NON-NLS-1$
 180                     String tempLine = line.substring(1);
 181                     String[] elements = tempLine.split(" ", 2); //$NON-NLS-1$
 182                     fEntityTable.put(Integer.parseInt(elements[0]), elements[1]);
 183                     pos = fFileInput.getFilePointer();
 184                     line = fFileInput.readLine();
 185                 }
 186                 fProperties.put(ENTITYTABLE, fEntityTable.toString());
 187                 fFileInput.seek(pos);
 188                 break;
 189             case lENTITYTYPETABLE:
 190                 pos = fFileInput.getFilePointer();
 191                 line = fFileInput.readLine();
 192                 while (line.startsWith("#-")) { //$NON-NLS-1$
 193                     String tempLine = line.substring(1);
 194                     String[] elements = tempLine.split(" ", 2); //$NON-NLS-1$
 195                     fEntityTypeTable.put(BtfEventTypeFactory.parse(elements[0]), elements[1]);
 196                     pos = fFileInput.getFilePointer();
 197                     line = fFileInput.readLine();
 198                 }
 199                 fFileInput.seek(pos);
 200                 fProperties.put(ENTITYTYPETABLE, fEntityTypeTable.toString());
 201                 break;
 202             default:
 203                 break;
 204             }
 205             fDataOffset = input.getFilePointer();
 206             line = input.readLine();
 207         }
 208         fTsOffset = (long) (fTsOffset * fTsFormat.getScaleFactor());
 209     }
 210
 211     @Override
 212     public void initTrace(IResource resource, String path, Class<? extends ITmfEvent> type) throws TmfTraceException {
 213         super.initTrace(resource, path, type);
 214         fFile = new File(path);
 215         try {
 216             fFileInput = new RandomAccessFile(fFile, "r"); //$NON-NLS-1$
 217             parseHeader(fFileInput);
 218         } catch (IOException e) {
 219             throw new TmfTraceException(e.getMessage(), e);
 220         }
 221
 222     }
 223
 224     private void initFile() throws TmfTraceException {
 225         closeFile();
 226         try {
 227             fFileInput = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
 228         } catch (IOException e) {
 229             throw new TmfTraceException(e.getMessage(), e);
 230         }
 231     }
 232
 233     private void closeFile() {
 234         if (fFileInput != null) {
 235             try {
 236                 fFileInput.close();
 237             } catch (IOException e) {
 238             } finally {
 239                 fFileInput = null;
 240             }
 241         }
 242     }
 243
 244     @Override
 245     public IStatus validate(IProject project, String path) {
 246         File file = new File(path);
 247         if (!file.exists()) {
 248             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "File not found: " + path); //$NON-NLS-1$
 249         }
 250         if (!file.isFile()) {
 251             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Not a file. It's a directory: " + path); //$NON-NLS-1$
 252         }
 253         int confidence = 0;
 254         try {
 255             if (!TmfTraceUtils.isText(file)) {
 256                 return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 257             }
 258         } catch (IOException e) {
 259             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 260             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 261         }
 262         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
 263             int lineCount = 0;
 264             int matches = 0;
 265             String line = rafile.getNextLine();
 266             while ((line != null) && line.startsWith("#")) { //$NON-NLS-1$
 267                 line = rafile.getNextLine();
 268             }
 269             while ((line != null) && (lineCount++ < MAX_LINES)) {
 270                 try {
 271                     ITmfEvent event = parseLine(0, line);
 272                     if (event != null) {
 273                         matches++;
 274                     }
 275                 } catch (RuntimeException e) {
 276                     confidence = Integer.MIN_VALUE;
 277                 }
 278
 279                 confidence = MAX_CONFIDENCE * matches / lineCount;
 280                 line = rafile.getNextLine();
 281             }
 282         } catch (IOException e) {
 283             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 284             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 285         }
 286
 287         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 288     }
 289
 290     @Override
 291     public ITmfLocation getCurrentLocation() {
 292         long temp = -1;
 293         try {
 294             temp = fFileInput.getFilePointer();
 295         } catch (IOException e) {
 296         }
 297         return new TmfLongLocation(temp);
 298     }
 299
 300     @Override
 301     public double getLocationRatio(ITmfLocation location) {
 302         long size = fFile.length() - fDataOffset;
 303         long pos;
 304         try {
 305             pos = fFileInput.getFilePointer() - fDataOffset;
 306         } catch (IOException e) {
 307             pos = 0;
 308         }
 309         return 1.0 / size * pos;
 310     }
 311
 312     @Override
 313     public ITmfContext seekEvent(ITmfLocation location) {
 314         final TmfContext context = new TmfContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 315         if (NULL_LOCATION.equals(location) || fFile == null) {
 316             return context;
 317         }
 318         try {
 319             if (location == null) {
 320                 fFileInput.seek(fDataOffset);
 321             } else if (location.getLocationInfo() instanceof Long) {
 322                 fFileInput.seek((Long) location.getLocationInfo());
 323             }
 324             context.setLocation(new TmfLongLocation(fFileInput.getFilePointer()));
 325             return context;
 326         } catch (final FileNotFoundException e) {
 327             Activator.logError("Error seeking event. File not found: " + getPath(), e); //$NON-NLS-1$
 328             return context;
 329         } catch (final IOException e) {
 330             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 331             return context;
 332         }
 333     }
 334
 335     @Override
 336     public ITmfContext seekEvent(double ratio) {
 337         if (fFile == null) {
 338             return new TmfContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 339         }
 340         try {
 341             long pos = Math.round(ratio * fFile.length()) - fDataOffset;
 342             while (pos > 0) {
 343                 fFileInput.seek(pos - 1);
 344                 if (fFileInput.read() == '\n') {
 345                     break;
 346                 }
 347                 pos--;
 348             }
 349             final ITmfLocation location = new TmfLongLocation(pos);
 350             final ITmfContext context = seekEvent(location);
 351             context.setRank(ITmfContext.UNKNOWN_RANK);
 352             return context;
 353         } catch (final IOException e) {
 354             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 355             return new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 356         }
 357     }
 358
 359     @Override
 360     public ITmfEvent parseEvent(ITmfContext tmfContext) {
 361         if (fFile == null || (!(tmfContext instanceof TmfContext))) {
 362             return null;
 363         }
 364
 365         final TmfContext context = (TmfContext) tmfContext;
 366         if (context.getLocation() == null
 367                 || !(context.getLocation().getLocationInfo() instanceof Long)
 368                 || NULL_LOCATION.equals(context.getLocation())) {
 369             return null;
 370         }
 371
 372         return parseLine(context);
 373
 374     }
 375
 376     /**
 377      * Parse a line with a context
 378      *
 379      * @param context
 380      *            the context, has a location
 381      * @return the event from a given line
 382      */
 383     private ITmfEvent parseLine(TmfContext context) {
 384         try {
 385             if (!context.getLocation().getLocationInfo().equals(fFileInput.getFilePointer())) {
 386                 seekEvent(context.getLocation());
 387             }
 388         } catch (IOException e1) {
 389             seekEvent(context.getLocation());
 390         }
 391         String line;
 392         try {
 393             line = fFileInput.readLine();
 394             return parseLine(context.getRank(), line);
 395
 396         } catch (IOException e) {
 397         }
 398
 399         return null;
 400     }
 401
 402     /**
 403      * Parse a line of text and make an event using it.
 404      *
 405      * @param rank
 406      *            the rank of the event
 407      * @param line
 408      *            the raw string of the event
 409      * @return the event
 410      */
 411     private ITmfEvent parseLine(long rank, String line) {
 412         if (line == null) {
 413             return null;
 414         }
 415         String[] tokens = line.split(",", MAX_FIELDS); //$NON-NLS-1$
 416         if (tokens.length < MAX_FIELDS) {
 417             return null;
 418         }
 419         int i = 0;
 420         long timestamp = Long.parseLong(tokens[i++]);
 421         String source = tokens[i++];
 422         long sourceInstance = -1;
 423         try {
 424             sourceInstance = Long.parseLong(tokens[i++]);
 425         } catch (NumberFormatException e) {
 426             // this field can be empty
 427         }
 428         BtfEventType type = BtfEventTypeFactory.parse(tokens[i++]);
 429         String target = tokens[i++];
 430         long targetInstance = -1;
 431         try {
 432             targetInstance = Long.parseLong(tokens[i++]);
 433         } catch (NumberFormatException e) {
 434             // this field can be empty
 435         }
 436         String event = tokens[i++];
 437
 438         ITmfEventField content = type.generateContent(event, sourceInstance, targetInstance);
 439
 440         return new BtfEvent(this, rank,
 441                 getTimestampTransform().transform(fTsFormat.createTimestamp(timestamp + fTsOffset)),
 442                 source,
 443                 type,
 444                 type.getDescription(),
 445                 content,
 446                 target);
 447     }
 448
 449     @Override
 450     public int getCheckpointSize() {
 451         synchronized (BtfTrace.class) {
 452             if (fCheckpointSize == -1) {
 453                 TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
 454                 ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
 455                 b.clear();
 456                 c.serialize(b);
 457                 fCheckpointSize = b.position();
 458             }
 459         }
 460
 461         return fCheckpointSize;
 462     }
 463
 464     @Override
 465     public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
 466         return new TmfLongLocation(bufferIn);
 467     }
 468
 469     @Override
 470     protected ITmfTraceIndexer createIndexer(int interval) {
 471         return new TmfBTreeTraceIndexer(this, interval);
 472     }
 473
 474     /**
 475      * @since 2.0
 476      */
 477     @Override
 478     public Map<String, String> getProperties() {
 479         return ImmutableMap.copyOf(fProperties);
 480     }
 481
 482     @Override
 483     public Iterable<ITmfEventAspect<?>> getEventAspects() {
 484         return BtfEventAspects.getAspects();
 485     }
 486
 487     @Override
 488     public synchronized void dispose() {
 489         RandomAccessFile raf = fFileInput;
 490         if (raf != null) {
 491             try {
 492                 raf.close();
 493             } catch (IOException e) {
 494             }
 495         }
 496         super.dispose();
 497     }
 498
 499     @TmfSignalHandler
 500     @Override
 501     public void traceRangeUpdated(TmfTraceRangeUpdatedSignal signal) {
 502         if (signal.getTrace() == this) {
 503             try {
 504                 synchronized (this) {
 505                     // Reset the file handle in case it has reached the end of the
 506                     // file already. Otherwise, it will not be able to read new data
 507                     // pass the previous end.
 508                     initFile();
 509                 }
 510             } catch (TmfTraceException e) {
 511                 Activator.logError(e.getLocalizedMessage(), e);
 512             }
 513         }
 514         super.traceRangeUpdated(signal);
 515     }
 516 }