tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/parsers/custom/CustomTxtTrace.java

   1 /*******************************************************************************
   2  * Copyright (c) 2010, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Patrick Tasse - Initial API and implementation
  11  *   Bernd Hufmann - Add trace type id handling
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.parsers.custom;
  15
  16 import java.io.File;
  17 import java.io.FileNotFoundException;
  18 import java.io.IOException;
  19 import java.nio.ByteBuffer;
  20 import java.util.HashMap;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map.Entry;
  24 import java.util.regex.Matcher;
  25
  26 import org.eclipse.core.resources.IProject;
  27 import org.eclipse.core.resources.IResource;
  28 import org.eclipse.core.runtime.IStatus;
  29 import org.eclipse.core.runtime.Status;
  30 import org.eclipse.jdt.annotation.NonNull;
  31 import org.eclipse.tracecompass.internal.tmf.core.Activator;
  32 import org.eclipse.tracecompass.internal.tmf.core.parsers.custom.CustomEventAspects;
  33 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  34 import org.eclipse.tracecompass.tmf.core.event.aspect.ITmfEventAspect;
  35 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  36 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
  37 import org.eclipse.tracecompass.tmf.core.parsers.custom.CustomTxtTraceDefinition.InputLine;
  38 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
  39 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
  40 import org.eclipse.tracecompass.tmf.core.trace.TmfContext;
  41 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
  42 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
  43 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
  44 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
  45 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
  46 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
  47 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
  48 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
  49 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  50
  51 /**
  52  * Base class for custom plain text traces.
  53  *
  54  * @author Patrick Tassé
  55  */
  56 public class CustomTxtTrace extends TmfTrace implements ITmfPersistentlyIndexable {
  57
  58     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
  59     private static final int DEFAULT_CACHE_SIZE = 100;
  60     private static final int MAX_LINES = 100;
  61     private static final int MAX_CONFIDENCE = 100;
  62
  63     private final CustomTxtTraceDefinition fDefinition;
  64     private final CustomTxtEventType fEventType;
  65     private BufferedRandomAccessFile fFile;
  66     private final String fTraceTypeId;
  67
  68     private static final char SEPARATOR = ':';
  69     private static final String CUSTOM_TXT_TRACE_TYPE_PREFIX = "custom.txt.trace" + SEPARATOR; //$NON-NLS-1$
  70     private static final String LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX = "org.eclipse.linuxtools.tmf.core.parsers.custom.CustomTxtTrace" + SEPARATOR; //$NON-NLS-1$
  71     private static final String EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX = "org.eclipse.tracecompass.tmf.core.parsers.custom.CustomTxtTrace" + SEPARATOR; //$NON-NLS-1$
  72
  73     /**
  74      * Basic constructor.
  75      *
  76      * @param definition
  77      *            Text trace definition
  78      */
  79     public CustomTxtTrace(final CustomTxtTraceDefinition definition) {
  80         fDefinition = definition;
  81         fEventType = new CustomTxtEventType(fDefinition);
  82         fTraceTypeId = buildTraceTypeId(definition.categoryName, definition.definitionName);
  83         setCacheSize(DEFAULT_CACHE_SIZE);
  84     }
  85
  86     /**
  87      * Full constructor.
  88      *
  89      * @param resource
  90      *            Trace's resource.
  91      * @param definition
  92      *            Text trace definition
  93      * @param path
  94      *            Path to the trace file
  95      * @param cacheSize
  96      *            Cache size to use
  97      * @throws TmfTraceException
  98      *             If we couldn't open the trace at 'path'
  99      */
 100     public CustomTxtTrace(final IResource resource,
 101             final CustomTxtTraceDefinition definition, final String path,
 102             final int cacheSize) throws TmfTraceException {
 103         this(definition);
 104         setCacheSize((cacheSize > 0) ? cacheSize : DEFAULT_CACHE_SIZE);
 105         initTrace(resource, path, CustomTxtEvent.class);
 106     }
 107
 108     @Override
 109     public void initTrace(final IResource resource, final String path, final Class<? extends ITmfEvent> eventType) throws TmfTraceException {
 110         super.initTrace(resource, path, eventType);
 111         try {
 112             fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
 113         } catch (IOException e) {
 114             throw new TmfTraceException(e.getMessage(), e);
 115         }
 116     }
 117
 118     @Override
 119     public synchronized void dispose() {
 120         super.dispose();
 121         if (fFile != null) {
 122             try {
 123                 fFile.close();
 124             } catch (IOException e) {
 125             } finally {
 126                 fFile = null;
 127             }
 128         }
 129     }
 130
 131     @Override
 132     public ITmfTraceIndexer getIndexer() {
 133         return super.getIndexer();
 134     }
 135
 136     @Override
 137     public Iterable<ITmfEventAspect> getEventAspects() {
 138         return CustomEventAspects.generateAspects(fDefinition);
 139     }
 140
 141     @Override
 142     public synchronized TmfContext seekEvent(final ITmfLocation location) {
 143         final CustomTxtTraceContext context = new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 144         if (NULL_LOCATION.equals(location) || fFile == null) {
 145             return context;
 146         }
 147         try {
 148             if (location == null) {
 149                 fFile.seek(0);
 150             } else if (location.getLocationInfo() instanceof Long) {
 151                 fFile.seek((Long) location.getLocationInfo());
 152             }
 153             long rawPos = fFile.getFilePointer();
 154             String line = fFile.getNextLine();
 155             while (line != null) {
 156                 for (final InputLine input : getFirstLines()) {
 157                     final Matcher matcher = input.getPattern().matcher(line);
 158                     if (matcher.matches()) {
 159                         context.setLocation(new TmfLongLocation(rawPos));
 160                         context.firstLineMatcher = matcher;
 161                         context.firstLine = line;
 162                         context.nextLineLocation = fFile.getFilePointer();
 163                         context.inputLine = input;
 164                         return context;
 165                     }
 166                 }
 167                 rawPos = fFile.getFilePointer();
 168                 line = fFile.getNextLine();
 169             }
 170             return context;
 171         } catch (final FileNotFoundException e) {
 172             Activator.logError("Error seeking event. File not found: " + getPath(), e); //$NON-NLS-1$
 173             return context;
 174         } catch (final IOException e) {
 175             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 176             return context;
 177         }
 178
 179     }
 180
 181     @Override
 182     public synchronized TmfContext seekEvent(final double ratio) {
 183         if (fFile == null) {
 184             return new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 185         }
 186         try {
 187             long pos = Math.round(ratio * fFile.length());
 188             while (pos > 0) {
 189                 fFile.seek(pos - 1);
 190                 if (fFile.read() == '\n') {
 191                     break;
 192                 }
 193                 pos--;
 194             }
 195             final ITmfLocation location = new TmfLongLocation(pos);
 196             final TmfContext context = seekEvent(location);
 197             context.setRank(ITmfContext.UNKNOWN_RANK);
 198             return context;
 199         } catch (final IOException e) {
 200             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 201             return new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 202         }
 203     }
 204
 205     @Override
 206     public synchronized double getLocationRatio(final ITmfLocation location) {
 207         if (fFile == null) {
 208             return 0;
 209         }
 210         try {
 211             if (location.getLocationInfo() instanceof Long) {
 212                 return ((Long) location.getLocationInfo()).doubleValue() / fFile.length();
 213             }
 214         } catch (final IOException e) {
 215             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 216         }
 217         return 0;
 218     }
 219
 220     @Override
 221     public ITmfLocation getCurrentLocation() {
 222         // TODO Auto-generated method stub
 223         return null;
 224     }
 225
 226     @Override
 227     public synchronized CustomTxtEvent parseEvent(final ITmfContext tmfContext) {
 228         ITmfContext context = seekEvent(tmfContext.getLocation());
 229         return parse(context);
 230     }
 231
 232     @Override
 233     public synchronized CustomTxtEvent getNext(final ITmfContext context) {
 234         final ITmfContext savedContext = new TmfContext(context.getLocation(), context.getRank());
 235         final CustomTxtEvent event = parse(context);
 236         if (event != null) {
 237             updateAttributes(savedContext, event);
 238             context.increaseRank();
 239         }
 240         return event;
 241     }
 242
 243     private synchronized CustomTxtEvent parse(final ITmfContext tmfContext) {
 244         if (fFile == null) {
 245             return null;
 246         }
 247         if (!(tmfContext instanceof CustomTxtTraceContext)) {
 248             return null;
 249         }
 250
 251         final CustomTxtTraceContext context = (CustomTxtTraceContext) tmfContext;
 252         if (context.getLocation() == null || !(context.getLocation().getLocationInfo() instanceof Long) || NULL_LOCATION.equals(context.getLocation())) {
 253             return null;
 254         }
 255
 256         CustomTxtEvent event = parseFirstLine(context);
 257
 258         final HashMap<InputLine, Integer> countMap = new HashMap<>();
 259         InputLine currentInput = null;
 260         if (context.inputLine.childrenInputs != null && context.inputLine.childrenInputs.size() > 0) {
 261             currentInput = context.inputLine.childrenInputs.get(0);
 262             countMap.put(currentInput, 0);
 263         }
 264
 265         try {
 266             if (fFile.getFilePointer() != context.nextLineLocation) {
 267                 fFile.seek(context.nextLineLocation);
 268             }
 269             long rawPos = fFile.getFilePointer();
 270             String line = fFile.getNextLine();
 271             while (line != null) {
 272                 boolean processed = false;
 273                 if (currentInput == null) {
 274                     for (final InputLine input : getFirstLines()) {
 275                         final Matcher matcher = input.getPattern().matcher(line);
 276                         if (matcher.matches()) {
 277                             context.setLocation(new TmfLongLocation(rawPos));
 278                             context.firstLineMatcher = matcher;
 279                             context.firstLine = line;
 280                             context.nextLineLocation = fFile.getFilePointer();
 281                             context.inputLine = input;
 282                             return event;
 283                         }
 284                     }
 285                 } else {
 286                     if (countMap.get(currentInput) >= currentInput.getMinCount()) {
 287                         final List<InputLine> nextInputs = currentInput.getNextInputs(countMap);
 288                         if (nextInputs.size() == 0 || nextInputs.get(nextInputs.size() - 1).getMinCount() == 0) {
 289                             for (final InputLine input : getFirstLines()) {
 290                                 final Matcher matcher = input.getPattern().matcher(line);
 291                                 if (matcher.matches()) {
 292                                     context.setLocation(new TmfLongLocation(rawPos));
 293                                     context.firstLineMatcher = matcher;
 294                                     context.firstLine = line;
 295                                     context.nextLineLocation = fFile.getFilePointer();
 296                                     context.inputLine = input;
 297                                     return event;
 298                                 }
 299                             }
 300                         }
 301                         for (final InputLine input : nextInputs) {
 302                             final Matcher matcher = input.getPattern().matcher(line);
 303                             if (matcher.matches()) {
 304                                 event.processGroups(input, matcher);
 305                                 currentInput = input;
 306                                 if (countMap.get(currentInput) == null) {
 307                                     countMap.put(currentInput, 1);
 308                                 } else {
 309                                     countMap.put(currentInput, countMap.get(currentInput) + 1);
 310                                 }
 311                                 Iterator<InputLine> iter = countMap.keySet().iterator();
 312                                 while (iter.hasNext()) {
 313                                     final InputLine inputLine = iter.next();
 314                                     if (inputLine.level > currentInput.level) {
 315                                         iter.remove();
 316                                     }
 317                                 }
 318                                 if (currentInput.childrenInputs != null && currentInput.childrenInputs.size() > 0) {
 319                                     currentInput = currentInput.childrenInputs.get(0);
 320                                     countMap.put(currentInput, 0);
 321                                 } else if (countMap.get(currentInput) >= currentInput.getMaxCount()) {
 322                                     if (currentInput.getNextInputs(countMap).size() > 0) {
 323                                         currentInput = currentInput.getNextInputs(countMap).get(0);
 324                                         if (countMap.get(currentInput) == null) {
 325                                             countMap.put(currentInput, 0);
 326                                         }
 327                                         iter = countMap.keySet().iterator();
 328                                         while (iter.hasNext()) {
 329                                             final InputLine inputLine = iter.next();
 330                                             if (inputLine.level > currentInput.level) {
 331                                                 iter.remove();
 332                                             }
 333                                         }
 334                                     } else {
 335                                         currentInput = null;
 336                                     }
 337                                 }
 338                                 processed = true;
 339                                 break;
 340                             }
 341                         }
 342                     }
 343                     if (!processed && currentInput != null) {
 344                         final Matcher matcher = currentInput.getPattern().matcher(line);
 345                         if (matcher.matches()) {
 346                             event.processGroups(currentInput, matcher);
 347                             countMap.put(currentInput, countMap.get(currentInput) + 1);
 348                             if (currentInput.childrenInputs != null && currentInput.childrenInputs.size() > 0) {
 349                                 currentInput = currentInput.childrenInputs.get(0);
 350                                 countMap.put(currentInput, 0);
 351                             } else if (countMap.get(currentInput) >= currentInput.getMaxCount()) {
 352                                 if (currentInput.getNextInputs(countMap).size() > 0) {
 353                                     currentInput = currentInput.getNextInputs(countMap).get(0);
 354                                     if (countMap.get(currentInput) == null) {
 355                                         countMap.put(currentInput, 0);
 356                                     }
 357                                     final Iterator<InputLine> iter = countMap.keySet().iterator();
 358                                     while (iter.hasNext()) {
 359                                         final InputLine inputLine = iter.next();
 360                                         if (inputLine.level > currentInput.level) {
 361                                             iter.remove();
 362                                         }
 363                                     }
 364                                 } else {
 365                                     currentInput = null;
 366                                 }
 367                             }
 368                         }
 369                         ((StringBuffer) event.getContent().getValue()).append("\n").append(line); //$NON-NLS-1$
 370                     }
 371                 }
 372                 rawPos = fFile.getFilePointer();
 373                 line = fFile.getNextLine();
 374             }
 375         } catch (final IOException e) {
 376             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 377         }
 378         for (final Entry<InputLine, Integer> entry : countMap.entrySet()) {
 379             if (entry.getValue() < entry.getKey().getMinCount()) {
 380                 event = null;
 381             }
 382         }
 383         context.setLocation(NULL_LOCATION);
 384         return event;
 385     }
 386
 387     /**
 388      * @return The first few lines of the text file
 389      */
 390     public List<InputLine> getFirstLines() {
 391         return fDefinition.inputs;
 392     }
 393
 394     /**
 395      * Parse the first line of the trace (to recognize the type).
 396      *
 397      * @param context
 398      *            Trace context
 399      * @return The first event
 400      */
 401     public CustomTxtEvent parseFirstLine(final CustomTxtTraceContext context) {
 402         final CustomTxtEvent event = new CustomTxtEvent(fDefinition, this, TmfTimestamp.ZERO, fEventType);
 403         event.processGroups(context.inputLine, context.firstLineMatcher);
 404         event.setContent(new CustomEventContent(event, new StringBuffer(context.firstLine)));
 405         return event;
 406     }
 407
 408     /**
 409      * Get the trace definition.
 410      *
 411      * @return The trace definition
 412      */
 413     public CustomTraceDefinition getDefinition() {
 414         return fDefinition;
 415     }
 416
 417     /**
 418      * {@inheritDoc}
 419      * <p>
 420      * The default implementation computes the confidence as the percentage of
 421      * lines in the first 100 lines of the file which match any of the root
 422      * input line patterns.
 423      */
 424     @Override
 425     public IStatus validate(IProject project, String path) {
 426         File file = new File(path);
 427         if (!file.exists() || !file.isFile() || !file.canRead()) {
 428             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, Messages.CustomTrace_FileNotFound + ": " + path); //$NON-NLS-1$
 429         }
 430         int confidence = 0;
 431         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
 432             int lineCount = 0;
 433             double matches = 0.0;
 434             String line = rafile.getNextLine();
 435             while ((line != null) && (lineCount++ < MAX_LINES)) {
 436                 for (InputLine inputLine : fDefinition.inputs) {
 437                     Matcher matcher = inputLine.getPattern().matcher(line);
 438                     if (matcher.matches()) {
 439                         int groupCount = matcher.groupCount();
 440                         matches += (1.0 + groupCount / ((double) groupCount + 1));
 441                         break;
 442                     }
 443                 }
 444                 confidence = (int) (MAX_CONFIDENCE * matches / lineCount);
 445                 line = rafile.getNextLine();
 446             }
 447         } catch (IOException e) {
 448             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 449         }
 450         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 451     }
 452
 453     private static int fCheckpointSize = -1;
 454
 455     @Override
 456     public synchronized int getCheckpointSize() {
 457         if (fCheckpointSize == -1) {
 458             TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
 459             ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
 460             b.clear();
 461             c.serialize(b);
 462             fCheckpointSize = b.position();
 463         }
 464
 465         return fCheckpointSize;
 466     }
 467
 468     @Override
 469     public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
 470         return new TmfLongLocation(bufferIn);
 471     }
 472
 473     @Override
 474     protected ITmfTraceIndexer createIndexer(int interval) {
 475         return new TmfBTreeTraceIndexer(this, interval);
 476     }
 477
 478     @Override
 479     public String getTraceTypeId() {
 480         return fTraceTypeId;
 481     }
 482
 483     /**
 484      * Build the trace type id for a custom text trace
 485      *
 486      * @param category
 487      *            the category
 488      * @param definitionName
 489      *            the definition name
 490      * @return the trace type id
 491      */
 492     public static @NonNull String buildTraceTypeId(String category, String definitionName) {
 493         return CUSTOM_TXT_TRACE_TYPE_PREFIX + category + SEPARATOR + definitionName;
 494     }
 495
 496     /**
 497      * Checks whether the given trace type ID is a custom text trace type ID
 498      *
 499      * @param traceTypeId
 500      *                the trace type ID to check
 501      * @return <code>true</code> if it's a custom text trace type ID else <code>false</code>
 502      */
 503     public static boolean isCustomTraceTypeId(@NonNull String traceTypeId) {
 504         return traceTypeId.startsWith(CUSTOM_TXT_TRACE_TYPE_PREFIX);
 505     }
 506
 507     /**
 508      * This methods builds a trace type ID from a given ID taking into
 509      * consideration any format changes that were done for the IDs of custom
 510      * text traces. For example, such format change took place when moving to
 511      * Trace Compass. Trace type IDs that are part of the plug-in extension for
 512      * trace types won't be changed.
 513      *
 514      * This method is useful for IDs that were persisted in the workspace before
 515      * the format changes (e.g. in the persistent properties of a trace
 516      * resource).
 517      *
 518      * It ensures backwards compatibility of the workspace for custom text
 519      * traces.
 520      *
 521      * @param traceTypeId
 522      *            the legacy trace type ID
 523      * @return the trace type id in Trace Compass format
 524      */
 525     public static @NonNull String buildCompatibilityTraceTypeId(@NonNull String traceTypeId) {
 526         // Handle early Trace Compass custom text trace type IDs
 527         if (traceTypeId.startsWith(EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX)) {
 528             return CUSTOM_TXT_TRACE_TYPE_PREFIX + traceTypeId.substring(EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX.length());
 529         }
 530
 531         // Handle Linux Tools custom text trace type IDs (with and without category)
 532         int index = traceTypeId.lastIndexOf(SEPARATOR);
 533         if ((index != -1) && (traceTypeId.startsWith(LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX))) {
 534             String definitionName = index < traceTypeId.length() ? traceTypeId.substring(index + 1) : ""; //$NON-NLS-1$
 535             if (traceTypeId.contains(CustomTxtTrace.class.getSimpleName() + SEPARATOR) && traceTypeId.indexOf(SEPARATOR) == index) {
 536                 return buildTraceTypeId(CustomTxtTraceDefinition.CUSTOM_TXT_CATEGORY, definitionName);
 537             }
 538             return CUSTOM_TXT_TRACE_TYPE_PREFIX + traceTypeId.substring(LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX.length());
 539         }
 540         return traceTypeId;
 541     }
 542 }