tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/parsers/custom/CustomTxtTrace.java

   1 /*******************************************************************************
   2  * Copyright (c) 2010, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Patrick Tasse - Initial API and implementation
  11  *   Bernd Hufmann - Add trace type id handling
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.parsers.custom;
  15
  16 import java.io.File;
  17 import java.io.FileNotFoundException;
  18 import java.io.IOException;
  19 import java.nio.ByteBuffer;
  20 import java.util.HashMap;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map.Entry;
  24 import java.util.regex.Matcher;
  25
  26 import org.eclipse.core.resources.IProject;
  27 import org.eclipse.core.resources.IResource;
  28 import org.eclipse.core.runtime.IStatus;
  29 import org.eclipse.core.runtime.Status;
  30 import org.eclipse.jdt.annotation.NonNull;
  31 import org.eclipse.tracecompass.internal.tmf.core.Activator;
  32 import org.eclipse.tracecompass.internal.tmf.core.parsers.custom.CustomEventAspects;
  33 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  34 import org.eclipse.tracecompass.tmf.core.event.aspect.ITmfEventAspect;
  35 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  36 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
  37 import org.eclipse.tracecompass.tmf.core.parsers.custom.CustomTxtTraceDefinition.InputLine;
  38 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
  39 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
  40 import org.eclipse.tracecompass.tmf.core.trace.TmfContext;
  41 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
  42 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  43 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
  44 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
  45 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
  46 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
  47 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
  48 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
  49 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
  50 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  51
  52 /**
  53  * Base class for custom plain text traces.
  54  *
  55  * @author Patrick Tassé
  56  */
  57 public class CustomTxtTrace extends TmfTrace implements ITmfPersistentlyIndexable {
  58
  59     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
  60     private static final int DEFAULT_CACHE_SIZE = 100;
  61     private static final int MAX_LINES = 100;
  62     private static final int MAX_CONFIDENCE = 100;
  63
  64     private final CustomTxtTraceDefinition fDefinition;
  65     private final CustomTxtEventType fEventType;
  66     private BufferedRandomAccessFile fFile;
  67     private final String fTraceTypeId;
  68
  69     private static final char SEPARATOR = ':';
  70     private static final String CUSTOM_TXT_TRACE_TYPE_PREFIX = "custom.txt.trace" + SEPARATOR; //$NON-NLS-1$
  71     private static final String LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX = "org.eclipse.linuxtools.tmf.core.parsers.custom.CustomTxtTrace" + SEPARATOR; //$NON-NLS-1$
  72     private static final String EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX = "org.eclipse.tracecompass.tmf.core.parsers.custom.CustomTxtTrace" + SEPARATOR; //$NON-NLS-1$
  73
  74     /**
  75      * Basic constructor.
  76      *
  77      * @param definition
  78      *            Text trace definition
  79      */
  80     public CustomTxtTrace(final CustomTxtTraceDefinition definition) {
  81         fDefinition = definition;
  82         fEventType = new CustomTxtEventType(fDefinition);
  83         fTraceTypeId = buildTraceTypeId(definition.categoryName, definition.definitionName);
  84         setCacheSize(DEFAULT_CACHE_SIZE);
  85     }
  86
  87     /**
  88      * Full constructor.
  89      *
  90      * @param resource
  91      *            Trace's resource.
  92      * @param definition
  93      *            Text trace definition
  94      * @param path
  95      *            Path to the trace file
  96      * @param cacheSize
  97      *            Cache size to use
  98      * @throws TmfTraceException
  99      *             If we couldn't open the trace at 'path'
 100      */
 101     public CustomTxtTrace(final IResource resource,
 102             final CustomTxtTraceDefinition definition, final String path,
 103             final int cacheSize) throws TmfTraceException {
 104         this(definition);
 105         setCacheSize((cacheSize > 0) ? cacheSize : DEFAULT_CACHE_SIZE);
 106         initTrace(resource, path, CustomTxtEvent.class);
 107     }
 108
 109     @Override
 110     public void initTrace(final IResource resource, final String path, final Class<? extends ITmfEvent> eventType) throws TmfTraceException {
 111         super.initTrace(resource, path, eventType);
 112         try {
 113             fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
 114         } catch (IOException e) {
 115             throw new TmfTraceException(e.getMessage(), e);
 116         }
 117     }
 118
 119     @Override
 120     public synchronized void dispose() {
 121         super.dispose();
 122         if (fFile != null) {
 123             try {
 124                 fFile.close();
 125             } catch (IOException e) {
 126             } finally {
 127                 fFile = null;
 128             }
 129         }
 130     }
 131
 132     @Override
 133     public ITmfTraceIndexer getIndexer() {
 134         return super.getIndexer();
 135     }
 136
 137     @Override
 138     public Iterable<ITmfEventAspect> getEventAspects() {
 139         return CustomEventAspects.generateAspects(fDefinition);
 140     }
 141
 142     @Override
 143     public synchronized TmfContext seekEvent(final ITmfLocation location) {
 144         final CustomTxtTraceContext context = new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 145         if (NULL_LOCATION.equals(location) || fFile == null) {
 146             return context;
 147         }
 148         try {
 149             if (location == null) {
 150                 fFile.seek(0);
 151             } else if (location.getLocationInfo() instanceof Long) {
 152                 fFile.seek((Long) location.getLocationInfo());
 153             }
 154             long rawPos = fFile.getFilePointer();
 155             String line = fFile.getNextLine();
 156             while (line != null) {
 157                 for (final InputLine input : getFirstLines()) {
 158                     final Matcher matcher = input.getPattern().matcher(line);
 159                     if (matcher.matches()) {
 160                         context.setLocation(new TmfLongLocation(rawPos));
 161                         context.firstLineMatcher = matcher;
 162                         context.firstLine = line;
 163                         context.nextLineLocation = fFile.getFilePointer();
 164                         context.inputLine = input;
 165                         return context;
 166                     }
 167                 }
 168                 rawPos = fFile.getFilePointer();
 169                 line = fFile.getNextLine();
 170             }
 171             return context;
 172         } catch (final FileNotFoundException e) {
 173             Activator.logError("Error seeking event. File not found: " + getPath(), e); //$NON-NLS-1$
 174             return context;
 175         } catch (final IOException e) {
 176             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 177             return context;
 178         }
 179
 180     }
 181
 182     @Override
 183     public synchronized TmfContext seekEvent(final double ratio) {
 184         if (fFile == null) {
 185             return new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 186         }
 187         try {
 188             long pos = Math.round(ratio * fFile.length());
 189             while (pos > 0) {
 190                 fFile.seek(pos - 1);
 191                 if (fFile.read() == '\n') {
 192                     break;
 193                 }
 194                 pos--;
 195             }
 196             final ITmfLocation location = new TmfLongLocation(pos);
 197             final TmfContext context = seekEvent(location);
 198             context.setRank(ITmfContext.UNKNOWN_RANK);
 199             return context;
 200         } catch (final IOException e) {
 201             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 202             return new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 203         }
 204     }
 205
 206     @Override
 207     public synchronized double getLocationRatio(final ITmfLocation location) {
 208         if (fFile == null) {
 209             return 0;
 210         }
 211         try {
 212             if (location.getLocationInfo() instanceof Long) {
 213                 return ((Long) location.getLocationInfo()).doubleValue() / fFile.length();
 214             }
 215         } catch (final IOException e) {
 216             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 217         }
 218         return 0;
 219     }
 220
 221     @Override
 222     public ITmfLocation getCurrentLocation() {
 223         // TODO Auto-generated method stub
 224         return null;
 225     }
 226
 227     @Override
 228     public synchronized CustomTxtEvent parseEvent(final ITmfContext tmfContext) {
 229         ITmfContext context = seekEvent(tmfContext.getLocation());
 230         return parse(context);
 231     }
 232
 233     @Override
 234     public synchronized CustomTxtEvent getNext(final ITmfContext context) {
 235         final ITmfContext savedContext = new TmfContext(context.getLocation(), context.getRank());
 236         final CustomTxtEvent event = parse(context);
 237         if (event != null) {
 238             updateAttributes(savedContext, event);
 239             context.increaseRank();
 240         }
 241         return event;
 242     }
 243
 244     private synchronized CustomTxtEvent parse(final ITmfContext tmfContext) {
 245         if (fFile == null) {
 246             return null;
 247         }
 248         if (!(tmfContext instanceof CustomTxtTraceContext)) {
 249             return null;
 250         }
 251
 252         final CustomTxtTraceContext context = (CustomTxtTraceContext) tmfContext;
 253         if (context.getLocation() == null || !(context.getLocation().getLocationInfo() instanceof Long) || NULL_LOCATION.equals(context.getLocation())) {
 254             return null;
 255         }
 256
 257         CustomTxtEvent event = parseFirstLine(context);
 258
 259         final HashMap<InputLine, Integer> countMap = new HashMap<>();
 260         InputLine currentInput = null;
 261         if (context.inputLine.childrenInputs != null && context.inputLine.childrenInputs.size() > 0) {
 262             currentInput = context.inputLine.childrenInputs.get(0);
 263             countMap.put(currentInput, 0);
 264         }
 265
 266         try {
 267             if (fFile.getFilePointer() != context.nextLineLocation) {
 268                 fFile.seek(context.nextLineLocation);
 269             }
 270             long rawPos = fFile.getFilePointer();
 271             String line = fFile.getNextLine();
 272             while (line != null) {
 273                 boolean processed = false;
 274                 if (currentInput == null) {
 275                     for (final InputLine input : getFirstLines()) {
 276                         final Matcher matcher = input.getPattern().matcher(line);
 277                         if (matcher.matches()) {
 278                             context.setLocation(new TmfLongLocation(rawPos));
 279                             context.firstLineMatcher = matcher;
 280                             context.firstLine = line;
 281                             context.nextLineLocation = fFile.getFilePointer();
 282                             context.inputLine = input;
 283                             return event;
 284                         }
 285                     }
 286                 } else {
 287                     if (countMap.get(currentInput) >= currentInput.getMinCount()) {
 288                         final List<InputLine> nextInputs = currentInput.getNextInputs(countMap);
 289                         if (nextInputs.size() == 0 || nextInputs.get(nextInputs.size() - 1).getMinCount() == 0) {
 290                             for (final InputLine input : getFirstLines()) {
 291                                 final Matcher matcher = input.getPattern().matcher(line);
 292                                 if (matcher.matches()) {
 293                                     context.setLocation(new TmfLongLocation(rawPos));
 294                                     context.firstLineMatcher = matcher;
 295                                     context.firstLine = line;
 296                                     context.nextLineLocation = fFile.getFilePointer();
 297                                     context.inputLine = input;
 298                                     return event;
 299                                 }
 300                             }
 301                         }
 302                         for (final InputLine input : nextInputs) {
 303                             final Matcher matcher = input.getPattern().matcher(line);
 304                             if (matcher.matches()) {
 305                                 event.processGroups(input, matcher);
 306                                 currentInput = input;
 307                                 if (countMap.get(currentInput) == null) {
 308                                     countMap.put(currentInput, 1);
 309                                 } else {
 310                                     countMap.put(currentInput, countMap.get(currentInput) + 1);
 311                                 }
 312                                 Iterator<InputLine> iter = countMap.keySet().iterator();
 313                                 while (iter.hasNext()) {
 314                                     final InputLine inputLine = iter.next();
 315                                     if (inputLine.level > currentInput.level) {
 316                                         iter.remove();
 317                                     }
 318                                 }
 319                                 if (currentInput.childrenInputs != null && currentInput.childrenInputs.size() > 0) {
 320                                     currentInput = currentInput.childrenInputs.get(0);
 321                                     countMap.put(currentInput, 0);
 322                                 } else if (countMap.get(currentInput) >= currentInput.getMaxCount()) {
 323                                     if (currentInput.getNextInputs(countMap).size() > 0) {
 324                                         currentInput = currentInput.getNextInputs(countMap).get(0);
 325                                         if (countMap.get(currentInput) == null) {
 326                                             countMap.put(currentInput, 0);
 327                                         }
 328                                         iter = countMap.keySet().iterator();
 329                                         while (iter.hasNext()) {
 330                                             final InputLine inputLine = iter.next();
 331                                             if (inputLine.level > currentInput.level) {
 332                                                 iter.remove();
 333                                             }
 334                                         }
 335                                     } else {
 336                                         currentInput = null;
 337                                     }
 338                                 }
 339                                 processed = true;
 340                                 break;
 341                             }
 342                         }
 343                     }
 344                     if (!processed && currentInput != null) {
 345                         final Matcher matcher = currentInput.getPattern().matcher(line);
 346                         if (matcher.matches()) {
 347                             event.processGroups(currentInput, matcher);
 348                             countMap.put(currentInput, countMap.get(currentInput) + 1);
 349                             if (currentInput.childrenInputs != null && currentInput.childrenInputs.size() > 0) {
 350                                 currentInput = currentInput.childrenInputs.get(0);
 351                                 countMap.put(currentInput, 0);
 352                             } else if (countMap.get(currentInput) >= currentInput.getMaxCount()) {
 353                                 if (currentInput.getNextInputs(countMap).size() > 0) {
 354                                     currentInput = currentInput.getNextInputs(countMap).get(0);
 355                                     if (countMap.get(currentInput) == null) {
 356                                         countMap.put(currentInput, 0);
 357                                     }
 358                                     final Iterator<InputLine> iter = countMap.keySet().iterator();
 359                                     while (iter.hasNext()) {
 360                                         final InputLine inputLine = iter.next();
 361                                         if (inputLine.level > currentInput.level) {
 362                                             iter.remove();
 363                                         }
 364                                     }
 365                                 } else {
 366                                     currentInput = null;
 367                                 }
 368                             }
 369                         }
 370                         ((StringBuffer) event.getContent().getValue()).append("\n").append(line); //$NON-NLS-1$
 371                     }
 372                 }
 373                 rawPos = fFile.getFilePointer();
 374                 line = fFile.getNextLine();
 375             }
 376         } catch (final IOException e) {
 377             Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
 378         }
 379         for (final Entry<InputLine, Integer> entry : countMap.entrySet()) {
 380             if (entry.getValue() < entry.getKey().getMinCount()) {
 381                 event = null;
 382             }
 383         }
 384         context.setLocation(NULL_LOCATION);
 385         return event;
 386     }
 387
 388     /**
 389      * @return The first few lines of the text file
 390      */
 391     public List<InputLine> getFirstLines() {
 392         return fDefinition.inputs;
 393     }
 394
 395     /**
 396      * Parse the first line of the trace (to recognize the type).
 397      *
 398      * @param context
 399      *            Trace context
 400      * @return The first event
 401      */
 402     public CustomTxtEvent parseFirstLine(final CustomTxtTraceContext context) {
 403         final CustomTxtEvent event = new CustomTxtEvent(fDefinition, this, TmfTimestamp.ZERO, fEventType);
 404         event.processGroups(context.inputLine, context.firstLineMatcher);
 405         event.setContent(new CustomEventContent(event, new StringBuffer(context.firstLine)));
 406         return event;
 407     }
 408
 409     /**
 410      * Get the trace definition.
 411      *
 412      * @return The trace definition
 413      */
 414     public CustomTraceDefinition getDefinition() {
 415         return fDefinition;
 416     }
 417
 418     /**
 419      * {@inheritDoc}
 420      * <p>
 421      * The default implementation computes the confidence as the percentage of
 422      * lines in the first 100 lines of the file which match any of the root
 423      * input line patterns.
 424      */
 425     @Override
 426     public IStatus validate(IProject project, String path) {
 427         File file = new File(path);
 428         if (!file.exists() || !file.isFile() || !file.canRead()) {
 429             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, Messages.CustomTrace_FileNotFound + ": " + path); //$NON-NLS-1$
 430         }
 431         int confidence = 0;
 432         try {
 433             if (!TmfTraceUtils.isText(file)) {
 434                 return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 435             }
 436         } catch (IOException e) {
 437             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 438             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 439         }
 440         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
 441             int lineCount = 0;
 442             double matches = 0.0;
 443             String line = rafile.getNextLine();
 444
 445             while ((line != null) && (lineCount++ < MAX_LINES)) {
 446                 for (InputLine inputLine : fDefinition.inputs) {
 447                     Matcher matcher = inputLine.getPattern().matcher(line);
 448                     if (matcher.matches()) {
 449                         int groupCount = matcher.groupCount();
 450                         matches += (1.0 + groupCount / ((double) groupCount + 1));
 451                         break;
 452                     }
 453                 }
 454                 confidence = (int) (MAX_CONFIDENCE * matches / lineCount);
 455                 line = rafile.getNextLine();
 456             }
 457         } catch (IOException e) {
 458             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 459         }
 460         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 461     }
 462
 463     private static int fCheckpointSize = -1;
 464
 465     @Override
 466     public synchronized int getCheckpointSize() {
 467         if (fCheckpointSize == -1) {
 468             TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
 469             ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
 470             b.clear();
 471             c.serialize(b);
 472             fCheckpointSize = b.position();
 473         }
 474
 475         return fCheckpointSize;
 476     }
 477
 478     @Override
 479     public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
 480         return new TmfLongLocation(bufferIn);
 481     }
 482
 483     @Override
 484     protected ITmfTraceIndexer createIndexer(int interval) {
 485         return new TmfBTreeTraceIndexer(this, interval);
 486     }
 487
 488     @Override
 489     public String getTraceTypeId() {
 490         return fTraceTypeId;
 491     }
 492
 493     /**
 494      * Build the trace type id for a custom text trace
 495      *
 496      * @param category
 497      *            the category
 498      * @param definitionName
 499      *            the definition name
 500      * @return the trace type id
 501      */
 502     public static @NonNull String buildTraceTypeId(String category, String definitionName) {
 503         return CUSTOM_TXT_TRACE_TYPE_PREFIX + category + SEPARATOR + definitionName;
 504     }
 505
 506     /**
 507      * Checks whether the given trace type ID is a custom text trace type ID
 508      *
 509      * @param traceTypeId
 510      *                the trace type ID to check
 511      * @return <code>true</code> if it's a custom text trace type ID else <code>false</code>
 512      */
 513     public static boolean isCustomTraceTypeId(@NonNull String traceTypeId) {
 514         return traceTypeId.startsWith(CUSTOM_TXT_TRACE_TYPE_PREFIX);
 515     }
 516
 517     /**
 518      * This methods builds a trace type ID from a given ID taking into
 519      * consideration any format changes that were done for the IDs of custom
 520      * text traces. For example, such format change took place when moving to
 521      * Trace Compass. Trace type IDs that are part of the plug-in extension for
 522      * trace types won't be changed.
 523      *
 524      * This method is useful for IDs that were persisted in the workspace before
 525      * the format changes (e.g. in the persistent properties of a trace
 526      * resource).
 527      *
 528      * It ensures backwards compatibility of the workspace for custom text
 529      * traces.
 530      *
 531      * @param traceTypeId
 532      *            the legacy trace type ID
 533      * @return the trace type id in Trace Compass format
 534      */
 535     public static @NonNull String buildCompatibilityTraceTypeId(@NonNull String traceTypeId) {
 536         // Handle early Trace Compass custom text trace type IDs
 537         if (traceTypeId.startsWith(EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX)) {
 538             return CUSTOM_TXT_TRACE_TYPE_PREFIX + traceTypeId.substring(EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX.length());
 539         }
 540
 541         // Handle Linux Tools custom text trace type IDs (with and without category)
 542         int index = traceTypeId.lastIndexOf(SEPARATOR);
 543         if ((index != -1) && (traceTypeId.startsWith(LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX))) {
 544             String definitionName = index < traceTypeId.length() ? traceTypeId.substring(index + 1) : ""; //$NON-NLS-1$
 545             if (traceTypeId.contains(CustomTxtTrace.class.getSimpleName() + SEPARATOR) && traceTypeId.indexOf(SEPARATOR) == index) {
 546                 return buildTraceTypeId(CustomTxtTraceDefinition.CUSTOM_TXT_CATEGORY, definitionName);
 547             }
 548             return CUSTOM_TXT_TRACE_TYPE_PREFIX + traceTypeId.substring(LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX.length());
 549         }
 550         return traceTypeId;
 551     }
 552 }