tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/text/TextTrace.java

   1 /*******************************************************************************
   2  * Copyright (c) 2012, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Patrick Tasse - Initial API and implementation
  11  *   Marc-Andre Laperle - Add persistent index support
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.trace.text;
  15
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.nio.ByteBuffer;
  19 import java.util.Collections;
  20 import java.util.List;
  21 import java.util.regex.Matcher;
  22 import java.util.regex.Pattern;
  23
  24 import org.eclipse.core.resources.IProject;
  25 import org.eclipse.core.resources.IResource;
  26 import org.eclipse.core.runtime.IStatus;
  27 import org.eclipse.core.runtime.Status;
  28 import org.eclipse.jdt.annotation.NonNull;
  29 import org.eclipse.jdt.annotation.Nullable;
  30 import org.eclipse.tracecompass.internal.tmf.core.Activator;
  31 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  32 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  33 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
  34 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
  35 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
  36 import org.eclipse.tracecompass.tmf.core.timestamp.ITmfTimestamp;
  37 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
  38 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
  39 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
  40 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  41 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
  42 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
  43 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
  44 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
  45 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
  46 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
  47 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
  48 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  49
  50 /**
  51  * Extension of TmfTrace for handling of line-based text traces parsed using
  52  * regular expressions. Each line that matches the first line pattern indicates
  53  * the start of a new event. The subsequent lines can contain additional
  54  * information that is added to the current event.
  55  *
  56  * @param <T>
  57  *            TmfEvent class returned by this trace
  58  */
  59 public abstract class TextTrace<T extends TextTraceEvent> extends TmfTrace implements ITmfPersistentlyIndexable {
  60
  61     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
  62     private static final int MAX_LINES = 100;
  63     private static final int MAX_CONFIDENCE = 100;
  64
  65     /** The default separator used for multi-line fields */
  66     protected static final String SEPARATOR = " | "; //$NON-NLS-1$
  67
  68     /** The text file */
  69     protected BufferedRandomAccessFile fFile;
  70
  71     /**
  72      * Constructor
  73      */
  74     public TextTrace() {
  75     }
  76
  77     /**
  78      * {@inheritDoc}
  79      * <p>
  80      * The default implementation computes the confidence as the sum of weighted
  81      * values of the first 100 lines of the file which match any of the provided
  82      * validation patterns. For each matching line a weighted value between 1.5
  83      * and 2.0 is assigned based on the group count of the matching patterns.
  84      * The higher the group count, the closer the weighted value will be to 2.0.
  85      */
  86     @Override
  87     public IStatus validate(IProject project, String path) {
  88         File file = new File(path);
  89         if (!file.exists()) {
  90             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "File not found: " + path); //$NON-NLS-1$
  91         }
  92         if (!file.isFile()) {
  93             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Not a file. It's a directory: " + path); //$NON-NLS-1$
  94         }
  95         try {
  96             if (!TmfTraceUtils.isText(file)) {
  97                 return new TraceValidationStatus(0, Activator.PLUGIN_ID);
  98             }
  99         } catch (IOException e) {
 100             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 101             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 102         }
 103         int confidence = 0;
 104         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
 105             int lineCount = 0;
 106             double matches = 0.0;
 107             String line = rafile.getNextLine();
 108             List<Pattern> validationPatterns = getValidationPatterns();
 109             while ((line != null) && (lineCount++ < MAX_LINES)) {
 110                 line = preProcessLine(line);
 111                 for(Pattern pattern : validationPatterns) {
 112                     Matcher matcher = pattern.matcher(line);
 113                     if (matcher.matches()) {
 114                         int groupCount = matcher.groupCount();
 115                         matches += (1.0 + groupCount / ((double) groupCount + 1));
 116                     }
 117                 }
 118                 confidence = (int) (MAX_CONFIDENCE * matches / lineCount);
 119                 line = rafile.getNextLine();
 120             }
 121         } catch (IOException e) {
 122             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 123             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 124         }
 125
 126         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 127
 128     }
 129     @Override
 130     public void initTrace(IResource resource, String path, Class<? extends ITmfEvent> type) throws TmfTraceException {
 131         super.initTrace(resource, path, type);
 132         initFile();
 133     }
 134
 135     private void initFile() throws TmfTraceException {
 136         closeFile();
 137         try {
 138             fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
 139         } catch (IOException e) {
 140             throw new TmfTraceException(e.getMessage(), e);
 141         }
 142     }
 143
 144     @Override
 145     public synchronized void dispose() {
 146         super.dispose();
 147         closeFile();
 148     }
 149
 150     private void closeFile() {
 151         if (fFile != null) {
 152             try {
 153                 fFile.close();
 154             } catch (IOException e) {
 155             } finally {
 156                 fFile = null;
 157             }
 158         }
 159     }
 160
 161     @Override
 162     public synchronized TextTraceContext seekEvent(ITmfLocation location) {
 163         TextTraceContext context = new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 164         if (NULL_LOCATION.equals(location) || fFile == null) {
 165             return context;
 166         }
 167         try {
 168             if (location == null) {
 169                 fFile.seek(0);
 170             } else if (location.getLocationInfo() instanceof Long) {
 171                 fFile.seek((Long) location.getLocationInfo());
 172             }
 173             long rawPos = fFile.getFilePointer();
 174             String line = fFile.getNextLine();
 175             while (line != null) {
 176                 line = preProcessLine(line);
 177                 Matcher matcher = getFirstLinePattern().matcher(line);
 178                 if (matcher.matches()) {
 179                     setupContext(context, rawPos, line, matcher);
 180                     return context;
 181                 }
 182                 rawPos = fFile.getFilePointer();
 183                 line = fFile.getNextLine();
 184             }
 185             return context;
 186         } catch (IOException e) {
 187             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 188             return context;
 189         }
 190     }
 191
 192     private void setupContext(TextTraceContext context, long rawPos, String line, Matcher matcher) throws IOException {
 193         context.setLocation(new TmfLongLocation(rawPos));
 194         context.firstLineMatcher = matcher;
 195         context.firstLine = line;
 196         context.nextLineLocation = fFile.getFilePointer();
 197     }
 198
 199     @Override
 200     public synchronized TextTraceContext seekEvent(double ratio) {
 201         if (fFile == null) {
 202             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 203         }
 204         try {
 205             long pos = Math.round(ratio * fFile.length());
 206             while (pos > 0) {
 207                 fFile.seek(pos - 1);
 208                 if (fFile.read() == '\n') {
 209                     break;
 210                 }
 211                 pos--;
 212             }
 213             ITmfLocation location = new TmfLongLocation(Long.valueOf(pos));
 214             TextTraceContext context = seekEvent(location);
 215             context.setRank(ITmfContext.UNKNOWN_RANK);
 216             return context;
 217         } catch (IOException e) {
 218             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 219             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 220         }
 221     }
 222
 223     @Override
 224     public double getLocationRatio(ITmfLocation location) {
 225         if (fFile == null) {
 226             return 0;
 227         }
 228         try {
 229             long length = fFile.length();
 230             if (length == 0) {
 231                 return 0;
 232             }
 233             if (location.getLocationInfo() instanceof Long) {
 234                 return (double) ((Long) location.getLocationInfo()) / length;
 235             }
 236         } catch (IOException e) {
 237             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 238         }
 239         return 0;
 240     }
 241
 242     @Override
 243     public ITmfLocation getCurrentLocation() {
 244         return null;
 245     }
 246
 247     @Override
 248     public TextTraceEvent parseEvent(ITmfContext tmfContext) {
 249         TextTraceContext context = seekEvent(tmfContext.getLocation());
 250         return parse(context);
 251     }
 252
 253     @Override
 254     public synchronized @Nullable T getNext(ITmfContext context) {
 255         if (!(context instanceof TextTraceContext)) {
 256             throw new IllegalArgumentException();
 257         }
 258         TextTraceContext savedContext = new TextTraceContext(context.getLocation(), context.getRank());
 259         @Nullable T event = parse((TextTraceContext) context);
 260         if (event != null) {
 261             updateAttributes(savedContext, event);
 262             context.increaseRank();
 263         }
 264         return event;
 265     }
 266
 267     /**
 268      * Parse the next event. The context is advanced.
 269      *
 270      * @param tmfContext
 271      *            the context
 272      * @return the next event or null
 273      */
 274     protected synchronized @Nullable T parse(TextTraceContext tmfContext) {
 275         if (fFile == null) {
 276             return null;
 277         }
 278         TextTraceContext context = tmfContext;
 279         ITmfLocation location = context.getLocation();
 280         if (location == null || !(location.getLocationInfo() instanceof Long) || NULL_LOCATION.equals(location)) {
 281             return null;
 282         }
 283
 284         T event = parseFirstLine(context.firstLineMatcher, context.firstLine);
 285
 286         try {
 287             if (fFile.getFilePointer() != context.nextLineLocation) {
 288                 fFile.seek(context.nextLineLocation);
 289             }
 290             long rawPos = fFile.getFilePointer();
 291             String line = fFile.getNextLine();
 292             while (line != null) {
 293                 line = preProcessLine(line);
 294                 Matcher matcher = getFirstLinePattern().matcher(line);
 295                 if (matcher.matches()) {
 296                     setupContext(context, rawPos, line, matcher);
 297                     return event;
 298                 }
 299                 parseNextLine(event, line);
 300                 rawPos = fFile.getFilePointer();
 301                 line = fFile.getNextLine();
 302             }
 303         } catch (IOException e) {
 304             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 305         }
 306
 307         context.setLocation(NULL_LOCATION);
 308         return event;
 309     }
 310
 311     /**
 312      * Pre-processes the input line. The default implementation returns the
 313      * input line.
 314      *
 315      * @param line
 316      *            non-null input string
 317      * @return the pre-processed input line
 318      */
 319     @NonNull
 320     protected String preProcessLine(@NonNull String line) {
 321         return line;
 322     }
 323
 324     /**
 325      * Gets the first line pattern.
 326      *
 327      * @return The first line pattern
 328      */
 329     protected abstract Pattern getFirstLinePattern();
 330
 331     /**
 332      * Parses the first line data and returns a new event. When constructing the
 333      * event, the concrete trace should use the trace's timestamp transform to
 334      * create the timestamp, by either transforming the parsed time value
 335      * directly or by using the method {@link #createTimestamp(long)}.
 336      *
 337      * @param matcher
 338      *            The matcher
 339      * @param line
 340      *            The line to parse
 341      * @return The parsed event
 342      */
 343     protected abstract T parseFirstLine(Matcher matcher, String line);
 344
 345     /**
 346      * Parses the next line data for the current event.
 347      *
 348      * @param event
 349      *            The current event being parsed
 350      * @param line
 351      *            The line to parse
 352      */
 353     protected abstract void parseNextLine(T event, String line);
 354
 355     /**
 356      * Returns a ordered list of validation patterns that will be used in
 357      * the default {@link #validate(IProject, String)} method to match
 358      * the first 100 to compute the confidence level
 359      *
 360      * @return collection of patterns to validate against
 361      */
 362     protected List<Pattern> getValidationPatterns() {
 363         return Collections.singletonList(getFirstLinePattern());
 364     }
 365
 366     // ------------------------------------------------------------------------
 367     // Helper methods
 368     // ------------------------------------------------------------------------
 369
 370     /**
 371      * Strip quotes surrounding a string
 372      *
 373      * @param input
 374      *            The input string
 375      * @return The string without quotes
 376      */
 377     protected static String replaceQuotes(String input) {
 378         String out = input.replaceAll("^\"|(\"\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 379         return out;
 380     }
 381
 382     /**
 383      * Strip brackets surrounding a string
 384      *
 385      * @param input
 386      *            The input string
 387      * @return The string without brackets
 388      */
 389     protected static String replaceBrackets(String input) {
 390         String out = input.replaceAll("^\\{|(\\}\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 391         return out;
 392     }
 393
 394     private static int fCheckpointSize = -1;
 395
 396     @Override
 397     public synchronized int getCheckpointSize() {
 398         if (fCheckpointSize == -1) {
 399             TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
 400             ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
 401             b.clear();
 402             c.serialize(b);
 403             fCheckpointSize = b.position();
 404         }
 405
 406         return fCheckpointSize;
 407     }
 408
 409     @Override
 410     protected ITmfTraceIndexer createIndexer(int interval) {
 411         return new TmfBTreeTraceIndexer(this, interval);
 412     }
 413
 414     @Override
 415     public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
 416         return new TmfLongLocation(bufferIn);
 417     }
 418
 419     @TmfSignalHandler
 420     @Override
 421     public void traceRangeUpdated(TmfTraceRangeUpdatedSignal signal) {
 422         if (signal.getTrace() == this) {
 423             try {
 424                 synchronized (this) {
 425                     // Reset the file handle in case it has reached the end of the
 426                     // file already. Otherwise, it will not be able to read new data
 427                     // pass the previous end.
 428                     initFile();
 429                 }
 430             } catch (TmfTraceException e) {
 431                 Activator.logError(e.getLocalizedMessage(), e);
 432             }
 433         }
 434         super.traceRangeUpdated(signal);
 435     }
 436
 437     /**
 438      * @since 3.0
 439      */
 440     @Override
 441     public synchronized ITmfTimestamp readEnd() {
 442         try {
 443             Long pos = fFile.length() - 1;
 444             /* Outer loop to find the first line of a matcher group. */
 445             while (pos > 0) {
 446                 /* Inner loop to find line beginning */
 447                 while (pos > 0) {
 448                     fFile.seek(pos - 1);
 449                     if (fFile.read() == '\n') {
 450                         break;
 451                     }
 452                     pos--;
 453                 }
 454                 ITmfLocation location = new TmfLongLocation(pos);
 455                 ITmfContext context = seekEvent(location);
 456                 ITmfEvent event = getNext(context);
 457                 context.dispose();
 458                 if (event != null) {
 459                     return event.getTimestamp();
 460                 }
 461                 pos--;
 462             }
 463         } catch (IOException e) {
 464             return null;
 465         }
 466
 467         /* Empty trace */
 468         return null;
 469     }
 470 }