tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/text/TextTrace.java

   1 /*******************************************************************************
   2  * Copyright (c) 2012, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Patrick Tasse - Initial API and implementation
  11  *   Marc-Andre Laperle - Add persistent index support
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.trace.text;
  15
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.nio.ByteBuffer;
  19 import java.util.Collections;
  20 import java.util.List;
  21 import java.util.regex.Matcher;
  22 import java.util.regex.Pattern;
  23
  24 import org.eclipse.core.resources.IProject;
  25 import org.eclipse.core.resources.IResource;
  26 import org.eclipse.core.runtime.IStatus;
  27 import org.eclipse.core.runtime.Status;
  28 import org.eclipse.jdt.annotation.NonNull;
  29 import org.eclipse.jdt.annotation.Nullable;
  30 import org.eclipse.tracecompass.internal.tmf.core.Activator;
  31 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  32 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  33 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
  34 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
  35 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
  36 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
  37 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
  38 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
  39 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  40 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
  41 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
  42 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
  43 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
  44 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
  45 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
  46 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
  47 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  48
  49 /**
  50  * Extension of TmfTrace for handling of line-based text traces parsed using
  51  * regular expressions. Each line that matches the first line pattern indicates
  52  * the start of a new event. The subsequent lines can contain additional
  53  * information that is added to the current event.
  54  *
  55  * @param <T>
  56  *            TmfEvent class returned by this trace
  57  */
  58 public abstract class TextTrace<T extends TextTraceEvent> extends TmfTrace implements ITmfPersistentlyIndexable {
  59
  60     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
  61     private static final int MAX_LINES = 100;
  62     private static final int MAX_CONFIDENCE = 100;
  63
  64     /** The default separator used for multi-line fields */
  65     protected static final String SEPARATOR = " | "; //$NON-NLS-1$
  66
  67     /** The text file */
  68     protected BufferedRandomAccessFile fFile;
  69
  70     /**
  71      * Constructor
  72      */
  73     public TextTrace() {
  74     }
  75
  76     /**
  77      * {@inheritDoc}
  78      * <p>
  79      * The default implementation computes the confidence as the sum of weighted
  80      * values of the first 100 lines of the file which match any of the provided
  81      * validation patterns. For each matching line a weighted value between 1.5
  82      * and 2.0 is assigned based on the group count of the matching patterns.
  83      * The higher the group count, the closer the weighted value will be to 2.0.
  84      */
  85     @Override
  86     public IStatus validate(IProject project, String path) {
  87         File file = new File(path);
  88         if (!file.exists()) {
  89             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "File not found: " + path); //$NON-NLS-1$
  90         }
  91         if (!file.isFile()) {
  92             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Not a file. It's a directory: " + path); //$NON-NLS-1$
  93         }
  94         try {
  95             if (!TmfTraceUtils.isText(file)) {
  96                 return new TraceValidationStatus(0, Activator.PLUGIN_ID);
  97             }
  98         } catch (IOException e) {
  99             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 100             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 101         }
 102         int confidence = 0;
 103         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
 104             int lineCount = 0;
 105             double matches = 0.0;
 106             String line = rafile.getNextLine();
 107             List<Pattern> validationPatterns = getValidationPatterns();
 108             while ((line != null) && (lineCount++ < MAX_LINES)) {
 109                 line = preProcessLine(line);
 110                 for(Pattern pattern : validationPatterns) {
 111                     Matcher matcher = pattern.matcher(line);
 112                     if (matcher.matches()) {
 113                         int groupCount = matcher.groupCount();
 114                         matches += (1.0 + groupCount / ((double) groupCount + 1));
 115                     }
 116                 }
 117                 confidence = (int) (MAX_CONFIDENCE * matches / lineCount);
 118                 line = rafile.getNextLine();
 119             }
 120         } catch (IOException e) {
 121             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 122             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 123         }
 124
 125         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 126
 127     }
 128     @Override
 129     public void initTrace(IResource resource, String path, Class<? extends ITmfEvent> type) throws TmfTraceException {
 130         super.initTrace(resource, path, type);
 131         initFile();
 132     }
 133
 134     private void initFile() throws TmfTraceException {
 135         closeFile();
 136         try {
 137             fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
 138         } catch (IOException e) {
 139             throw new TmfTraceException(e.getMessage(), e);
 140         }
 141     }
 142
 143     @Override
 144     public synchronized void dispose() {
 145         super.dispose();
 146         closeFile();
 147     }
 148
 149     private void closeFile() {
 150         if (fFile != null) {
 151             try {
 152                 fFile.close();
 153             } catch (IOException e) {
 154             } finally {
 155                 fFile = null;
 156             }
 157         }
 158     }
 159
 160     @Override
 161     public synchronized TextTraceContext seekEvent(ITmfLocation location) {
 162         TextTraceContext context = new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 163         if (NULL_LOCATION.equals(location) || fFile == null) {
 164             return context;
 165         }
 166         try {
 167             if (location == null) {
 168                 fFile.seek(0);
 169             } else if (location.getLocationInfo() instanceof Long) {
 170                 fFile.seek((Long) location.getLocationInfo());
 171             }
 172             long rawPos = fFile.getFilePointer();
 173             String line = fFile.getNextLine();
 174             while (line != null) {
 175                 line = preProcessLine(line);
 176                 Matcher matcher = getFirstLinePattern().matcher(line);
 177                 if (matcher.matches()) {
 178                     setupContext(context, rawPos, line, matcher);
 179                     return context;
 180                 }
 181                 rawPos = fFile.getFilePointer();
 182                 line = fFile.getNextLine();
 183             }
 184             return context;
 185         } catch (IOException e) {
 186             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 187             return context;
 188         }
 189     }
 190
 191     private void setupContext(TextTraceContext context, long rawPos, String line, Matcher matcher) throws IOException {
 192         context.setLocation(new TmfLongLocation(rawPos));
 193         context.firstLineMatcher = matcher;
 194         context.firstLine = line;
 195         context.nextLineLocation = fFile.getFilePointer();
 196     }
 197
 198     @Override
 199     public synchronized TextTraceContext seekEvent(double ratio) {
 200         if (fFile == null) {
 201             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 202         }
 203         try {
 204             long pos = Math.round(ratio * fFile.length());
 205             while (pos > 0) {
 206                 fFile.seek(pos - 1);
 207                 if (fFile.read() == '\n') {
 208                     break;
 209                 }
 210                 pos--;
 211             }
 212             ITmfLocation location = new TmfLongLocation(Long.valueOf(pos));
 213             TextTraceContext context = seekEvent(location);
 214             context.setRank(ITmfContext.UNKNOWN_RANK);
 215             return context;
 216         } catch (IOException e) {
 217             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 218             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 219         }
 220     }
 221
 222     @Override
 223     public double getLocationRatio(ITmfLocation location) {
 224         if (fFile == null) {
 225             return 0;
 226         }
 227         try {
 228             long length = fFile.length();
 229             if (length == 0) {
 230                 return 0;
 231             }
 232             if (location.getLocationInfo() instanceof Long) {
 233                 return (double) ((Long) location.getLocationInfo()) / length;
 234             }
 235         } catch (IOException e) {
 236             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 237         }
 238         return 0;
 239     }
 240
 241     @Override
 242     public ITmfLocation getCurrentLocation() {
 243         return null;
 244     }
 245
 246     @Override
 247     public TextTraceEvent parseEvent(ITmfContext tmfContext) {
 248         TextTraceContext context = seekEvent(tmfContext.getLocation());
 249         return parse(context);
 250     }
 251
 252     @Override
 253     public synchronized @Nullable T getNext(ITmfContext context) {
 254         if (!(context instanceof TextTraceContext)) {
 255             throw new IllegalArgumentException();
 256         }
 257         TextTraceContext savedContext = new TextTraceContext(context.getLocation(), context.getRank());
 258         @Nullable T event = parse((TextTraceContext) context);
 259         if (event != null) {
 260             updateAttributes(savedContext, event);
 261             context.increaseRank();
 262         }
 263         return event;
 264     }
 265
 266     /**
 267      * Parse the next event. The context is advanced.
 268      *
 269      * @param tmfContext
 270      *            the context
 271      * @return the next event or null
 272      */
 273     protected synchronized @Nullable T parse(TextTraceContext tmfContext) {
 274         if (fFile == null) {
 275             return null;
 276         }
 277         TextTraceContext context = tmfContext;
 278         ITmfLocation location = context.getLocation();
 279         if (location == null || !(location.getLocationInfo() instanceof Long) || NULL_LOCATION.equals(location)) {
 280             return null;
 281         }
 282
 283         T event = parseFirstLine(context.firstLineMatcher, context.firstLine);
 284
 285         try {
 286             if (fFile.getFilePointer() != context.nextLineLocation) {
 287                 fFile.seek(context.nextLineLocation);
 288             }
 289             long rawPos = fFile.getFilePointer();
 290             String line = fFile.getNextLine();
 291             while (line != null) {
 292                 line = preProcessLine(line);
 293                 Matcher matcher = getFirstLinePattern().matcher(line);
 294                 if (matcher.matches()) {
 295                     setupContext(context, rawPos, line, matcher);
 296                     return event;
 297                 }
 298                 parseNextLine(event, line);
 299                 rawPos = fFile.getFilePointer();
 300                 line = fFile.getNextLine();
 301             }
 302         } catch (IOException e) {
 303             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 304         }
 305
 306         context.setLocation(NULL_LOCATION);
 307         return event;
 308     }
 309
 310     /**
 311      * Pre-processes the input line. The default implementation returns the
 312      * input line.
 313      *
 314      * @param line
 315      *            non-null input string
 316      * @return the pre-processed input line
 317      */
 318     @NonNull
 319     protected String preProcessLine(@NonNull String line) {
 320         return line;
 321     }
 322
 323     /**
 324      * Gets the first line pattern.
 325      *
 326      * @return The first line pattern
 327      */
 328     protected abstract Pattern getFirstLinePattern();
 329
 330     /**
 331      * Parses the first line data and returns a new event. When constructing the
 332      * event, the concrete trace should use the trace's timestamp transform to
 333      * create the timestamp, by either transforming the parsed time value
 334      * directly or by using the method {@link #createTimestamp(long)}.
 335      *
 336      * @param matcher
 337      *            The matcher
 338      * @param line
 339      *            The line to parse
 340      * @return The parsed event
 341      */
 342     protected abstract T parseFirstLine(Matcher matcher, String line);
 343
 344     /**
 345      * Parses the next line data for the current event.
 346      *
 347      * @param event
 348      *            The current event being parsed
 349      * @param line
 350      *            The line to parse
 351      */
 352     protected abstract void parseNextLine(T event, String line);
 353
 354     /**
 355      * Returns a ordered list of validation patterns that will be used in
 356      * the default {@link #validate(IProject, String)} method to match
 357      * the first 100 to compute the confidence level
 358      *
 359      * @return collection of patterns to validate against
 360      */
 361     protected List<Pattern> getValidationPatterns() {
 362         return Collections.singletonList(getFirstLinePattern());
 363     }
 364
 365     // ------------------------------------------------------------------------
 366     // Helper methods
 367     // ------------------------------------------------------------------------
 368
 369     /**
 370      * Strip quotes surrounding a string
 371      *
 372      * @param input
 373      *            The input string
 374      * @return The string without quotes
 375      */
 376     protected static String replaceQuotes(String input) {
 377         String out = input.replaceAll("^\"|(\"\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 378         return out;
 379     }
 380
 381     /**
 382      * Strip brackets surrounding a string
 383      *
 384      * @param input
 385      *            The input string
 386      * @return The string without brackets
 387      */
 388     protected static String replaceBrackets(String input) {
 389         String out = input.replaceAll("^\\{|(\\}\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 390         return out;
 391     }
 392
 393     private static int fCheckpointSize = -1;
 394
 395     @Override
 396     public synchronized int getCheckpointSize() {
 397         if (fCheckpointSize == -1) {
 398             TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
 399             ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
 400             b.clear();
 401             c.serialize(b);
 402             fCheckpointSize = b.position();
 403         }
 404
 405         return fCheckpointSize;
 406     }
 407
 408     @Override
 409     protected ITmfTraceIndexer createIndexer(int interval) {
 410         return new TmfBTreeTraceIndexer(this, interval);
 411     }
 412
 413     @Override
 414     public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
 415         return new TmfLongLocation(bufferIn);
 416     }
 417
 418     @TmfSignalHandler
 419     @Override
 420     public void traceRangeUpdated(TmfTraceRangeUpdatedSignal signal) {
 421         if (signal.getTrace() == this) {
 422             try {
 423                 synchronized (this) {
 424                     // Reset the file handle in case it has reached the end of the
 425                     // file already. Otherwise, it will not be able to read new data
 426                     // pass the previous end.
 427                     initFile();
 428                 }
 429             } catch (TmfTraceException e) {
 430                 Activator.logError(e.getLocalizedMessage(), e);
 431             }
 432         }
 433         super.traceRangeUpdated(signal);
 434     }
 435 }