tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/text/TextTrace.java

   1 /*******************************************************************************
   2  * Copyright (c) 2012, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Patrick Tasse - Initial API and implementation
  11  *   Marc-Andre Laperle - Add persistent index support
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.trace.text;
  15
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.nio.ByteBuffer;
  19 import java.util.Collections;
  20 import java.util.List;
  21 import java.util.regex.Matcher;
  22 import java.util.regex.Pattern;
  23
  24 import org.eclipse.core.resources.IProject;
  25 import org.eclipse.core.resources.IResource;
  26 import org.eclipse.core.runtime.IStatus;
  27 import org.eclipse.core.runtime.Status;
  28 import org.eclipse.jdt.annotation.NonNull;
  29 import org.eclipse.tracecompass.internal.tmf.core.Activator;
  30 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  31 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  32 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
  33 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
  34 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
  35 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
  36 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
  37 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
  38 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  39 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
  40 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
  41 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
  42 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
  43 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
  44 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
  45 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
  46 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  47
  48 /**
  49  * Extension of TmfTrace for handling of line-based text traces parsed using
  50  * regular expressions. Each line that matches the first line pattern indicates
  51  * the start of a new event. The subsequent lines can contain additional
  52  * information that is added to the current event.
  53  *
  54  * @param <T>
  55  *            TmfEvent class returned by this trace
  56  */
  57 public abstract class TextTrace<T extends TextTraceEvent> extends TmfTrace implements ITmfPersistentlyIndexable {
  58
  59     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
  60     private static final int MAX_LINES = 100;
  61     private static final int MAX_CONFIDENCE = 100;
  62
  63     /** The default separator used for multi-line fields */
  64     protected static final String SEPARATOR = " | "; //$NON-NLS-1$
  65
  66     /** The text file */
  67     protected BufferedRandomAccessFile fFile;
  68
  69     /**
  70      * Constructor
  71      */
  72     public TextTrace() {
  73     }
  74
  75     /**
  76      * {@inheritDoc}
  77      * <p>
  78      * The default implementation computes the confidence as the sum of weighted
  79      * values of the first 100 lines of the file which match any of the provided
  80      * validation patterns. For each matching line a weighted value between 1.5
  81      * and 2.0 is assigned based on the group count of the matching patterns.
  82      * The higher the group count, the closer the weighted value will be to 2.0.
  83      */
  84     @Override
  85     public IStatus validate(IProject project, String path) {
  86         File file = new File(path);
  87         if (!file.exists()) {
  88             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "File not found: " + path); //$NON-NLS-1$
  89         }
  90         if (!file.isFile()) {
  91             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Not a file. It's a directory: " + path); //$NON-NLS-1$
  92         }
  93         try {
  94             if (!TmfTraceUtils.isText(file)) {
  95                 return new TraceValidationStatus(0, Activator.PLUGIN_ID);
  96             }
  97         } catch (IOException e) {
  98             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
  99             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 100         }
 101         int confidence = 0;
 102         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
 103             int lineCount = 0;
 104             double matches = 0.0;
 105             String line = rafile.getNextLine();
 106             List<Pattern> validationPatterns = getValidationPatterns();
 107             while ((line != null) && (lineCount++ < MAX_LINES)) {
 108                 line = preProcessLine(line);
 109                 for(Pattern pattern : validationPatterns) {
 110                     Matcher matcher = pattern.matcher(line);
 111                     if (matcher.matches()) {
 112                         int groupCount = matcher.groupCount();
 113                         matches += (1.0 + groupCount / ((double) groupCount + 1));
 114                     }
 115                 }
 116                 confidence = (int) (MAX_CONFIDENCE * matches / lineCount);
 117                 line = rafile.getNextLine();
 118             }
 119         } catch (IOException e) {
 120             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 121             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 122         }
 123
 124         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 125
 126     }
 127     @Override
 128     public void initTrace(IResource resource, String path, Class<? extends ITmfEvent> type) throws TmfTraceException {
 129         super.initTrace(resource, path, type);
 130         initFile();
 131     }
 132
 133     private void initFile() throws TmfTraceException {
 134         closeFile();
 135         try {
 136             fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
 137         } catch (IOException e) {
 138             throw new TmfTraceException(e.getMessage(), e);
 139         }
 140     }
 141
 142     @Override
 143     public synchronized void dispose() {
 144         super.dispose();
 145         closeFile();
 146     }
 147
 148     private void closeFile() {
 149         if (fFile != null) {
 150             try {
 151                 fFile.close();
 152             } catch (IOException e) {
 153             } finally {
 154                 fFile = null;
 155             }
 156         }
 157     }
 158
 159     @Override
 160     public synchronized TextTraceContext seekEvent(ITmfLocation location) {
 161         TextTraceContext context = new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 162         if (NULL_LOCATION.equals(location) || fFile == null) {
 163             return context;
 164         }
 165         try {
 166             if (location == null) {
 167                 fFile.seek(0);
 168             } else if (location.getLocationInfo() instanceof Long) {
 169                 fFile.seek((Long) location.getLocationInfo());
 170             }
 171             long rawPos = fFile.getFilePointer();
 172             String line = fFile.getNextLine();
 173             while (line != null) {
 174                 line = preProcessLine(line);
 175                 Matcher matcher = getFirstLinePattern().matcher(line);
 176                 if (matcher.matches()) {
 177                     setupContext(context, rawPos, line, matcher);
 178                     return context;
 179                 }
 180                 rawPos = fFile.getFilePointer();
 181                 line = fFile.getNextLine();
 182             }
 183             return context;
 184         } catch (IOException e) {
 185             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 186             return context;
 187         }
 188     }
 189
 190     private void setupContext(TextTraceContext context, long rawPos, String line, Matcher matcher) throws IOException {
 191         context.setLocation(new TmfLongLocation(rawPos));
 192         context.firstLineMatcher = matcher;
 193         context.firstLine = line;
 194         context.nextLineLocation = fFile.getFilePointer();
 195     }
 196
 197     @Override
 198     public synchronized TextTraceContext seekEvent(double ratio) {
 199         if (fFile == null) {
 200             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 201         }
 202         try {
 203             long pos = Math.round(ratio * fFile.length());
 204             while (pos > 0) {
 205                 fFile.seek(pos - 1);
 206                 if (fFile.read() == '\n') {
 207                     break;
 208                 }
 209                 pos--;
 210             }
 211             ITmfLocation location = new TmfLongLocation(Long.valueOf(pos));
 212             TextTraceContext context = seekEvent(location);
 213             context.setRank(ITmfContext.UNKNOWN_RANK);
 214             return context;
 215         } catch (IOException e) {
 216             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 217             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 218         }
 219     }
 220
 221     @Override
 222     public double getLocationRatio(ITmfLocation location) {
 223         if (fFile == null) {
 224             return 0;
 225         }
 226         try {
 227             long length = fFile.length();
 228             if (length == 0) {
 229                 return 0;
 230             }
 231             if (location.getLocationInfo() instanceof Long) {
 232                 return (double) ((Long) location.getLocationInfo()) / length;
 233             }
 234         } catch (IOException e) {
 235             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 236         }
 237         return 0;
 238     }
 239
 240     @Override
 241     public ITmfLocation getCurrentLocation() {
 242         return null;
 243     }
 244
 245     @Override
 246     public TextTraceEvent parseEvent(ITmfContext tmfContext) {
 247         TextTraceContext context = seekEvent(tmfContext.getLocation());
 248         return parse(context);
 249     }
 250
 251     @Override
 252     public synchronized T getNext(ITmfContext context) {
 253         if (!(context instanceof TextTraceContext)) {
 254             throw new IllegalArgumentException();
 255         }
 256         TextTraceContext savedContext = new TextTraceContext(context.getLocation(), context.getRank());
 257         T event = parse((TextTraceContext) context);
 258         if (event != null) {
 259             updateAttributes(savedContext, event);
 260             context.increaseRank();
 261         }
 262         return event;
 263     }
 264
 265     /**
 266      * Parse the next event. The context is advanced.
 267      *
 268      * @param tmfContext
 269      *            the context
 270      * @return the next event or null
 271      */
 272     protected synchronized T parse(TextTraceContext tmfContext) {
 273         if (fFile == null) {
 274             return null;
 275         }
 276         TextTraceContext context = tmfContext;
 277         if (context.getLocation() == null || !(context.getLocation().getLocationInfo() instanceof Long) || NULL_LOCATION.equals(context.getLocation())) {
 278             return null;
 279         }
 280
 281         T event = parseFirstLine(context.firstLineMatcher, context.firstLine);
 282
 283         try {
 284             if (fFile.getFilePointer() != context.nextLineLocation) {
 285                 fFile.seek(context.nextLineLocation);
 286             }
 287             long rawPos = fFile.getFilePointer();
 288             String line = fFile.getNextLine();
 289             while (line != null) {
 290                 line = preProcessLine(line);
 291                 Matcher matcher = getFirstLinePattern().matcher(line);
 292                 if (matcher.matches()) {
 293                     setupContext(context, rawPos, line, matcher);
 294                     return event;
 295                 }
 296                 parseNextLine(event, line);
 297                 rawPos = fFile.getFilePointer();
 298                 line = fFile.getNextLine();
 299             }
 300         } catch (IOException e) {
 301             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 302         }
 303
 304         context.setLocation(NULL_LOCATION);
 305         return event;
 306     }
 307
 308     /**
 309      * Pre-processes the input line. The default implementation returns the
 310      * input line.
 311      *
 312      * @param line
 313      *            non-null input string
 314      * @return the pre-processed input line
 315      */
 316     @NonNull
 317     protected String preProcessLine(@NonNull String line) {
 318         return line;
 319     }
 320
 321     /**
 322      * Gets the first line pattern.
 323      *
 324      * @return The first line pattern
 325      */
 326     protected abstract Pattern getFirstLinePattern();
 327
 328     /**
 329      * Parses the first line data and returns a new event. When constructing the
 330      * event, the concrete trace should use the trace's timestamp transform to
 331      * create the timestamp, by either transforming the parsed time value
 332      * directly or by using the method {@link #createTimestamp(long)}.
 333      *
 334      * @param matcher
 335      *            The matcher
 336      * @param line
 337      *            The line to parse
 338      * @return The parsed event
 339      */
 340     protected abstract T parseFirstLine(Matcher matcher, String line);
 341
 342     /**
 343      * Parses the next line data for the current event.
 344      *
 345      * @param event
 346      *            The current event being parsed
 347      * @param line
 348      *            The line to parse
 349      */
 350     protected abstract void parseNextLine(T event, String line);
 351
 352     /**
 353      * Returns a ordered list of validation patterns that will be used in
 354      * the default {@link #validate(IProject, String)} method to match
 355      * the first 100 to compute the confidence level
 356      *
 357      * @return collection of patterns to validate against
 358      */
 359     protected List<Pattern> getValidationPatterns() {
 360         return Collections.singletonList(getFirstLinePattern());
 361     }
 362
 363     // ------------------------------------------------------------------------
 364     // Helper methods
 365     // ------------------------------------------------------------------------
 366
 367     /**
 368      * Strip quotes surrounding a string
 369      *
 370      * @param input
 371      *            The input string
 372      * @return The string without quotes
 373      */
 374     protected static String replaceQuotes(String input) {
 375         String out = input.replaceAll("^\"|(\"\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 376         return out;
 377     }
 378
 379     /**
 380      * Strip brackets surrounding a string
 381      *
 382      * @param input
 383      *            The input string
 384      * @return The string without brackets
 385      */
 386     protected static String replaceBrackets(String input) {
 387         String out = input.replaceAll("^\\{|(\\}\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 388         return out;
 389     }
 390
 391     private static int fCheckpointSize = -1;
 392
 393     @Override
 394     public synchronized int getCheckpointSize() {
 395         if (fCheckpointSize == -1) {
 396             TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
 397             ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
 398             b.clear();
 399             c.serialize(b);
 400             fCheckpointSize = b.position();
 401         }
 402
 403         return fCheckpointSize;
 404     }
 405
 406     @Override
 407     protected ITmfTraceIndexer createIndexer(int interval) {
 408         return new TmfBTreeTraceIndexer(this, interval);
 409     }
 410
 411     @Override
 412     public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
 413         return new TmfLongLocation(bufferIn);
 414     }
 415
 416     @TmfSignalHandler
 417     @Override
 418     public void traceRangeUpdated(TmfTraceRangeUpdatedSignal signal) {
 419         if (signal.getTrace() == this) {
 420             try {
 421                 synchronized (this) {
 422                     // Reset the file handle in case it has reached the end of the
 423                     // file already. Otherwise, it will not be able to read new data
 424                     // pass the previous end.
 425                     initFile();
 426                 }
 427             } catch (TmfTraceException e) {
 428                 Activator.logError(e.getLocalizedMessage(), e);
 429             }
 430         }
 431         super.traceRangeUpdated(signal);
 432     }
 433 }