tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/text/TextTrace.java

   1 /*******************************************************************************
   2  * Copyright (c) 2012, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Patrick Tasse - Initial API and implementation
  11  *   Marc-Andre Laperle - Add persistent index support
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.trace.text;
  15
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.nio.ByteBuffer;
  19 import java.util.Collections;
  20 import java.util.List;
  21 import java.util.regex.Matcher;
  22 import java.util.regex.Pattern;
  23
  24 import org.eclipse.core.resources.IProject;
  25 import org.eclipse.core.resources.IResource;
  26 import org.eclipse.core.runtime.IStatus;
  27 import org.eclipse.core.runtime.Status;
  28 import org.eclipse.jdt.annotation.NonNull;
  29 import org.eclipse.tracecompass.internal.tmf.core.Activator;
  30 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  31 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  32 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
  33 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
  34 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
  35 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
  36 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
  37 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
  38 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
  39 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
  40 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
  41 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
  42 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
  43 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
  44 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  45
  46 /**
  47  * Extension of TmfTrace for handling of line-based text traces parsed using
  48  * regular expressions. Each line that matches the first line pattern indicates
  49  * the start of a new event. The subsequent lines can contain additional
  50  * information that is added to the current event.
  51  *
  52  * @param <T>
  53  *            TmfEvent class returned by this trace
  54  */
  55 public abstract class TextTrace<T extends TextTraceEvent> extends TmfTrace implements ITmfPersistentlyIndexable {
  56
  57     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
  58     private static final int MAX_LINES = 100;
  59     private static final int MAX_CONFIDENCE = 100;
  60
  61     /** The default separator used for multi-line fields */
  62     protected static final String SEPARATOR = " | "; //$NON-NLS-1$
  63
  64     /** The text file */
  65     protected BufferedRandomAccessFile fFile;
  66
  67     /**
  68      * Constructor
  69      */
  70     public TextTrace() {
  71     }
  72
  73     /**
  74      * {@inheritDoc}
  75      * <p>
  76      * The default implementation computes the confidence as the sum of weighted
  77      * values of the first 100 lines of the file which match any of the provided
  78      * validation patterns. For each matching line a weighted value between 1.5
  79      * and 2.0 is assigned based on the group count of the matching patterns.
  80      * The higher the group count, the closer the weighted value will be to 2.0.
  81      */
  82     @Override
  83     public IStatus validate(IProject project, String path) {
  84         File file = new File(path);
  85         if (!file.exists()) {
  86             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "File not found: " + path); //$NON-NLS-1$
  87         }
  88         if (!file.isFile()) {
  89             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Not a file. It's a directory: " + path); //$NON-NLS-1$
  90         }
  91         try {
  92             if (!TmfTraceUtils.isText(file)) {
  93                 return new TraceValidationStatus(0, Activator.PLUGIN_ID);
  94             }
  95         } catch (IOException e) {
  96             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
  97             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
  98         }
  99         int confidence = 0;
 100         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
 101             int lineCount = 0;
 102             double matches = 0.0;
 103             String line = rafile.getNextLine();
 104             List<Pattern> validationPatterns = getValidationPatterns();
 105             while ((line != null) && (lineCount++ < MAX_LINES)) {
 106                 line = preProcessLine(line);
 107                 for(Pattern pattern : validationPatterns) {
 108                     Matcher matcher = pattern.matcher(line);
 109                     if (matcher.matches()) {
 110                         int groupCount = matcher.groupCount();
 111                         matches += (1.0 + groupCount / ((double) groupCount + 1));
 112                     }
 113                 }
 114                 confidence = (int) (MAX_CONFIDENCE * matches / lineCount);
 115                 line = rafile.getNextLine();
 116             }
 117         } catch (IOException e) {
 118             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 119             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 120         }
 121
 122         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 123
 124     }
 125     @Override
 126     public void initTrace(IResource resource, String path, Class<? extends ITmfEvent> type) throws TmfTraceException {
 127         super.initTrace(resource, path, type);
 128         try {
 129             fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
 130         } catch (IOException e) {
 131             throw new TmfTraceException(e.getMessage(), e);
 132         }
 133     }
 134
 135     @Override
 136     public synchronized void dispose() {
 137         super.dispose();
 138         if (fFile != null) {
 139             try {
 140                 fFile.close();
 141             } catch (IOException e) {
 142             } finally {
 143                 fFile = null;
 144             }
 145         }
 146     }
 147
 148     @Override
 149     public synchronized TextTraceContext seekEvent(ITmfLocation location) {
 150         TextTraceContext context = new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 151         if (NULL_LOCATION.equals(location) || fFile == null) {
 152             return context;
 153         }
 154         try {
 155             if (location == null) {
 156                 fFile.seek(0);
 157             } else if (location.getLocationInfo() instanceof Long) {
 158                 fFile.seek((Long) location.getLocationInfo());
 159             }
 160             long rawPos = fFile.getFilePointer();
 161             String line = fFile.getNextLine();
 162             while (line != null) {
 163                 line = preProcessLine(line);
 164                 Matcher matcher = getFirstLinePattern().matcher(line);
 165                 if (matcher.matches()) {
 166                     setupContext(context, rawPos, line, matcher);
 167                     return context;
 168                 }
 169                 rawPos = fFile.getFilePointer();
 170                 line = fFile.getNextLine();
 171             }
 172             return context;
 173         } catch (IOException e) {
 174             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 175             return context;
 176         }
 177     }
 178
 179     private void setupContext(TextTraceContext context, long rawPos, String line, Matcher matcher) throws IOException {
 180         context.setLocation(new TmfLongLocation(rawPos));
 181         context.firstLineMatcher = matcher;
 182         context.firstLine = line;
 183         context.nextLineLocation = fFile.getFilePointer();
 184     }
 185
 186     @Override
 187     public synchronized TextTraceContext seekEvent(double ratio) {
 188         if (fFile == null) {
 189             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 190         }
 191         try {
 192             long pos = Math.round(ratio * fFile.length());
 193             while (pos > 0) {
 194                 fFile.seek(pos - 1);
 195                 if (fFile.read() == '\n') {
 196                     break;
 197                 }
 198                 pos--;
 199             }
 200             ITmfLocation location = new TmfLongLocation(Long.valueOf(pos));
 201             TextTraceContext context = seekEvent(location);
 202             context.setRank(ITmfContext.UNKNOWN_RANK);
 203             return context;
 204         } catch (IOException e) {
 205             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 206             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 207         }
 208     }
 209
 210     @Override
 211     public double getLocationRatio(ITmfLocation location) {
 212         if (fFile == null) {
 213             return 0;
 214         }
 215         try {
 216             long length = fFile.length();
 217             if (length == 0) {
 218                 return 0;
 219             }
 220             if (location.getLocationInfo() instanceof Long) {
 221                 return (double) ((Long) location.getLocationInfo()) / length;
 222             }
 223         } catch (IOException e) {
 224             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 225         }
 226         return 0;
 227     }
 228
 229     @Override
 230     public ITmfLocation getCurrentLocation() {
 231         return null;
 232     }
 233
 234     @Override
 235     public TextTraceEvent parseEvent(ITmfContext tmfContext) {
 236         TextTraceContext context = seekEvent(tmfContext.getLocation());
 237         return parse(context);
 238     }
 239
 240     @Override
 241     public synchronized T getNext(ITmfContext context) {
 242         if (!(context instanceof TextTraceContext)) {
 243             throw new IllegalArgumentException();
 244         }
 245         TextTraceContext savedContext = new TextTraceContext(context.getLocation(), context.getRank());
 246         T event = parse((TextTraceContext) context);
 247         if (event != null) {
 248             updateAttributes(savedContext, event);
 249             context.increaseRank();
 250         }
 251         return event;
 252     }
 253
 254     /**
 255      * Parse the next event. The context is advanced.
 256      *
 257      * @param tmfContext
 258      *            the context
 259      * @return the next event or null
 260      */
 261     protected synchronized T parse(TextTraceContext tmfContext) {
 262         if (fFile == null) {
 263             return null;
 264         }
 265         TextTraceContext context = tmfContext;
 266         if (context.getLocation() == null || !(context.getLocation().getLocationInfo() instanceof Long) || NULL_LOCATION.equals(context.getLocation())) {
 267             return null;
 268         }
 269
 270         T event = parseFirstLine(context.firstLineMatcher, context.firstLine);
 271
 272         try {
 273             if (fFile.getFilePointer() != context.nextLineLocation) {
 274                 fFile.seek(context.nextLineLocation);
 275             }
 276             long rawPos = fFile.getFilePointer();
 277             String line = fFile.getNextLine();
 278             while (line != null) {
 279                 line = preProcessLine(line);
 280                 Matcher matcher = getFirstLinePattern().matcher(line);
 281                 if (matcher.matches()) {
 282                     setupContext(context, rawPos, line, matcher);
 283                     return event;
 284                 }
 285                 parseNextLine(event, line);
 286                 rawPos = fFile.getFilePointer();
 287                 line = fFile.getNextLine();
 288             }
 289         } catch (IOException e) {
 290             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 291         }
 292
 293         context.setLocation(NULL_LOCATION);
 294         return event;
 295     }
 296
 297     /**
 298      * Pre-processes the input line. The default implementation returns the
 299      * input line.
 300      *
 301      * @param line
 302      *            non-null input string
 303      * @return the pre-processed input line
 304      */
 305     @NonNull
 306     protected String preProcessLine(@NonNull String line) {
 307         return line;
 308     }
 309
 310     /**
 311      * Gets the first line pattern.
 312      *
 313      * @return The first line pattern
 314      */
 315     protected abstract Pattern getFirstLinePattern();
 316
 317     /**
 318      * Parses the first line data and returns a new event. When constructing the
 319      * event, the concrete trace should use the trace's timestamp transform to
 320      * create the timestamp, by either transforming the parsed time value
 321      * directly or by using the method {@link #createTimestamp(long)}.
 322      *
 323      * @param matcher
 324      *            The matcher
 325      * @param line
 326      *            The line to parse
 327      * @return The parsed event
 328      */
 329     protected abstract T parseFirstLine(Matcher matcher, String line);
 330
 331     /**
 332      * Parses the next line data for the current event.
 333      *
 334      * @param event
 335      *            The current event being parsed
 336      * @param line
 337      *            The line to parse
 338      */
 339     protected abstract void parseNextLine(T event, String line);
 340
 341     /**
 342      * Returns a ordered list of validation patterns that will be used in
 343      * the default {@link #validate(IProject, String)} method to match
 344      * the first 100 to compute the confidence level
 345      *
 346      * @return collection of patterns to validate against
 347      */
 348     protected List<Pattern> getValidationPatterns() {
 349         return Collections.singletonList(getFirstLinePattern());
 350     }
 351
 352     // ------------------------------------------------------------------------
 353     // Helper methods
 354     // ------------------------------------------------------------------------
 355
 356     /**
 357      * Strip quotes surrounding a string
 358      *
 359      * @param input
 360      *            The input string
 361      * @return The string without quotes
 362      */
 363     protected static String replaceQuotes(String input) {
 364         String out = input.replaceAll("^\"|(\"\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 365         return out;
 366     }
 367
 368     /**
 369      * Strip brackets surrounding a string
 370      *
 371      * @param input
 372      *            The input string
 373      * @return The string without brackets
 374      */
 375     protected static String replaceBrackets(String input) {
 376         String out = input.replaceAll("^\\{|(\\}\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 377         return out;
 378     }
 379
 380     private static int fCheckpointSize = -1;
 381
 382     @Override
 383     public synchronized int getCheckpointSize() {
 384         if (fCheckpointSize == -1) {
 385             TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
 386             ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
 387             b.clear();
 388             c.serialize(b);
 389             fCheckpointSize = b.position();
 390         }
 391
 392         return fCheckpointSize;
 393     }
 394
 395     @Override
 396     protected ITmfTraceIndexer createIndexer(int interval) {
 397         return new TmfBTreeTraceIndexer(this, interval);
 398     }
 399
 400     @Override
 401     public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
 402         return new TmfLongLocation(bufferIn);
 403     }
 404 }