tmf/org.eclipse.tracecompass.tmf.core/src/org/eclipse/tracecompass/tmf/core/trace/text/TextTrace.java

   1 /*******************************************************************************
   2  * Copyright (c) 2012, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Patrick Tasse - Initial API and implementation
  11  *   Marc-Andre Laperle - Add persistent index support
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.tmf.core.trace.text;
  15
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.nio.ByteBuffer;
  19 import java.util.Collections;
  20 import java.util.List;
  21 import java.util.regex.Matcher;
  22 import java.util.regex.Pattern;
  23
  24 import org.eclipse.core.resources.IProject;
  25 import org.eclipse.core.resources.IResource;
  26 import org.eclipse.core.runtime.IStatus;
  27 import org.eclipse.core.runtime.Status;
  28 import org.eclipse.jdt.annotation.NonNull;
  29 import org.eclipse.tracecompass.internal.tmf.core.Activator;
  30 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
  31 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
  32 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
  33 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
  34 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
  35 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
  36 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
  37 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
  38 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
  39 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
  40 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
  41 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
  42 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
  43 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  44
  45 /**
  46  * Extension of TmfTrace for handling of line-based text traces parsed using
  47  * regular expressions. Each line that matches the first line pattern indicates
  48  * the start of a new event. The subsequent lines can contain additional
  49  * information that is added to the current event.
  50  *
  51  * @param <T>
  52  *            TmfEvent class returned by this trace
  53  */
  54 public abstract class TextTrace<T extends TextTraceEvent> extends TmfTrace implements ITmfPersistentlyIndexable {
  55
  56     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
  57     private static final int MAX_LINES = 100;
  58     private static final int MAX_CONFIDENCE = 100;
  59
  60     /** The default separator used for multi-line fields */
  61     protected static final String SEPARATOR = " | "; //$NON-NLS-1$
  62
  63     /** The text file */
  64     protected BufferedRandomAccessFile fFile;
  65
  66     /**
  67      * Constructor
  68      */
  69     public TextTrace() {
  70     }
  71
  72     /**
  73      * {@inheritDoc}
  74      * <p>
  75      * The default implementation computes the confidence as the sum of weighted
  76      * values of the first 100 lines of the file which match any of the provided
  77      * validation patterns. For each matching line a weighted value between 1.5
  78      * and 2.0 is assigned based on the group count of the matching patterns.
  79      * The higher the group count, the closer the weighted value will be to 2.0.
  80      */
  81     @Override
  82     public IStatus validate(IProject project, String path) {
  83         File file = new File(path);
  84         if (!file.exists()) {
  85             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "File not found: " + path); //$NON-NLS-1$
  86         }
  87         if (!file.isFile()) {
  88             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Not a file. It's a directory: " + path); //$NON-NLS-1$
  89         }
  90         int confidence = 0;
  91         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
  92             int lineCount = 0;
  93             double matches = 0.0;
  94             String line = rafile.getNextLine();
  95             List<Pattern> validationPatterns = getValidationPatterns();
  96             while ((line != null) && (lineCount++ < MAX_LINES)) {
  97                 line = preProcessLine(line);
  98                 for(Pattern pattern : validationPatterns) {
  99                     Matcher matcher = pattern.matcher(line);
 100                     if (matcher.matches()) {
 101                         int groupCount = matcher.groupCount();
 102                         matches += (1.0 + groupCount / ((double) groupCount + 1));
 103                     }
 104                 }
 105                 confidence = (int) (MAX_CONFIDENCE * matches / lineCount);
 106                 line = rafile.getNextLine();
 107             }
 108         } catch (IOException e) {
 109             Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
 110             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
 111         }
 112
 113         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
 114
 115     }
 116     @Override
 117     public void initTrace(IResource resource, String path, Class<? extends ITmfEvent> type) throws TmfTraceException {
 118         super.initTrace(resource, path, type);
 119         try {
 120             fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
 121         } catch (IOException e) {
 122             throw new TmfTraceException(e.getMessage(), e);
 123         }
 124     }
 125
 126     @Override
 127     public synchronized void dispose() {
 128         super.dispose();
 129         if (fFile != null) {
 130             try {
 131                 fFile.close();
 132             } catch (IOException e) {
 133             } finally {
 134                 fFile = null;
 135             }
 136         }
 137     }
 138
 139     @Override
 140     public synchronized TextTraceContext seekEvent(ITmfLocation location) {
 141         TextTraceContext context = new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 142         if (NULL_LOCATION.equals(location) || fFile == null) {
 143             return context;
 144         }
 145         try {
 146             if (location == null) {
 147                 fFile.seek(0);
 148             } else if (location.getLocationInfo() instanceof Long) {
 149                 fFile.seek((Long) location.getLocationInfo());
 150             }
 151             long rawPos = fFile.getFilePointer();
 152             String line = fFile.getNextLine();
 153             while (line != null) {
 154                 line = preProcessLine(line);
 155                 Matcher matcher = getFirstLinePattern().matcher(line);
 156                 if (matcher.matches()) {
 157                     setupContext(context, rawPos, line, matcher);
 158                     return context;
 159                 }
 160                 rawPos = fFile.getFilePointer();
 161                 line = fFile.getNextLine();
 162             }
 163             return context;
 164         } catch (IOException e) {
 165             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 166             return context;
 167         }
 168     }
 169
 170     private void setupContext(TextTraceContext context, long rawPos, String line, Matcher matcher) throws IOException {
 171         context.setLocation(new TmfLongLocation(rawPos));
 172         context.firstLineMatcher = matcher;
 173         context.firstLine = line;
 174         context.nextLineLocation = fFile.getFilePointer();
 175     }
 176
 177     @Override
 178     public synchronized TextTraceContext seekEvent(double ratio) {
 179         if (fFile == null) {
 180             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 181         }
 182         try {
 183             long pos = Math.round(ratio * fFile.length());
 184             while (pos > 0) {
 185                 fFile.seek(pos - 1);
 186                 if (fFile.read() == '\n') {
 187                     break;
 188                 }
 189                 pos--;
 190             }
 191             ITmfLocation location = new TmfLongLocation(Long.valueOf(pos));
 192             TextTraceContext context = seekEvent(location);
 193             context.setRank(ITmfContext.UNKNOWN_RANK);
 194             return context;
 195         } catch (IOException e) {
 196             Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$
 197             return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
 198         }
 199     }
 200
 201     @Override
 202     public double getLocationRatio(ITmfLocation location) {
 203         if (fFile == null) {
 204             return 0;
 205         }
 206         try {
 207             long length = fFile.length();
 208             if (length == 0) {
 209                 return 0;
 210             }
 211             if (location.getLocationInfo() instanceof Long) {
 212                 return (double) ((Long) location.getLocationInfo()) / length;
 213             }
 214         } catch (IOException e) {
 215             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 216         }
 217         return 0;
 218     }
 219
 220     @Override
 221     public ITmfLocation getCurrentLocation() {
 222         return null;
 223     }
 224
 225     @Override
 226     public TextTraceEvent parseEvent(ITmfContext tmfContext) {
 227         TextTraceContext context = seekEvent(tmfContext.getLocation());
 228         return parse(context);
 229     }
 230
 231     @Override
 232     public synchronized T getNext(ITmfContext context) {
 233         if (!(context instanceof TextTraceContext)) {
 234             throw new IllegalArgumentException();
 235         }
 236         TextTraceContext savedContext = new TextTraceContext(context.getLocation(), context.getRank());
 237         T event = parse((TextTraceContext) context);
 238         if (event != null) {
 239             updateAttributes(savedContext, event);
 240             context.increaseRank();
 241         }
 242         return event;
 243     }
 244
 245     /**
 246      * Parse the next event. The context is advanced.
 247      *
 248      * @param tmfContext
 249      *            the context
 250      * @return the next event or null
 251      */
 252     protected synchronized T parse(TextTraceContext tmfContext) {
 253         if (fFile == null) {
 254             return null;
 255         }
 256         TextTraceContext context = tmfContext;
 257         if (context.getLocation() == null || !(context.getLocation().getLocationInfo() instanceof Long) || NULL_LOCATION.equals(context.getLocation())) {
 258             return null;
 259         }
 260
 261         T event = parseFirstLine(context.firstLineMatcher, context.firstLine);
 262
 263         try {
 264             if (fFile.getFilePointer() != context.nextLineLocation) {
 265                 fFile.seek(context.nextLineLocation);
 266             }
 267             long rawPos = fFile.getFilePointer();
 268             String line = fFile.getNextLine();
 269             while (line != null) {
 270                 line = preProcessLine(line);
 271                 Matcher matcher = getFirstLinePattern().matcher(line);
 272                 if (matcher.matches()) {
 273                     setupContext(context, rawPos, line, matcher);
 274                     return event;
 275                 }
 276                 parseNextLine(event, line);
 277                 rawPos = fFile.getFilePointer();
 278                 line = fFile.getNextLine();
 279             }
 280         } catch (IOException e) {
 281             Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$
 282         }
 283
 284         context.setLocation(NULL_LOCATION);
 285         return event;
 286     }
 287
 288     /**
 289      * Pre-processes the input line. The default implementation returns the
 290      * input line.
 291      *
 292      * @param line
 293      *            non-null input string
 294      * @return the pre-processed input line
 295      */
 296     @NonNull
 297     protected String preProcessLine(@NonNull String line) {
 298         return line;
 299     }
 300
 301     /**
 302      * Gets the first line pattern.
 303      *
 304      * @return The first line pattern
 305      */
 306     protected abstract Pattern getFirstLinePattern();
 307
 308     /**
 309      * Parses the first line data and returns a new event. When constructing the
 310      * event, the concrete trace should use the trace's timestamp transform to
 311      * create the timestamp, by either transforming the parsed time value
 312      * directly or by using the method {@link #createTimestamp(long)}.
 313      *
 314      * @param matcher
 315      *            The matcher
 316      * @param line
 317      *            The line to parse
 318      * @return The parsed event
 319      */
 320     protected abstract T parseFirstLine(Matcher matcher, String line);
 321
 322     /**
 323      * Parses the next line data for the current event.
 324      *
 325      * @param event
 326      *            The current event being parsed
 327      * @param line
 328      *            The line to parse
 329      */
 330     protected abstract void parseNextLine(T event, String line);
 331
 332     /**
 333      * Returns a ordered list of validation patterns that will be used in
 334      * the default {@link #validate(IProject, String)} method to match
 335      * the first 100 to compute the confidence level
 336      *
 337      * @return collection of patterns to validate against
 338      */
 339     protected List<Pattern> getValidationPatterns() {
 340         return Collections.singletonList(getFirstLinePattern());
 341     }
 342
 343     // ------------------------------------------------------------------------
 344     // Helper methods
 345     // ------------------------------------------------------------------------
 346
 347     /**
 348      * Strip quotes surrounding a string
 349      *
 350      * @param input
 351      *            The input string
 352      * @return The string without quotes
 353      */
 354     protected static String replaceQuotes(String input) {
 355         String out = input.replaceAll("^\"|(\"\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 356         return out;
 357     }
 358
 359     /**
 360      * Strip brackets surrounding a string
 361      *
 362      * @param input
 363      *            The input string
 364      * @return The string without brackets
 365      */
 366     protected static String replaceBrackets(String input) {
 367         String out = input.replaceAll("^\\{|(\\}\\s*)$", "");  //$NON-NLS-1$//$NON-NLS-2$
 368         return out;
 369     }
 370
 371     private static int fCheckpointSize = -1;
 372
 373     @Override
 374     public synchronized int getCheckpointSize() {
 375         if (fCheckpointSize == -1) {
 376             TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
 377             ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
 378             b.clear();
 379             c.serialize(b);
 380             fCheckpointSize = b.position();
 381         }
 382
 383         return fCheckpointSize;
 384     }
 385
 386     @Override
 387     protected ITmfTraceIndexer createIndexer(int interval) {
 388         return new TmfBTreeTraceIndexer(this, interval);
 389     }
 390
 391     @Override
 392     public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
 393         return new TmfLongLocation(bufferIn);
 394     }
 395 }