| 1 | /******************************************************************************* |
| 2 | * Copyright (c) 2012, 2015 Ericsson |
| 3 | * |
| 4 | * All rights reserved. This program and the accompanying materials are |
| 5 | * made available under the terms of the Eclipse Public License v1.0 which |
| 6 | * accompanies this distribution, and is available at |
| 7 | * http://www.eclipse.org/legal/epl-v10.html |
| 8 | * |
| 9 | * Contributors: |
| 10 | * Patrick Tasse - Initial API and implementation |
| 11 | * Marc-Andre Laperle - Add persistent index support |
| 12 | *******************************************************************************/ |
| 13 | |
| 14 | package org.eclipse.tracecompass.tmf.core.trace.text; |
| 15 | |
| 16 | import java.io.File; |
| 17 | import java.io.IOException; |
| 18 | import java.nio.ByteBuffer; |
| 19 | import java.util.Collections; |
| 20 | import java.util.List; |
| 21 | import java.util.regex.Matcher; |
| 22 | import java.util.regex.Pattern; |
| 23 | |
| 24 | import org.eclipse.core.resources.IProject; |
| 25 | import org.eclipse.core.resources.IResource; |
| 26 | import org.eclipse.core.runtime.IStatus; |
| 27 | import org.eclipse.core.runtime.Status; |
| 28 | import org.eclipse.jdt.annotation.NonNull; |
| 29 | import org.eclipse.jdt.annotation.Nullable; |
| 30 | import org.eclipse.tracecompass.internal.tmf.core.Activator; |
| 31 | import org.eclipse.tracecompass.tmf.core.event.ITmfEvent; |
| 32 | import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException; |
| 33 | import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile; |
| 34 | import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler; |
| 35 | import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal; |
| 36 | import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp; |
| 37 | import org.eclipse.tracecompass.tmf.core.trace.ITmfContext; |
| 38 | import org.eclipse.tracecompass.tmf.core.trace.TmfTrace; |
| 39 | import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils; |
| 40 | import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus; |
| 41 | import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable; |
| 42 | import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer; |
| 43 | import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer; |
| 44 | import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint; |
| 45 | import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint; |
| 46 | import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation; |
| 47 | import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation; |
| 48 | |
| 49 | /** |
| 50 | * Extension of TmfTrace for handling of line-based text traces parsed using |
| 51 | * regular expressions. Each line that matches the first line pattern indicates |
| 52 | * the start of a new event. The subsequent lines can contain additional |
| 53 | * information that is added to the current event. |
| 54 | * |
| 55 | * @param <T> |
| 56 | * TmfEvent class returned by this trace |
| 57 | */ |
| 58 | public abstract class TextTrace<T extends TextTraceEvent> extends TmfTrace implements ITmfPersistentlyIndexable { |
| 59 | |
| 60 | private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L); |
| 61 | private static final int MAX_LINES = 100; |
| 62 | private static final int MAX_CONFIDENCE = 100; |
| 63 | |
| 64 | /** The default separator used for multi-line fields */ |
| 65 | protected static final String SEPARATOR = " | "; //$NON-NLS-1$ |
| 66 | |
| 67 | /** The text file */ |
| 68 | protected BufferedRandomAccessFile fFile; |
| 69 | |
| 70 | /** |
| 71 | * Constructor |
| 72 | */ |
| 73 | public TextTrace() { |
| 74 | } |
| 75 | |
| 76 | /** |
| 77 | * {@inheritDoc} |
| 78 | * <p> |
| 79 | * The default implementation computes the confidence as the sum of weighted |
| 80 | * values of the first 100 lines of the file which match any of the provided |
| 81 | * validation patterns. For each matching line a weighted value between 1.5 |
| 82 | * and 2.0 is assigned based on the group count of the matching patterns. |
| 83 | * The higher the group count, the closer the weighted value will be to 2.0. |
| 84 | */ |
| 85 | @Override |
| 86 | public IStatus validate(IProject project, String path) { |
| 87 | File file = new File(path); |
| 88 | if (!file.exists()) { |
| 89 | return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "File not found: " + path); //$NON-NLS-1$ |
| 90 | } |
| 91 | if (!file.isFile()) { |
| 92 | return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Not a file. It's a directory: " + path); //$NON-NLS-1$ |
| 93 | } |
| 94 | try { |
| 95 | if (!TmfTraceUtils.isText(file)) { |
| 96 | return new TraceValidationStatus(0, Activator.PLUGIN_ID); |
| 97 | } |
| 98 | } catch (IOException e) { |
| 99 | Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$ |
| 100 | return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$ |
| 101 | } |
| 102 | int confidence = 0; |
| 103 | try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$ |
| 104 | int lineCount = 0; |
| 105 | double matches = 0.0; |
| 106 | String line = rafile.getNextLine(); |
| 107 | List<Pattern> validationPatterns = getValidationPatterns(); |
| 108 | while ((line != null) && (lineCount++ < MAX_LINES)) { |
| 109 | line = preProcessLine(line); |
| 110 | for(Pattern pattern : validationPatterns) { |
| 111 | Matcher matcher = pattern.matcher(line); |
| 112 | if (matcher.matches()) { |
| 113 | int groupCount = matcher.groupCount(); |
| 114 | matches += (1.0 + groupCount / ((double) groupCount + 1)); |
| 115 | } |
| 116 | } |
| 117 | confidence = (int) (MAX_CONFIDENCE * matches / lineCount); |
| 118 | line = rafile.getNextLine(); |
| 119 | } |
| 120 | } catch (IOException e) { |
| 121 | Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$ |
| 122 | return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$ |
| 123 | } |
| 124 | |
| 125 | return new TraceValidationStatus(confidence, Activator.PLUGIN_ID); |
| 126 | |
| 127 | } |
| 128 | @Override |
| 129 | public void initTrace(IResource resource, String path, Class<? extends ITmfEvent> type) throws TmfTraceException { |
| 130 | super.initTrace(resource, path, type); |
| 131 | initFile(); |
| 132 | } |
| 133 | |
| 134 | private void initFile() throws TmfTraceException { |
| 135 | closeFile(); |
| 136 | try { |
| 137 | fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$ |
| 138 | } catch (IOException e) { |
| 139 | throw new TmfTraceException(e.getMessage(), e); |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | @Override |
| 144 | public synchronized void dispose() { |
| 145 | super.dispose(); |
| 146 | closeFile(); |
| 147 | } |
| 148 | |
| 149 | private void closeFile() { |
| 150 | if (fFile != null) { |
| 151 | try { |
| 152 | fFile.close(); |
| 153 | } catch (IOException e) { |
| 154 | } finally { |
| 155 | fFile = null; |
| 156 | } |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | @Override |
| 161 | public synchronized TextTraceContext seekEvent(ITmfLocation location) { |
| 162 | TextTraceContext context = new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK); |
| 163 | if (NULL_LOCATION.equals(location) || fFile == null) { |
| 164 | return context; |
| 165 | } |
| 166 | try { |
| 167 | if (location == null) { |
| 168 | fFile.seek(0); |
| 169 | } else if (location.getLocationInfo() instanceof Long) { |
| 170 | fFile.seek((Long) location.getLocationInfo()); |
| 171 | } |
| 172 | long rawPos = fFile.getFilePointer(); |
| 173 | String line = fFile.getNextLine(); |
| 174 | while (line != null) { |
| 175 | line = preProcessLine(line); |
| 176 | Matcher matcher = getFirstLinePattern().matcher(line); |
| 177 | if (matcher.matches()) { |
| 178 | setupContext(context, rawPos, line, matcher); |
| 179 | return context; |
| 180 | } |
| 181 | rawPos = fFile.getFilePointer(); |
| 182 | line = fFile.getNextLine(); |
| 183 | } |
| 184 | return context; |
| 185 | } catch (IOException e) { |
| 186 | Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$ |
| 187 | return context; |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | private void setupContext(TextTraceContext context, long rawPos, String line, Matcher matcher) throws IOException { |
| 192 | context.setLocation(new TmfLongLocation(rawPos)); |
| 193 | context.firstLineMatcher = matcher; |
| 194 | context.firstLine = line; |
| 195 | context.nextLineLocation = fFile.getFilePointer(); |
| 196 | } |
| 197 | |
| 198 | @Override |
| 199 | public synchronized TextTraceContext seekEvent(double ratio) { |
| 200 | if (fFile == null) { |
| 201 | return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK); |
| 202 | } |
| 203 | try { |
| 204 | long pos = Math.round(ratio * fFile.length()); |
| 205 | while (pos > 0) { |
| 206 | fFile.seek(pos - 1); |
| 207 | if (fFile.read() == '\n') { |
| 208 | break; |
| 209 | } |
| 210 | pos--; |
| 211 | } |
| 212 | ITmfLocation location = new TmfLongLocation(Long.valueOf(pos)); |
| 213 | TextTraceContext context = seekEvent(location); |
| 214 | context.setRank(ITmfContext.UNKNOWN_RANK); |
| 215 | return context; |
| 216 | } catch (IOException e) { |
| 217 | Activator.logError("Error seeking file: " + getPath(), e); //$NON-NLS-1$ |
| 218 | return new TextTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK); |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | @Override |
| 223 | public double getLocationRatio(ITmfLocation location) { |
| 224 | if (fFile == null) { |
| 225 | return 0; |
| 226 | } |
| 227 | try { |
| 228 | long length = fFile.length(); |
| 229 | if (length == 0) { |
| 230 | return 0; |
| 231 | } |
| 232 | if (location.getLocationInfo() instanceof Long) { |
| 233 | return (double) ((Long) location.getLocationInfo()) / length; |
| 234 | } |
| 235 | } catch (IOException e) { |
| 236 | Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$ |
| 237 | } |
| 238 | return 0; |
| 239 | } |
| 240 | |
| 241 | @Override |
| 242 | public ITmfLocation getCurrentLocation() { |
| 243 | return null; |
| 244 | } |
| 245 | |
| 246 | @Override |
| 247 | public TextTraceEvent parseEvent(ITmfContext tmfContext) { |
| 248 | TextTraceContext context = seekEvent(tmfContext.getLocation()); |
| 249 | return parse(context); |
| 250 | } |
| 251 | |
| 252 | @Override |
| 253 | public synchronized @Nullable T getNext(ITmfContext context) { |
| 254 | if (!(context instanceof TextTraceContext)) { |
| 255 | throw new IllegalArgumentException(); |
| 256 | } |
| 257 | TextTraceContext savedContext = new TextTraceContext(context.getLocation(), context.getRank()); |
| 258 | @Nullable T event = parse((TextTraceContext) context); |
| 259 | if (event != null) { |
| 260 | updateAttributes(savedContext, event); |
| 261 | context.increaseRank(); |
| 262 | } |
| 263 | return event; |
| 264 | } |
| 265 | |
| 266 | /** |
| 267 | * Parse the next event. The context is advanced. |
| 268 | * |
| 269 | * @param tmfContext |
| 270 | * the context |
| 271 | * @return the next event or null |
| 272 | */ |
| 273 | protected synchronized @Nullable T parse(TextTraceContext tmfContext) { |
| 274 | if (fFile == null) { |
| 275 | return null; |
| 276 | } |
| 277 | TextTraceContext context = tmfContext; |
| 278 | ITmfLocation location = context.getLocation(); |
| 279 | if (location == null || !(location.getLocationInfo() instanceof Long) || NULL_LOCATION.equals(location)) { |
| 280 | return null; |
| 281 | } |
| 282 | |
| 283 | T event = parseFirstLine(context.firstLineMatcher, context.firstLine); |
| 284 | |
| 285 | try { |
| 286 | if (fFile.getFilePointer() != context.nextLineLocation) { |
| 287 | fFile.seek(context.nextLineLocation); |
| 288 | } |
| 289 | long rawPos = fFile.getFilePointer(); |
| 290 | String line = fFile.getNextLine(); |
| 291 | while (line != null) { |
| 292 | line = preProcessLine(line); |
| 293 | Matcher matcher = getFirstLinePattern().matcher(line); |
| 294 | if (matcher.matches()) { |
| 295 | setupContext(context, rawPos, line, matcher); |
| 296 | return event; |
| 297 | } |
| 298 | parseNextLine(event, line); |
| 299 | rawPos = fFile.getFilePointer(); |
| 300 | line = fFile.getNextLine(); |
| 301 | } |
| 302 | } catch (IOException e) { |
| 303 | Activator.logError("Error reading file: " + getPath(), e); //$NON-NLS-1$ |
| 304 | } |
| 305 | |
| 306 | context.setLocation(NULL_LOCATION); |
| 307 | return event; |
| 308 | } |
| 309 | |
| 310 | /** |
| 311 | * Pre-processes the input line. The default implementation returns the |
| 312 | * input line. |
| 313 | * |
| 314 | * @param line |
| 315 | * non-null input string |
| 316 | * @return the pre-processed input line |
| 317 | */ |
| 318 | @NonNull |
| 319 | protected String preProcessLine(@NonNull String line) { |
| 320 | return line; |
| 321 | } |
| 322 | |
| 323 | /** |
| 324 | * Gets the first line pattern. |
| 325 | * |
| 326 | * @return The first line pattern |
| 327 | */ |
| 328 | protected abstract Pattern getFirstLinePattern(); |
| 329 | |
| 330 | /** |
| 331 | * Parses the first line data and returns a new event. When constructing the |
| 332 | * event, the concrete trace should use the trace's timestamp transform to |
| 333 | * create the timestamp, by either transforming the parsed time value |
| 334 | * directly or by using the method {@link #createTimestamp(long)}. |
| 335 | * |
| 336 | * @param matcher |
| 337 | * The matcher |
| 338 | * @param line |
| 339 | * The line to parse |
| 340 | * @return The parsed event |
| 341 | */ |
| 342 | protected abstract T parseFirstLine(Matcher matcher, String line); |
| 343 | |
| 344 | /** |
| 345 | * Parses the next line data for the current event. |
| 346 | * |
| 347 | * @param event |
| 348 | * The current event being parsed |
| 349 | * @param line |
| 350 | * The line to parse |
| 351 | */ |
| 352 | protected abstract void parseNextLine(T event, String line); |
| 353 | |
| 354 | /** |
| 355 | * Returns a ordered list of validation patterns that will be used in |
| 356 | * the default {@link #validate(IProject, String)} method to match |
| 357 | * the first 100 to compute the confidence level |
| 358 | * |
| 359 | * @return collection of patterns to validate against |
| 360 | */ |
| 361 | protected List<Pattern> getValidationPatterns() { |
| 362 | return Collections.singletonList(getFirstLinePattern()); |
| 363 | } |
| 364 | |
| 365 | // ------------------------------------------------------------------------ |
| 366 | // Helper methods |
| 367 | // ------------------------------------------------------------------------ |
| 368 | |
| 369 | /** |
| 370 | * Strip quotes surrounding a string |
| 371 | * |
| 372 | * @param input |
| 373 | * The input string |
| 374 | * @return The string without quotes |
| 375 | */ |
| 376 | protected static String replaceQuotes(String input) { |
| 377 | String out = input.replaceAll("^\"|(\"\\s*)$", ""); //$NON-NLS-1$//$NON-NLS-2$ |
| 378 | return out; |
| 379 | } |
| 380 | |
| 381 | /** |
| 382 | * Strip brackets surrounding a string |
| 383 | * |
| 384 | * @param input |
| 385 | * The input string |
| 386 | * @return The string without brackets |
| 387 | */ |
| 388 | protected static String replaceBrackets(String input) { |
| 389 | String out = input.replaceAll("^\\{|(\\}\\s*)$", ""); //$NON-NLS-1$//$NON-NLS-2$ |
| 390 | return out; |
| 391 | } |
| 392 | |
| 393 | private static int fCheckpointSize = -1; |
| 394 | |
| 395 | @Override |
| 396 | public synchronized int getCheckpointSize() { |
| 397 | if (fCheckpointSize == -1) { |
| 398 | TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0); |
| 399 | ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE); |
| 400 | b.clear(); |
| 401 | c.serialize(b); |
| 402 | fCheckpointSize = b.position(); |
| 403 | } |
| 404 | |
| 405 | return fCheckpointSize; |
| 406 | } |
| 407 | |
| 408 | @Override |
| 409 | protected ITmfTraceIndexer createIndexer(int interval) { |
| 410 | return new TmfBTreeTraceIndexer(this, interval); |
| 411 | } |
| 412 | |
| 413 | @Override |
| 414 | public ITmfLocation restoreLocation(ByteBuffer bufferIn) { |
| 415 | return new TmfLongLocation(bufferIn); |
| 416 | } |
| 417 | |
| 418 | @TmfSignalHandler |
| 419 | @Override |
| 420 | public void traceRangeUpdated(TmfTraceRangeUpdatedSignal signal) { |
| 421 | if (signal.getTrace() == this) { |
| 422 | try { |
| 423 | synchronized (this) { |
| 424 | // Reset the file handle in case it has reached the end of the |
| 425 | // file already. Otherwise, it will not be able to read new data |
| 426 | // pass the previous end. |
| 427 | initFile(); |
| 428 | } |
| 429 | } catch (TmfTraceException e) { |
| 430 | Activator.logError(e.getLocalizedMessage(), e); |
| 431 | } |
| 432 | } |
| 433 | super.traceRangeUpdated(signal); |
| 434 | } |
| 435 | } |