tmf: Bug 489971: Premature processing of payload in custom parser
[deliverable/tracecompass.git] / tmf / org.eclipse.tracecompass.tmf.core / src / org / eclipse / tracecompass / tmf / core / parsers / custom / CustomTxtTrace.java
1 /*******************************************************************************
2 * Copyright (c) 2010, 2015 Ericsson
3 *
4 * All rights reserved. This program and the accompanying materials are
5 * made available under the terms of the Eclipse Public License v1.0 which
6 * accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
8 *
9 * Contributors:
10 * Patrick Tasse - Initial API and implementation
11 * Bernd Hufmann - Add trace type id handling
12 *******************************************************************************/
13
14 package org.eclipse.tracecompass.tmf.core.parsers.custom;
15
16 import static org.eclipse.tracecompass.common.core.NonNullUtils.checkNotNull;
17
18 import java.io.File;
19 import java.io.FileNotFoundException;
20 import java.io.IOException;
21 import java.nio.ByteBuffer;
22 import java.util.HashMap;
23 import java.util.Iterator;
24 import java.util.List;
25 import java.util.Map.Entry;
26 import java.util.regex.Matcher;
27
28 import org.eclipse.core.resources.IProject;
29 import org.eclipse.core.resources.IResource;
30 import org.eclipse.core.runtime.IStatus;
31 import org.eclipse.core.runtime.Status;
32 import org.eclipse.jdt.annotation.NonNull;
33 import org.eclipse.tracecompass.internal.tmf.core.Activator;
34 import org.eclipse.tracecompass.internal.tmf.core.parsers.custom.CustomEventAspects;
35 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
36 import org.eclipse.tracecompass.tmf.core.event.aspect.ITmfEventAspect;
37 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
38 import org.eclipse.tracecompass.tmf.core.io.BufferedRandomAccessFile;
39 import org.eclipse.tracecompass.tmf.core.parsers.custom.CustomTxtTraceDefinition.InputLine;
40 import org.eclipse.tracecompass.tmf.core.signal.TmfSignalHandler;
41 import org.eclipse.tracecompass.tmf.core.signal.TmfTraceRangeUpdatedSignal;
42 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
43 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
44 import org.eclipse.tracecompass.tmf.core.trace.TmfContext;
45 import org.eclipse.tracecompass.tmf.core.trace.TmfTrace;
46 import org.eclipse.tracecompass.tmf.core.trace.TmfTraceUtils;
47 import org.eclipse.tracecompass.tmf.core.trace.TraceValidationStatus;
48 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable;
49 import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfTraceIndexer;
50 import org.eclipse.tracecompass.tmf.core.trace.indexer.TmfBTreeTraceIndexer;
51 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.ITmfCheckpoint;
52 import org.eclipse.tracecompass.tmf.core.trace.indexer.checkpoint.TmfCheckpoint;
53 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
54 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
55
56 /**
57 * Base class for custom plain text traces.
58 *
59 * @author Patrick Tassé
60 */
61 public class CustomTxtTrace extends TmfTrace implements ITmfPersistentlyIndexable {
62
63 private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
64 private static final int DEFAULT_CACHE_SIZE = 100;
65 private static final int MAX_LINES = 100;
66 private static final int MAX_CONFIDENCE = 100;
67
68 private final CustomTxtTraceDefinition fDefinition;
69 private final CustomTxtEventType fEventType;
70 private BufferedRandomAccessFile fFile;
71 private final String fTraceTypeId;
72
73 private static final char SEPARATOR = ':';
74 private static final String CUSTOM_TXT_TRACE_TYPE_PREFIX = "custom.txt.trace" + SEPARATOR; //$NON-NLS-1$
75 private static final String LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX = "org.eclipse.linuxtools.tmf.core.parsers.custom.CustomTxtTrace" + SEPARATOR; //$NON-NLS-1$
76 private static final String EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX = "org.eclipse.tracecompass.tmf.core.parsers.custom.CustomTxtTrace" + SEPARATOR; //$NON-NLS-1$
77
78 /**
79 * Basic constructor.
80 *
81 * @param definition
82 * Text trace definition
83 */
84 public CustomTxtTrace(final CustomTxtTraceDefinition definition) {
85 fDefinition = definition;
86 fEventType = new CustomTxtEventType(fDefinition);
87 fTraceTypeId = buildTraceTypeId(definition.categoryName, definition.definitionName);
88 setCacheSize(DEFAULT_CACHE_SIZE);
89 }
90
91 /**
92 * Full constructor.
93 *
94 * @param resource
95 * Trace's resource.
96 * @param definition
97 * Text trace definition
98 * @param path
99 * Path to the trace file
100 * @param cacheSize
101 * Cache size to use
102 * @throws TmfTraceException
103 * If we couldn't open the trace at 'path'
104 */
105 public CustomTxtTrace(final IResource resource,
106 final CustomTxtTraceDefinition definition, final String path,
107 final int cacheSize) throws TmfTraceException {
108 this(definition);
109 setCacheSize((cacheSize > 0) ? cacheSize : DEFAULT_CACHE_SIZE);
110 initTrace(resource, path, CustomTxtEvent.class);
111 }
112
113 @Override
114 public void initTrace(final IResource resource, final String path, final Class<? extends ITmfEvent> eventType) throws TmfTraceException {
115 super.initTrace(resource, path, eventType);
116 initFile();
117 }
118
119 private void initFile() throws TmfTraceException {
120 closeFile();
121 try {
122 fFile = new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
123 } catch (IOException e) {
124 throw new TmfTraceException(e.getMessage(), e);
125 }
126 }
127
128 @Override
129 public synchronized void dispose() {
130 super.dispose();
131 closeFile();
132 }
133
134 private void closeFile() {
135 if (fFile != null) {
136 try {
137 fFile.close();
138 } catch (IOException e) {
139 } finally {
140 fFile = null;
141 }
142 }
143 }
144
145 @Override
146 public ITmfTraceIndexer getIndexer() {
147 return super.getIndexer();
148 }
149
150 @Override
151 public Iterable<ITmfEventAspect> getEventAspects() {
152 return CustomEventAspects.generateAspects(fDefinition);
153 }
154
155 @Override
156 public synchronized TmfContext seekEvent(final ITmfLocation location) {
157 final CustomTxtTraceContext context = new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
158 if (NULL_LOCATION.equals(location) || fFile == null) {
159 return context;
160 }
161 try {
162 if (location == null) {
163 fFile.seek(0);
164 } else if (location.getLocationInfo() instanceof Long) {
165 fFile.seek((Long) location.getLocationInfo());
166 }
167 long rawPos = fFile.getFilePointer();
168 String line = fFile.getNextLine();
169 while (line != null) {
170 for (final InputLine input : getFirstLines()) {
171 final Matcher matcher = input.getPattern().matcher(line);
172 if (matcher.matches()) {
173 context.setLocation(new TmfLongLocation(rawPos));
174 context.firstLineMatcher = matcher;
175 context.firstLine = line;
176 context.nextLineLocation = fFile.getFilePointer();
177 context.inputLine = input;
178 return context;
179 }
180 }
181 rawPos = fFile.getFilePointer();
182 line = fFile.getNextLine();
183 }
184 return context;
185 } catch (final FileNotFoundException e) {
186 Activator.logError("Error seeking event. File not found: " + getPath(), e); //$NON-NLS-1$
187 return context;
188 } catch (final IOException e) {
189 Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
190 return context;
191 }
192
193 }
194
195 @Override
196 public synchronized TmfContext seekEvent(final double ratio) {
197 if (fFile == null) {
198 return new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
199 }
200 try {
201 long pos = Math.round(ratio * fFile.length());
202 while (pos > 0) {
203 fFile.seek(pos - 1);
204 if (fFile.read() == '\n') {
205 break;
206 }
207 pos--;
208 }
209 final ITmfLocation location = new TmfLongLocation(pos);
210 final TmfContext context = seekEvent(location);
211 context.setRank(ITmfContext.UNKNOWN_RANK);
212 return context;
213 } catch (final IOException e) {
214 Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
215 return new CustomTxtTraceContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
216 }
217 }
218
219 @Override
220 public synchronized double getLocationRatio(final ITmfLocation location) {
221 if (fFile == null) {
222 return 0;
223 }
224 try {
225 if (location.getLocationInfo() instanceof Long) {
226 return ((Long) location.getLocationInfo()).doubleValue() / fFile.length();
227 }
228 } catch (final IOException e) {
229 Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
230 }
231 return 0;
232 }
233
234 @Override
235 public ITmfLocation getCurrentLocation() {
236 // TODO Auto-generated method stub
237 return null;
238 }
239
240 @Override
241 public synchronized CustomTxtEvent parseEvent(final ITmfContext tmfContext) {
242 ITmfContext context = seekEvent(tmfContext.getLocation());
243 return parse(context);
244 }
245
246 @Override
247 public synchronized CustomTxtEvent getNext(final ITmfContext context) {
248 final ITmfContext savedContext = new TmfContext(context.getLocation(), context.getRank());
249 final CustomTxtEvent event = parse(context);
250 if (event != null) {
251 updateAttributes(savedContext, event);
252 context.increaseRank();
253 }
254 return event;
255 }
256
257 private synchronized CustomTxtEvent parse(final ITmfContext tmfContext) {
258 if (fFile == null) {
259 return null;
260 }
261 if (!(tmfContext instanceof CustomTxtTraceContext)) {
262 return null;
263 }
264
265 final CustomTxtTraceContext context = (CustomTxtTraceContext) tmfContext;
266 if (context.getLocation() == null || !(context.getLocation().getLocationInfo() instanceof Long) || NULL_LOCATION.equals(context.getLocation())) {
267 return null;
268 }
269
270 CustomTxtEvent event = parseFirstLine(context);
271
272 final HashMap<InputLine, Integer> countMap = new HashMap<>();
273 InputLine currentInput = null;
274 if (context.inputLine.childrenInputs != null && context.inputLine.childrenInputs.size() > 0) {
275 currentInput = context.inputLine.childrenInputs.get(0);
276 countMap.put(currentInput, 0);
277 }
278
279 try {
280 if (fFile.getFilePointer() != context.nextLineLocation) {
281 fFile.seek(context.nextLineLocation);
282 }
283 long rawPos = fFile.getFilePointer();
284 String line = fFile.getNextLine();
285 while (line != null) {
286 boolean processed = false;
287 if (currentInput == null) {
288 for (final InputLine input : getFirstLines()) {
289 final Matcher matcher = input.getPattern().matcher(line);
290 if (matcher.matches()) {
291 context.setLocation(new TmfLongLocation(rawPos));
292 context.firstLineMatcher = matcher;
293 context.firstLine = line;
294 context.nextLineLocation = fFile.getFilePointer();
295 context.inputLine = input;
296 return event;
297 }
298 }
299 } else {
300 if (checkNotNull(countMap.get(currentInput)) >= currentInput.getMinCount()) {
301 final List<InputLine> nextInputs = currentInput.getNextInputs(countMap);
302 if (nextInputs.size() == 0 || nextInputs.get(nextInputs.size() - 1).getMinCount() == 0) {
303 for (final InputLine input : getFirstLines()) {
304 final Matcher matcher = input.getPattern().matcher(line);
305 if (matcher.matches()) {
306 context.setLocation(new TmfLongLocation(rawPos));
307 context.firstLineMatcher = matcher;
308 context.firstLine = line;
309 context.nextLineLocation = fFile.getFilePointer();
310 context.inputLine = input;
311 return event;
312 }
313 }
314 }
315 for (final InputLine input : nextInputs) {
316 final Matcher matcher = input.getPattern().matcher(line);
317 if (matcher.matches()) {
318 event.processGroups(input, matcher);
319 currentInput = input;
320 if (countMap.get(currentInput) == null) {
321 countMap.put(currentInput, 1);
322 } else {
323 countMap.put(currentInput, checkNotNull(countMap.get(currentInput)) + 1);
324 }
325 Iterator<InputLine> iter = countMap.keySet().iterator();
326 while (iter.hasNext()) {
327 final InputLine inputLine = iter.next();
328 if (inputLine.level > currentInput.level) {
329 iter.remove();
330 }
331 }
332 if (currentInput.childrenInputs != null && currentInput.childrenInputs.size() > 0) {
333 currentInput = currentInput.childrenInputs.get(0);
334 countMap.put(currentInput, 0);
335 } else if (checkNotNull(countMap.get(currentInput)) >= currentInput.getMaxCount()) {
336 if (currentInput.getNextInputs(countMap).size() > 0) {
337 currentInput = currentInput.getNextInputs(countMap).get(0);
338 if (countMap.get(currentInput) == null) {
339 countMap.put(currentInput, 0);
340 }
341 iter = countMap.keySet().iterator();
342 while (iter.hasNext()) {
343 final InputLine inputLine = iter.next();
344 if (inputLine.level > currentInput.level) {
345 iter.remove();
346 }
347 }
348 } else {
349 currentInput = null;
350 }
351 }
352 processed = true;
353 break;
354 }
355 }
356 }
357 if (!processed && currentInput != null) {
358 final Matcher matcher = currentInput.getPattern().matcher(line);
359 if (matcher.matches()) {
360 event.processGroups(currentInput, matcher);
361 countMap.put(currentInput, checkNotNull(countMap.get(currentInput)) + 1);
362 if (currentInput.childrenInputs != null && currentInput.childrenInputs.size() > 0) {
363 currentInput = currentInput.childrenInputs.get(0);
364 countMap.put(currentInput, 0);
365 } else if (checkNotNull(countMap.get(currentInput)) >= currentInput.getMaxCount()) {
366 if (currentInput.getNextInputs(countMap).size() > 0) {
367 currentInput = currentInput.getNextInputs(countMap).get(0);
368 if (countMap.get(currentInput) == null) {
369 countMap.put(currentInput, 0);
370 }
371 final Iterator<InputLine> iter = countMap.keySet().iterator();
372 while (iter.hasNext()) {
373 final InputLine inputLine = iter.next();
374 if (inputLine.level > currentInput.level) {
375 iter.remove();
376 }
377 }
378 } else {
379 currentInput = null;
380 }
381 }
382 }
383 ((StringBuffer) event.getContentValue()).append("\n").append(line); //$NON-NLS-1$
384 }
385 }
386 rawPos = fFile.getFilePointer();
387 line = fFile.getNextLine();
388 }
389 } catch (final IOException e) {
390 Activator.logError("Error seeking event. File: " + getPath(), e); //$NON-NLS-1$
391 }
392 for (final Entry<InputLine, Integer> entry : countMap.entrySet()) {
393 if (entry.getValue() < entry.getKey().getMinCount()) {
394 event = null;
395 }
396 }
397 context.setLocation(NULL_LOCATION);
398 return event;
399 }
400
401 /**
402 * @return The first few lines of the text file
403 */
404 public List<InputLine> getFirstLines() {
405 return fDefinition.inputs;
406 }
407
408 /**
409 * Parse the first line of the trace (to recognize the type).
410 *
411 * @param context
412 * Trace context
413 * @return The first event
414 */
415 public CustomTxtEvent parseFirstLine(final CustomTxtTraceContext context) {
416 final CustomTxtEvent event = new CustomTxtEvent(fDefinition, this, TmfTimestamp.ZERO, fEventType);
417 event.processGroups(context.inputLine, context.firstLineMatcher);
418 event.setContent(new CustomEventContent(event, new StringBuffer(context.firstLine)));
419 return event;
420 }
421
422 /**
423 * Get the trace definition.
424 *
425 * @return The trace definition
426 */
427 public CustomTraceDefinition getDefinition() {
428 return fDefinition;
429 }
430
431 /**
432 * {@inheritDoc}
433 * <p>
434 * The default implementation computes the confidence as the percentage of
435 * lines in the first 100 lines of the file which match any of the root
436 * input line patterns.
437 */
438 @Override
439 public IStatus validate(IProject project, String path) {
440 File file = new File(path);
441 if (!file.exists() || !file.isFile() || !file.canRead()) {
442 return new Status(IStatus.ERROR, Activator.PLUGIN_ID, Messages.CustomTrace_FileNotFound + ": " + path); //$NON-NLS-1$
443 }
444 int confidence = 0;
445 try {
446 if (!TmfTraceUtils.isText(file)) {
447 return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
448 }
449 } catch (IOException e) {
450 Activator.logError("Error validating file: " + path, e); //$NON-NLS-1$
451 return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
452 }
453 try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
454 int lineCount = 0;
455 double matches = 0.0;
456 String line = rafile.getNextLine();
457
458 while ((line != null) && (lineCount++ < MAX_LINES)) {
459 for (InputLine inputLine : fDefinition.inputs) {
460 Matcher matcher = inputLine.getPattern().matcher(line);
461 if (matcher.matches()) {
462 int groupCount = matcher.groupCount();
463 matches += (1.0 + groupCount / ((double) groupCount + 1));
464 break;
465 }
466 }
467 confidence = (int) (MAX_CONFIDENCE * matches / lineCount);
468 line = rafile.getNextLine();
469 }
470 } catch (IOException e) {
471 return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
472 }
473 return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
474 }
475
476 private static int fCheckpointSize = -1;
477
478 @Override
479 public synchronized int getCheckpointSize() {
480 if (fCheckpointSize == -1) {
481 TmfCheckpoint c = new TmfCheckpoint(TmfTimestamp.ZERO, new TmfLongLocation(0L), 0);
482 ByteBuffer b = ByteBuffer.allocate(ITmfCheckpoint.MAX_SERIALIZE_SIZE);
483 b.clear();
484 c.serialize(b);
485 fCheckpointSize = b.position();
486 }
487
488 return fCheckpointSize;
489 }
490
491 @Override
492 public ITmfLocation restoreLocation(ByteBuffer bufferIn) {
493 return new TmfLongLocation(bufferIn);
494 }
495
496 @Override
497 protected ITmfTraceIndexer createIndexer(int interval) {
498 return new TmfBTreeTraceIndexer(this, interval);
499 }
500
501 @Override
502 public String getTraceTypeId() {
503 return fTraceTypeId;
504 }
505
506 /**
507 * Build the trace type id for a custom text trace
508 *
509 * @param category
510 * the category
511 * @param definitionName
512 * the definition name
513 * @return the trace type id
514 */
515 public static @NonNull String buildTraceTypeId(String category, String definitionName) {
516 return CUSTOM_TXT_TRACE_TYPE_PREFIX + category + SEPARATOR + definitionName;
517 }
518
519 /**
520 * Checks whether the given trace type ID is a custom text trace type ID
521 *
522 * @param traceTypeId
523 * the trace type ID to check
524 * @return <code>true</code> if it's a custom text trace type ID else <code>false</code>
525 */
526 public static boolean isCustomTraceTypeId(@NonNull String traceTypeId) {
527 return traceTypeId.startsWith(CUSTOM_TXT_TRACE_TYPE_PREFIX);
528 }
529
530 /**
531 * This methods builds a trace type ID from a given ID taking into
532 * consideration any format changes that were done for the IDs of custom
533 * text traces. For example, such format change took place when moving to
534 * Trace Compass. Trace type IDs that are part of the plug-in extension for
535 * trace types won't be changed.
536 *
537 * This method is useful for IDs that were persisted in the workspace before
538 * the format changes (e.g. in the persistent properties of a trace
539 * resource).
540 *
541 * It ensures backwards compatibility of the workspace for custom text
542 * traces.
543 *
544 * @param traceTypeId
545 * the legacy trace type ID
546 * @return the trace type id in Trace Compass format
547 */
548 public static @NonNull String buildCompatibilityTraceTypeId(@NonNull String traceTypeId) {
549 // Handle early Trace Compass custom text trace type IDs
550 if (traceTypeId.startsWith(EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX)) {
551 return CUSTOM_TXT_TRACE_TYPE_PREFIX + traceTypeId.substring(EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX.length());
552 }
553
554 // Handle Linux Tools custom text trace type IDs (with and without category)
555 int index = traceTypeId.lastIndexOf(SEPARATOR);
556 if ((index != -1) && (traceTypeId.startsWith(LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX))) {
557 String definitionName = index < traceTypeId.length() ? traceTypeId.substring(index + 1) : ""; //$NON-NLS-1$
558 if (traceTypeId.contains(CustomTxtTrace.class.getSimpleName() + SEPARATOR) && traceTypeId.indexOf(SEPARATOR) == index) {
559 return buildTraceTypeId(CustomTxtTraceDefinition.CUSTOM_TXT_CATEGORY, definitionName);
560 }
561 return CUSTOM_TXT_TRACE_TYPE_PREFIX + traceTypeId.substring(LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX.length());
562 }
563 return traceTypeId;
564 }
565
566 @TmfSignalHandler
567 @Override
568 public void traceRangeUpdated(TmfTraceRangeUpdatedSignal signal) {
569 if (signal.getTrace() == this) {
570 try {
571 synchronized (this) {
572 // Reset the file handle in case it has reached the end of the
573 // file already. Otherwise, it will not be able to read new data
574 // pass the previous end.
575 initFile();
576 }
577 } catch (TmfTraceException e) {
578 Activator.logError(e.getLocalizedMessage(), e);
579 }
580 }
581 super.traceRangeUpdated(signal);
582 }
583 }
This page took 0.046276 seconds and 5 git commands to generate.