1 /*******************************************************************************
2 * Copyright (c) 2010, 2016 Ericsson
4 * All rights reserved. This program and the accompanying materials are
5 * made available under the terms of the Eclipse Public License v1.0 which
6 * accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * Patrick Tasse - Initial API and implementation
11 * Bernd Hufmann - Add trace type id handling
12 *******************************************************************************/
14 package org
.eclipse
.tracecompass
.tmf
.core
.parsers
.custom
;
16 import static org
.eclipse
.tracecompass
.common
.core
.NonNullUtils
.checkNotNull
;
19 import java
.io
.FileNotFoundException
;
20 import java
.io
.IOException
;
21 import java
.nio
.ByteBuffer
;
22 import java
.util
.HashMap
;
23 import java
.util
.Iterator
;
24 import java
.util
.List
;
25 import java
.util
.Map
.Entry
;
26 import java
.util
.regex
.Matcher
;
28 import org
.eclipse
.core
.resources
.IProject
;
29 import org
.eclipse
.core
.resources
.IResource
;
30 import org
.eclipse
.core
.runtime
.IStatus
;
31 import org
.eclipse
.core
.runtime
.Status
;
32 import org
.eclipse
.jdt
.annotation
.NonNull
;
33 import org
.eclipse
.tracecompass
.internal
.tmf
.core
.Activator
;
34 import org
.eclipse
.tracecompass
.internal
.tmf
.core
.parsers
.custom
.CustomEventAspects
;
35 import org
.eclipse
.tracecompass
.tmf
.core
.event
.ITmfEvent
;
36 import org
.eclipse
.tracecompass
.tmf
.core
.event
.ITmfEventField
;
37 import org
.eclipse
.tracecompass
.tmf
.core
.event
.aspect
.ITmfEventAspect
;
38 import org
.eclipse
.tracecompass
.tmf
.core
.exceptions
.TmfTraceException
;
39 import org
.eclipse
.tracecompass
.tmf
.core
.io
.BufferedRandomAccessFile
;
40 import org
.eclipse
.tracecompass
.tmf
.core
.parsers
.custom
.CustomTxtTraceDefinition
.InputLine
;
41 import org
.eclipse
.tracecompass
.tmf
.core
.signal
.TmfSignalHandler
;
42 import org
.eclipse
.tracecompass
.tmf
.core
.signal
.TmfTraceRangeUpdatedSignal
;
43 import org
.eclipse
.tracecompass
.tmf
.core
.timestamp
.ITmfTimestamp
;
44 import org
.eclipse
.tracecompass
.tmf
.core
.timestamp
.TmfTimestamp
;
45 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.ITmfContext
;
46 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.TmfContext
;
47 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.TmfTrace
;
48 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.TmfTraceUtils
;
49 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.TraceValidationStatus
;
50 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.ITmfPersistentlyIndexable
;
51 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.ITmfTraceIndexer
;
52 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.TmfBTreeTraceIndexer
;
53 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.checkpoint
.ITmfCheckpoint
;
54 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.checkpoint
.TmfCheckpoint
;
55 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.location
.ITmfLocation
;
56 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.location
.TmfLongLocation
;
59 * Base class for custom plain text traces.
61 * @author Patrick Tassé
63 public class CustomTxtTrace
extends TmfTrace
implements ITmfPersistentlyIndexable
{
65 private static final TmfLongLocation NULL_LOCATION
= new TmfLongLocation(-1L);
66 private static final int DEFAULT_CACHE_SIZE
= 100;
67 private static final int MAX_LINES
= 100;
68 private static final int MAX_CONFIDENCE
= 100;
70 private final CustomTxtTraceDefinition fDefinition
;
71 private final ITmfEventField fRootField
;
72 private BufferedRandomAccessFile fFile
;
73 private final @NonNull String fTraceTypeId
;
75 private static final char SEPARATOR
= ':';
76 private static final String CUSTOM_TXT_TRACE_TYPE_PREFIX
= "custom.txt.trace" + SEPARATOR
; //$NON-NLS-1$
77 private static final String LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX
= "org.eclipse.linuxtools.tmf.core.parsers.custom.CustomTxtTrace" + SEPARATOR
; //$NON-NLS-1$
78 private static final String EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX
= "org.eclipse.tracecompass.tmf.core.parsers.custom.CustomTxtTrace" + SEPARATOR
; //$NON-NLS-1$
84 * Text trace definition
86 public CustomTxtTrace(final CustomTxtTraceDefinition definition
) {
87 fDefinition
= definition
;
88 fRootField
= CustomEventType
.getRootField(definition
);
89 fTraceTypeId
= buildTraceTypeId(definition
.categoryName
, definition
.definitionName
);
90 setCacheSize(DEFAULT_CACHE_SIZE
);
99 * Text trace definition
101 * Path to the trace file
104 * @throws TmfTraceException
105 * If we couldn't open the trace at 'path'
107 public CustomTxtTrace(final IResource resource
,
108 final CustomTxtTraceDefinition definition
, final String path
,
109 final int cacheSize
) throws TmfTraceException
{
111 setCacheSize((cacheSize
> 0) ? cacheSize
: DEFAULT_CACHE_SIZE
);
112 initTrace(resource
, path
, CustomTxtEvent
.class);
116 public void initTrace(final IResource resource
, final String path
, final Class
<?
extends ITmfEvent
> eventType
) throws TmfTraceException
{
117 super.initTrace(resource
, path
, eventType
);
121 private void initFile() throws TmfTraceException
{
124 fFile
= new BufferedRandomAccessFile(getPath(), "r"); //$NON-NLS-1$
125 } catch (IOException e
) {
126 throw new TmfTraceException(e
.getMessage(), e
);
131 public synchronized void dispose() {
136 private void closeFile() {
140 } catch (IOException e
) {
148 public ITmfTraceIndexer
getIndexer() {
149 return super.getIndexer();
153 public Iterable
<ITmfEventAspect
<?
>> getEventAspects() {
154 return CustomEventAspects
.generateAspects(fDefinition
);
158 public synchronized TmfContext
seekEvent(final ITmfLocation location
) {
159 final CustomTxtTraceContext context
= new CustomTxtTraceContext(NULL_LOCATION
, ITmfContext
.UNKNOWN_RANK
);
160 if (NULL_LOCATION
.equals(location
) || fFile
== null) {
164 if (location
== null) {
166 } else if (location
.getLocationInfo() instanceof Long
) {
167 fFile
.seek((Long
) location
.getLocationInfo());
169 long rawPos
= fFile
.getFilePointer();
170 String line
= fFile
.getNextLine();
171 while (line
!= null) {
172 for (final InputLine input
: getFirstLines()) {
173 final Matcher matcher
= input
.getPattern().matcher(line
);
174 if (matcher
.matches()) {
175 context
.setLocation(new TmfLongLocation(rawPos
));
176 context
.firstLineMatcher
= matcher
;
177 context
.firstLine
= line
;
178 context
.nextLineLocation
= fFile
.getFilePointer();
179 context
.inputLine
= input
;
183 rawPos
= fFile
.getFilePointer();
184 line
= fFile
.getNextLine();
187 } catch (final FileNotFoundException e
) {
188 Activator
.logError("Error seeking event. File not found: " + getPath(), e
); //$NON-NLS-1$
190 } catch (final IOException e
) {
191 Activator
.logError("Error seeking event. File: " + getPath(), e
); //$NON-NLS-1$
198 public synchronized TmfContext
seekEvent(final double ratio
) {
200 return new CustomTxtTraceContext(NULL_LOCATION
, ITmfContext
.UNKNOWN_RANK
);
203 long pos
= Math
.round(ratio
* fFile
.length());
206 if (fFile
.read() == '\n') {
211 final ITmfLocation location
= new TmfLongLocation(pos
);
212 final TmfContext context
= seekEvent(location
);
213 context
.setRank(ITmfContext
.UNKNOWN_RANK
);
215 } catch (final IOException e
) {
216 Activator
.logError("Error seeking event. File: " + getPath(), e
); //$NON-NLS-1$
217 return new CustomTxtTraceContext(NULL_LOCATION
, ITmfContext
.UNKNOWN_RANK
);
222 public synchronized double getLocationRatio(final ITmfLocation location
) {
227 if (location
.getLocationInfo() instanceof Long
) {
228 return ((Long
) location
.getLocationInfo()).doubleValue() / fFile
.length();
230 } catch (final IOException e
) {
231 Activator
.logError("Error seeking event. File: " + getPath(), e
); //$NON-NLS-1$
237 public ITmfLocation
getCurrentLocation() {
238 // TODO Auto-generated method stub
243 public synchronized CustomTxtEvent
parseEvent(final ITmfContext tmfContext
) {
244 ITmfContext context
= seekEvent(tmfContext
.getLocation());
245 return parse(context
);
249 public synchronized CustomTxtEvent
getNext(final ITmfContext context
) {
250 final ITmfContext savedContext
= new TmfContext(context
.getLocation(), context
.getRank());
251 final CustomTxtEvent event
= parse(context
);
253 updateAttributes(savedContext
, event
);
254 context
.increaseRank();
259 private synchronized CustomTxtEvent
parse(final ITmfContext tmfContext
) {
263 if (!(tmfContext
instanceof CustomTxtTraceContext
)) {
267 final CustomTxtTraceContext context
= (CustomTxtTraceContext
) tmfContext
;
268 ITmfLocation location
= context
.getLocation();
269 if (location
== null || !(location
.getLocationInfo() instanceof Long
) || NULL_LOCATION
.equals(location
)) {
273 CustomTxtEvent event
= parseFirstLine(context
);
275 final HashMap
<InputLine
, Integer
> countMap
= new HashMap
<>();
276 InputLine currentInput
= null;
277 if (context
.inputLine
.childrenInputs
!= null && context
.inputLine
.childrenInputs
.size() > 0) {
278 currentInput
= context
.inputLine
.childrenInputs
.get(0);
279 countMap
.put(currentInput
, 0);
283 if (fFile
.getFilePointer() != context
.nextLineLocation
) {
284 fFile
.seek(context
.nextLineLocation
);
286 long rawPos
= fFile
.getFilePointer();
287 String line
= fFile
.getNextLine();
288 while (line
!= null) {
289 boolean processed
= false;
290 if (currentInput
== null) {
291 for (final InputLine input
: getFirstLines()) {
292 final Matcher matcher
= input
.getPattern().matcher(line
);
293 if (matcher
.matches()) {
294 context
.setLocation(new TmfLongLocation(rawPos
));
295 context
.firstLineMatcher
= matcher
;
296 context
.firstLine
= line
;
297 context
.nextLineLocation
= fFile
.getFilePointer();
298 context
.inputLine
= input
;
303 if (checkNotNull(countMap
.get(currentInput
)) >= currentInput
.getMinCount()) {
304 final List
<InputLine
> nextInputs
= currentInput
.getNextInputs(countMap
);
305 if (nextInputs
.size() == 0 || nextInputs
.get(nextInputs
.size() - 1).getMinCount() == 0) {
306 for (final InputLine input
: getFirstLines()) {
307 final Matcher matcher
= input
.getPattern().matcher(line
);
308 if (matcher
.matches()) {
309 context
.setLocation(new TmfLongLocation(rawPos
));
310 context
.firstLineMatcher
= matcher
;
311 context
.firstLine
= line
;
312 context
.nextLineLocation
= fFile
.getFilePointer();
313 context
.inputLine
= input
;
318 for (final InputLine input
: nextInputs
) {
319 final Matcher matcher
= input
.getPattern().matcher(line
);
320 if (matcher
.matches()) {
321 event
.processGroups(input
, matcher
);
322 currentInput
= input
;
323 if (countMap
.get(currentInput
) == null) {
324 countMap
.put(currentInput
, 1);
326 countMap
.put(currentInput
, checkNotNull(countMap
.get(currentInput
)) + 1);
328 Iterator
<InputLine
> iter
= countMap
.keySet().iterator();
329 while (iter
.hasNext()) {
330 final InputLine inputLine
= iter
.next();
331 if (inputLine
.level
> currentInput
.level
) {
335 if (currentInput
.childrenInputs
!= null && currentInput
.childrenInputs
.size() > 0) {
336 currentInput
= currentInput
.childrenInputs
.get(0);
337 countMap
.put(currentInput
, 0);
338 } else if (checkNotNull(countMap
.get(currentInput
)) >= currentInput
.getMaxCount()) {
339 if (currentInput
.getNextInputs(countMap
).size() > 0) {
340 currentInput
= currentInput
.getNextInputs(countMap
).get(0);
341 if (countMap
.get(currentInput
) == null) {
342 countMap
.put(currentInput
, 0);
344 iter
= countMap
.keySet().iterator();
345 while (iter
.hasNext()) {
346 final InputLine inputLine
= iter
.next();
347 if (inputLine
.level
> currentInput
.level
) {
360 if (!processed
&& currentInput
!= null) {
361 final Matcher matcher
= currentInput
.getPattern().matcher(line
);
362 if (matcher
.matches()) {
363 event
.processGroups(currentInput
, matcher
);
364 countMap
.put(currentInput
, checkNotNull(countMap
.get(currentInput
)) + 1);
365 if (currentInput
.childrenInputs
!= null && currentInput
.childrenInputs
.size() > 0) {
366 currentInput
= currentInput
.childrenInputs
.get(0);
367 countMap
.put(currentInput
, 0);
368 } else if (checkNotNull(countMap
.get(currentInput
)) >= currentInput
.getMaxCount()) {
369 if (currentInput
.getNextInputs(countMap
).size() > 0) {
370 currentInput
= currentInput
.getNextInputs(countMap
).get(0);
371 if (countMap
.get(currentInput
) == null) {
372 countMap
.put(currentInput
, 0);
374 final Iterator
<InputLine
> iter
= countMap
.keySet().iterator();
375 while (iter
.hasNext()) {
376 final InputLine inputLine
= iter
.next();
377 if (inputLine
.level
> currentInput
.level
) {
386 ((StringBuffer
) event
.getContentValue()).append("\n").append(line
); //$NON-NLS-1$
389 rawPos
= fFile
.getFilePointer();
390 line
= fFile
.getNextLine();
392 } catch (final IOException e
) {
393 Activator
.logError("Error seeking event. File: " + getPath(), e
); //$NON-NLS-1$
395 for (final Entry
<InputLine
, Integer
> entry
: countMap
.entrySet()) {
396 if (entry
.getValue() < entry
.getKey().getMinCount()) {
400 context
.setLocation(NULL_LOCATION
);
405 * @return The first few lines of the text file
407 public List
<InputLine
> getFirstLines() {
408 return fDefinition
.inputs
;
412 * Parse the first line of the trace (to recognize the type).
416 * @return The first event
418 public CustomTxtEvent
parseFirstLine(final CustomTxtTraceContext context
) {
419 CustomTxtEventType eventType
= new CustomTxtEventType(checkNotNull(fDefinition
.definitionName
), fRootField
);
420 final CustomTxtEvent event
= new CustomTxtEvent(fDefinition
, this, TmfTimestamp
.ZERO
, eventType
);
421 event
.processGroups(context
.inputLine
, context
.firstLineMatcher
);
422 event
.setContent(new CustomEventContent(event
, new StringBuffer(context
.firstLine
)));
427 * Get the trace definition.
429 * @return The trace definition
431 public CustomTraceDefinition
getDefinition() {
438 * The default implementation computes the confidence as the percentage of
439 * lines in the first 100 lines of the file which match any of the root
440 * input line patterns.
443 public IStatus
validate(IProject project
, String path
) {
444 File file
= new File(path
);
445 if (!file
.exists() || !file
.isFile() || !file
.canRead()) {
446 return new Status(IStatus
.ERROR
, Activator
.PLUGIN_ID
, Messages
.CustomTrace_FileNotFound
+ ": " + path
); //$NON-NLS-1$
450 if (!TmfTraceUtils
.isText(file
)) {
451 return new TraceValidationStatus(confidence
, Activator
.PLUGIN_ID
);
453 } catch (IOException e
) {
454 Activator
.logError("Error validating file: " + path
, e
); //$NON-NLS-1$
455 return new Status(IStatus
.ERROR
, Activator
.PLUGIN_ID
, "IOException validating file: " + path
, e
); //$NON-NLS-1$
457 try (BufferedRandomAccessFile rafile
= new BufferedRandomAccessFile(path
, "r")) { //$NON-NLS-1$
459 double matches
= 0.0;
460 String line
= rafile
.getNextLine();
462 while ((line
!= null) && (lineCount
++ < MAX_LINES
)) {
463 for (InputLine inputLine
: fDefinition
.inputs
) {
464 Matcher matcher
= inputLine
.getPattern().matcher(line
);
465 if (matcher
.matches()) {
466 int groupCount
= matcher
.groupCount();
467 matches
+= (1.0 + groupCount
/ ((double) groupCount
+ 1));
471 confidence
= (int) (MAX_CONFIDENCE
* matches
/ lineCount
);
472 line
= rafile
.getNextLine();
474 } catch (IOException e
) {
475 return new Status(IStatus
.ERROR
, Activator
.PLUGIN_ID
, "IOException validating file: " + path
, e
); //$NON-NLS-1$
477 return new TraceValidationStatus(confidence
, Activator
.PLUGIN_ID
);
480 private static int fCheckpointSize
= -1;
483 public synchronized int getCheckpointSize() {
484 if (fCheckpointSize
== -1) {
485 TmfCheckpoint c
= new TmfCheckpoint(TmfTimestamp
.ZERO
, new TmfLongLocation(0L), 0);
486 ByteBuffer b
= ByteBuffer
.allocate(ITmfCheckpoint
.MAX_SERIALIZE_SIZE
);
489 fCheckpointSize
= b
.position();
492 return fCheckpointSize
;
496 public ITmfLocation
restoreLocation(ByteBuffer bufferIn
) {
497 return new TmfLongLocation(bufferIn
);
501 protected ITmfTraceIndexer
createIndexer(int interval
) {
502 return new TmfBTreeTraceIndexer(this, interval
);
506 public String
getTraceTypeId() {
511 * Build the trace type id for a custom text trace
515 * @param definitionName
516 * the definition name
517 * @return the trace type id
519 public static @NonNull String
buildTraceTypeId(String category
, String definitionName
) {
520 return CUSTOM_TXT_TRACE_TYPE_PREFIX
+ category
+ SEPARATOR
+ definitionName
;
524 * Checks whether the given trace type ID is a custom text trace type ID
527 * the trace type ID to check
528 * @return <code>true</code> if it's a custom text trace type ID else <code>false</code>
530 public static boolean isCustomTraceTypeId(@NonNull String traceTypeId
) {
531 return traceTypeId
.startsWith(CUSTOM_TXT_TRACE_TYPE_PREFIX
);
535 * This methods builds a trace type ID from a given ID taking into
536 * consideration any format changes that were done for the IDs of custom
537 * text traces. For example, such format change took place when moving to
538 * Trace Compass. Trace type IDs that are part of the plug-in extension for
539 * trace types won't be changed.
541 * This method is useful for IDs that were persisted in the workspace before
542 * the format changes (e.g. in the persistent properties of a trace
545 * It ensures backwards compatibility of the workspace for custom text
549 * the legacy trace type ID
550 * @return the trace type id in Trace Compass format
552 public static @NonNull String
buildCompatibilityTraceTypeId(@NonNull String traceTypeId
) {
553 // Handle early Trace Compass custom text trace type IDs
554 if (traceTypeId
.startsWith(EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX
)) {
555 return CUSTOM_TXT_TRACE_TYPE_PREFIX
+ traceTypeId
.substring(EARLY_TRACE_COMPASS_CUSTOM_TXT_TRACE_TYPE_PREFIX
.length());
558 // Handle Linux Tools custom text trace type IDs (with and without category)
559 int index
= traceTypeId
.lastIndexOf(SEPARATOR
);
560 if ((index
!= -1) && (traceTypeId
.startsWith(LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX
))) {
561 String definitionName
= index
< traceTypeId
.length() ? traceTypeId
.substring(index
+ 1) : ""; //$NON-NLS-1$
562 if (traceTypeId
.contains(CustomTxtTrace
.class.getSimpleName() + SEPARATOR
) && traceTypeId
.indexOf(SEPARATOR
) == index
) {
563 return buildTraceTypeId(CustomTxtTraceDefinition
.CUSTOM_TXT_CATEGORY
, definitionName
);
565 return CUSTOM_TXT_TRACE_TYPE_PREFIX
+ traceTypeId
.substring(LINUX_TOOLS_CUSTOM_TXT_TRACE_TYPE_PREFIX
.length());
572 public void traceRangeUpdated(TmfTraceRangeUpdatedSignal signal
) {
573 if (signal
.getTrace() == this) {
575 synchronized (this) {
576 // Reset the file handle in case it has reached the end of the
577 // file already. Otherwise, it will not be able to read new data
578 // pass the previous end.
581 } catch (TmfTraceException e
) {
582 Activator
.logError(e
.getLocalizedMessage(), e
);
585 super.traceRangeUpdated(signal
);
592 public synchronized ITmfTimestamp
readEnd() {
594 Long pos
= fFile
.length() - 1;
595 /* Outer loop to find the first line of a matcher group. */
597 /* Inner loop to find line beginning */
600 if (fFile
.read() == '\n') {
605 ITmfLocation location
= new TmfLongLocation(pos
);
606 ITmfContext context
= seekEvent(location
);
607 ITmfEvent event
= parseEvent(context
);
610 /* The last event in the trace was successfully parsed. */
611 return event
.getTimestamp();
613 /* pos was after the beginning of the lines of the last event. */
616 } catch (IOException e
) {
617 Activator
.logError("Error seeking last event. File: " + getPath(), e
); //$NON-NLS-1$