1 /*******************************************************************************
2 * Copyright (c) 2012, 2015 Ericsson, École Polytechnique de Montréal
4 * All rights reserved. This program and the accompanying materials are made
5 * available under the terms of the Eclipse Public License v1.0 which
6 * accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * Matthew Khouzam - Initial API and implementation
11 * Patrick Tasse - Updated for removal of context clone
12 * Geneviève Bastien - Added the createTimestamp function
13 *******************************************************************************/
15 package org
.eclipse
.tracecompass
.tmf
.ctf
.core
.trace
;
17 import static org
.eclipse
.tracecompass
.common
.core
.NonNullUtils
.checkNotNull
;
19 import java
.nio
.BufferOverflowException
;
20 import java
.nio
.ByteBuffer
;
21 import java
.util
.ArrayList
;
22 import java
.util
.Collection
;
23 import java
.util
.Collections
;
24 import java
.util
.HashMap
;
25 import java
.util
.HashSet
;
26 import java
.util
.List
;
30 import org
.eclipse
.core
.resources
.IProject
;
31 import org
.eclipse
.core
.resources
.IResource
;
32 import org
.eclipse
.core
.runtime
.CoreException
;
33 import org
.eclipse
.core
.runtime
.IStatus
;
34 import org
.eclipse
.core
.runtime
.Status
;
35 import org
.eclipse
.jdt
.annotation
.NonNull
;
36 import org
.eclipse
.tracecompass
.ctf
.core
.CTFException
;
37 import org
.eclipse
.tracecompass
.ctf
.core
.event
.CTFClock
;
38 import org
.eclipse
.tracecompass
.ctf
.core
.event
.IEventDeclaration
;
39 import org
.eclipse
.tracecompass
.ctf
.core
.event
.types
.StructDeclaration
;
40 import org
.eclipse
.tracecompass
.ctf
.core
.trace
.CTFStreamInput
;
41 import org
.eclipse
.tracecompass
.ctf
.core
.trace
.CTFTrace
;
42 import org
.eclipse
.tracecompass
.ctf
.core
.trace
.CTFTraceReader
;
43 import org
.eclipse
.tracecompass
.ctf
.core
.trace
.ICTFStream
;
44 import org
.eclipse
.tracecompass
.ctf
.core
.trace
.Metadata
;
45 import org
.eclipse
.tracecompass
.internal
.tmf
.ctf
.core
.Activator
;
46 import org
.eclipse
.tracecompass
.internal
.tmf
.ctf
.core
.trace
.iterator
.CtfIterator
;
47 import org
.eclipse
.tracecompass
.internal
.tmf
.ctf
.core
.trace
.iterator
.CtfIteratorManager
;
48 import org
.eclipse
.tracecompass
.tmf
.core
.event
.ITmfEvent
;
49 import org
.eclipse
.tracecompass
.tmf
.core
.event
.ITmfEventField
;
50 import org
.eclipse
.tracecompass
.tmf
.core
.event
.TmfEventField
;
51 import org
.eclipse
.tracecompass
.tmf
.core
.event
.aspect
.TmfBaseAspects
;
52 import org
.eclipse
.tracecompass
.tmf
.core
.event
.aspect
.ITmfEventAspect
;
53 import org
.eclipse
.tracecompass
.tmf
.core
.exceptions
.TmfTraceException
;
54 import org
.eclipse
.tracecompass
.tmf
.core
.project
.model
.ITmfPropertiesProvider
;
55 import org
.eclipse
.tracecompass
.tmf
.core
.timestamp
.ITmfTimestamp
;
56 import org
.eclipse
.tracecompass
.tmf
.core
.timestamp
.TmfTimestamp
;
57 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.ITmfContext
;
58 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.ITmfTraceKnownSize
;
59 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.ITmfTraceWithPreDefinedEvents
;
60 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.TmfTrace
;
61 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.TraceValidationStatus
;
62 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.ITmfPersistentlyIndexable
;
63 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.ITmfTraceIndexer
;
64 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.TmfBTreeTraceIndexer
;
65 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.checkpoint
.ITmfCheckpoint
;
66 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.indexer
.checkpoint
.TmfCheckpoint
;
67 import org
.eclipse
.tracecompass
.tmf
.core
.trace
.location
.ITmfLocation
;
68 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.CtfConstants
;
69 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.context
.CtfLocation
;
70 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.context
.CtfLocationInfo
;
71 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.context
.CtfTmfContext
;
72 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.event
.CtfTmfEvent
;
73 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.event
.CtfTmfEventFactory
;
74 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.event
.CtfTmfEventType
;
75 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.event
.aspect
.CtfChannelAspect
;
76 import org
.eclipse
.tracecompass
.tmf
.ctf
.core
.event
.aspect
.CtfCpuAspect
;
78 import com
.google
.common
.collect
.ImmutableList
;
79 import com
.google
.common
.collect
.ImmutableSet
;
82 * The CTf trace handler
85 * @author Matthew khouzam
87 public class CtfTmfTrace
extends TmfTrace
88 implements ITmfPropertiesProvider
, ITmfPersistentlyIndexable
,
89 ITmfTraceWithPreDefinedEvents
, ITmfTraceKnownSize
{
91 // -------------------------------------------
93 // -------------------------------------------
96 * Clock offset property
100 public static final String CLOCK_OFFSET
= "clock_offset"; //$NON-NLS-1$
103 * Default cache size for CTF traces
105 protected static final int DEFAULT_CACHE_SIZE
= 50000;
108 * Event aspects available for all CTF traces
112 protected static final @NonNull Collection
<@NonNull ITmfEventAspect
<?
>> CTF_ASPECTS
= ImmutableList
.of(
113 TmfBaseAspects
.getTimestampAspect(),
114 new CtfChannelAspect(),
116 TmfBaseAspects
.getEventTypeAspect(),
117 TmfBaseAspects
.getContentsAspect());
120 * The Ctf clock unique identifier field
122 private static final String CLOCK_HOST_PROPERTY
= "uuid"; //$NON-NLS-1$
123 private static final int CONFIDENCE
= 10;
124 private static final int MIN_CONFIDENCE
= 1;
127 * This is a reduction factor to avoid overflows.
129 private static final long REDUCTION_FACTOR
= 4096;
132 * Average CTF event size, used to estimate the trace size. (Inspired by
133 * empirical observations with LTTng kernel traces, to avoid hanging at 100%
136 * TODO: Find a more suitable approximation, perhaps per concrete trace type
137 * or per trace directly with the metadata
139 private static final int CTF_AVG_EVENT_SIZE
= 16;
141 // -------------------------------------------
143 // -------------------------------------------
145 private final Map
<@NonNull String
, @NonNull CtfTmfEventType
> fContainedEventTypes
= Collections
.synchronizedMap(new HashMap
<>());
147 private final CtfIteratorManager fIteratorManager
= new CtfIteratorManager(this);
149 private final @NonNull CtfTmfEventFactory fEventFactory
;
151 /** Reference to the CTF Trace */
152 private CTFTrace fTrace
;
154 // -------------------------------------------
156 // -------------------------------------------
159 * Default constructor
161 public CtfTmfTrace() {
164 /* Use default event factory */
165 fEventFactory
= CtfTmfEventFactory
.instance();
169 * Constructor for sub-classes to specify their own event factory.
171 * @param eventFactory
172 * The event factory to use to generate trace events
175 protected CtfTmfTrace(@NonNull CtfTmfEventFactory eventFactory
) {
177 fEventFactory
= eventFactory
;
180 // -------------------------------------------
181 // TmfTrace Overrides
182 // -------------------------------------------
187 * The resource associated with this trace
189 * The path to the trace file
191 * The type of events that will be read from this trace
192 * @throws TmfTraceException
193 * If something went wrong while reading the trace
196 public void initTrace(final IResource resource
, final String path
, final Class
<?
extends ITmfEvent
> eventType
)
197 throws TmfTraceException
{
199 * Set the cache size. This has to be done before the call to super()
200 * because the super needs to know the cache size.
204 super.initTrace(resource
, path
, eventType
);
207 this.fTrace
= new CTFTrace(path
);
209 /* Set the start and (current) end times for this trace */
210 ctx
= (CtfTmfContext
) seekEvent(0L);
211 CtfTmfEvent event
= getNext(ctx
);
212 if ((ctx
.getLocation().equals(CtfIterator
.NULL_LOCATION
)) || (ctx
.getCurrentEvent() == null)) {
213 /* Handle the case where the trace is empty */
214 this.setStartTime(TmfTimestamp
.BIG_BANG
);
216 final ITmfTimestamp curTime
= event
.getTimestamp();
217 this.setStartTime(curTime
);
218 this.setEndTime(curTime
);
221 * Register every event type. When you call getType, it will
222 * register a trace to that type in the TmfEventTypeManager
224 try (CtfIterator iter
= fIteratorManager
.getIterator(ctx
)) {
225 Set
<@NonNull ITmfEventField
> streamContextNames
= new HashSet
<>();
226 for (IEventDeclaration ied
: iter
.getEventDeclarations()) {
227 CtfTmfEventType ctfTmfEventType
= fContainedEventTypes
.get(ied
.getName());
228 if (ctfTmfEventType
== null) {
229 List
<ITmfEventField
> content
= new ArrayList
<>();
231 /* Should only return null the first time */
232 final StructDeclaration fields
= ied
.getFields();
233 if (fields
!= null) {
234 for (String fieldName
: fields
.getFieldsList()) {
235 content
.add(new TmfEventField(checkNotNull(fieldName
), null, null));
239 /* Add stream contexts */
240 final StructDeclaration streamContexts
= ied
.getStream().getEventContextDecl();
241 if (streamContextNames
.isEmpty()) {
242 if (streamContexts
!= null) {
243 for (String fieldName
: streamContexts
.getFieldsList()) {
244 streamContextNames
.add(new TmfEventField(checkNotNull(CtfConstants
.CONTEXT_FIELD_PREFIX
+ fieldName
), null, null));
248 content
.addAll(streamContextNames
);
250 if (!content
.isEmpty()) {
251 ITmfEventField contentTree
= new TmfEventField(
252 ITmfEventField
.ROOT_FIELD_ID
,
254 content
.toArray(new ITmfEventField
[content
.size()]));
256 ctfTmfEventType
= new CtfTmfEventType(checkNotNull(ied
.getName()), contentTree
);
257 fContainedEventTypes
.put(ctfTmfEventType
.getName(), ctfTmfEventType
);
263 } catch (final CTFException e
) {
265 * If it failed at the init(), we can assume it's because the file
266 * was not found or was not recognized as a CTF trace. Throw into
267 * the new type of exception expected by the rest of TMF.
269 throw new TmfTraceException(e
.getMessage(), e
);
274 public synchronized void dispose() {
275 fIteratorManager
.dispose();
276 if (fTrace
!= null) {
285 * The default implementation of a CTF trace.
287 * Firstly a weak validation of the metadata is done to determine if the
288 * path is actually for a CTF trace. After that a full validation is done.
290 * If the weak and full validation are successful the confidence is set to
293 * If the weak validation was successful, but the full validation fails a
294 * TraceValidationStatus with severity warning and confidence of 1 is
297 * If both weak and full validation fails an error status is returned.
300 public IStatus
validate(final IProject project
, final String path
) {
301 boolean isMetadataFile
= false;
303 isMetadataFile
= Metadata
.preValidate(path
);
304 } catch (final CTFException e
) {
305 return new Status(IStatus
.ERROR
, Activator
.PLUGIN_ID
, Messages
.CtfTmfTrace_ReadingError
+ ": " + e
.toString(), e
); //$NON-NLS-1$
308 if (isMetadataFile
) {
309 // Trace is pre-validated, continue will full validation
311 final CTFTrace trace
= new CTFTrace(path
);
312 if (!trace
.majorIsSet()) {
313 if (isMetadataFile
) {
314 return new TraceValidationStatus(MIN_CONFIDENCE
, IStatus
.WARNING
, Activator
.PLUGIN_ID
, Messages
.CtfTmfTrace_MajorNotSet
, null);
316 return new Status(IStatus
.ERROR
, Activator
.PLUGIN_ID
, Messages
.CtfTmfTrace_MajorNotSet
);
319 // Validate using reader initialization
320 try (CTFTraceReader ctfTraceReader
= new CTFTraceReader(trace
)) {
324 // Trace is validated, return with confidence
325 return new CtfTraceValidationStatus(CONFIDENCE
, Activator
.PLUGIN_ID
, trace
.getEnvironment());
327 } catch (final CTFException
| BufferOverflowException e
) {
328 // return warning since it's a CTF trace but with errors in it
329 return new TraceValidationStatus(MIN_CONFIDENCE
, IStatus
.WARNING
, Activator
.PLUGIN_ID
, Messages
.CtfTmfTrace_ReadingError
+ ": " + e
.toString(), e
); //$NON-NLS-1$
332 return new Status(IStatus
.ERROR
, Activator
.PLUGIN_ID
, Messages
.CtfTmfTrace_ReadingError
);
336 public Iterable
<ITmfEventAspect
<?
>> getEventAspects() {
341 * Method getCurrentLocation. This is not applicable in CTF
343 * @return null, since the trace has no knowledge of the current location
344 * @see org.eclipse.tracecompass.tmf.core.trace.ITmfTrace#getCurrentLocation()
347 public ITmfLocation
getCurrentLocation() {
352 public double getLocationRatio(ITmfLocation location
) {
353 final CtfLocation curLocation
= (CtfLocation
) location
;
354 final long startTime
= getStartTime().getValue();
355 final double diff
= curLocation
.getLocationInfo().getTimestamp() - startTime
;
356 final double total
= getEndTime().getValue() - startTime
;
357 return Math
.max(0.0, Math
.min(1.0, diff
/ total
));
365 * @return ITmfContext
368 public synchronized ITmfContext
seekEvent(final ITmfLocation location
) {
369 CtfLocation currentLocation
= (CtfLocation
) location
;
370 CtfTmfContext context
= new CtfTmfContext(this);
371 if (fTrace
== null) {
372 context
.setLocation(null);
373 context
.setRank(ITmfContext
.UNKNOWN_RANK
);
377 * The rank is set to 0 if the iterator seeks the beginning. If not, it
378 * will be set to UNKNOWN_RANK, since CTF traces don't support seeking
381 if (currentLocation
== null) {
382 currentLocation
= new CtfLocation(new CtfLocationInfo(0L, 0L));
385 context
.setRank(ITmfContext
.UNKNOWN_RANK
);
387 /* This will seek and update the location after the seek */
388 context
.setLocation(currentLocation
);
393 public synchronized ITmfContext
seekEvent(double ratio
) {
394 CtfTmfContext context
= new CtfTmfContext(this);
395 if (fTrace
== null) {
396 context
.setLocation(null);
397 context
.setRank(ITmfContext
.UNKNOWN_RANK
);
400 final long end
= getEndTime().getValue();
401 final long start
= getStartTime().getValue();
402 final long diff
= end
- start
;
403 final long ratioTs
= Math
.round(diff
* ratio
) + start
;
404 context
.seek(ratioTs
);
405 context
.setRank(ITmfContext
.UNKNOWN_RANK
);
410 * Method readNextEvent.
414 * @return CtfTmfEvent
415 * @see org.eclipse.tracecompass.tmf.core.trace.ITmfTrace#getNext(ITmfContext)
418 public synchronized CtfTmfEvent
getNext(final ITmfContext context
) {
419 if (fTrace
== null) {
422 CtfTmfEvent event
= null;
423 if (context
instanceof CtfTmfContext
) {
424 if (context
.getLocation() == null || CtfLocation
.INVALID_LOCATION
.equals(context
.getLocation().getLocationInfo())) {
427 CtfTmfContext ctfContext
= (CtfTmfContext
) context
;
428 event
= ctfContext
.getCurrentEvent();
431 updateAttributes(context
, event
);
432 ctfContext
.advance();
433 ctfContext
.increaseRank();
441 * Ctf traces have a clock with a unique uuid that will be used to identify
442 * the host. Traces with the same clock uuid will be known to have been made
443 * on the same machine.
445 * Note: uuid is an optional field, it may not be there for a clock.
448 public String
getHostId() {
449 CTFClock clock
= fTrace
.getClock();
451 String clockHost
= (String
) clock
.getProperty(CLOCK_HOST_PROPERTY
);
452 if (clockHost
!= null) {
456 return super.getHostId();
460 * Get the CTF environment variables defined in this CTF trace, in <name,
461 * value> form. This comes from the trace's CTF metadata.
463 * @return The CTF environment
465 public Map
<String
, String
> getEnvironment() {
466 return fTrace
.getEnvironment();
469 // -------------------------------------------
470 // ITmfPropertiesProvider
471 // -------------------------------------------
477 public Map
<String
, String
> getProperties() {
478 Map
<String
, String
> properties
= new HashMap
<>();
479 properties
.putAll(fTrace
.getEnvironment());
480 properties
.put(CLOCK_OFFSET
, Long
.toUnsignedString(fTrace
.getOffset()));
481 properties
.put(Messages
.CtfTmfTrace_HostID
, getHostId());
485 // -------------------------------------------
487 // -------------------------------------------
490 * gets the clock offset
492 * @return the clock offset in ns
494 public long getOffset() {
495 if (fTrace
!= null) {
496 return fTrace
.getOffset();
502 * Convert a CTF timestamp in CPU cycles to its equivalent in nanoseconds
506 * The timestamp in cycles
507 * @return The timestamp in nanoseconds
509 public long timestampCyclesToNanos(long cycles
) {
510 return fTrace
.timestampCyclesToNanos(cycles
);
514 * Convert a CTF timestamp in nanoseconds to its equivalent in CPU cycles
518 * The timestamp in nanoseconds
519 * @return The timestamp in cycles
521 public long timestampNanoToCycles(long nanos
) {
522 return fTrace
.timestampNanoToCycles(nanos
);
526 * Gets the list of declared events
529 public Set
<@NonNull CtfTmfEventType
> getContainedEventTypes() {
530 return ImmutableSet
.copyOf(fContainedEventTypes
.values());
534 * Register an event type to this trace.
536 * Public visibility so that {@link CtfTmfEvent#getType} can call it.
538 * FIXME This could probably be made cleaner?
541 * The event type to register
543 public void registerEventType(CtfTmfEventType eventType
) {
544 fContainedEventTypes
.put(eventType
.getName(), eventType
);
547 // -------------------------------------------
549 // -------------------------------------------
552 public CtfTmfEvent
parseEvent(ITmfContext context
) {
553 CtfTmfEvent event
= null;
554 if (context
instanceof CtfTmfContext
) {
555 final ITmfContext tmpContext
= seekEvent(context
.getLocation());
556 event
= getNext(tmpContext
);
562 * Sets the cache size for a CtfTmfTrace.
564 protected void setCacheSize() {
565 setCacheSize(DEFAULT_CACHE_SIZE
);
568 // -------------------------------------------
569 // CtfIterator factory methods
570 // -------------------------------------------
573 * Get the event factory for this trace to generate new events for it.
575 * @return The event factory
578 public @NonNull CtfTmfEventFactory
getEventFactory() {
579 return fEventFactory
;
583 * Get an iterator to the trace
585 * @return an iterator to the trace
587 public ITmfContext
createIterator() {
589 return new CtfIterator(fTrace
, this);
590 } catch (CTFException e
) {
591 Activator
.getDefault().logError(e
.getMessage(), e
);
597 * Get an iterator to the trace, , which will initially point to the given
600 * @param ctfLocationData
601 * The initial timestamp the iterator will be pointing to
604 * @return The new iterator
606 public ITmfContext
createIterator(CtfLocationInfo ctfLocationData
, long rank
) {
608 return new CtfIterator(fTrace
, this, ctfLocationData
, rank
);
609 } catch (CTFException e
) {
610 Activator
.getDefault().logError(e
.getMessage(), e
);
616 * Create the 'CtfIterator' object from a CtfTmfContext.
619 * The iterator will initially be pointing to this context
620 * @return A new CtfIterator object
623 public ITmfContext
createIteratorFromContext(CtfTmfContext context
) {
624 return fIteratorManager
.getIterator(context
);
628 * Dispose an iterator that was create with
629 * {@link #createIteratorFromContext}
632 * The last context that was pointed to by the iterator (this is
633 * the 'key' to find the correct iterator to dispose).
636 public void disposeContext(CtfTmfContext context
) {
637 fIteratorManager
.removeIterator(context
);
640 // ------------------------------------------------------------------------
641 // Timestamp transformation functions
642 // ------------------------------------------------------------------------
648 public @NonNull ITmfTimestamp
createTimestamp(long ts
) {
649 return TmfTimestamp
.fromNanos(getTimestampTransform().transform(ts
));
652 private static int fCheckpointSize
= -1;
655 public synchronized int getCheckpointSize() {
656 if (fCheckpointSize
== -1) {
657 TmfCheckpoint c
= new TmfCheckpoint(TmfTimestamp
.fromNanos(0), new CtfLocation(0, 0), 0);
658 ByteBuffer b
= ByteBuffer
.allocate(ITmfCheckpoint
.MAX_SERIALIZE_SIZE
);
661 fCheckpointSize
= b
.position();
664 return fCheckpointSize
;
668 protected ITmfTraceIndexer
createIndexer(int interval
) {
669 return new TmfBTreeTraceIndexer(this, interval
);
673 public ITmfLocation
restoreLocation(ByteBuffer bufferIn
) {
674 return new CtfLocation(bufferIn
);
678 public boolean isComplete() {
679 if (getResource() == null) {
685 String sessionName
= null;
687 host
= getResource().getPersistentProperty(CtfConstants
.LIVE_HOST
);
688 port
= getResource().getPersistentProperty(CtfConstants
.LIVE_PORT
);
689 sessionName
= getResource().getPersistentProperty(CtfConstants
.LIVE_SESSION_NAME
);
690 } catch (CoreException e
) {
691 Activator
.getDefault().logError(e
.getMessage(), e
);
692 // Something happened to the resource, assume we won't get any more
696 return host
== null || port
== null || sessionName
== null;
700 public void setComplete(final boolean isComplete
) {
701 super.setComplete(isComplete
);
704 getResource().setPersistentProperty(CtfConstants
.LIVE_HOST
, null);
705 getResource().setPersistentProperty(CtfConstants
.LIVE_PORT
, null);
706 getResource().setPersistentProperty(CtfConstants
.LIVE_SESSION_NAME
, null);
708 } catch (CoreException e
) {
709 Activator
.getDefault().logError(e
.getMessage(), e
);
714 * @return the number of estimated chunks of events read. This reads the
715 * file size of the trace and divides it by a factor and the average
716 * event size, this is not accurate but can give a ball park figure
717 * of how much is done.
723 CTFTrace trace
= fTrace
;
725 Iterable
<ICTFStream
> streams
= trace
.getStreams();
726 for (ICTFStream stream
: streams
) {
727 for (CTFStreamInput si
: stream
.getStreamInputs()) {
728 size
+= si
.getFile().length();
732 return (int) (size
/ REDUCTION_FACTOR
/ CTF_AVG_EVENT_SIZE
);
736 * @return the number of events divided a reduction factor. Is monotonic.
740 public int progress() {
741 return (int) (getNbEvents() / REDUCTION_FACTOR
);