ctf/org.eclipse.tracecompass.ctf.core.tests/shared/org/eclipse/tracecompass/ctf/core/tests/shared/LttngKernelTraceGenerator.java

   1 /*******************************************************************************
   2  * Copyright (c) 2013, 2015 Ericsson
   3  *
   4  * All rights reserved. This program and the accompanying materials are
   5  * made available under the terms of the Eclipse Public License v1.0 which
   6  * accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *   Matthew Khouzam - Initial API and implementation
  11  *   Marc-Andre Laperle - Move generation to traces folder
  12  *******************************************************************************/
  13
  14 package org.eclipse.tracecompass.ctf.core.tests.shared;
  15
  16 import java.io.File;
  17 import java.io.FileNotFoundException;
  18 import java.io.FileOutputStream;
  19 import java.io.IOException;
  20 import java.nio.ByteBuffer;
  21 import java.nio.ByteOrder;
  22 import java.nio.channels.FileChannel;
  23 import java.nio.file.Path;
  24 import java.nio.file.Paths;
  25 import java.util.Arrays;
  26 import java.util.List;
  27 import java.util.Random;
  28
  29 import org.eclipse.tracecompass.ctf.core.tests.CtfCoreTestPlugin;
  30
  31 /**
  32  * Generate a kernel trace
  33  *
  34  * @author Matthew Khouzam
  35  */
  36 public class LttngKernelTraceGenerator {
  37
  38     private static final String metadata = "/* CTF 1.8 */ \n" +
  39             "typealias integer { size = 8; align = 8; signed = false; } := uint8_t;\n" +
  40             "typealias integer { size = 16; align = 8; signed = false; } := uint16_t;\n" +
  41             "typealias integer { size = 32; align = 8; signed = false; } := uint32_t;\n" +
  42             "typealias integer { size = 64; align = 8; signed = false; } := uint64_t;\n" +
  43             "typealias integer { size = 32; align = 8; signed = false; } := unsigned long;\n" +
  44             "typealias integer { size = 5; align = 1; signed = false; } := uint5_t;\n" +
  45             "typealias integer { size = 27; align = 1; signed = false; } := uint27_t;\n" +
  46             "\n" +
  47             "trace {\n" +
  48             "   major = 1;\n" +
  49             "   minor = 8;\n" +
  50             "   uuid = \"11111111-1111-1111-1111-111111111111\";\n" +
  51             "   byte_order = le;\n" +
  52             "   packet.header := struct {\n" +
  53             "       uint32_t magic;\n" +
  54             "       uint8_t  uuid[16];\n" +
  55             "       uint32_t stream_id;\n" +
  56             "   };\n" +
  57             "};\n" +
  58             "\n" +
  59             "env {\n" +
  60             "   hostname = \"synthetic-host\";\n" +
  61             "   domain = \"kernel\";\n" +
  62             "   sysname = \"FakeLinux\";\n" +
  63             "   kernel_release = \"1.0\";\n" +
  64             "   kernel_version = \"Fake Os Synthetic Trace\";\n" +
  65             "   tracer_name = \"lttng-modules\";\n" +
  66             "   tracer_major = 2;\n" +
  67             "   tracer_minor = 1;\n" +
  68             "   tracer_patchlevel = 0;\n" +
  69             "};\n" +
  70             "\n" +
  71             "clock {\n" +
  72             "   name = monotonic;\n" +
  73             "   uuid = \"bbff68f0-c633-4ea1-92cd-bd11024ec4de\";\n" +
  74             "   description = \"Monotonic Clock\";\n" +
  75             "   freq = 1000000000; /* Frequency, in Hz */\n" +
  76             "   /* clock value offset from Epoch is: offset * (1/freq) */\n" +
  77             "   offset = 1368000272650993664;\n" +
  78             "};\n" +
  79             "\n" +
  80             "typealias integer {\n" +
  81             "   size = 27; align = 1; signed = false;\n" +
  82             "   map = clock.monotonic.value;\n" +
  83             "} := uint27_clock_monotonic_t;\n" +
  84             "\n" +
  85             "typealias integer {\n" +
  86             "   size = 32; align = 8; signed = false;\n" +
  87             "   map = clock.monotonic.value;\n" +
  88             "} := uint32_clock_monotonic_t;\n" +
  89             "\n" +
  90             "typealias integer {\n" +
  91             "   size = 64; align = 8; signed = false;\n" +
  92             "   map = clock.monotonic.value;\n" +
  93             "} := uint64_clock_monotonic_t;\n" +
  94             "\n" +
  95             "struct packet_context {\n" +
  96             "   uint64_clock_monotonic_t timestamp_begin;\n" +
  97             "   uint64_clock_monotonic_t timestamp_end;\n" +
  98             "   uint64_t content_size;\n" +
  99             "   uint64_t packet_size;\n" +
 100             "   unsigned long events_discarded;\n" +
 101             "   uint32_t cpu_id;\n" +
 102             "};\n" +
 103             "\n" +
 104             "struct event_header_compact {\n" +
 105             "   enum : uint5_t { compact = 0 ... 30, extended = 31 } id;\n" +
 106             "   variant <id> {\n" +
 107             "       struct {\n" +
 108             "           uint27_clock_monotonic_t timestamp;\n" +
 109             "       } compact;\n" +
 110             "       struct {\n" +
 111             "           uint32_t id;\n" +
 112             "           uint64_clock_monotonic_t timestamp;\n" +
 113             "       } extended;\n" +
 114             "   } v;\n" +
 115             "} align(8);\n" +
 116             "\n" +
 117             "struct event_header_large {\n" +
 118             "   enum : uint16_t { compact = 0 ... 65534, extended = 65535 } id;\n" +
 119             "   variant <id> {\n" +
 120             "       struct {\n" +
 121             "           uint32_clock_monotonic_t timestamp;\n" +
 122             "       } compact;\n" +
 123             "       struct {\n" +
 124             "           uint32_t id;\n" +
 125             "           uint64_clock_monotonic_t timestamp;\n" +
 126             "       } extended;\n" +
 127             "   } v;\n" +
 128             "} align(8);\n" +
 129             "\n" +
 130             "stream {\n" +
 131             "   id = 0;\n" +
 132             "   event.header := struct event_header_compact;\n" +
 133             "   packet.context := struct packet_context;\n" +
 134             "};\n" +
 135             "\n" +
 136             "event {\n" +
 137             "   name = sched_switch;\n" +
 138             "   id = 0;\n" +
 139             "   stream_id = 0;\n" +
 140             "   fields := struct {\n" +
 141             "       integer { size = 8; align = 8; signed = 1; encoding = UTF8; base = 10; } _prev_comm[16];\n" +
 142             "       integer { size = 32; align = 8; signed = 1; encoding = none; base = 10; } _prev_tid;\n" +
 143             "       integer { size = 32; align = 8; signed = 1; encoding = none; base = 10; } _prev_prio;\n" +
 144             "       integer { size = 32; align = 8; signed = 1; encoding = none; base = 10; } _prev_state;\n" +
 145             "       integer { size = 8; align = 8; signed = 1; encoding = UTF8; base = 10; } _next_comm[16];\n" +
 146             "       integer { size = 32; align = 8; signed = 1; encoding = none; base = 10; } _next_tid;\n" +
 147             "       integer { size = 32; align = 8; signed = 1; encoding = none; base = 10; } _next_prio;\n" +
 148             "   };\n" +
 149             "};\n" +
 150             "\n";
 151
 152     private final List<String> fProcesses;
 153     private final long fDuration;
 154     private final long fNbEvents;
 155     private final int fNbChans;
 156
 157     private static final String[] sfProcesses = {
 158             "IDLE",
 159             "gnuplot",
 160             "starcraft 2:pt3",
 161             "bash",
 162             "smash",
 163             "thrash",
 164             "fireball",
 165             "Half-life 3",
 166             "ST: The game"
 167     };
 168
 169
 170     private static final String TRACES_DIRECTORY = "traces";
 171     private static final String TRACE_NAME = "synthetic-trace";
 172
 173     /**
 174      * Main, not always needed
 175      *
 176      * @param args
 177      *            args
 178      */
 179     public static void main(String[] args) {
 180         // not using createTempFile as this is a directory
 181         String path = CtfCoreTestPlugin.getTemporaryDirPath() + File.separator + TRACE_NAME;
 182         generateLttngKernelTrace(new File(path));
 183     }
 184
 185     /**
 186      * Gets the name of the trace (top directory name)
 187      *
 188      * @return the name of the trace
 189      */
 190     public static String getName() {
 191         return TRACE_NAME;
 192     }
 193
 194     /**
 195      * Get the path
 196      *
 197      * @return the path
 198      */
 199     public static String getPath() {
 200         CtfCoreTestPlugin plugin = CtfCoreTestPlugin.getDefault();
 201         if (plugin == null) {
 202             return null;
 203         }
 204         Path tracePath = Paths.get("..", "..", "ctf", "org.eclipse.tracecompass.ctf.core.tests", TRACES_DIRECTORY, TRACE_NAME);
 205         tracePath = tracePath.toAbsolutePath();
 206         File file = tracePath.toFile();
 207
 208         if (!file.exists()) {
 209             generateLttngKernelTrace(file);
 210         }
 211         return file.getAbsolutePath();
 212     }
 213
 214     /**
 215      * Generate a trace
 216      *
 217      * @param file
 218      *            the file to write the trace to
 219      */
 220     public static void generateLttngKernelTrace(File file) {
 221         final int cpus = 25;
 222         LttngKernelTraceGenerator gt = new LttngKernelTraceGenerator(2l * Integer.MAX_VALUE - 100, 500000, cpus);
 223         gt.writeTrace(file);
 224     }
 225
 226     /**
 227      * Make a kernel trace
 228      *
 229      * @param duration
 230      *            the duration of the trace
 231      * @param events
 232      *            the number of events in a trace
 233      * @param nbChannels
 234      *            the number of channels in the trace
 235      */
 236     public LttngKernelTraceGenerator(long duration, long events, int nbChannels) {
 237         fProcesses = Arrays.asList(sfProcesses);
 238         fDuration = duration;
 239         fNbEvents = events;
 240         fNbChans = nbChannels;
 241     }
 242
 243     /**
 244      * Write the trace to a file
 245      *
 246      * @param file
 247      *            the file to write the trace to
 248      */
 249     public void writeTrace(File file) {
 250
 251         if (!file.exists()) {
 252             file.mkdir();
 253         } else {
 254             if (file.isFile()) {
 255                 file.delete();
 256                 file.mkdir();
 257             } else {
 258                 // the ctf parser doesn't recurse, so we don't need to.
 259                 final File[] listFiles = file.listFiles();
 260                 for (File child : listFiles) {
 261                     child.delete();
 262                 }
 263             }
 264         }
 265
 266         File metadataFile = new File(file.getPath() + File.separator + "metadata");
 267         File[] streams = new File[fNbChans];
 268         FileChannel[] channels = new FileChannel[fNbChans];
 269
 270         try {
 271             for (int i = 0; i < fNbChans; i++) {
 272                 streams[i] = new File(file.getPath() + File.separator + "channel" + i);
 273                 channels[i] = new FileOutputStream(streams[i]).getChannel();
 274             }
 275         } catch (FileNotFoundException e) {
 276         }
 277         // determine the number of events per channel
 278         long evPerChan = fNbEvents / fNbChans;
 279         final int evPerPacket = PacketWriter.CONTENT_SIZE / EventWriter.SIZE;
 280         long delta = (int) (fDuration / evPerChan);
 281         long offsetTime = 0;
 282         Random rndLost = new Random(1337);
 283         for (int chan = 0; chan < fNbChans; chan++) {
 284             int currentSpace = 0;
 285             ByteBuffer bb = ByteBuffer.allocate(65536);
 286             bb.order(ByteOrder.LITTLE_ENDIAN);
 287             Random rnd = new Random(1337);
 288             int rnd0 = rnd.nextInt(fProcesses.size());
 289             String prevComm = fProcesses.get(rnd0);
 290             int prevPID = rnd0 + chan * fProcesses.size();
 291             if (rnd0 == 0) {
 292                 prevPID = 0;
 293             }
 294             int prevPrio = 0;
 295             int prevPos = -1;
 296             int discarded = 0;
 297             int discardedTotal = 0;
 298             for (int eventNb = 0; eventNb < evPerChan; eventNb++) {
 299                 if (EventWriter.SIZE > currentSpace) {
 300                     eventNb += discarded;
 301                 }
 302                 long ts = eventNb * delta + delta / (fNbChans + 1) * chan;
 303
 304                 int pos = rnd.nextInt((int) (fProcesses.size() * 1.5));
 305                 if (pos >= fProcesses.size()) {
 306                     pos = 0;
 307                 }
 308                 while (pos == prevPos) {
 309                     pos = rnd.nextInt((int) (fProcesses.size() * 1.5));
 310                     if (pos >= fProcesses.size()) {
 311                         pos = 0;
 312                     }
 313                 }
 314                 String nextComm = fProcesses.get(pos);
 315                 int nextPID = pos + fProcesses.size() * chan;
 316                 if (pos == 0) {
 317                     nextPID = 0;
 318                 }
 319                 int nextPrio = 0;
 320                 if (EventWriter.SIZE > currentSpace) {
 321                     // pad to end
 322                     for (int i = 0; i < currentSpace; i++) {
 323                         bb.put((byte) 0x00);
 324                     }
 325                     // write new packet
 326                     PacketWriter pw = new PacketWriter(bb);
 327                     long tsBegin = ts;
 328                     offsetTime = ts;
 329                     int eventCount = Math.min(evPerPacket, (int) evPerChan - eventNb);
 330                     discarded = rndLost.nextInt(10 * fNbChans) == 0 ? rndLost.nextInt(evPerPacket) : 0;
 331                     discarded = Math.min(discarded, (int) evPerChan - eventNb - eventCount);
 332                     discardedTotal += discarded;
 333                     long tsEnd = (eventNb + eventCount + discarded) * delta;
 334                     pw.writeNewHeader(tsBegin, tsEnd, chan, eventCount, discardedTotal);
 335                     currentSpace = PacketWriter.CONTENT_SIZE;
 336                 }
 337                 EventWriter ew = new EventWriter(bb);
 338                 int prev_state = rnd.nextInt(100);
 339                 if (prev_state != 0) {
 340                     prev_state = 1;
 341                 }
 342                 final long shrunkenTimestamp = ts - offsetTime;
 343                 final int tsMask = (1 << 27) - 1;
 344                 if (shrunkenTimestamp > ((1 << 27) + tsMask)) {
 345                     /* allow only one compact timestamp overflow per packet */
 346                     throw new IllegalStateException("Invalid timestamp overflow:" + shrunkenTimestamp);
 347                 }
 348                 final int clampedTs = (int) (ts & tsMask);
 349                 int evSize = ew.writeEvent(clampedTs, prevComm, prevPID, prevPrio, prev_state, nextComm, nextPID, nextPrio);
 350                 currentSpace -= evSize;
 351                 prevComm = nextComm;
 352                 prevPID = nextPID;
 353                 prevPrio = nextPrio;
 354                 if (bb.position() > 63000) {
 355                     writeToDisk(channels, chan, bb);
 356                 }
 357             }
 358             for (int i = 0; i < currentSpace; i++) {
 359                 bb.put((byte) 0x00);
 360             }
 361             writeToDisk(channels, chan, bb);
 362             try {
 363                 channels[chan].close();
 364             } catch (IOException e) {
 365                 e.printStackTrace();
 366             }
 367         }
 368         try (FileOutputStream fos = new FileOutputStream(metadataFile);) {
 369             fos.write(metadata.getBytes());
 370         } catch (IOException e) {
 371         }
 372     }
 373
 374     private static void writeToDisk(FileChannel[] channels, int chan, ByteBuffer bb) {
 375         try {
 376             bb.flip();
 377             channels[chan].write(bb);
 378             bb.clear();
 379         } catch (IOException e) {
 380             e.printStackTrace();
 381         }
 382     }
 383
 384     private class EventWriter {
 385         public static final int SIZE =
 386                 4 +  // timestamp
 387                 16 + // prev_comm
 388                 4 +  // prev_tid
 389                 4 +  // prev_prio
 390                 4 +  // prev_state
 391                 16 + // current_comm
 392                 4 +  // next_tid
 393                 4;   // next_prio
 394         private final ByteBuffer data;
 395
 396         public EventWriter(ByteBuffer bb) {
 397             data = bb;
 398         }
 399
 400         public int writeEvent(int ts, String prev_comm, int prev_tid, int prev_prio, int prev_state, String next_comm, int next_tid, int next_prio) {
 401             byte[] bOut = new byte[16];
 402             byte[] bIn = new byte[16];
 403             byte[] temp = prev_comm.getBytes();
 404             for (int i = 0; i < Math.min(temp.length, 16); i++) {
 405                 bOut[i] = temp[i];
 406             }
 407             temp = next_comm.getBytes();
 408             for (int i = 0; i < Math.min(temp.length, 16); i++) {
 409                 bIn[i] = temp[i];
 410             }
 411
 412             int timestamp = ts << 5;
 413
 414             data.putInt(timestamp);
 415             data.put(bOut);
 416             data.putInt(prev_tid);
 417             data.putInt(prev_prio);
 418             data.putInt(prev_state);
 419             data.put(bIn);
 420             data.putInt(next_tid);
 421             data.putInt(next_prio);
 422             return SIZE;
 423         }
 424
 425     }
 426
 427     private class PacketWriter {
 428         private static final int SIZE = 4096;
 429         private static final int HEADER_SIZE = 64;
 430         private static final int CONTENT_SIZE = SIZE - HEADER_SIZE;
 431
 432         private final ByteBuffer data;
 433
 434         public PacketWriter(ByteBuffer bb) {
 435             data = bb;
 436         }
 437
 438         public void writeNewHeader(long tsBegin, long tsEnd, int cpu, int eventCount, int discarded) {
 439             final int magicLE = 0xC1FC1FC1;
 440             byte uuid[] = {
 441                     0x11, 0x11, 0x11, 0x11,
 442                     0x11, 0x11, 0x11, 0x11,
 443                     0x11, 0x11, 0x11, 0x11,
 444                     0x11, 0x11, 0x11, 0x11 };
 445             // packet header
 446
 447             // magic number 4
 448             data.putInt(magicLE);
 449             // uuid 16
 450             data.put(uuid);
 451             // stream ID 4
 452             data.putInt(0);
 453
 454             // packet context
 455             // timestamp_begin 8
 456             data.putLong(tsBegin);
 457
 458             // timestamp_end 8
 459             data.putLong(tsEnd);
 460
 461             // content_size 8
 462             data.putLong((eventCount * EventWriter.SIZE + HEADER_SIZE)* 8);
 463
 464             // packet_size 8
 465             data.putLong((SIZE) * 8);
 466
 467             // events_discarded 4
 468             data.putInt(discarded);
 469
 470             // cpu_id 4
 471             data.putInt(cpu);
 472
 473         }
 474
 475     }
 476
 477
 478
 479 }