Build CTF stream indexes from LTTng index files
authorJérémie Galarneau <jeremie.galarneau@efficios.com>
Mon, 12 Dec 2016 16:55:05 +0000 (11:55 -0500)
committerJérémie Galarneau <jeremie.galarneau@efficios.com>
Sat, 27 May 2017 18:09:08 +0000 (14:09 -0400)
Signed-off-by: Jérémie Galarneau <jeremie.galarneau@efficios.com>
plugins/ctf/fs/data-stream.c
plugins/ctf/fs/data-stream.h
plugins/ctf/fs/fs.h
plugins/ctf/fs/lttng-index.h [new file with mode: 0644]

index ca5b47e8d3383064b72b7d44277891cf3a7457a6..e56e9f7a73352a71092e4da2d8b1b820f4410024 100644 (file)
@@ -176,10 +176,162 @@ static struct bt_ctf_notif_iter_medium_ops medops = {
        .get_stream = medop_get_stream,
 };
 
+static
+int build_index_from_idx_file(struct ctf_fs_stream *stream)
+{
+       int ret = 0;
+       gchar *directory = NULL;
+       gchar *basename = NULL;
+       GString *index_basename = NULL;
+       gchar *index_file_path = NULL;
+       GMappedFile *mapped_file = NULL;
+       gsize filesize;
+       const struct ctf_packet_index_file_hdr *header;
+       const char *mmap_begin, *file_pos;
+       struct index_entry *index;
+       uint64_t total_packets_size = 0;
+       size_t file_index_entry_size;
+       size_t file_entry_count;
+       size_t i;
+
+       /* Look for index file in relative path index/name.idx. */
+       basename = g_path_get_basename(stream->file->path->str);
+       if (!basename) {
+               ret = -1;
+               goto end;
+       }
+
+       directory = g_path_get_dirname(stream->file->path->str);
+       if (!directory) {
+               ret = -1;
+               goto end;
+       }
+
+       index_basename = g_string_new(basename);
+       if (!index_basename) {
+               ret = -1;
+               goto end;
+       }
+
+       g_string_append(index_basename, ".idx");
+       index_file_path = g_build_filename(directory, "index",
+                       index_basename->str, NULL);
+       mapped_file = g_mapped_file_new(index_file_path, FALSE, NULL);
+       filesize = g_mapped_file_get_length(mapped_file);
+       if (filesize < sizeof(*header)) {
+               printf_error("Invalid LTTng trace index: file size < header size");
+               ret = -1;
+               goto end;
+       }
+
+       mmap_begin = g_mapped_file_get_contents(mapped_file);
+       header = (struct ctf_packet_index_file_hdr *) mmap_begin;
+
+       file_pos = g_mapped_file_get_contents(mapped_file) + sizeof(*header);
+       if (be32toh(header->magic) != CTF_INDEX_MAGIC) {
+               printf_error("Invalid LTTng trace index: \"magic\" validation failed");
+               ret = -1;
+               goto end;
+       }
+       stream->index.version.major = be32toh(header->index_major);
+       stream->index.version.minor = be32toh(header->index_minor);
+
+       file_index_entry_size = be32toh(header->packet_index_len);
+       file_entry_count = (filesize - sizeof(*header)) / file_index_entry_size;
+       if ((filesize - sizeof(*header)) % (file_entry_count * file_index_entry_size)) {
+               printf_error("Invalid index file size; not a multiple of index entry size");
+               ret = -1;
+               goto end;
+       }
+
+       stream->index.entries = g_array_sized_new(FALSE, TRUE,
+                       sizeof(struct index_entry), file_entry_count);
+       if (!stream->index.entries) {
+               ret = -1;
+               goto end;
+       }
+       index = (struct index_entry *) stream->index.entries->data;
+       for (i = 0; i < file_entry_count; i++) {
+               struct ctf_packet_index *file_index =
+                               (struct ctf_packet_index *) file_pos;
+               uint64_t packet_size = be64toh(file_index->packet_size);
+
+               if (packet_size % CHAR_BIT) {
+                       ret = -1;
+                       printf_error("Invalid packet size encountered in index file");
+                       goto invalid_index;
+               }
+
+               /* Convert size in bits to bytes. */
+               packet_size /= CHAR_BIT;
+               index->packet_size = packet_size;
+
+               index->offset = be64toh(file_index->offset);
+               if (i != 0 && index->offset < (index - 1)->offset) {
+                       printf_error("Invalid, non-monotonic, packet offset encountered in index file");
+                       ret = -1;
+                       goto invalid_index;
+               }
+
+               index->timestamp_begin = be64toh(file_index->timestamp_begin);
+               index->timestamp_end = be64toh(file_index->timestamp_end);
+               if (index->timestamp_end < index->timestamp_begin) {
+                       printf_error("Invalid packet time bounds encountered in index file");
+                       ret = -1;
+                       goto invalid_index;
+               }
+
+               total_packets_size += packet_size;
+               file_pos += file_index_entry_size;
+               index++;
+       }
+
+       /* Validate that the index addresses the complete stream. */
+       if (stream->file->size != total_packets_size) {
+               printf_error("Invalid index; indexed size != stream file size");
+               ret = -1;
+               goto invalid_index;
+       }
+end:
+       g_free(directory);
+       g_free(basename);
+       g_free(index_file_path);
+       g_string_free(index_basename, TRUE);
+       g_mapped_file_unref(mapped_file);
+       return ret;
+invalid_index:
+       g_array_free(stream->index.entries, TRUE);
+       goto end;
+}
+
+static
+int build_index_from_stream(struct ctf_fs_stream *stream)
+{
+       int ret = 0;
+end:
+       return ret;
+}
+
+static
+int init_stream_index(struct ctf_fs_stream *stream)
+{
+       int ret;
+
+       ret = build_index_from_idx_file(stream);
+       if (!ret) {
+               goto end;
+       }
+
+       ret = build_index_from_stream(stream);
+end:
+       return ret;
+}
+
 BT_HIDDEN
 struct ctf_fs_stream *ctf_fs_stream_create(
                struct ctf_fs_component *ctf_fs, struct ctf_fs_file *file)
 {
+       int ret;
        struct ctf_fs_stream *stream = g_new0(struct ctf_fs_stream, 1);
 
        if (!stream) {
@@ -194,6 +346,10 @@ struct ctf_fs_stream *ctf_fs_stream_create(
        }
 
        stream->mmap_max_len = ctf_fs->page_size * 2048;
+       ret = init_stream_index(stream);
+       if (ret) {
+               goto error;
+       }
        goto end;
 error:
        /* Do not touch "borrowed" file. */
@@ -223,5 +379,9 @@ void ctf_fs_stream_destroy(struct ctf_fs_stream *stream)
                bt_ctf_notif_iter_destroy(stream->notif_iter);
        }
 
+       if (stream->index.entries) {
+               g_array_free(stream->index.entries, TRUE);
+       }
+
        g_free(stream);
 }
index 6e7d0e728bff8a48c6c9c5ff9af2dc3540c7f30b..11c149ccd90d8fd7ce0d1e185c10e20ef7b0f8ef 100644 (file)
 #include <babeltrace/ctf-ir/trace.h>
 
 #include "../common/notif-iter/notif-iter.h"
+#include "lttng-index.h"
+
+struct ctf_fs_component;
+struct ctf_fs_file;
+struct ctf_fs_stream;
+
+struct index_entry {
+       uint64_t offset; /* in bytes. */
+       uint64_t packet_size; /* in bytes. */
+       /* relative to the packet context field's mapped clock. */
+       uint64_t timestamp_begin, timestamp_end;
+};
+
+struct index {
+        struct {
+               unsigned int major, minor;
+       } version;
+       GArray *entries; /* Array of struct index_entry. */
+};
 
 BT_HIDDEN
 struct ctf_fs_stream *ctf_fs_stream_create(
index b3bd3544245e74a2bb1da5ce97ca6cf861c775a7..80bd6d04efc864deabba09a8121cf77ebab63e49 100644 (file)
@@ -30,6 +30,7 @@
 
 #include <babeltrace/babeltrace-internal.h>
 #include <babeltrace/plugin/component.h>
+#include "data-stream.h"
 
 #define CTF_FS_COMPONENT_NAME "fs"
 #define CTF_FS_COMPONENT_DESCRIPTION \
@@ -60,6 +61,8 @@ struct ctf_fs_stream {
        struct bt_ctf_stream *stream;
        /* FIXME There should be many and ctf_fs_stream should not own them. */
        struct bt_ctf_notif_iter *notif_iter;
+       /* A stream is assumed to be indexed. */
+       struct index index;
        void *mmap_addr;
        /* Max length of chunk to mmap() when updating the current mapping. */
        size_t mmap_max_len;
diff --git a/plugins/ctf/fs/lttng-index.h b/plugins/ctf/fs/lttng-index.h
new file mode 100644 (file)
index 0000000..b26f313
--- /dev/null
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2013 - Julien Desfossez <jdesfossez@efficios.com>
+ *                      Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *                      David Goulet <dgoulet@efficios.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef LTTNG_INDEX_H
+#define LTTNG_INDEX_H
+
+#include <babeltrace/compat/limits.h>
+
+#define CTF_INDEX_MAGIC 0xC1F1DCC1
+#define CTF_INDEX_MAJOR 1
+#define CTF_INDEX_MINOR 1
+
+/*
+ * Header at the beginning of each index file.
+ * All integer fields are stored in big endian.
+ */
+struct ctf_packet_index_file_hdr {
+       uint32_t magic;
+       uint32_t index_major;
+       uint32_t index_minor;
+       /* size of struct ctf_packet_index, in bytes. */
+       uint32_t packet_index_len;
+} __attribute__((__packed__));
+
+/*
+ * Packet index generated for each trace packet store in a trace file.
+ * All integer fields are stored in big endian.
+ */
+struct ctf_packet_index {
+       uint64_t offset;                /* offset of the packet in the file, in bytes */
+       uint64_t packet_size;           /* packet size, in bits */
+       uint64_t content_size;          /* content size, in bits */
+       uint64_t timestamp_begin;
+       uint64_t timestamp_end;
+       uint64_t events_discarded;
+       uint64_t stream_id;
+       /* CTF_INDEX 1.0 limit */
+       uint64_t stream_instance_id;    /* ID of the channel instance */
+       uint64_t packet_seq_num;        /* packet sequence number */
+} __attribute__((__packed__));
+
+#endif /* LTTNG_INDEX_H */
This page took 0.029807 seconds and 4 git commands to generate.