From b6c3dcb2881ae8bf5115cf72be2834e88ff1ce1b Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=A9mie=20Galarneau?= Date: Mon, 12 Dec 2016 11:55:05 -0500 Subject: [PATCH] Build CTF stream indexes from LTTng index files MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérémie Galarneau --- plugins/ctf/fs/data-stream.c | 160 +++++++++++++++++++++++++++++++++++ plugins/ctf/fs/data-stream.h | 19 +++++ plugins/ctf/fs/fs.h | 3 + plugins/ctf/fs/lttng-index.h | 63 ++++++++++++++ 4 files changed, 245 insertions(+) create mode 100644 plugins/ctf/fs/lttng-index.h diff --git a/plugins/ctf/fs/data-stream.c b/plugins/ctf/fs/data-stream.c index ca5b47e8..e56e9f7a 100644 --- a/plugins/ctf/fs/data-stream.c +++ b/plugins/ctf/fs/data-stream.c @@ -176,10 +176,162 @@ static struct bt_ctf_notif_iter_medium_ops medops = { .get_stream = medop_get_stream, }; +static +int build_index_from_idx_file(struct ctf_fs_stream *stream) +{ + int ret = 0; + gchar *directory = NULL; + gchar *basename = NULL; + GString *index_basename = NULL; + gchar *index_file_path = NULL; + GMappedFile *mapped_file = NULL; + gsize filesize; + const struct ctf_packet_index_file_hdr *header; + const char *mmap_begin, *file_pos; + struct index_entry *index; + uint64_t total_packets_size = 0; + size_t file_index_entry_size; + size_t file_entry_count; + size_t i; + + /* Look for index file in relative path index/name.idx. */ + basename = g_path_get_basename(stream->file->path->str); + if (!basename) { + ret = -1; + goto end; + } + + directory = g_path_get_dirname(stream->file->path->str); + if (!directory) { + ret = -1; + goto end; + } + + index_basename = g_string_new(basename); + if (!index_basename) { + ret = -1; + goto end; + } + + g_string_append(index_basename, ".idx"); + index_file_path = g_build_filename(directory, "index", + index_basename->str, NULL); + mapped_file = g_mapped_file_new(index_file_path, FALSE, NULL); + filesize = g_mapped_file_get_length(mapped_file); + if (filesize < sizeof(*header)) { + printf_error("Invalid LTTng trace index: file size < header size"); + ret = -1; + goto end; + } + + mmap_begin = g_mapped_file_get_contents(mapped_file); + header = (struct ctf_packet_index_file_hdr *) mmap_begin; + + file_pos = g_mapped_file_get_contents(mapped_file) + sizeof(*header); + if (be32toh(header->magic) != CTF_INDEX_MAGIC) { + printf_error("Invalid LTTng trace index: \"magic\" validation failed"); + ret = -1; + goto end; + } + stream->index.version.major = be32toh(header->index_major); + stream->index.version.minor = be32toh(header->index_minor); + + file_index_entry_size = be32toh(header->packet_index_len); + file_entry_count = (filesize - sizeof(*header)) / file_index_entry_size; + if ((filesize - sizeof(*header)) % (file_entry_count * file_index_entry_size)) { + printf_error("Invalid index file size; not a multiple of index entry size"); + ret = -1; + goto end; + } + + stream->index.entries = g_array_sized_new(FALSE, TRUE, + sizeof(struct index_entry), file_entry_count); + if (!stream->index.entries) { + ret = -1; + goto end; + } + index = (struct index_entry *) stream->index.entries->data; + for (i = 0; i < file_entry_count; i++) { + struct ctf_packet_index *file_index = + (struct ctf_packet_index *) file_pos; + uint64_t packet_size = be64toh(file_index->packet_size); + + if (packet_size % CHAR_BIT) { + ret = -1; + printf_error("Invalid packet size encountered in index file"); + goto invalid_index; + } + + /* Convert size in bits to bytes. */ + packet_size /= CHAR_BIT; + index->packet_size = packet_size; + + index->offset = be64toh(file_index->offset); + if (i != 0 && index->offset < (index - 1)->offset) { + printf_error("Invalid, non-monotonic, packet offset encountered in index file"); + ret = -1; + goto invalid_index; + } + + index->timestamp_begin = be64toh(file_index->timestamp_begin); + index->timestamp_end = be64toh(file_index->timestamp_end); + if (index->timestamp_end < index->timestamp_begin) { + printf_error("Invalid packet time bounds encountered in index file"); + ret = -1; + goto invalid_index; + } + + total_packets_size += packet_size; + file_pos += file_index_entry_size; + index++; + } + + /* Validate that the index addresses the complete stream. */ + if (stream->file->size != total_packets_size) { + printf_error("Invalid index; indexed size != stream file size"); + ret = -1; + goto invalid_index; + } +end: + g_free(directory); + g_free(basename); + g_free(index_file_path); + g_string_free(index_basename, TRUE); + g_mapped_file_unref(mapped_file); + return ret; +invalid_index: + g_array_free(stream->index.entries, TRUE); + goto end; +} + +static +int build_index_from_stream(struct ctf_fs_stream *stream) +{ + int ret = 0; +end: + return ret; +} + +static +int init_stream_index(struct ctf_fs_stream *stream) +{ + int ret; + + ret = build_index_from_idx_file(stream); + if (!ret) { + goto end; + } + + ret = build_index_from_stream(stream); +end: + return ret; +} + BT_HIDDEN struct ctf_fs_stream *ctf_fs_stream_create( struct ctf_fs_component *ctf_fs, struct ctf_fs_file *file) { + int ret; struct ctf_fs_stream *stream = g_new0(struct ctf_fs_stream, 1); if (!stream) { @@ -194,6 +346,10 @@ struct ctf_fs_stream *ctf_fs_stream_create( } stream->mmap_max_len = ctf_fs->page_size * 2048; + ret = init_stream_index(stream); + if (ret) { + goto error; + } goto end; error: /* Do not touch "borrowed" file. */ @@ -223,5 +379,9 @@ void ctf_fs_stream_destroy(struct ctf_fs_stream *stream) bt_ctf_notif_iter_destroy(stream->notif_iter); } + if (stream->index.entries) { + g_array_free(stream->index.entries, TRUE); + } + g_free(stream); } diff --git a/plugins/ctf/fs/data-stream.h b/plugins/ctf/fs/data-stream.h index 6e7d0e72..11c149cc 100644 --- a/plugins/ctf/fs/data-stream.h +++ b/plugins/ctf/fs/data-stream.h @@ -29,6 +29,25 @@ #include #include "../common/notif-iter/notif-iter.h" +#include "lttng-index.h" + +struct ctf_fs_component; +struct ctf_fs_file; +struct ctf_fs_stream; + +struct index_entry { + uint64_t offset; /* in bytes. */ + uint64_t packet_size; /* in bytes. */ + /* relative to the packet context field's mapped clock. */ + uint64_t timestamp_begin, timestamp_end; +}; + +struct index { + struct { + unsigned int major, minor; + } version; + GArray *entries; /* Array of struct index_entry. */ +}; BT_HIDDEN struct ctf_fs_stream *ctf_fs_stream_create( diff --git a/plugins/ctf/fs/fs.h b/plugins/ctf/fs/fs.h index b3bd3544..80bd6d04 100644 --- a/plugins/ctf/fs/fs.h +++ b/plugins/ctf/fs/fs.h @@ -30,6 +30,7 @@ #include #include +#include "data-stream.h" #define CTF_FS_COMPONENT_NAME "fs" #define CTF_FS_COMPONENT_DESCRIPTION \ @@ -60,6 +61,8 @@ struct ctf_fs_stream { struct bt_ctf_stream *stream; /* FIXME There should be many and ctf_fs_stream should not own them. */ struct bt_ctf_notif_iter *notif_iter; + /* A stream is assumed to be indexed. */ + struct index index; void *mmap_addr; /* Max length of chunk to mmap() when updating the current mapping. */ size_t mmap_max_len; diff --git a/plugins/ctf/fs/lttng-index.h b/plugins/ctf/fs/lttng-index.h new file mode 100644 index 00000000..b26f3132 --- /dev/null +++ b/plugins/ctf/fs/lttng-index.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2013 - Julien Desfossez + * Mathieu Desnoyers + * David Goulet + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef LTTNG_INDEX_H +#define LTTNG_INDEX_H + +#include + +#define CTF_INDEX_MAGIC 0xC1F1DCC1 +#define CTF_INDEX_MAJOR 1 +#define CTF_INDEX_MINOR 1 + +/* + * Header at the beginning of each index file. + * All integer fields are stored in big endian. + */ +struct ctf_packet_index_file_hdr { + uint32_t magic; + uint32_t index_major; + uint32_t index_minor; + /* size of struct ctf_packet_index, in bytes. */ + uint32_t packet_index_len; +} __attribute__((__packed__)); + +/* + * Packet index generated for each trace packet store in a trace file. + * All integer fields are stored in big endian. + */ +struct ctf_packet_index { + uint64_t offset; /* offset of the packet in the file, in bytes */ + uint64_t packet_size; /* packet size, in bits */ + uint64_t content_size; /* content size, in bits */ + uint64_t timestamp_begin; + uint64_t timestamp_end; + uint64_t events_discarded; + uint64_t stream_id; + /* CTF_INDEX 1.0 limit */ + uint64_t stream_instance_id; /* ID of the channel instance */ + uint64_t packet_seq_num; /* packet sequence number */ +} __attribute__((__packed__)); + +#endif /* LTTNG_INDEX_H */ -- 2.34.1