2 * Copyright 2016-2017 - Philippe Proulx <pproulx@efficios.com>
3 * Copyright 2016 - Jérémie Galarneau <jeremie.galarneau@efficios.com>
4 * Copyright 2010-2011 - EfficiOS Inc. and Linux Foundation
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25 #define BT_COMP_LOG_SELF_COMP (self_comp)
26 #define BT_LOG_OUTPUT_LEVEL (log_level)
27 #define BT_LOG_TAG "PLUGIN/SRC.CTF.FS/DS"
28 #include "logging/comp-logging.h"
36 #include "compat/mman.h"
37 #include "compat/endian.h"
38 #include <babeltrace2/babeltrace.h>
39 #include "common/common.h"
42 #include "../common/msg-iter/msg-iter.h"
43 #include "common/assert.h"
44 #include "data-stream-file.h"
48 size_t remaining_mmap_bytes(struct ctf_fs_ds_file
*ds_file
)
50 BT_ASSERT_DBG(ds_file
->mmap_len
>= ds_file
->request_offset_in_mapping
);
51 return ds_file
->mmap_len
- ds_file
->request_offset_in_mapping
;
55 * Return true if `offset_in_file` is in the current mapping.
59 bool offset_ist_mapped(struct ctf_fs_ds_file
*ds_file
, off_t offset_in_file
)
61 return offset_in_file
>= ds_file
->mmap_offset_in_file
&&
62 offset_in_file
< (ds_file
->mmap_offset_in_file
+ ds_file
->mmap_len
);
66 enum ctf_msg_iter_medium_status
ds_file_munmap(
67 struct ctf_fs_ds_file
*ds_file
)
69 enum ctf_msg_iter_medium_status status
;
70 bt_self_component
*self_comp
= ds_file
->self_comp
;
71 bt_logging_level log_level
= ds_file
->log_level
;
75 if (!ds_file
->mmap_addr
) {
76 status
= CTF_MSG_ITER_MEDIUM_STATUS_OK
;
80 if (bt_munmap(ds_file
->mmap_addr
, ds_file
->mmap_len
)) {
81 BT_COMP_LOGE_ERRNO("Cannot memory-unmap file",
82 ": address=%p, size=%zu, file_path=\"%s\", file=%p",
83 ds_file
->mmap_addr
, ds_file
->mmap_len
,
84 ds_file
->file
? ds_file
->file
->path
->str
: "NULL",
85 ds_file
->file
? ds_file
->file
->fp
: NULL
);
86 status
= CTF_MSG_ITER_MEDIUM_STATUS_ERROR
;
90 ds_file
->mmap_addr
= NULL
;
92 status
= CTF_MSG_ITER_MEDIUM_STATUS_OK
;
98 * mmap a region of `ds_file` such that `requested_offset_in_file` is in the
99 * mapping. If the currently mmap-ed region already contains
100 * `requested_offset_in_file`, the mapping is kept.
102 * Set `ds_file->requested_offset_in_mapping` based on `request_offset_in_file`,
103 * such that the next call to `request_bytes` will return bytes starting at that
106 * `requested_offset_in_file` must be a valid offset in the file.
109 enum ctf_msg_iter_medium_status
ds_file_mmap(
110 struct ctf_fs_ds_file
*ds_file
, off_t requested_offset_in_file
)
112 enum ctf_msg_iter_medium_status status
;
113 bt_self_component
*self_comp
= ds_file
->self_comp
;
114 bt_logging_level log_level
= ds_file
->log_level
;
116 /* Ensure the requested offset is in the file range. */
117 BT_ASSERT(requested_offset_in_file
>= 0);
118 BT_ASSERT(requested_offset_in_file
< ds_file
->file
->size
);
121 * If the mapping already contains the requested offset, just adjust
122 * requested_offset_in_mapping.
124 if (offset_ist_mapped(ds_file
, requested_offset_in_file
)) {
125 ds_file
->request_offset_in_mapping
=
126 requested_offset_in_file
- ds_file
->mmap_offset_in_file
;
127 status
= CTF_MSG_ITER_MEDIUM_STATUS_OK
;
131 /* Unmap old region */
132 status
= ds_file_munmap(ds_file
);
133 if (status
!= CTF_MSG_ITER_MEDIUM_STATUS_OK
) {
138 * Compute a mapping that has the required alignment properties and
139 * contains `requested_offset_in_file`.
141 ds_file
->request_offset_in_mapping
=
142 requested_offset_in_file
% bt_mmap_get_offset_align_size(ds_file
->log_level
);
143 ds_file
->mmap_offset_in_file
=
144 requested_offset_in_file
- ds_file
->request_offset_in_mapping
;
145 ds_file
->mmap_len
= MIN(ds_file
->file
->size
- ds_file
->mmap_offset_in_file
,
146 ds_file
->mmap_max_len
);
148 BT_ASSERT(ds_file
->mmap_len
> 0);
150 ds_file
->mmap_addr
= bt_mmap((void *) 0, ds_file
->mmap_len
,
151 PROT_READ
, MAP_PRIVATE
, fileno(ds_file
->file
->fp
),
152 ds_file
->mmap_offset_in_file
, ds_file
->log_level
);
153 if (ds_file
->mmap_addr
== MAP_FAILED
) {
154 BT_COMP_LOGE("Cannot memory-map address (size %zu) of file \"%s\" (%p) at offset %jd: %s",
155 ds_file
->mmap_len
, ds_file
->file
->path
->str
,
156 ds_file
->file
->fp
, (intmax_t) ds_file
->mmap_offset_in_file
,
158 status
= CTF_MSG_ITER_MEDIUM_STATUS_ERROR
;
162 status
= CTF_MSG_ITER_MEDIUM_STATUS_OK
;
169 * Change the mapping of the file to read the region that follows the current
172 * If the file hasn't been mapped yet, then everything (mmap_offset_in_file,
173 * mmap_len, request_offset_in_mapping) should have the value 0, which will
174 * result in the beginning of the file getting mapped.
176 * return _EOF if the current mapping is the end of the file.
180 enum ctf_msg_iter_medium_status
ds_file_mmap_next(
181 struct ctf_fs_ds_file
*ds_file
)
183 enum ctf_msg_iter_medium_status status
;
186 * If we're called, it's because more bytes are requested but we have
187 * given all the bytes of the current mapping.
189 BT_ASSERT(ds_file
->request_offset_in_mapping
== ds_file
->mmap_len
);
192 * If the current mapping coincides with the end of the file, there is
195 if (ds_file
->mmap_offset_in_file
+ ds_file
->mmap_len
== ds_file
->file
->size
) {
196 status
= CTF_MSG_ITER_MEDIUM_STATUS_EOF
;
200 status
= ds_file_mmap(ds_file
,
201 ds_file
->mmap_offset_in_file
+ ds_file
->mmap_len
);
208 enum ctf_msg_iter_medium_status
medop_request_bytes(
209 size_t request_sz
, uint8_t **buffer_addr
,
210 size_t *buffer_sz
, void *data
)
212 enum ctf_msg_iter_medium_status status
=
213 CTF_MSG_ITER_MEDIUM_STATUS_OK
;
214 struct ctf_fs_ds_file
*ds_file
= data
;
215 bt_self_component
*self_comp
= ds_file
->self_comp
;
216 bt_logging_level log_level
= ds_file
->log_level
;
218 BT_ASSERT(request_sz
> 0);
221 * Check if we have at least one memory-mapped byte left. If we don't,
222 * mmap the next file.
224 if (remaining_mmap_bytes(ds_file
) == 0) {
225 /* Are we at the end of the file? */
226 if (ds_file
->mmap_offset_in_file
>= ds_file
->file
->size
) {
227 BT_COMP_LOGD("Reached end of file \"%s\" (%p)",
228 ds_file
->file
->path
->str
, ds_file
->file
->fp
);
229 status
= CTF_MSG_ITER_MEDIUM_STATUS_EOF
;
233 status
= ds_file_mmap_next(ds_file
);
235 case CTF_MSG_ITER_MEDIUM_STATUS_OK
:
237 case CTF_MSG_ITER_MEDIUM_STATUS_EOF
:
240 BT_COMP_LOGE("Cannot memory-map next region of file \"%s\" (%p)",
241 ds_file
->file
->path
->str
,
247 BT_ASSERT(remaining_mmap_bytes(ds_file
) > 0);
248 *buffer_sz
= MIN(remaining_mmap_bytes(ds_file
), request_sz
);
250 BT_ASSERT(ds_file
->mmap_addr
);
251 *buffer_addr
= ((uint8_t *) ds_file
->mmap_addr
) + ds_file
->request_offset_in_mapping
;
253 ds_file
->request_offset_in_mapping
+= *buffer_sz
;
257 status
= CTF_MSG_ITER_MEDIUM_STATUS_ERROR
;
264 bt_stream
*medop_borrow_stream(bt_stream_class
*stream_class
, int64_t stream_id
,
267 struct ctf_fs_ds_file
*ds_file
= data
;
268 bt_stream_class
*ds_file_stream_class
;
269 bt_stream
*stream
= NULL
;
271 ds_file_stream_class
= bt_stream_borrow_class(
274 if (stream_class
!= ds_file_stream_class
) {
276 * Not supported: two packets described by two different
277 * stream classes within the same data stream file.
282 stream
= ds_file
->stream
;
289 enum ctf_msg_iter_medium_status
medop_seek(off_t offset
, void *data
)
291 struct ctf_fs_ds_file
*ds_file
= data
;
293 BT_ASSERT(offset
>= 0);
294 BT_ASSERT(offset
< ds_file
->file
->size
);
296 return ds_file_mmap(ds_file
, offset
);
300 struct ctf_msg_iter_medium_ops ctf_fs_ds_file_medops
= {
301 .request_bytes
= medop_request_bytes
,
302 .borrow_stream
= medop_borrow_stream
,
306 struct ctf_fs_ds_group_medops_data
{
307 /* Weak, set once at creation time. */
308 struct ctf_fs_ds_file_group
*ds_file_group
;
311 * Index (as in element rank) of the index entry of ds_file_groups'
312 * index we will read next (so, the one after the one we are reading
315 guint next_index_entry_index
;
318 * File we are currently reading. Changes whenever we switch to
319 * reading another data file.
323 struct ctf_fs_ds_file
*file
;
325 /* Weak, for context / logging / appending causes. */
326 bt_self_message_iterator
*self_msg_iter
;
327 bt_logging_level log_level
;
331 enum ctf_msg_iter_medium_status
medop_group_request_bytes(
333 uint8_t **buffer_addr
,
337 struct ctf_fs_ds_group_medops_data
*data
= void_data
;
339 /* Return bytes from the current file. */
340 return medop_request_bytes(request_sz
, buffer_addr
, buffer_sz
, data
->file
);
344 bt_stream
*medop_group_borrow_stream(
345 bt_stream_class
*stream_class
,
349 struct ctf_fs_ds_group_medops_data
*data
= void_data
;
351 return medop_borrow_stream(stream_class
, stream_id
, data
->file
);
355 * Set `data->file` to prepare it to read the packet described
360 enum ctf_msg_iter_medium_status
ctf_fs_ds_group_medops_set_file(
361 struct ctf_fs_ds_group_medops_data
*data
,
362 struct ctf_fs_ds_index_entry
*index_entry
,
363 bt_self_message_iterator
*self_msg_iter
,
364 bt_logging_level log_level
)
366 enum ctf_msg_iter_medium_status status
;
369 BT_ASSERT(index_entry
);
371 /* Check if that file is already the one mapped. */
372 if (!data
->file
|| strcmp(index_entry
->path
, data
->file
->file
->path
->str
) != 0) {
373 /* Destroy the previously used file. */
374 ctf_fs_ds_file_destroy(data
->file
);
376 /* Create the new file. */
377 data
->file
= ctf_fs_ds_file_create(
378 data
->ds_file_group
->ctf_fs_trace
,
380 data
->ds_file_group
->stream
,
384 BT_MSG_ITER_LOGE_APPEND_CAUSE(self_msg_iter
,
385 "failed to create ctf_fs_ds_file.");
386 status
= CTF_MSG_ITER_MEDIUM_STATUS_ERROR
;
392 * Ensure the right portion of the file will be returned on the next
393 * request_bytes call.
395 status
= ds_file_mmap(data
->file
, index_entry
->offset
);
396 if (status
!= CTF_MSG_ITER_MEDIUM_STATUS_OK
) {
400 status
= CTF_MSG_ITER_MEDIUM_STATUS_OK
;
407 enum ctf_msg_iter_medium_status
medop_group_switch_packet(void *void_data
)
409 struct ctf_fs_ds_group_medops_data
*data
= void_data
;
410 struct ctf_fs_ds_index_entry
*index_entry
;
411 enum ctf_msg_iter_medium_status status
;
413 /* If we have gone through all index entries, we are done. */
414 if (data
->next_index_entry_index
>=
415 data
->ds_file_group
->index
->entries
->len
) {
416 status
= CTF_MSG_ITER_MEDIUM_STATUS_EOF
;
421 * Otherwise, look up the next index entry / packet and prepare it
424 index_entry
= g_ptr_array_index(
425 data
->ds_file_group
->index
->entries
,
426 data
->next_index_entry_index
);
428 status
= ctf_fs_ds_group_medops_set_file(
429 data
, index_entry
, data
->self_msg_iter
, data
->log_level
);
430 if (status
!= CTF_MSG_ITER_MEDIUM_STATUS_OK
) {
434 data
->next_index_entry_index
++;
436 status
= CTF_MSG_ITER_MEDIUM_STATUS_OK
;
442 void ctf_fs_ds_group_medops_data_destroy(
443 struct ctf_fs_ds_group_medops_data
*data
)
449 ctf_fs_ds_file_destroy(data
->file
);
457 enum ctf_msg_iter_medium_status
ctf_fs_ds_group_medops_data_create(
458 struct ctf_fs_ds_file_group
*ds_file_group
,
459 bt_self_message_iterator
*self_msg_iter
,
460 bt_logging_level log_level
,
461 struct ctf_fs_ds_group_medops_data
**out
)
463 struct ctf_fs_ds_group_medops_data
*data
;
464 enum ctf_msg_iter_medium_status status
;
466 BT_ASSERT(self_msg_iter
);
467 BT_ASSERT(ds_file_group
);
468 BT_ASSERT(ds_file_group
->index
);
469 BT_ASSERT(ds_file_group
->index
->entries
->len
> 0);
471 data
= g_new0(struct ctf_fs_ds_group_medops_data
, 1);
473 BT_MSG_ITER_LOGE_APPEND_CAUSE(self_msg_iter
,
474 "Failed to allocate a struct ctf_fs_ds_group_medops_data");
475 status
= CTF_MSG_ITER_MEDIUM_STATUS_MEMORY_ERROR
;
479 data
->ds_file_group
= ds_file_group
;
480 data
->self_msg_iter
= self_msg_iter
;
481 data
->log_level
= log_level
;
484 * No need to prepare the first file. ctf_msg_iter will call
485 * switch_packet before reading the first packet, it will be
490 status
= CTF_MSG_ITER_MEDIUM_STATUS_OK
;
494 ctf_fs_ds_group_medops_data_destroy(data
);
500 void ctf_fs_ds_group_medops_data_reset(struct ctf_fs_ds_group_medops_data
*data
)
502 data
->next_index_entry_index
= 0;
505 struct ctf_msg_iter_medium_ops ctf_fs_ds_group_medops
= {
506 .request_bytes
= medop_group_request_bytes
,
507 .borrow_stream
= medop_group_borrow_stream
,
508 .switch_packet
= medop_group_switch_packet
,
511 * We don't support seeking using this medops. It would probably be
512 * possible, but it's not needed at the moment.
518 struct ctf_fs_ds_index_entry
*ctf_fs_ds_index_entry_create(
519 bt_self_component
*self_comp
, bt_logging_level log_level
)
521 struct ctf_fs_ds_index_entry
*entry
;
523 entry
= g_new0(struct ctf_fs_ds_index_entry
, 1);
525 BT_COMP_LOGE_APPEND_CAUSE(self_comp
, "Failed to allocate a ctf_fs_ds_index_entry.");
529 entry
->packet_seq_num
= UINT64_MAX
;
536 int convert_cycles_to_ns(struct ctf_clock_class
*clock_class
,
537 uint64_t cycles
, int64_t *ns
)
539 return bt_util_clock_cycles_to_ns_from_origin(cycles
,
540 clock_class
->frequency
, clock_class
->offset_seconds
,
541 clock_class
->offset_cycles
, ns
);
545 struct ctf_fs_ds_index
*build_index_from_idx_file(
546 struct ctf_fs_ds_file
*ds_file
,
547 struct ctf_fs_ds_file_info
*file_info
,
548 struct ctf_msg_iter
*msg_iter
)
551 gchar
*directory
= NULL
;
552 gchar
*basename
= NULL
;
553 GString
*index_basename
= NULL
;
554 gchar
*index_file_path
= NULL
;
555 GMappedFile
*mapped_file
= NULL
;
557 const char *mmap_begin
= NULL
, *file_pos
= NULL
;
558 const struct ctf_packet_index_file_hdr
*header
= NULL
;
559 struct ctf_fs_ds_index
*index
= NULL
;
560 struct ctf_fs_ds_index_entry
*index_entry
= NULL
, *prev_index_entry
= NULL
;
561 uint64_t total_packets_size
= 0;
562 size_t file_index_entry_size
;
563 size_t file_entry_count
;
565 struct ctf_stream_class
*sc
;
566 struct ctf_msg_iter_packet_properties props
;
567 uint32_t version_major
, version_minor
;
568 bt_self_component
*self_comp
= ds_file
->self_comp
;
569 bt_logging_level log_level
= ds_file
->log_level
;
571 BT_COMP_LOGI("Building index from .idx file of stream file %s",
572 ds_file
->file
->path
->str
);
573 ret
= ctf_msg_iter_get_packet_properties(msg_iter
, &props
);
575 BT_COMP_LOGI_STR("Cannot read first packet's header and context fields.");
579 sc
= ctf_trace_class_borrow_stream_class_by_id(ds_file
->metadata
->tc
,
580 props
.stream_class_id
);
582 if (!sc
->default_clock_class
) {
583 BT_COMP_LOGI_STR("Cannot find stream class's default clock class.");
587 /* Look for index file in relative path index/name.idx. */
588 basename
= g_path_get_basename(ds_file
->file
->path
->str
);
590 BT_COMP_LOGE("Cannot get the basename of datastream file %s",
591 ds_file
->file
->path
->str
);
595 directory
= g_path_get_dirname(ds_file
->file
->path
->str
);
597 BT_COMP_LOGE("Cannot get dirname of datastream file %s",
598 ds_file
->file
->path
->str
);
602 index_basename
= g_string_new(basename
);
603 if (!index_basename
) {
604 BT_COMP_LOGE_STR("Cannot allocate index file basename string");
608 g_string_append(index_basename
, ".idx");
609 index_file_path
= g_build_filename(directory
, "index",
610 index_basename
->str
, NULL
);
611 mapped_file
= g_mapped_file_new(index_file_path
, FALSE
, NULL
);
613 BT_COMP_LOGD("Cannot create new mapped file %s",
619 * The g_mapped_file API limits us to 4GB files on 32-bit.
620 * Traces with such large indexes have never been seen in the wild,
621 * but this would need to be adjusted to support them.
623 filesize
= g_mapped_file_get_length(mapped_file
);
624 if (filesize
< sizeof(*header
)) {
625 BT_COMP_LOGW("Invalid LTTng trace index file: "
626 "file size (%zu bytes) < header size (%zu bytes)",
627 filesize
, sizeof(*header
));
631 mmap_begin
= g_mapped_file_get_contents(mapped_file
);
632 header
= (struct ctf_packet_index_file_hdr
*) mmap_begin
;
634 file_pos
= g_mapped_file_get_contents(mapped_file
) + sizeof(*header
);
635 if (be32toh(header
->magic
) != CTF_INDEX_MAGIC
) {
636 BT_COMP_LOGW_STR("Invalid LTTng trace index: \"magic\" field validation failed");
640 version_major
= be32toh(header
->index_major
);
641 version_minor
= be32toh(header
->index_minor
);
642 if (version_major
!= 1) {
644 "Unknown LTTng trace index version: "
645 "major=%" PRIu32
", minor=%" PRIu32
,
646 version_major
, version_minor
);
650 file_index_entry_size
= be32toh(header
->packet_index_len
);
651 if (file_index_entry_size
< CTF_INDEX_1_0_SIZE
) {
652 BT_COMP_LOGW("Invalid `packet_index_len` in LTTng trace index file (`packet_index_len` < CTF index 1.0 index entry size): "
653 "packet_index_len=%zu, CTF_INDEX_1_0_SIZE=%zu",
654 file_index_entry_size
, CTF_INDEX_1_0_SIZE
);
658 file_entry_count
= (filesize
- sizeof(*header
)) / file_index_entry_size
;
659 if ((filesize
- sizeof(*header
)) % file_index_entry_size
) {
660 BT_COMP_LOGW("Invalid LTTng trace index: the index's size after the header "
661 "(%zu bytes) is not a multiple of the index entry size "
662 "(%zu bytes)", (filesize
- sizeof(*header
)),
667 index
= ctf_fs_ds_index_create(ds_file
->log_level
, ds_file
->self_comp
);
672 for (i
= 0; i
< file_entry_count
; i
++) {
673 struct ctf_packet_index
*file_index
=
674 (struct ctf_packet_index
*) file_pos
;
675 uint64_t packet_size
= be64toh(file_index
->packet_size
);
677 if (packet_size
% CHAR_BIT
) {
678 BT_COMP_LOGW("Invalid packet size encountered in LTTng trace index file");
682 index_entry
= ctf_fs_ds_index_entry_create(
683 ds_file
->self_comp
, ds_file
->log_level
);
685 BT_COMP_LOGE_APPEND_CAUSE(ds_file
->self_comp
,
686 "Failed to create a ctf_fs_ds_index_entry.");
690 /* Set path to stream file. */
691 index_entry
->path
= file_info
->path
->str
;
693 /* Convert size in bits to bytes. */
694 packet_size
/= CHAR_BIT
;
695 index_entry
->packet_size
= packet_size
;
697 index_entry
->offset
= be64toh(file_index
->offset
);
698 if (i
!= 0 && index_entry
->offset
< prev_index_entry
->offset
) {
699 BT_COMP_LOGW("Invalid, non-monotonic, packet offset encountered in LTTng trace index file: "
700 "previous offset=%" PRIu64
", current offset=%" PRIu64
,
701 prev_index_entry
->offset
, index_entry
->offset
);
705 index_entry
->timestamp_begin
= be64toh(file_index
->timestamp_begin
);
706 index_entry
->timestamp_end
= be64toh(file_index
->timestamp_end
);
707 if (index_entry
->timestamp_end
< index_entry
->timestamp_begin
) {
708 BT_COMP_LOGW("Invalid packet time bounds encountered in LTTng trace index file (begin > end): "
709 "timestamp_begin=%" PRIu64
"timestamp_end=%" PRIu64
,
710 index_entry
->timestamp_begin
,
711 index_entry
->timestamp_end
);
715 /* Convert the packet's bound to nanoseconds since Epoch. */
716 ret
= convert_cycles_to_ns(sc
->default_clock_class
,
717 index_entry
->timestamp_begin
,
718 &index_entry
->timestamp_begin_ns
);
720 BT_COMP_LOGI_STR("Failed to convert raw timestamp to nanoseconds since Epoch during index parsing");
723 ret
= convert_cycles_to_ns(sc
->default_clock_class
,
724 index_entry
->timestamp_end
,
725 &index_entry
->timestamp_end_ns
);
727 BT_COMP_LOGI_STR("Failed to convert raw timestamp to nanoseconds since Epoch during LTTng trace index parsing");
731 if (version_minor
>= 1) {
732 index_entry
->packet_seq_num
= be64toh(file_index
->packet_seq_num
);
735 total_packets_size
+= packet_size
;
736 file_pos
+= file_index_entry_size
;
738 prev_index_entry
= index_entry
;
740 /* Give ownership of `index_entry` to `index->entries`. */
741 g_ptr_array_add(index
->entries
, index_entry
);
745 /* Validate that the index addresses the complete stream. */
746 if (ds_file
->file
->size
!= total_packets_size
) {
747 BT_COMP_LOGW("Invalid LTTng trace index file; indexed size != stream file size: "
748 "file-size=%" PRIu64
", total-packets-size=%" PRIu64
,
749 ds_file
->file
->size
, total_packets_size
);
755 g_free(index_file_path
);
756 if (index_basename
) {
757 g_string_free(index_basename
, TRUE
);
760 g_mapped_file_unref(mapped_file
);
764 ctf_fs_ds_index_destroy(index
);
771 int init_index_entry(struct ctf_fs_ds_index_entry
*entry
,
772 struct ctf_fs_ds_file
*ds_file
,
773 struct ctf_msg_iter_packet_properties
*props
,
774 off_t packet_size
, off_t packet_offset
)
777 struct ctf_stream_class
*sc
;
778 bt_self_component
*self_comp
= ds_file
->self_comp
;
779 bt_logging_level log_level
= ds_file
->log_level
;
781 sc
= ctf_trace_class_borrow_stream_class_by_id(ds_file
->metadata
->tc
,
782 props
->stream_class_id
);
784 BT_ASSERT(packet_offset
>= 0);
785 entry
->offset
= packet_offset
;
786 BT_ASSERT(packet_size
>= 0);
787 entry
->packet_size
= packet_size
;
789 if (props
->snapshots
.beginning_clock
!= UINT64_C(-1)) {
790 entry
->timestamp_begin
= props
->snapshots
.beginning_clock
;
792 /* Convert the packet's bound to nanoseconds since Epoch. */
793 ret
= convert_cycles_to_ns(sc
->default_clock_class
,
794 props
->snapshots
.beginning_clock
,
795 &entry
->timestamp_begin_ns
);
797 BT_COMP_LOGI_STR("Failed to convert raw timestamp to nanoseconds since Epoch.");
801 entry
->timestamp_begin
= UINT64_C(-1);
802 entry
->timestamp_begin_ns
= UINT64_C(-1);
805 if (props
->snapshots
.end_clock
!= UINT64_C(-1)) {
806 entry
->timestamp_end
= props
->snapshots
.end_clock
;
808 /* Convert the packet's bound to nanoseconds since Epoch. */
809 ret
= convert_cycles_to_ns(sc
->default_clock_class
,
810 props
->snapshots
.end_clock
,
811 &entry
->timestamp_end_ns
);
813 BT_COMP_LOGI_STR("Failed to convert raw timestamp to nanoseconds since Epoch.");
817 entry
->timestamp_end
= UINT64_C(-1);
818 entry
->timestamp_end_ns
= UINT64_C(-1);
826 struct ctf_fs_ds_index
*build_index_from_stream_file(
827 struct ctf_fs_ds_file
*ds_file
,
828 struct ctf_fs_ds_file_info
*file_info
,
829 struct ctf_msg_iter
*msg_iter
)
832 struct ctf_fs_ds_index
*index
= NULL
;
833 enum ctf_msg_iter_status iter_status
= CTF_MSG_ITER_STATUS_OK
;
834 off_t current_packet_offset_bytes
= 0;
835 bt_self_component
*self_comp
= ds_file
->self_comp
;
836 bt_logging_level log_level
= ds_file
->log_level
;
838 BT_COMP_LOGI("Indexing stream file %s", ds_file
->file
->path
->str
);
840 index
= ctf_fs_ds_index_create(ds_file
->log_level
, ds_file
->self_comp
);
846 off_t current_packet_size_bytes
;
847 struct ctf_fs_ds_index_entry
*index_entry
;
848 struct ctf_msg_iter_packet_properties props
;
850 if (current_packet_offset_bytes
< 0) {
851 BT_COMP_LOGE_STR("Cannot get the current packet's offset.");
853 } else if (current_packet_offset_bytes
> ds_file
->file
->size
) {
854 BT_COMP_LOGE_STR("Unexpected current packet's offset (larger than file).");
856 } else if (current_packet_offset_bytes
== ds_file
->file
->size
) {
861 iter_status
= ctf_msg_iter_seek(msg_iter
,
862 current_packet_offset_bytes
);
863 if (iter_status
!= CTF_MSG_ITER_STATUS_OK
) {
867 iter_status
= ctf_msg_iter_get_packet_properties(
869 if (iter_status
!= CTF_MSG_ITER_STATUS_OK
) {
873 if (props
.exp_packet_total_size
>= 0) {
874 current_packet_size_bytes
=
875 (uint64_t) props
.exp_packet_total_size
/ 8;
877 current_packet_size_bytes
= ds_file
->file
->size
;
880 if (current_packet_offset_bytes
+ current_packet_size_bytes
>
881 ds_file
->file
->size
) {
882 BT_COMP_LOGW("Invalid packet size reported in file: stream=\"%s\", "
883 "packet-offset=%jd, packet-size-bytes=%jd, "
885 ds_file
->file
->path
->str
,
886 (intmax_t) current_packet_offset_bytes
,
887 (intmax_t) current_packet_size_bytes
,
888 (intmax_t) ds_file
->file
->size
);
892 index_entry
= ctf_fs_ds_index_entry_create(
893 ds_file
->self_comp
, ds_file
->log_level
);
895 BT_COMP_LOGE_APPEND_CAUSE(ds_file
->self_comp
,
896 "Failed to create a ctf_fs_ds_index_entry.");
900 /* Set path to stream file. */
901 index_entry
->path
= file_info
->path
->str
;
903 ret
= init_index_entry(index_entry
, ds_file
, &props
,
904 current_packet_size_bytes
, current_packet_offset_bytes
);
910 g_ptr_array_add(index
->entries
, index_entry
);
912 current_packet_offset_bytes
+= current_packet_size_bytes
;
913 BT_COMP_LOGD("Seeking to next packet: current-packet-offset=%jd, "
914 "next-packet-offset=%jd",
915 (intmax_t) (current_packet_offset_bytes
- current_packet_size_bytes
),
916 (intmax_t) current_packet_offset_bytes
);
923 ctf_fs_ds_index_destroy(index
);
929 struct ctf_fs_ds_file
*ctf_fs_ds_file_create(
930 struct ctf_fs_trace
*ctf_fs_trace
,
931 bt_self_message_iterator
*self_msg_iter
,
932 bt_stream
*stream
, const char *path
,
933 bt_logging_level log_level
)
936 const size_t offset_align
= bt_mmap_get_offset_align_size(log_level
);
937 struct ctf_fs_ds_file
*ds_file
= g_new0(struct ctf_fs_ds_file
, 1);
943 ds_file
->log_level
= log_level
;
944 ds_file
->self_comp
= ctf_fs_trace
->self_comp
;
945 ds_file
->self_msg_iter
= self_msg_iter
;
946 ds_file
->file
= ctf_fs_file_create(log_level
, ds_file
->self_comp
);
947 if (!ds_file
->file
) {
951 ds_file
->stream
= stream
;
952 bt_stream_get_ref(ds_file
->stream
);
953 ds_file
->metadata
= ctf_fs_trace
->metadata
;
954 g_string_assign(ds_file
->file
->path
, path
);
955 ret
= ctf_fs_file_open(ds_file
->file
, "rb");
960 ds_file
->mmap_max_len
= offset_align
* 2048;
965 /* Do not touch "borrowed" file. */
966 ctf_fs_ds_file_destroy(ds_file
);
974 struct ctf_fs_ds_index
*ctf_fs_ds_file_build_index(
975 struct ctf_fs_ds_file
*ds_file
,
976 struct ctf_fs_ds_file_info
*file_info
,
977 struct ctf_msg_iter
*msg_iter
)
979 struct ctf_fs_ds_index
*index
;
980 bt_self_component
*self_comp
= ds_file
->self_comp
;
981 bt_logging_level log_level
= ds_file
->log_level
;
983 index
= build_index_from_idx_file(ds_file
, file_info
, msg_iter
);
988 BT_COMP_LOGI("Failed to build index from .index file; "
989 "falling back to stream indexing.");
990 index
= build_index_from_stream_file(ds_file
, file_info
, msg_iter
);
996 struct ctf_fs_ds_index
*ctf_fs_ds_index_create(bt_logging_level log_level
,
997 bt_self_component
*self_comp
)
999 struct ctf_fs_ds_index
*index
= g_new0(struct ctf_fs_ds_index
, 1);
1002 BT_COMP_LOG_CUR_LVL(BT_LOG_ERROR
, log_level
, self_comp
,
1003 "Failed to allocate index");
1007 index
->entries
= g_ptr_array_new_with_free_func((GDestroyNotify
) g_free
);
1008 if (!index
->entries
) {
1009 BT_COMP_LOG_CUR_LVL(BT_LOG_ERROR
, log_level
, self_comp
,
1010 "Failed to allocate index entries.");
1017 ctf_fs_ds_index_destroy(index
);
1024 void ctf_fs_ds_file_destroy(struct ctf_fs_ds_file
*ds_file
)
1030 bt_stream_put_ref(ds_file
->stream
);
1031 (void) ds_file_munmap(ds_file
);
1033 if (ds_file
->file
) {
1034 ctf_fs_file_destroy(ds_file
->file
);
1041 void ctf_fs_ds_index_destroy(struct ctf_fs_ds_index
*index
)
1047 if (index
->entries
) {
1048 g_ptr_array_free(index
->entries
, TRUE
);