plugins/ctf/fs-src/fs.c

   1 /*
   2  * fs.c
   3  *
   4  * Babeltrace CTF file system Reader Component
   5  *
   6  * Copyright 2015-2017 Philippe Proulx <pproulx@efficios.com>
   7  * Copyright 2016 Jérémie Galarneau <jeremie.galarneau@efficios.com>
   8  *
   9  * Permission is hereby granted, free of charge, to any person obtaining a copy
  10  * of this software and associated documentation files (the "Software"), to deal
  11  * in the Software without restriction, including without limitation the rights
  12  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  13  * copies of the Software, and to permit persons to whom the Software is
  14  * furnished to do so, subject to the following conditions:
  15  *
  16  * The above copyright notice and this permission notice shall be included in
  17  * all copies or substantial portions of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  22  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  24  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  25  * SOFTWARE.
  26  */
  27
  28 #include <babeltrace/common-internal.h>
  29 #include <babeltrace/ctf-ir/packet.h>
  30 #include <babeltrace/ctf-ir/clock-class.h>
  31 #include <babeltrace/ctf-ir/stream.h>
  32 #include <babeltrace/ctf-ir/fields.h>
  33 #include <babeltrace/graph/private-port.h>
  34 #include <babeltrace/graph/private-component.h>
  35 #include <babeltrace/graph/private-component-source.h>
  36 #include <babeltrace/graph/private-notification-iterator.h>
  37 #include <babeltrace/graph/component.h>
  38 #include <babeltrace/graph/notification-iterator.h>
  39 #include <babeltrace/graph/clock-class-priority-map.h>
  40 #include <plugins-common.h>
  41 #include <glib.h>
  42 #include <assert.h>
  43 #include <inttypes.h>
  44 #include <stdbool.h>
  45 #include "fs.h"
  46 #include "metadata.h"
  47 #include "data-stream-file.h"
  48 #include "file.h"
  49 #include "../common/metadata/decoder.h"
  50 #include "query.h"
  51
  52 #define BT_LOG_TAG "PLUGIN-CTF-FS-SRC"
  53 #include "logging.h"
  54
  55 BT_HIDDEN
  56 bool ctf_fs_debug;
  57
  58 static
  59 int notif_iter_data_set_current_ds_file(struct ctf_fs_notif_iter_data *notif_iter_data)
  60 {
  61         struct ctf_fs_ds_file_info *ds_file_info;
  62         int ret = 0;
  63
  64         assert(notif_iter_data->ds_file_info_index <
  65                 notif_iter_data->ds_file_group->ds_file_infos->len);
  66         ds_file_info = g_ptr_array_index(
  67                 notif_iter_data->ds_file_group->ds_file_infos,
  68                 notif_iter_data->ds_file_info_index);
  69
  70         ctf_fs_ds_file_destroy(notif_iter_data->ds_file);
  71         notif_iter_data->ds_file = ctf_fs_ds_file_create(
  72                 notif_iter_data->ds_file_group->ctf_fs_trace,
  73                 notif_iter_data->ds_file_group->stream,
  74                 ds_file_info->path->str, true);
  75         if (!notif_iter_data->ds_file) {
  76                 ret = -1;
  77         }
  78
  79         return ret;
  80 }
  81
  82 static
  83 void ctf_fs_notif_iter_data_destroy(
  84                 struct ctf_fs_notif_iter_data *notif_iter_data)
  85 {
  86         if (!notif_iter_data) {
  87                 return;
  88         }
  89
  90         ctf_fs_ds_file_destroy(notif_iter_data->ds_file);
  91         g_free(notif_iter_data);
  92 }
  93
  94 struct bt_notification_iterator_next_return ctf_fs_iterator_next(
  95                 struct bt_private_notification_iterator *iterator)
  96 {
  97         struct bt_notification_iterator_next_return next_ret;
  98         struct ctf_fs_notif_iter_data *notif_iter_data =
  99                 bt_private_notification_iterator_get_user_data(iterator);
 100         int ret;
 101
 102         assert(notif_iter_data->ds_file);
 103         next_ret = ctf_fs_ds_file_next(notif_iter_data->ds_file);
 104         if (next_ret.status == BT_NOTIFICATION_ITERATOR_STATUS_END) {
 105                 assert(!next_ret.notification);
 106                 notif_iter_data->ds_file_info_index++;
 107
 108                 if (notif_iter_data->ds_file_info_index ==
 109                                 notif_iter_data->ds_file_group->ds_file_infos->len) {
 110                         /*
 111                          * No more stream files to read: we reached the
 112                          * real end.
 113                          */
 114                         goto end;
 115                 }
 116
 117                 /*
 118                  * Open and start reading the next stream file within
 119                  * our stream file group.
 120                  */
 121                 ret = notif_iter_data_set_current_ds_file(notif_iter_data);
 122                 if (ret) {
 123                         next_ret.status = BT_NOTIFICATION_ITERATOR_STATUS_ERROR;
 124                         goto end;
 125                 }
 126
 127                 next_ret = ctf_fs_ds_file_next(notif_iter_data->ds_file);
 128
 129                 /*
 130                  * We should not get BT_NOTIFICATION_ITERATOR_STATUS_END
 131                  * with a brand new stream file because empty stream
 132                  * files are not even part of stream file groups, which
 133                  * means we're sure to get at least one pair of "packet
 134                  * begin" and "packet end" notifications in the case of
 135                  * a single, empty packet.
 136                  */
 137                 assert(next_ret.status != BT_NOTIFICATION_ITERATOR_STATUS_END);
 138         }
 139
 140 end:
 141         return next_ret;
 142 }
 143
 144 void ctf_fs_iterator_finalize(struct bt_private_notification_iterator *it)
 145 {
 146         void *notif_iter_data =
 147                 bt_private_notification_iterator_get_user_data(it);
 148
 149         ctf_fs_notif_iter_data_destroy(notif_iter_data);
 150 }
 151
 152 enum bt_notification_iterator_status ctf_fs_iterator_init(
 153                 struct bt_private_notification_iterator *it,
 154                 struct bt_private_port *port)
 155 {
 156         struct ctf_fs_port_data *port_data;
 157         struct ctf_fs_notif_iter_data *notif_iter_data = NULL;
 158         enum bt_notification_iterator_status ret =
 159                 BT_NOTIFICATION_ITERATOR_STATUS_OK;
 160         int iret;
 161
 162         port_data = bt_private_port_get_user_data(port);
 163         if (!port_data) {
 164                 ret = BT_NOTIFICATION_ITERATOR_STATUS_INVALID;
 165                 goto error;
 166         }
 167
 168         notif_iter_data = g_new0(struct ctf_fs_notif_iter_data, 1);
 169         if (!notif_iter_data) {
 170                 ret = BT_NOTIFICATION_ITERATOR_STATUS_NOMEM;
 171                 goto error;
 172         }
 173
 174         notif_iter_data->ds_file_group = port_data->ds_file_group;
 175         iret = notif_iter_data_set_current_ds_file(notif_iter_data);
 176         if (iret) {
 177                 ret = BT_NOTIFICATION_ITERATOR_STATUS_ERROR;
 178                 goto error;
 179         }
 180
 181         ret = bt_private_notification_iterator_set_user_data(it, notif_iter_data);
 182         if (ret) {
 183                 goto error;
 184         }
 185
 186         notif_iter_data = NULL;
 187         goto end;
 188
 189 error:
 190         (void) bt_private_notification_iterator_set_user_data(it, NULL);
 191
 192         if (ret == BT_NOTIFICATION_ITERATOR_STATUS_OK) {
 193                 ret = BT_NOTIFICATION_ITERATOR_STATUS_ERROR;
 194         }
 195
 196 end:
 197         ctf_fs_notif_iter_data_destroy(notif_iter_data);
 198         return ret;
 199 }
 200
 201 static
 202 void ctf_fs_destroy(struct ctf_fs_component *ctf_fs)
 203 {
 204         if (!ctf_fs) {
 205                 return;
 206         }
 207
 208         if (ctf_fs->traces) {
 209                 g_ptr_array_free(ctf_fs->traces, TRUE);
 210         }
 211
 212         if (ctf_fs->port_data) {
 213                 g_ptr_array_free(ctf_fs->port_data, TRUE);
 214         }
 215
 216         g_free(ctf_fs);
 217 }
 218
 219 static
 220 void ctf_fs_trace_destroy(void *data)
 221 {
 222         struct ctf_fs_trace *ctf_fs_trace = data;
 223
 224         if (!ctf_fs_trace) {
 225                 return;
 226         }
 227
 228         if (ctf_fs_trace->ds_file_groups) {
 229                 g_ptr_array_free(ctf_fs_trace->ds_file_groups, TRUE);
 230         }
 231
 232         if (ctf_fs_trace->path) {
 233                 g_string_free(ctf_fs_trace->path, TRUE);
 234         }
 235
 236         if (ctf_fs_trace->name) {
 237                 g_string_free(ctf_fs_trace->name, TRUE);
 238         }
 239
 240         if (ctf_fs_trace->metadata) {
 241                 ctf_fs_metadata_fini(ctf_fs_trace->metadata);
 242                 g_free(ctf_fs_trace->metadata);
 243         }
 244
 245         bt_put(ctf_fs_trace->cc_prio_map);
 246         g_free(ctf_fs_trace);
 247 }
 248
 249 void ctf_fs_finalize(struct bt_private_component *component)
 250 {
 251         void *data = bt_private_component_get_user_data(component);
 252
 253         ctf_fs_destroy(data);
 254 }
 255
 256 static
 257 void port_data_destroy(void *data) {
 258         struct ctf_fs_port_data *port_data = data;
 259
 260         if (!port_data) {
 261                 return;
 262         }
 263
 264         g_free(port_data);
 265 }
 266
 267 static
 268 int create_one_port_for_trace(struct ctf_fs_component *ctf_fs,
 269                 struct ctf_fs_trace *ctf_fs_trace,
 270                 struct ctf_fs_ds_file_group *ds_file_group)
 271 {
 272         int ret = 0;
 273         struct ctf_fs_port_data *port_data = NULL;
 274         GString *port_name = NULL;
 275         struct ctf_fs_ds_file_info *ds_file_info =
 276                 g_ptr_array_index(ds_file_group->ds_file_infos, 0);
 277
 278         port_name = g_string_new(NULL);
 279         if (!port_name) {
 280                 goto error;
 281         }
 282
 283         /*
 284          * Assign the name for the new output port. If there's more than
 285          * one stream file in the stream file group, the first
 286          * (earliest) stream file's path is used.
 287          */
 288         assert(ds_file_group->ds_file_infos->len > 0);
 289         ds_file_info = g_ptr_array_index(ds_file_group->ds_file_infos, 0);
 290         g_string_assign(port_name, ds_file_info->path->str);
 291         BT_LOGD("Creating one port named `%s`", port_name->str);
 292
 293         /* Create output port for this file */
 294         port_data = g_new0(struct ctf_fs_port_data, 1);
 295         if (!port_data) {
 296                 goto error;
 297         }
 298
 299         port_data->ds_file_group = ds_file_group;
 300         ret = bt_private_component_source_add_output_private_port(
 301                 ctf_fs->priv_comp, port_name->str, port_data, NULL);
 302         if (ret) {
 303                 goto error;
 304         }
 305
 306         g_ptr_array_add(ctf_fs->port_data, port_data);
 307         port_data = NULL;
 308         goto end;
 309
 310 error:
 311         ret = -1;
 312
 313 end:
 314         if (port_name) {
 315                 g_string_free(port_name, TRUE);
 316         }
 317
 318         port_data_destroy(port_data);
 319         return ret;
 320 }
 321
 322 static
 323 int create_ports_for_trace(struct ctf_fs_component *ctf_fs,
 324                 struct ctf_fs_trace *ctf_fs_trace)
 325 {
 326         int ret = 0;
 327         size_t i;
 328
 329         /* Create one output port for each stream file group */
 330         for (i = 0; i < ctf_fs_trace->ds_file_groups->len; i++) {
 331                 struct ctf_fs_ds_file_group *ds_file_group =
 332                         g_ptr_array_index(ctf_fs_trace->ds_file_groups, i);
 333
 334                 ret = create_one_port_for_trace(ctf_fs, ctf_fs_trace,
 335                         ds_file_group);
 336                 if (ret) {
 337                         BT_LOGE("Cannot create output port.");
 338                         goto end;
 339                 }
 340         }
 341
 342 end:
 343         return ret;
 344 }
 345
 346 uint64_t get_packet_header_stream_instance_id(struct ctf_fs_trace *ctf_fs_trace,
 347                 struct bt_ctf_field *packet_header_field)
 348 {
 349         struct bt_ctf_field *stream_instance_id_field = NULL;
 350         uint64_t stream_instance_id = -1ULL;
 351         int ret;
 352
 353         if (!packet_header_field) {
 354                 goto end;
 355         }
 356
 357         stream_instance_id_field = bt_ctf_field_structure_get_field_by_name(
 358                 packet_header_field, "stream_instance_id");
 359         if (!stream_instance_id_field) {
 360                 goto end;
 361         }
 362
 363         ret = bt_ctf_field_unsigned_integer_get_value(stream_instance_id_field,
 364                 &stream_instance_id);
 365         if (ret) {
 366                 stream_instance_id = -1ULL;
 367                 goto end;
 368         }
 369
 370 end:
 371         bt_put(stream_instance_id_field);
 372         return stream_instance_id;
 373 }
 374
 375 struct bt_ctf_stream_class *stream_class_from_packet_header(
 376                 struct ctf_fs_trace *ctf_fs_trace,
 377                 struct bt_ctf_field *packet_header_field)
 378 {
 379         struct bt_ctf_field *stream_id_field = NULL;
 380         struct bt_ctf_stream_class *stream_class = NULL;
 381         uint64_t stream_id = -1ULL;
 382         int ret;
 383
 384         if (!packet_header_field) {
 385                 goto single_stream_class;
 386         }
 387
 388         stream_id_field = bt_ctf_field_structure_get_field_by_name(
 389                 packet_header_field, "stream_id");
 390         if (!stream_id_field) {
 391                 goto end;
 392         }
 393
 394         ret = bt_ctf_field_unsigned_integer_get_value(stream_id_field,
 395                 &stream_id);
 396         if (ret) {
 397                 stream_id = -1ULL;
 398         }
 399
 400         if (stream_id == -1ULL) {
 401 single_stream_class:
 402                 /* Single stream class */
 403                 if (bt_ctf_trace_get_stream_class_count(
 404                                 ctf_fs_trace->metadata->trace) == 0) {
 405                         goto end;
 406                 }
 407
 408                 stream_class = bt_ctf_trace_get_stream_class_by_index(
 409                         ctf_fs_trace->metadata->trace, 0);
 410         } else {
 411                 stream_class = bt_ctf_trace_get_stream_class_by_id(
 412                         ctf_fs_trace->metadata->trace, stream_id);
 413         }
 414
 415 end:
 416         bt_put(stream_id_field);
 417         return stream_class;
 418 }
 419
 420 uint64_t get_packet_context_timestamp_begin_ns(
 421                 struct ctf_fs_trace *ctf_fs_trace,
 422                 struct bt_ctf_field *packet_context_field)
 423 {
 424         int ret;
 425         struct bt_ctf_field *timestamp_begin_field = NULL;
 426         struct bt_ctf_field_type *timestamp_begin_ft = NULL;
 427         uint64_t timestamp_begin_raw_value = -1ULL;
 428         uint64_t timestamp_begin_ns = -1ULL;
 429         int64_t timestamp_begin_ns_signed;
 430         struct bt_ctf_clock_class *timestamp_begin_clock_class = NULL;
 431         struct bt_ctf_clock_value *clock_value = NULL;
 432
 433         if (!packet_context_field) {
 434                 goto end;
 435         }
 436
 437         timestamp_begin_field = bt_ctf_field_structure_get_field_by_name(
 438                 packet_context_field, "timestamp_begin");
 439         if (!timestamp_begin_field) {
 440                 goto end;
 441         }
 442
 443         timestamp_begin_ft = bt_ctf_field_get_type(timestamp_begin_field);
 444         assert(timestamp_begin_ft);
 445         timestamp_begin_clock_class =
 446                 bt_ctf_field_type_integer_get_mapped_clock_class(timestamp_begin_ft);
 447         if (!timestamp_begin_clock_class) {
 448                 goto end;
 449         }
 450
 451         ret = bt_ctf_field_unsigned_integer_get_value(timestamp_begin_field,
 452                 &timestamp_begin_raw_value);
 453         if (ret) {
 454                 goto end;
 455         }
 456
 457         clock_value = bt_ctf_clock_value_create(timestamp_begin_clock_class,
 458                 timestamp_begin_raw_value);
 459         if (!clock_value) {
 460                 goto end;
 461         }
 462
 463         ret = bt_ctf_clock_value_get_value_ns_from_epoch(clock_value,
 464                 &timestamp_begin_ns_signed);
 465         if (ret) {
 466                 goto end;
 467         }
 468
 469         timestamp_begin_ns = (uint64_t) timestamp_begin_ns_signed;
 470
 471 end:
 472         bt_put(timestamp_begin_field);
 473         bt_put(timestamp_begin_ft);
 474         bt_put(timestamp_begin_clock_class);
 475         bt_put(clock_value);
 476         return timestamp_begin_ns;
 477 }
 478
 479 static
 480 void ctf_fs_ds_file_info_destroy(struct ctf_fs_ds_file_info *ds_file_info)
 481 {
 482         if (!ds_file_info) {
 483                 return;
 484         }
 485
 486         if (ds_file_info->path) {
 487                 g_string_free(ds_file_info->path, TRUE);
 488         }
 489
 490         g_free(ds_file_info);
 491 }
 492
 493 static
 494 struct ctf_fs_ds_file_info *ctf_fs_ds_file_info_create(const char *path,
 495                 uint64_t begin_ns)
 496 {
 497         struct ctf_fs_ds_file_info *ds_file_info;
 498
 499         ds_file_info = g_new0(struct ctf_fs_ds_file_info, 1);
 500         if (!ds_file_info) {
 501                 goto end;
 502         }
 503
 504         ds_file_info->path = g_string_new(path);
 505         if (!ds_file_info->path) {
 506                 ctf_fs_ds_file_info_destroy(ds_file_info);
 507                 ds_file_info = NULL;
 508                 goto end;
 509         }
 510
 511         ds_file_info->begin_ns = begin_ns;
 512
 513 end:
 514         return ds_file_info;
 515 }
 516
 517 static
 518 void ctf_fs_ds_file_group_destroy(struct ctf_fs_ds_file_group *ds_file_group)
 519 {
 520         if (!ds_file_group) {
 521                 return;
 522         }
 523
 524         if (ds_file_group->ds_file_infos) {
 525                 g_ptr_array_free(ds_file_group->ds_file_infos, TRUE);
 526         }
 527
 528         bt_put(ds_file_group->stream);
 529         g_free(ds_file_group);
 530 }
 531
 532 static
 533 struct ctf_fs_ds_file_group *ctf_fs_ds_file_group_create(
 534                 struct ctf_fs_trace *ctf_fs_trace,
 535                 struct bt_ctf_stream_class *stream_class,
 536                 uint64_t stream_instance_id)
 537 {
 538         struct ctf_fs_ds_file_group *ds_file_group;
 539
 540         assert(stream_class);
 541         ds_file_group = g_new0(struct ctf_fs_ds_file_group, 1);
 542         if (!ds_file_group) {
 543                 goto error;
 544         }
 545
 546         ds_file_group->ds_file_infos = g_ptr_array_new_with_free_func(
 547                 (GDestroyNotify) ctf_fs_ds_file_info_destroy);
 548         if (!ds_file_group->ds_file_infos) {
 549                 goto error;
 550         }
 551
 552         if (stream_instance_id == -1ULL) {
 553                 ds_file_group->stream = bt_ctf_stream_create(
 554                         stream_class, NULL);
 555         } else {
 556                 ds_file_group->stream = bt_ctf_stream_create_with_id(
 557                         stream_class, NULL, stream_instance_id);
 558         }
 559
 560         if (!ds_file_group->stream) {
 561                 goto error;
 562         }
 563
 564         ds_file_group->ctf_fs_trace = ctf_fs_trace;
 565
 566         goto end;
 567
 568 error:
 569         ctf_fs_ds_file_group_destroy(ds_file_group);
 570         ds_file_group = NULL;
 571
 572 end:
 573         return ds_file_group;
 574 }
 575
 576 static
 577 int ctf_fs_ds_file_group_add_ds_file_info(
 578                 struct ctf_fs_ds_file_group *ds_file_group,
 579                 const char *path, uint64_t begin_ns)
 580 {
 581         struct ctf_fs_ds_file_info *ds_file_info;
 582         gint i = 0;
 583         int ret = 0;
 584
 585         ds_file_info = ctf_fs_ds_file_info_create(path, begin_ns);
 586         if (!ds_file_info) {
 587                 goto error;
 588         }
 589
 590         /* Find a spot to insert this one */
 591         for (i = 0; i < ds_file_group->ds_file_infos->len; i++) {
 592                 struct ctf_fs_ds_file_info *other_ds_file_info =
 593                         g_ptr_array_index(ds_file_group->ds_file_infos, i);
 594
 595                 if (begin_ns < other_ds_file_info->begin_ns) {
 596                         break;
 597                 }
 598         }
 599
 600         if (i == ds_file_group->ds_file_infos->len) {
 601                 /* Append instead */
 602                 i = -1;
 603         }
 604
 605         g_ptr_array_insert(ds_file_group->ds_file_infos, i, ds_file_info);
 606         ds_file_info = NULL;
 607         goto end;
 608
 609 error:
 610         ctf_fs_ds_file_info_destroy(ds_file_info);
 611         ret = -1;
 612
 613 end:
 614         return ret;
 615 }
 616
 617 static
 618 int add_ds_file_to_ds_file_group(struct ctf_fs_trace *ctf_fs_trace,
 619                 const char *path)
 620 {
 621         struct bt_ctf_field *packet_header_field = NULL;
 622         struct bt_ctf_field *packet_context_field = NULL;
 623         struct bt_ctf_stream_class *stream_class = NULL;
 624         uint64_t stream_instance_id = -1ULL;
 625         uint64_t begin_ns = -1ULL;
 626         struct ctf_fs_ds_file_group *ds_file_group = NULL;
 627         bool add_group = false;
 628         int ret;
 629         size_t i;
 630
 631         ret = ctf_fs_ds_file_get_packet_header_context_fields(
 632                 ctf_fs_trace, path, &packet_header_field,
 633                 &packet_context_field);
 634         if (ret) {
 635                 BT_LOGE("Cannot get stream file's first packet's header and context fields (`%s`).",
 636                         path);
 637                 goto error;
 638         }
 639
 640         stream_instance_id = get_packet_header_stream_instance_id(ctf_fs_trace,
 641                 packet_header_field);
 642         begin_ns = get_packet_context_timestamp_begin_ns(ctf_fs_trace,
 643                 packet_context_field);
 644         stream_class = stream_class_from_packet_header(ctf_fs_trace,
 645                 packet_header_field);
 646         if (!stream_class) {
 647                 goto error;
 648         }
 649
 650         if (begin_ns == -1ULL) {
 651                 /*
 652                  * No beggining timestamp to sort the stream files
 653                  * within a stream file group, so consider that this
 654                  * file must be the only one within its group.
 655                  */
 656                 stream_instance_id = -1ULL;
 657         }
 658
 659         if (stream_instance_id == -1ULL) {
 660                 /*
 661                  * No stream instance ID or no beginning timestamp:
 662                  * create a unique stream file group for this stream
 663                  * file because, even if there's a stream instance ID,
 664                  * there's no timestamp to order the file within its
 665                  * group.
 666                  */
 667
 668                 ds_file_group = ctf_fs_ds_file_group_create(ctf_fs_trace,
 669                         stream_class, stream_instance_id);
 670                 if (!ds_file_group) {
 671                         goto error;
 672                 }
 673
 674                 ret = ctf_fs_ds_file_group_add_ds_file_info(ds_file_group,
 675                                 path, begin_ns);
 676                 if (ret) {
 677                         goto error;
 678                 }
 679
 680                 add_group = true;
 681                 goto end;
 682         }
 683
 684         assert(stream_instance_id != -1ULL);
 685         assert(begin_ns != -1ULL);
 686
 687         /* Find an existing stream file group with this ID */
 688         for (i = 0; i < ctf_fs_trace->ds_file_groups->len; i++) {
 689                 int64_t id;
 690                 struct bt_ctf_stream_class *cand_stream_class;
 691
 692                 ds_file_group = g_ptr_array_index(
 693                         ctf_fs_trace->ds_file_groups, i);
 694                 id = bt_ctf_stream_get_id(ds_file_group->stream);
 695                 cand_stream_class = bt_ctf_stream_get_class(
 696                         ds_file_group->stream);
 697
 698                 assert(cand_stream_class);
 699                 bt_put(cand_stream_class);
 700
 701                 if (cand_stream_class == stream_class &&
 702                                 (uint64_t) id == stream_instance_id) {
 703                         break;
 704                 }
 705
 706                 ds_file_group = NULL;
 707         }
 708
 709         if (!ds_file_group) {
 710                 ds_file_group = ctf_fs_ds_file_group_create(ctf_fs_trace,
 711                         stream_class, stream_instance_id);
 712                 if (!ds_file_group) {
 713                         goto error;
 714                 }
 715
 716                 add_group = true;
 717         }
 718
 719         ret = ctf_fs_ds_file_group_add_ds_file_info(ds_file_group,
 720                         path, begin_ns);
 721         if (ret) {
 722                 goto error;
 723         }
 724
 725         goto end;
 726
 727 error:
 728         ctf_fs_ds_file_group_destroy(ds_file_group);
 729         ret = -1;
 730
 731 end:
 732         if (add_group && ds_file_group) {
 733                 g_ptr_array_add(ctf_fs_trace->ds_file_groups, ds_file_group);
 734         }
 735
 736         bt_put(packet_header_field);
 737         bt_put(packet_context_field);
 738         bt_put(stream_class);
 739         return ret;
 740 }
 741
 742 static
 743 int create_ds_file_groups(struct ctf_fs_trace *ctf_fs_trace)
 744 {
 745         int ret = 0;
 746         const char *basename;
 747         GError *error = NULL;
 748         GDir *dir = NULL;
 749
 750         /* Check each file in the path directory, except specific ones */
 751         dir = g_dir_open(ctf_fs_trace->path->str, 0, &error);
 752         if (!dir) {
 753                 BT_LOGE("Cannot open directory `%s`: %s (code %d)",
 754                         ctf_fs_trace->path->str, error->message,
 755                         error->code);
 756                 goto error;
 757         }
 758
 759         while ((basename = g_dir_read_name(dir))) {
 760                 struct ctf_fs_file *file;
 761
 762                 if (!strcmp(basename, CTF_FS_METADATA_FILENAME)) {
 763                         /* Ignore the metadata stream. */
 764                         BT_LOGD("Ignoring metadata file `%s/%s`",
 765                                 ctf_fs_trace->path->str, basename);
 766                         continue;
 767                 }
 768
 769                 if (basename[0] == '.') {
 770                         BT_LOGD("Ignoring hidden file `%s/%s`",
 771                                 ctf_fs_trace->path->str, basename);
 772                         continue;
 773                 }
 774
 775                 /* Create the file. */
 776                 file = ctf_fs_file_create();
 777                 if (!file) {
 778                         BT_LOGE("Cannot create stream file object for file `%s/%s`",
 779                                 ctf_fs_trace->path->str, basename);
 780                         goto error;
 781                 }
 782
 783                 /* Create full path string. */
 784                 g_string_append_printf(file->path, "%s/%s",
 785                                 ctf_fs_trace->path->str, basename);
 786                 if (!g_file_test(file->path->str, G_FILE_TEST_IS_REGULAR)) {
 787                         BT_LOGD("Ignoring non-regular file `%s`",
 788                                 file->path->str);
 789                         ctf_fs_file_destroy(file);
 790                         file = NULL;
 791                         continue;
 792                 }
 793
 794                 ret = ctf_fs_file_open(file, "rb");
 795                 if (ret) {
 796                         BT_LOGE("Cannot open stream file `%s`", file->path->str);
 797                         goto error;
 798                 }
 799
 800                 if (file->size == 0) {
 801                         /* Skip empty stream. */
 802                         BT_LOGD("Ignoring empty file `%s`", file->path->str);
 803                         ctf_fs_file_destroy(file);
 804                         continue;
 805                 }
 806
 807                 ret = add_ds_file_to_ds_file_group(ctf_fs_trace,
 808                         file->path->str);
 809                 if (ret) {
 810                         BT_LOGD("Cannot add stream file `%s` to stream file group",
 811                                 file->path->str);
 812                         ctf_fs_file_destroy(file);
 813                         goto error;
 814                 }
 815
 816                 ctf_fs_file_destroy(file);
 817         }
 818
 819         goto end;
 820
 821 error:
 822         ret = -1;
 823
 824 end:
 825         if (dir) {
 826                 g_dir_close(dir);
 827                 dir = NULL;
 828         }
 829
 830         if (error) {
 831                 g_error_free(error);
 832         }
 833
 834         return ret;
 835 }
 836
 837 static
 838 int create_cc_prio_map(struct ctf_fs_trace *ctf_fs_trace)
 839 {
 840         int ret = 0;
 841         size_t i;
 842         int count;
 843
 844         assert(ctf_fs_trace);
 845         ctf_fs_trace->cc_prio_map = bt_clock_class_priority_map_create();
 846         if (!ctf_fs_trace->cc_prio_map) {
 847                 ret = -1;
 848                 goto end;
 849         }
 850
 851         count = bt_ctf_trace_get_clock_class_count(
 852                 ctf_fs_trace->metadata->trace);
 853         assert(count >= 0);
 854
 855         for (i = 0; i < count; i++) {
 856                 struct bt_ctf_clock_class *clock_class =
 857                         bt_ctf_trace_get_clock_class_by_index(
 858                                 ctf_fs_trace->metadata->trace, i);
 859
 860                 assert(clock_class);
 861                 ret = bt_clock_class_priority_map_add_clock_class(
 862                         ctf_fs_trace->cc_prio_map, clock_class, 0);
 863                 BT_PUT(clock_class);
 864
 865                 if (ret) {
 866                         goto end;
 867                 }
 868         }
 869
 870 end:
 871         return ret;
 872 }
 873
 874 static
 875 struct ctf_fs_trace *ctf_fs_trace_create(const char *path, const char *name,
 876                 struct metadata_overrides *overrides)
 877 {
 878         struct ctf_fs_trace *ctf_fs_trace;
 879         int ret;
 880
 881         ctf_fs_trace = g_new0(struct ctf_fs_trace, 1);
 882         if (!ctf_fs_trace) {
 883                 goto end;
 884         }
 885
 886         ctf_fs_trace->path = g_string_new(path);
 887         if (!ctf_fs_trace->path) {
 888                 goto error;
 889         }
 890
 891         ctf_fs_trace->name = g_string_new(name);
 892         if (!ctf_fs_trace->name) {
 893                 goto error;
 894         }
 895
 896         ctf_fs_trace->metadata = g_new0(struct ctf_fs_metadata, 1);
 897         if (!ctf_fs_trace->metadata) {
 898                 goto error;
 899         }
 900
 901         ctf_fs_trace->ds_file_groups = g_ptr_array_new_with_free_func(
 902                 (GDestroyNotify) ctf_fs_ds_file_group_destroy);
 903         if (!ctf_fs_trace->ds_file_groups) {
 904                 goto error;
 905         }
 906
 907         ret = ctf_fs_metadata_set_trace(ctf_fs_trace, overrides);
 908         if (ret) {
 909                 goto error;
 910         }
 911
 912         ret = create_ds_file_groups(ctf_fs_trace);
 913         if (ret) {
 914                 goto error;
 915         }
 916
 917         ret = create_cc_prio_map(ctf_fs_trace);
 918         if (ret) {
 919                 goto error;
 920         }
 921
 922         /*
 923          * create_ds_file_groups() created all the streams that this
 924          * trace needs. There won't be any more. Therefore it is safe to
 925          * make this trace static.
 926          */
 927         (void) bt_ctf_trace_set_is_static(ctf_fs_trace->metadata->trace);
 928
 929         goto end;
 930
 931 error:
 932         ctf_fs_trace_destroy(ctf_fs_trace);
 933         ctf_fs_trace = NULL;
 934 end:
 935         return ctf_fs_trace;
 936 }
 937
 938 static
 939 int path_is_ctf_trace(const char *path)
 940 {
 941         GString *metadata_path = g_string_new(NULL);
 942         int ret = 0;
 943
 944         if (!metadata_path) {
 945                 ret = -1;
 946                 goto end;
 947         }
 948
 949         g_string_printf(metadata_path, "%s/%s", path, CTF_FS_METADATA_FILENAME);
 950
 951         if (g_file_test(metadata_path->str, G_FILE_TEST_IS_REGULAR)) {
 952                 ret = 1;
 953                 goto end;
 954         }
 955
 956 end:
 957         g_string_free(metadata_path, TRUE);
 958         return ret;
 959 }
 960
 961 static
 962 int add_trace_path(GList **trace_paths, const char *path)
 963 {
 964         GString *norm_path = NULL;
 965         int ret = 0;
 966
 967         norm_path = bt_common_normalize_path(path, NULL);
 968         if (!norm_path) {
 969                 BT_LOGE("Failed to normalize path `%s`.", path);
 970                 ret = -1;
 971                 goto end;
 972         }
 973
 974         if (strcmp(norm_path->str, "/") == 0) {
 975                 BT_LOGE("Opening a trace in `/` is not supported.");
 976                 ret = -1;
 977                 goto end;
 978         }
 979
 980         *trace_paths = g_list_prepend(*trace_paths, norm_path);
 981         assert(*trace_paths);
 982         norm_path = NULL;
 983
 984 end:
 985         if (norm_path) {
 986                 g_string_free(norm_path, TRUE);
 987         }
 988
 989         return ret;
 990 }
 991
 992 BT_HIDDEN
 993 int ctf_fs_find_traces(GList **trace_paths, const char *start_path)
 994 {
 995         int ret;
 996         GError *error = NULL;
 997         GDir *dir = NULL;
 998         const char *basename = NULL;
 999
1000         /* Check if the starting path is a CTF trace itself */
1001         ret = path_is_ctf_trace(start_path);
1002         if (ret < 0) {
1003                 goto end;
1004         }
1005
1006         if (ret) {
1007                 /*
1008                  * Stop recursion: a CTF trace cannot contain another
1009                  * CTF trace.
1010                  */
1011                 ret = add_trace_path(trace_paths, start_path);
1012                 goto end;
1013         }
1014
1015         /* Look for subdirectories */
1016         if (!g_file_test(start_path, G_FILE_TEST_IS_DIR)) {
1017                 /* Starting path is not a directory: end of recursion */
1018                 goto end;
1019         }
1020
1021         dir = g_dir_open(start_path, 0, &error);
1022         if (!dir) {
1023                 if (error->code == G_FILE_ERROR_ACCES) {
1024                         BT_LOGD("Cannot open directory `%s`: %s (code %d): continuing",
1025                                 start_path, error->message, error->code);
1026                         goto end;
1027                 }
1028
1029                 BT_LOGE("Cannot open directory `%s`: %s (code %d)",
1030                         start_path, error->message, error->code);
1031                 ret = -1;
1032                 goto end;
1033         }
1034
1035         while ((basename = g_dir_read_name(dir))) {
1036                 GString *sub_path = g_string_new(NULL);
1037
1038                 if (!sub_path) {
1039                         ret = -1;
1040                         goto end;
1041                 }
1042
1043                 g_string_printf(sub_path, "%s/%s", start_path, basename);
1044                 ret = ctf_fs_find_traces(trace_paths, sub_path->str);
1045                 g_string_free(sub_path, TRUE);
1046                 if (ret) {
1047                         goto end;
1048                 }
1049         }
1050
1051 end:
1052         if (dir) {
1053                 g_dir_close(dir);
1054         }
1055
1056         if (error) {
1057                 g_error_free(error);
1058         }
1059
1060         return ret;
1061 }
1062
1063 BT_HIDDEN
1064 GList *ctf_fs_create_trace_names(GList *trace_paths, const char *base_path) {
1065         GList *trace_names = NULL;
1066         GList *node;
1067         const char *last_sep;
1068         size_t base_dist;
1069
1070         /*
1071          * At this point we know that all the trace paths are
1072          * normalized, and so is the base path. This means that
1073          * they are absolute and they don't end with a separator.
1074          * We can simply find the location of the last separator
1075          * in the base path, which gives us the name of the actual
1076          * directory to look into, and use this location as the
1077          * start of each trace name within each trace path.
1078          *
1079          * For example:
1080          *
1081          *     Base path: /home/user/my-traces/some-trace
1082          *     Trace paths:
1083          *       - /home/user/my-traces/some-trace/host1/trace1
1084          *       - /home/user/my-traces/some-trace/host1/trace2
1085          *       - /home/user/my-traces/some-trace/host2/trace
1086          *       - /home/user/my-traces/some-trace/other-trace
1087          *
1088          * In this case the trace names are:
1089          *
1090          *       - some-trace/host1/trace1
1091          *       - some-trace/host1/trace2
1092          *       - some-trace/host2/trace
1093          *       - some-trace/other-trace
1094          */
1095         last_sep = strrchr(base_path, G_DIR_SEPARATOR);
1096
1097         /* We know there's at least one separator */
1098         assert(last_sep);
1099
1100         /* Distance to base */
1101         base_dist = last_sep - base_path + 1;
1102
1103         /* Create the trace names */
1104         for (node = trace_paths; node; node = g_list_next(node)) {
1105                 GString *trace_name = g_string_new(NULL);
1106                 GString *trace_path = node->data;
1107
1108                 assert(trace_name);
1109                 g_string_assign(trace_name, &trace_path->str[base_dist]);
1110                 trace_names = g_list_append(trace_names, trace_name);
1111         }
1112
1113         return trace_names;
1114 }
1115
1116 static
1117 int create_ctf_fs_traces(struct ctf_fs_component *ctf_fs,
1118                 const char *path_param)
1119 {
1120         struct ctf_fs_trace *ctf_fs_trace = NULL;
1121         int ret = 0;
1122         GString *norm_path = NULL;
1123         GList *trace_paths = NULL;
1124         GList *trace_names = NULL;
1125         GList *tp_node;
1126         GList *tn_node;
1127         struct metadata_overrides metadata_overrides = {
1128                 .clock_offset_s = ctf_fs->options.clock_offset,
1129                 .clock_offset_ns = ctf_fs->options.clock_offset_ns,
1130         };
1131
1132         norm_path = bt_common_normalize_path(path_param, NULL);
1133         if (!norm_path) {
1134                 BT_LOGE("Failed to normalize path: `%s`.",
1135                         path_param);
1136                 goto error;
1137         }
1138
1139         ret = ctf_fs_find_traces(&trace_paths, norm_path->str);
1140         if (ret) {
1141                 goto error;
1142         }
1143
1144         if (!trace_paths) {
1145                 BT_LOGE("No CTF traces recursively found in `%s`.",
1146                         path_param);
1147                 goto error;
1148         }
1149
1150         trace_names = ctf_fs_create_trace_names(trace_paths, norm_path->str);
1151         if (!trace_names) {
1152                 BT_LOGE("Cannot create trace names from trace paths.");
1153                 goto error;
1154         }
1155
1156         for (tp_node = trace_paths, tn_node = trace_names; tp_node;
1157                         tp_node = g_list_next(tp_node),
1158                         tn_node = g_list_next(tn_node)) {
1159                 GString *trace_path = tp_node->data;
1160                 GString *trace_name = tn_node->data;
1161
1162                 ctf_fs_trace = ctf_fs_trace_create(trace_path->str,
1163                                 trace_name->str, &metadata_overrides);
1164                 if (!ctf_fs_trace) {
1165                         BT_LOGE("Cannot create trace for `%s`.",
1166                                 trace_path->str);
1167                         goto error;
1168                 }
1169
1170                 ret = create_ports_for_trace(ctf_fs, ctf_fs_trace);
1171                 if (ret) {
1172                         goto error;
1173                 }
1174
1175                 g_ptr_array_add(ctf_fs->traces, ctf_fs_trace);
1176                 ctf_fs_trace = NULL;
1177         }
1178
1179         goto end;
1180
1181 error:
1182         ret = -1;
1183         ctf_fs_trace_destroy(ctf_fs_trace);
1184
1185 end:
1186         for (tp_node = trace_paths; tp_node; tp_node = g_list_next(tp_node)) {
1187                 if (tp_node->data) {
1188                         g_string_free(tp_node->data, TRUE);
1189                 }
1190         }
1191
1192         for (tn_node = trace_names; tn_node; tn_node = g_list_next(tn_node)) {
1193                 if (tn_node->data) {
1194                         g_string_free(tn_node->data, TRUE);
1195                 }
1196         }
1197
1198         if (trace_paths) {
1199                 g_list_free(trace_paths);
1200         }
1201
1202         if (trace_names) {
1203                 g_list_free(trace_names);
1204         }
1205
1206         if (norm_path) {
1207                 g_string_free(norm_path, TRUE);
1208         }
1209
1210         return ret;
1211 }
1212
1213 static
1214 struct ctf_fs_component *ctf_fs_create(struct bt_private_component *priv_comp,
1215                 struct bt_value *params)
1216 {
1217         struct ctf_fs_component *ctf_fs;
1218         struct bt_value *value = NULL;
1219         const char *path_param;
1220         int ret;
1221
1222         ctf_fs = g_new0(struct ctf_fs_component, 1);
1223         if (!ctf_fs) {
1224                 goto end;
1225         }
1226
1227         ret = bt_private_component_set_user_data(priv_comp, ctf_fs);
1228         assert(ret == 0);
1229
1230         /*
1231          * We don't need to get a new reference here because as long as
1232          * our private ctf_fs_component object exists, the containing
1233          * private component should also exist.
1234          */
1235         ctf_fs->priv_comp = priv_comp;
1236         value = bt_value_map_get(params, "path");
1237         if (!bt_value_is_string(value)) {
1238                 goto error;
1239         }
1240
1241         ret = bt_value_string_get(value, &path_param);
1242         assert(ret == 0);
1243         BT_PUT(value);
1244         value = bt_value_map_get(params, "offset-s");
1245         if (value) {
1246                 int64_t offset;
1247
1248                 if (!bt_value_is_integer(value)) {
1249                         BT_LOGE("offset-s should be an integer");
1250                         goto error;
1251                 }
1252                 ret = bt_value_integer_get(value, &offset);
1253                 assert(ret == 0);
1254                 ctf_fs->options.clock_offset = offset;
1255                 BT_PUT(value);
1256         }
1257
1258         value = bt_value_map_get(params, "offset-ns");
1259         if (value) {
1260                 int64_t offset;
1261
1262                 if (!bt_value_is_integer(value)) {
1263                         BT_LOGE("offset-ns should be an integer");
1264                         goto error;
1265                 }
1266                 ret = bt_value_integer_get(value, &offset);
1267                 assert(ret == 0);
1268                 ctf_fs->options.clock_offset_ns = offset;
1269                 BT_PUT(value);
1270         }
1271
1272         ctf_fs->port_data = g_ptr_array_new_with_free_func(port_data_destroy);
1273         if (!ctf_fs->port_data) {
1274                 goto error;
1275         }
1276
1277         ctf_fs->traces = g_ptr_array_new_with_free_func(ctf_fs_trace_destroy);
1278         if (!ctf_fs->traces) {
1279                 goto error;
1280         }
1281
1282         ret = create_ctf_fs_traces(ctf_fs, path_param);
1283         if (ret) {
1284                 goto error;
1285         }
1286
1287         goto end;
1288
1289 error:
1290         ctf_fs_destroy(ctf_fs);
1291         ctf_fs = NULL;
1292         ret = bt_private_component_set_user_data(priv_comp, NULL);
1293         assert(ret == 0);
1294
1295 end:
1296         bt_put(value);
1297         return ctf_fs;
1298 }
1299
1300 BT_HIDDEN
1301 enum bt_component_status ctf_fs_init(struct bt_private_component *priv_comp,
1302                 struct bt_value *params, UNUSED_VAR void *init_method_data)
1303 {
1304         struct ctf_fs_component *ctf_fs;
1305         enum bt_component_status ret = BT_COMPONENT_STATUS_OK;
1306
1307         ctf_fs_debug = g_strcmp0(getenv("CTF_FS_DEBUG"), "1") == 0;
1308         ctf_fs = ctf_fs_create(priv_comp, params);
1309         if (!ctf_fs) {
1310                 ret = BT_COMPONENT_STATUS_ERROR;
1311         }
1312
1313         return ret;
1314 }
1315
1316 BT_HIDDEN
1317 struct bt_value *ctf_fs_query(struct bt_component_class *comp_class,
1318                 const char *object, struct bt_value *params)
1319 {
1320         struct bt_value *result = NULL;
1321
1322         if (!strcmp(object, "metadata-info")) {
1323                 result = metadata_info_query(comp_class, params);
1324         } else {
1325                 BT_LOGE("Unknown query object `%s`", object);
1326                 goto end;
1327         }
1328 end:
1329         return result;
1330 }