formats/ctf/ctf.c

   1 /*
   2  * BabelTrace - Common Trace Format (CTF)
   3  *
   4  * Format registration.
   5  *
   6  * Copyright 2010, 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a copy
   9  * of this software and associated documentation files (the "Software"), to deal
  10  * in the Software without restriction, including without limitation the rights
  11  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12  * copies of the Software, and to permit persons to whom the Software is
  13  * furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice shall be included in
  16  * all copies or substantial portions of the Software.
  17  */
  18
  19 #include <babeltrace/format.h>
  20 #include <babeltrace/ctf/types.h>
  21 #include <babeltrace/ctf/metadata.h>
  22 #include <babeltrace/babeltrace.h>
  23 #include <inttypes.h>
  24 #include <uuid/uuid.h>
  25 #include <sys/mman.h>
  26 #include <errno.h>
  27 #include <sys/types.h>
  28 #include <sys/stat.h>
  29 #include <fcntl.h>
  30 #include <dirent.h>
  31 #include <glib.h>
  32 #include <unistd.h>
  33 #include <stdlib.h>
  34
  35 #include "metadata/ctf-scanner.h"
  36 #include "metadata/ctf-parser.h"
  37 #include "metadata/ctf-ast.h"
  38
  39 /*
  40  * We currently simply map a page to read the packet header and packet
  41  * context to get the packet length and content length.
  42  */
  43 #define MAX_PACKET_HEADER_LEN   getpagesize()
  44 #define UUID_LEN 16     /* uuid by value len */
  45
  46 extern int yydebug;
  47
  48 struct trace_descriptor {
  49         struct ctf_trace ctf_trace;
  50 };
  51
  52 struct trace_descriptor *ctf_open_trace(const char *path, int flags);
  53 void ctf_close_trace(struct trace_descriptor *descriptor);
  54
  55 static struct format ctf_format = {
  56         .uint_read = ctf_uint_read,
  57         .int_read = ctf_int_read,
  58         .uint_write = ctf_uint_write,
  59         .int_write = ctf_int_write,
  60         .double_read = ctf_double_read,
  61         .double_write = ctf_double_write,
  62         .float_copy = ctf_float_copy,
  63         .string_copy = ctf_string_copy,
  64         .string_read = ctf_string_read,
  65         .string_write = ctf_string_write,
  66         .string_free_temp = ctf_string_free_temp,
  67         .enum_read = ctf_enum_read,
  68         .enum_write = ctf_enum_write,
  69         .struct_begin = ctf_struct_begin,
  70         .struct_end = ctf_struct_end,
  71         .variant_begin = ctf_variant_begin,
  72         .variant_end = ctf_variant_end,
  73         .array_begin = ctf_array_begin,
  74         .array_end = ctf_array_end,
  75         .sequence_begin = ctf_sequence_begin,
  76         .sequence_end = ctf_sequence_end,
  77         .open_trace = ctf_open_trace,
  78         .close_trace = ctf_close_trace,
  79 };
  80
  81 void move_pos_slow(struct stream_pos *pos, size_t offset)
  82 {
  83         int ret;
  84
  85         /*
  86          * The caller should never ask for move_pos across packets,
  87          * except to get exactly at the beginning of the next packet.
  88          */
  89         assert(pos->offset + offset == pos->content_size);
  90
  91         if (pos->base) {
  92                 /* unmap old base */
  93                 ret = munmap(pos->base, pos->packet_size);
  94                 if (ret) {
  95                         fprintf(stdout, "Unable to unmap old base: %s.\n",
  96                                 strerror(errno));
  97                         assert(0);
  98                 }
  99         }
 100
 101         pos->mmap_offset += pos->packet_size / CHAR_BIT;
 102         /* map new base. Need mapping length from header. */
 103         pos->base = mmap(NULL, MAX_PACKET_HEADER_LEN, PROT_READ,
 104                          MAP_PRIVATE, pos->fd, pos->mmap_offset);
 105         pos->content_size = 0;  /* Unknown at this point */
 106         pos->packet_size = 0;   /* Unknown at this point */
 107
 108 }
 109
 110 /*
 111  * TODO: for now, we treat the metadata file as a simple text file
 112  * (without any header nor packets nor padding).
 113  */
 114 static
 115 int ctf_open_trace_metadata_read(struct trace_descriptor *td)
 116 {
 117         struct ctf_scanner *scanner;
 118         FILE *fp;
 119         int ret = 0;
 120
 121         td->ctf_trace.metadata.pos.fd = openat(td->ctf_trace.dirfd,
 122                         "metadata", O_RDONLY);
 123         if (td->ctf_trace.metadata.pos.fd < 0) {
 124                 fprintf(stdout, "Unable to open metadata.\n");
 125                 return td->ctf_trace.metadata.pos.fd;
 126         }
 127
 128         if (babeltrace_debug)
 129                 yydebug = 1;
 130
 131         fp = fdopen(td->ctf_trace.metadata.pos.fd, "r");
 132         if (!fp) {
 133                 fprintf(stdout, "Unable to open metadata stream.\n");
 134                 ret = -errno;
 135                 goto end_stream;
 136         }
 137
 138         scanner = ctf_scanner_alloc(fp);
 139         if (!scanner) {
 140                 fprintf(stdout, "Error allocating scanner\n");
 141                 ret = -ENOMEM;
 142                 goto end_scanner_alloc;
 143         }
 144         ret = ctf_scanner_append_ast(scanner);
 145         if (ret) {
 146                 fprintf(stdout, "Error creating AST\n");
 147                 goto end;
 148         }
 149
 150         if (babeltrace_debug) {
 151                 ret = ctf_visitor_print_xml(stdout, 0, &scanner->ast->root);
 152                 if (ret) {
 153                         fprintf(stdout, "Error visiting AST for XML output\n");
 154                         goto end;
 155                 }
 156         }
 157
 158         ret = ctf_visitor_semantic_check(stdout, 0, &scanner->ast->root);
 159         if (ret) {
 160                 fprintf(stdout, "Error in CTF semantic validation %d\n", ret);
 161                 goto end;
 162         }
 163         ret = ctf_visitor_construct_metadata(stdout, 0, &scanner->ast->root,
 164                         &td->ctf_trace, BYTE_ORDER);
 165         if (ret) {
 166                 fprintf(stdout, "Error in CTF metadata constructor %d\n", ret);
 167                 goto end;
 168         }
 169 end:
 170         ctf_scanner_free(scanner);
 171 end_scanner_alloc:
 172         fclose(fp);
 173 end_stream:
 174         close(td->ctf_trace.metadata.pos.fd);
 175         return ret;
 176 }
 177
 178
 179 static
 180 int create_stream_packet_index(struct trace_descriptor *td,
 181                                struct ctf_file_stream *file_stream)
 182 {
 183         struct ctf_stream *stream;
 184         int len_index;
 185         struct stream_pos *pos;
 186         struct stat filestats;
 187         struct packet_index packet_index;
 188         int first_packet = 1;
 189         int ret;
 190
 191         pos = &file_stream->pos;
 192
 193         ret = fstat(pos->fd, &filestats);
 194         if (ret < 0)
 195                 return ret;
 196
 197         for (pos->mmap_offset = 0; pos->mmap_offset < filestats.st_size; ) {
 198                 uint64_t stream_id = 0;
 199
 200                 if (pos->base) {
 201                         /* unmap old base */
 202                         ret = munmap(pos->base, pos->packet_size);
 203                         if (ret) {
 204                                 fprintf(stdout, "Unable to unmap old base: %s.\n",
 205                                         strerror(errno));
 206                                 return ret;
 207                         }
 208                 }
 209                 /* map new base. Need mapping length from header. */
 210                 pos->base = mmap(NULL, MAX_PACKET_HEADER_LEN, PROT_READ,
 211                                  MAP_PRIVATE, pos->fd, pos->mmap_offset);
 212                 pos->content_size = 0;  /* Unknown at this point */
 213                 pos->packet_size = 0;   /* Unknown at this point */
 214                 pos->offset = 0;        /* Position of the packet header */
 215
 216                 /* read and check header, set stream id (and check) */
 217                 if (td->ctf_trace.packet_header) {
 218                         /* Read packet header */
 219                         td->ctf_trace.packet_header->p.declaration->copy(NULL, NULL,
 220                                 pos, &ctf_format, &td->ctf_trace.packet_header->p);
 221
 222                         len_index = struct_declaration_lookup_field_index(td->ctf_trace.packet_header->declaration, g_quark_from_static_string("magic"));
 223                         if (len_index >= 0) {
 224                                 struct definition_integer *defint;
 225                                 struct field *field;
 226
 227                                 field = struct_definition_get_field_from_index(td->ctf_trace.packet_header, len_index);
 228                                 assert(field->definition->declaration->id == CTF_TYPE_INTEGER);
 229                                 defint = container_of(field->definition, struct definition_integer, p);
 230                                 assert(defint->declaration->signedness == FALSE);
 231                                 if (defint->value._unsigned != CTF_MAGIC) {
 232                                         fprintf(stdout, "[error] Invalid magic number %" PRIX64 ".\n",
 233                                                         defint->value._unsigned);
 234                                         return -EINVAL;
 235                                 }
 236                         }
 237
 238                         /* check uuid */
 239                         len_index = struct_declaration_lookup_field_index(td->ctf_trace.packet_header->declaration, g_quark_from_static_string("trace_uuid"));
 240                         if (len_index >= 0) {
 241                                 struct definition_array *defarray;
 242                                 struct field *field;
 243                                 uint64_t i;
 244                                 uint8_t uuidval[UUID_LEN];
 245
 246
 247                                 field = struct_definition_get_field_from_index(td->ctf_trace.packet_header, len_index);
 248                                 assert(field->definition->declaration->id == CTF_TYPE_ARRAY);
 249                                 defarray = container_of(field->definition, struct definition_array, p);
 250                                 assert(defarray->declaration->len == UUID_LEN);
 251                                 assert(defarray->declaration->elem->id == CTF_TYPE_INTEGER);
 252
 253                                 for (i = 0; i < UUID_LEN; i++) {
 254                                         struct definition *elem;
 255                                         struct definition_integer *defint;
 256
 257                                         elem = array_index(defarray, i);
 258                                         assert(elem);
 259                                         defint = container_of(elem, struct definition_integer, p);
 260                                         uuidval[i] = defint->value._unsigned;
 261                                 }
 262                                 ret = uuid_compare(td->ctf_trace.uuid, uuidval);
 263                                 if (ret) {
 264                                         fprintf(stdout, "[error] Unique Universal Identifiers do not match.\n");
 265                                         return -EINVAL;
 266                                 }
 267                         }
 268
 269
 270                         len_index = struct_declaration_lookup_field_index(td->ctf_trace.packet_header->declaration, g_quark_from_static_string("stream_id"));
 271                         if (len_index >= 0) {
 272                                 struct definition_integer *defint;
 273                                 struct field *field;
 274
 275                                 field = struct_definition_get_field_from_index(td->ctf_trace.packet_header, len_index);
 276                                 assert(field->definition->declaration->id == CTF_TYPE_INTEGER);
 277                                 defint = container_of(field->definition, struct definition_integer, p);
 278                                 assert(defint->declaration->signedness == FALSE);
 279                                 stream_id = defint->value._unsigned;
 280                         }
 281                 }
 282
 283                 if (!first_packet && file_stream->stream_id != stream_id) {
 284                         fprintf(stdout, "[error] Stream ID is changing within a stream.\n");
 285                         return -EINVAL;
 286                 }
 287                 if (first_packet) {
 288                         file_stream->stream_id = stream_id;
 289                         if (stream_id >= td->ctf_trace.streams->len) {
 290                                 fprintf(stdout, "[error] Stream %" PRIu64 " is not declared in metadata.\n", stream_id);
 291                                 return -EINVAL;
 292                         }
 293                         stream = g_ptr_array_index(td->ctf_trace.streams, stream_id);
 294                         if (!stream) {
 295                                 fprintf(stdout, "[error] Stream %" PRIu64 " is not declared in metadata.\n", stream_id);
 296                                 return -EINVAL;
 297                         }
 298                         file_stream->stream = stream;
 299                 }
 300                 first_packet = 0;
 301
 302                 /* Read packet context */
 303                 stream->packet_context->p.declaration->copy(NULL, NULL,
 304                                 pos, &ctf_format, &stream->packet_context->p);
 305
 306                 /* read content size from header */
 307                 len_index = struct_declaration_lookup_field_index(stream->packet_context->declaration, g_quark_from_static_string("content_size"));
 308                 if (len_index >= 0) {
 309                         struct definition_integer *defint;
 310                         struct field *field;
 311
 312                         field = struct_definition_get_field_from_index(stream->packet_context, len_index);
 313                         assert(field->definition->declaration->id == CTF_TYPE_INTEGER);
 314                         defint = container_of(field->definition, struct definition_integer, p);
 315                         assert(defint->declaration->signedness == FALSE);
 316                         pos->content_size = defint->value._unsigned;
 317                 } else {
 318                         /* Use file size for packet size */
 319                         pos->content_size = filestats.st_size * CHAR_BIT;
 320                 }
 321
 322                 /* read packet size from header */
 323                 len_index = struct_declaration_lookup_field_index(stream->packet_context->declaration, g_quark_from_static_string("packet_size"));
 324                 if (len_index >= 0) {
 325                         struct definition_integer *defint;
 326                         struct field *field;
 327
 328                         field = struct_definition_get_field_from_index(stream->packet_context, len_index);
 329                         assert(field->definition->declaration->id == CTF_TYPE_INTEGER);
 330                         defint = container_of(field->definition, struct definition_integer, p);
 331                         assert(defint->declaration->signedness == FALSE);
 332                         pos->packet_size = defint->value._unsigned;
 333                 } else {
 334                         /* Use content size if non-zero, else file size */
 335                         pos->packet_size = pos->content_size ? : filestats.st_size * CHAR_BIT;
 336                 }
 337
 338                 packet_index.offset = pos->mmap_offset;
 339                 packet_index.content_size = pos->content_size;
 340                 packet_index.packet_size = pos->packet_size;
 341                 /* add index to packet array */
 342                 g_array_append_val(file_stream->pos.packet_index, packet_index);
 343
 344                 pos->mmap_offset += pos->packet_size / CHAR_BIT;
 345         }
 346
 347         return 0;
 348 }
 349
 350 /*
 351  * Note: many file streams can inherit from the same stream class
 352  * description (metadata).
 353  */
 354 static
 355 int ctf_open_file_stream_read(struct trace_descriptor *td, const char *path, int flags)
 356 {
 357         int ret;
 358         struct ctf_file_stream *file_stream;
 359
 360         ret = openat(td->ctf_trace.dirfd, path, flags);
 361         if (ret < 0)
 362                 goto error;
 363         file_stream = g_new0(struct ctf_file_stream, 1);
 364         file_stream->pos.fd = ret;
 365         file_stream->pos.packet_index = g_array_new(FALSE, TRUE,
 366                                                 sizeof(struct packet_index));
 367         ret = create_stream_packet_index(td, file_stream);
 368         if (ret)
 369                 goto error_index;
 370         /* Add stream file to stream class */
 371         g_ptr_array_add(file_stream->stream->files, file_stream);
 372         return 0;
 373
 374 error_index:
 375         (void) g_array_free(file_stream->pos.packet_index, TRUE);
 376         close(file_stream->pos.fd);
 377         g_free(file_stream);
 378 error:
 379         return ret;
 380 }
 381
 382 static
 383 int ctf_open_trace_read(struct trace_descriptor *td, const char *path, int flags)
 384 {
 385         int ret;
 386         struct dirent *dirent;
 387         struct dirent *diriter;
 388         size_t dirent_len;
 389
 390         td->ctf_trace.flags = flags;
 391
 392         /* Open trace directory */
 393         td->ctf_trace.dir = opendir(path);
 394         if (!td->ctf_trace.dir) {
 395                 fprintf(stdout, "Unable to open trace directory.\n");
 396                 ret = -ENOENT;
 397                 goto error;
 398         }
 399
 400         td->ctf_trace.dirfd = open(path, 0);
 401         if (td->ctf_trace.dirfd < 0) {
 402                 fprintf(stdout, "Unable to open trace directory file descriptor.\n");
 403                 ret = -ENOENT;
 404                 goto error_dirfd;
 405         }
 406
 407         td->ctf_trace.streams = g_ptr_array_new();
 408
 409         /*
 410          * Keep the metadata file separate.
 411          */
 412
 413         ret = ctf_open_trace_metadata_read(td);
 414         if (ret) {
 415                 goto error_metadata;
 416         }
 417
 418         /*
 419          * Open each stream: for each file, try to open, check magic
 420          * number, and get the stream ID to add to the right location in
 421          * the stream array.
 422          */
 423
 424         dirent_len = offsetof(struct dirent, d_name) +
 425                         fpathconf(td->ctf_trace.dirfd, _PC_NAME_MAX) + 1;
 426
 427         dirent = malloc(dirent_len);
 428
 429         for (;;) {
 430                 ret = readdir_r(td->ctf_trace.dir, dirent, &diriter);
 431                 if (ret) {
 432                         fprintf(stdout, "Readdir error.\n");
 433                         goto readdir_error;
 434                 }
 435                 if (!diriter)
 436                         break;
 437                 if (!strcmp(diriter->d_name, ".")
 438                                 || !strcmp(diriter->d_name, "..")
 439                                 || !strcmp(diriter->d_name, "metadata"))
 440                         continue;
 441                 /* TODO: open file stream */
 442         }
 443
 444         free(dirent);
 445         return 0;
 446
 447 readdir_error:
 448         free(dirent);
 449 error_metadata:
 450         g_ptr_array_free(td->ctf_trace.streams, TRUE);
 451         close(td->ctf_trace.dirfd);
 452 error_dirfd:
 453         closedir(td->ctf_trace.dir);
 454 error:
 455         return ret;
 456 }
 457
 458 static
 459 int ctf_open_trace_write(struct trace_descriptor *td, const char *path, int flags)
 460 {
 461         int ret;
 462
 463         ret = mkdir(path, S_IRWXU|S_IRWXG);
 464         if (ret)
 465                 return ret;
 466
 467         /* Open trace directory */
 468         td->ctf_trace.dir = opendir(path);
 469         if (!td->ctf_trace.dir) {
 470                 fprintf(stdout, "Unable to open trace directory.\n");
 471                 ret = -ENOENT;
 472                 goto error;
 473         }
 474
 475
 476         return 0;
 477
 478 error:
 479         return ret;
 480 }
 481
 482 struct trace_descriptor *ctf_open_trace(const char *path, int flags)
 483 {
 484         struct trace_descriptor *td;
 485         int ret;
 486
 487         td = g_new0(struct trace_descriptor, 1);
 488
 489         switch (flags) {
 490         case O_RDONLY:
 491                 ret = ctf_open_trace_read(td, path, flags);
 492                 if (ret)
 493                         goto error;
 494                 break;
 495         case O_WRONLY:
 496                 ret = ctf_open_trace_write(td, path, flags);
 497                 if (ret)
 498                         goto error;
 499                 break;
 500         default:
 501                 fprintf(stdout, "Incorrect open flags.\n");
 502                 goto error;
 503         }
 504
 505         return td;
 506 error:
 507         g_free(td);
 508         return NULL;
 509 }
 510
 511 static
 512 void ctf_close_file_stream(struct ctf_file_stream *file_stream)
 513 {
 514         (void) g_array_free(file_stream->pos.packet_index, TRUE);
 515         close(file_stream->pos.fd);
 516 }
 517
 518 void ctf_close_trace(struct trace_descriptor *td)
 519 {
 520         int i;
 521
 522         if (td->ctf_trace.streams) {
 523                 for (i = 0; i < td->ctf_trace.streams->len; i++) {
 524                         struct ctf_stream *stream;
 525                         int j;
 526
 527                         stream = g_ptr_array_index(td->ctf_trace.streams, i);
 528                         for (j = 0; j < stream->files->len; j++) {
 529                                 struct ctf_file_stream *file_stream;
 530                                 file_stream = g_ptr_array_index(td->ctf_trace.streams, j);
 531                                 ctf_close_file_stream(file_stream);
 532                         }
 533
 534                 }
 535                 g_ptr_array_free(td->ctf_trace.streams, TRUE);
 536         }
 537         closedir(td->ctf_trace.dir);
 538         g_free(td);
 539 }
 540
 541 void __attribute__((constructor)) ctf_init(void)
 542 {
 543         int ret;
 544
 545         ctf_format.name = g_quark_from_static_string("ctf");
 546         ret = bt_register_format(&ctf_format);
 547         assert(!ret);
 548 }
 549
 550 /* TODO: finalize */