Use O_RDWR for open write mode (for mmap)
[babeltrace.git] / formats / ctf / ctf.c
1 /*
2 * BabelTrace - Common Trace Format (CTF)
3 *
4 * Format registration.
5 *
6 * Copyright 2010, 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a copy
9 * of this software and associated documentation files (the "Software"), to deal
10 * in the Software without restriction, including without limitation the rights
11 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12 * copies of the Software, and to permit persons to whom the Software is
13 * furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice shall be included in
16 * all copies or substantial portions of the Software.
17 */
18
19 #include <babeltrace/format.h>
20 #include <babeltrace/ctf/types.h>
21 #include <babeltrace/ctf/metadata.h>
22 #include <babeltrace/babeltrace.h>
23 #include <inttypes.h>
24 #include <uuid/uuid.h>
25 #include <sys/mman.h>
26 #include <errno.h>
27 #include <sys/types.h>
28 #include <sys/stat.h>
29 #include <fcntl.h>
30 #include <dirent.h>
31 #include <glib.h>
32 #include <unistd.h>
33 #include <stdlib.h>
34
35 #include "metadata/ctf-scanner.h"
36 #include "metadata/ctf-parser.h"
37 #include "metadata/ctf-ast.h"
38
39 /*
40 * We currently simply map a page to read the packet header and packet
41 * context to get the packet length and content length. (in bits)
42 */
43 #define MAX_PACKET_HEADER_LEN (getpagesize() * CHAR_BIT)
44 #define WRITE_PACKET_LEN (getpagesize() * 8 * CHAR_BIT)
45 #define UUID_LEN 16 /* uuid by value len */
46
47 extern int yydebug;
48
49 struct trace_descriptor *ctf_open_trace(const char *path, int flags);
50 void ctf_close_trace(struct trace_descriptor *descriptor);
51
52 static
53 rw_dispatch read_dispatch_table[] = {
54 [ CTF_TYPE_INTEGER ] = ctf_integer_read,
55 [ CTF_TYPE_FLOAT ] = ctf_float_read,
56 [ CTF_TYPE_ENUM ] = ctf_enum_read,
57 [ CTF_TYPE_STRING ] = ctf_string_read,
58 [ CTF_TYPE_STRUCT ] = ctf_struct_rw,
59 [ CTF_TYPE_VARIANT ] = ctf_variant_rw,
60 [ CTF_TYPE_ARRAY ] = ctf_array_rw,
61 [ CTF_TYPE_SEQUENCE ] = ctf_sequence_rw,
62 };
63
64 static
65 rw_dispatch write_dispatch_table[] = {
66 [ CTF_TYPE_INTEGER ] = ctf_integer_write,
67 [ CTF_TYPE_FLOAT ] = ctf_float_write,
68 [ CTF_TYPE_ENUM ] = ctf_enum_write,
69 [ CTF_TYPE_STRING ] = ctf_string_write,
70 [ CTF_TYPE_STRUCT ] = ctf_struct_rw,
71 [ CTF_TYPE_VARIANT ] = ctf_variant_rw,
72 [ CTF_TYPE_ARRAY ] = ctf_array_rw,
73 [ CTF_TYPE_SEQUENCE ] = ctf_sequence_rw,
74 };
75
76 static
77 struct format ctf_format = {
78 .open_trace = ctf_open_trace,
79 .close_trace = ctf_close_trace,
80 };
81
82 void ctf_init_pos(struct ctf_stream_pos *pos, int fd, int open_flags)
83 {
84 pos->fd = fd;
85 pos->mmap_offset = 0;
86 pos->packet_size = 0;
87 pos->content_size = 0;
88 pos->content_size_loc = NULL;
89 pos->base = NULL;
90 pos->offset = 0;
91 pos->dummy = false;
92 pos->cur_index = 0;
93 if (fd >= 0)
94 pos->packet_index = g_array_new(FALSE, TRUE,
95 sizeof(struct packet_index));
96 else
97 pos->packet_index = NULL;
98 switch (open_flags & O_ACCMODE) {
99 case O_RDONLY:
100 pos->prot = PROT_READ;
101 pos->flags = MAP_PRIVATE;
102 pos->parent.rw_table = read_dispatch_table;
103 break;
104 case O_RDWR:
105 pos->prot = PROT_WRITE; /* Write has priority */
106 pos->flags = MAP_SHARED;
107 pos->parent.rw_table = write_dispatch_table;
108 if (fd >= 0)
109 ctf_move_pos_slow(pos, 0, SEEK_SET); /* position for write */
110 break;
111 default:
112 assert(0);
113 }
114 }
115
116 void ctf_fini_pos(struct ctf_stream_pos *pos)
117 {
118 int ret;
119
120 if (pos->prot == PROT_WRITE && pos->content_size_loc)
121 *pos->content_size_loc = pos->offset;
122 if (pos->base) {
123 /* unmap old base */
124 ret = munmap(pos->base, pos->packet_size / CHAR_BIT);
125 if (ret) {
126 fprintf(stdout, "[error] Unable to unmap old base: %s.\n",
127 strerror(errno));
128 assert(0);
129 }
130 }
131 (void) g_array_free(pos->packet_index, TRUE);
132 }
133
134 void ctf_move_pos_slow(struct ctf_stream_pos *pos, size_t offset, int whence)
135 {
136 int ret;
137 off_t off;
138 struct packet_index *index;
139
140 if (pos->prot == PROT_WRITE && pos->content_size_loc)
141 *pos->content_size_loc = pos->offset;
142
143 if (pos->base) {
144 /* unmap old base */
145 ret = munmap(pos->base, pos->packet_size / CHAR_BIT);
146 if (ret) {
147 fprintf(stdout, "[error] Unable to unmap old base: %s.\n",
148 strerror(errno));
149 assert(0);
150 }
151 pos->base = NULL;
152 }
153
154 /*
155 * The caller should never ask for ctf_move_pos across packets,
156 * except to get exactly at the beginning of the next packet.
157 */
158 if (pos->prot == PROT_WRITE) {
159 switch (whence) {
160 case SEEK_CUR:
161 /* The writer will add padding */
162 assert(pos->offset + offset == pos->packet_size);
163 pos->mmap_offset += WRITE_PACKET_LEN / CHAR_BIT;
164 break;
165 case SEEK_SET:
166 assert(offset == 0); /* only seek supported for now */
167 pos->cur_index = 0;
168 break;
169 default:
170 assert(0);
171 }
172 pos->content_size = -1U; /* Unknown at this point */
173 pos->packet_size = WRITE_PACKET_LEN;
174 off = posix_fallocate(pos->fd, pos->mmap_offset,
175 pos->packet_size / CHAR_BIT);
176 assert(off >= 0);
177 pos->offset = 0;
178 } else {
179 switch (whence) {
180 case SEEK_CUR:
181 /* The reader will expect us to skip padding */
182 assert(pos->offset + offset == pos->content_size);
183 ++pos->cur_index;
184 break;
185 case SEEK_SET:
186 assert(offset == 0); /* only seek supported for now */
187 pos->cur_index = 0;
188 break;
189 default:
190 assert(0);
191 }
192 if (pos->cur_index >= pos->packet_index->len) {
193 pos->offset = EOF;
194 return;
195 }
196 index = &g_array_index(pos->packet_index, struct packet_index,
197 pos->cur_index);
198 pos->mmap_offset = index->offset;
199
200 /* Lookup context/packet size in index */
201 pos->content_size = index->content_size;
202 pos->packet_size = index->packet_size;
203 pos->offset = index->data_offset;
204 }
205 /* map new base. Need mapping length from header. */
206 pos->base = mmap(NULL, pos->packet_size / CHAR_BIT, pos->prot,
207 pos->flags, pos->fd, pos->mmap_offset);
208 if (pos->base == MAP_FAILED) {
209 fprintf(stdout, "[error] mmap error %s.\n",
210 strerror(errno));
211 assert(0);
212 }
213 }
214
215 /*
216 * TODO: for now, we treat the metadata file as a simple text file
217 * (without any header nor packets nor padding).
218 */
219 static
220 int ctf_open_trace_metadata_read(struct ctf_trace *td)
221 {
222 struct ctf_scanner *scanner;
223 FILE *fp;
224 int ret = 0;
225
226 td->metadata.pos.fd = openat(td->dirfd, "metadata", O_RDONLY);
227 if (td->metadata.pos.fd < 0) {
228 fprintf(stdout, "Unable to open metadata.\n");
229 return td->metadata.pos.fd;
230 }
231
232 if (babeltrace_debug)
233 yydebug = 1;
234
235 fp = fdopen(td->metadata.pos.fd, "r");
236 if (!fp) {
237 fprintf(stdout, "[error] Unable to open metadata stream.\n");
238 ret = -errno;
239 goto end_stream;
240 }
241
242 scanner = ctf_scanner_alloc(fp);
243 if (!scanner) {
244 fprintf(stdout, "[error] Error allocating scanner\n");
245 ret = -ENOMEM;
246 goto end_scanner_alloc;
247 }
248 ret = ctf_scanner_append_ast(scanner);
249 if (ret) {
250 fprintf(stdout, "[error] Error creating AST\n");
251 goto end;
252 }
253
254 if (babeltrace_debug) {
255 ret = ctf_visitor_print_xml(stdout, 0, &scanner->ast->root);
256 if (ret) {
257 fprintf(stdout, "[error] Error visiting AST for XML output\n");
258 goto end;
259 }
260 }
261
262 ret = ctf_visitor_semantic_check(stdout, 0, &scanner->ast->root);
263 if (ret) {
264 fprintf(stdout, "[error] Error in CTF semantic validation %d\n", ret);
265 goto end;
266 }
267 ret = ctf_visitor_construct_metadata(stdout, 0, &scanner->ast->root,
268 td, BYTE_ORDER);
269 if (ret) {
270 fprintf(stdout, "[error] Error in CTF metadata constructor %d\n", ret);
271 goto end;
272 }
273 end:
274 ctf_scanner_free(scanner);
275 end_scanner_alloc:
276 fclose(fp);
277 end_stream:
278 close(td->metadata.pos.fd);
279 return ret;
280 }
281
282
283 static
284 int create_stream_packet_index(struct ctf_trace *td,
285 struct ctf_file_stream *file_stream)
286 {
287 struct ctf_stream *stream;
288 int len_index;
289 struct ctf_stream_pos *pos;
290 struct stat filestats;
291 struct packet_index packet_index;
292 int first_packet = 1;
293 int ret;
294
295 pos = &file_stream->pos;
296
297 ret = fstat(pos->fd, &filestats);
298 if (ret < 0)
299 return ret;
300
301 for (pos->mmap_offset = 0; pos->mmap_offset < filestats.st_size; ) {
302 uint64_t stream_id = 0;
303
304 if (pos->base) {
305 /* unmap old base */
306 ret = munmap(pos->base, pos->packet_size / CHAR_BIT);
307 if (ret) {
308 fprintf(stdout, "[error] Unable to unmap old base: %s.\n",
309 strerror(errno));
310 return ret;
311 }
312 pos->base = NULL;
313 }
314 /* map new base. Need mapping length from header. */
315 pos->base = mmap(NULL, MAX_PACKET_HEADER_LEN / CHAR_BIT, PROT_READ,
316 MAP_PRIVATE, pos->fd, pos->mmap_offset);
317 pos->content_size = MAX_PACKET_HEADER_LEN; /* Unknown at this point */
318 pos->packet_size = MAX_PACKET_HEADER_LEN; /* Unknown at this point */
319 pos->offset = 0; /* Position of the packet header */
320
321 packet_index.offset = pos->mmap_offset;
322 packet_index.content_size = 0;
323 packet_index.packet_size = 0;
324
325 /* read and check header, set stream id (and check) */
326 if (td->packet_header) {
327 /* Read packet header */
328 ret = generic_rw(&pos->parent, &td->packet_header->p);
329 if (ret)
330 return ret;
331 len_index = struct_declaration_lookup_field_index(td->packet_header->declaration, g_quark_from_static_string("magic"));
332 if (len_index >= 0) {
333 struct definition_integer *defint;
334 struct field *field;
335
336 field = struct_definition_get_field_from_index(td->packet_header, len_index);
337 assert(field->definition->declaration->id == CTF_TYPE_INTEGER);
338 defint = container_of(field->definition, struct definition_integer, p);
339 assert(defint->declaration->signedness == FALSE);
340 if (defint->value._unsigned != CTF_MAGIC) {
341 fprintf(stdout, "[error] Invalid magic number 0x%" PRIX64 " at packet %u (file offset %zd).\n",
342 defint->value._unsigned,
343 file_stream->pos.packet_index->len,
344 (ssize_t) pos->mmap_offset);
345 return -EINVAL;
346 }
347 }
348
349 /* check uuid */
350 len_index = struct_declaration_lookup_field_index(td->packet_header->declaration, g_quark_from_static_string("trace_uuid"));
351 if (len_index >= 0) {
352 struct definition_array *defarray;
353 struct field *field;
354 uint64_t i;
355 uint8_t uuidval[UUID_LEN];
356
357 field = struct_definition_get_field_from_index(td->packet_header, len_index);
358 assert(field->definition->declaration->id == CTF_TYPE_ARRAY);
359 defarray = container_of(field->definition, struct definition_array, p);
360 assert(array_len(defarray) == UUID_LEN);
361 assert(defarray->declaration->elem->id == CTF_TYPE_INTEGER);
362
363 for (i = 0; i < UUID_LEN; i++) {
364 struct definition *elem;
365 struct definition_integer *defint;
366
367 elem = array_index(defarray, i);
368 assert(elem);
369 defint = container_of(elem, struct definition_integer, p);
370 uuidval[i] = defint->value._unsigned;
371 }
372 ret = uuid_compare(td->uuid, uuidval);
373 if (ret) {
374 fprintf(stdout, "[error] Unique Universal Identifiers do not match.\n");
375 return -EINVAL;
376 }
377 }
378
379
380 len_index = struct_declaration_lookup_field_index(td->packet_header->declaration, g_quark_from_static_string("stream_id"));
381 if (len_index >= 0) {
382 struct definition_integer *defint;
383 struct field *field;
384
385 field = struct_definition_get_field_from_index(td->packet_header, len_index);
386 assert(field->definition->declaration->id == CTF_TYPE_INTEGER);
387 defint = container_of(field->definition, struct definition_integer, p);
388 assert(defint->declaration->signedness == FALSE);
389 stream_id = defint->value._unsigned;
390 }
391 }
392
393 if (!first_packet && file_stream->stream_id != stream_id) {
394 fprintf(stdout, "[error] Stream ID is changing within a stream.\n");
395 return -EINVAL;
396 }
397 if (first_packet) {
398 file_stream->stream_id = stream_id;
399 if (stream_id >= td->streams->len) {
400 fprintf(stdout, "[error] Stream %" PRIu64 " is not declared in metadata.\n", stream_id);
401 return -EINVAL;
402 }
403 stream = g_ptr_array_index(td->streams, stream_id);
404 if (!stream) {
405 fprintf(stdout, "[error] Stream %" PRIu64 " is not declared in metadata.\n", stream_id);
406 return -EINVAL;
407 }
408 file_stream->stream = stream;
409 }
410 first_packet = 0;
411
412 if (stream->packet_context) {
413 /* Read packet context */
414 ret = generic_rw(&pos->parent, &stream->packet_context->p);
415 if (ret)
416 return ret;
417 /* read content size from header */
418 len_index = struct_declaration_lookup_field_index(stream->packet_context->declaration, g_quark_from_static_string("content_size"));
419 if (len_index >= 0) {
420 struct definition_integer *defint;
421 struct field *field;
422
423 field = struct_definition_get_field_from_index(stream->packet_context, len_index);
424 assert(field->definition->declaration->id == CTF_TYPE_INTEGER);
425 defint = container_of(field->definition, struct definition_integer, p);
426 assert(defint->declaration->signedness == FALSE);
427 packet_index.content_size = defint->value._unsigned;
428 } else {
429 /* Use file size for packet size */
430 packet_index.content_size = filestats.st_size * CHAR_BIT;
431 }
432
433 /* read packet size from header */
434 len_index = struct_declaration_lookup_field_index(stream->packet_context->declaration, g_quark_from_static_string("packet_size"));
435 if (len_index >= 0) {
436 struct definition_integer *defint;
437 struct field *field;
438
439 field = struct_definition_get_field_from_index(stream->packet_context, len_index);
440 assert(field->definition->declaration->id == CTF_TYPE_INTEGER);
441 defint = container_of(field->definition, struct definition_integer, p);
442 assert(defint->declaration->signedness == FALSE);
443 packet_index.packet_size = defint->value._unsigned;
444 } else {
445 /* Use content size if non-zero, else file size */
446 packet_index.packet_size = packet_index.content_size ? : filestats.st_size * CHAR_BIT;
447 }
448 } else {
449 /* Use file size for packet size */
450 packet_index.content_size = filestats.st_size * CHAR_BIT;
451 /* Use content size if non-zero, else file size */
452 packet_index.packet_size = packet_index.content_size ? : filestats.st_size * CHAR_BIT;
453 }
454
455 /* Validate content size and packet size values */
456 if (packet_index.content_size > packet_index.packet_size) {
457 fprintf(stdout, "[error] Content size (%zu bits) is larger than packet size (%zu bits).\n",
458 packet_index.content_size, packet_index.packet_size);
459 return -EINVAL;
460 }
461
462 if (packet_index.packet_size > (filestats.st_size - packet_index.offset) * CHAR_BIT) {
463 fprintf(stdout, "[error] Packet size (%zu bits) is larger than remaining file size (%zu bits).\n",
464 packet_index.content_size, (filestats.st_size - packet_index.offset) * CHAR_BIT);
465 return -EINVAL;
466 }
467
468 /* Save position after header and context */
469 packet_index.data_offset = pos->offset;
470
471 /* add index to packet array */
472 g_array_append_val(file_stream->pos.packet_index, packet_index);
473
474 pos->mmap_offset += packet_index.packet_size / CHAR_BIT;
475 }
476
477 /* Move pos back to beginning of file */
478 ctf_move_pos_slow(pos, 0, SEEK_SET); /* position for write */
479
480 return 0;
481 }
482
483 /*
484 * Note: many file streams can inherit from the same stream class
485 * description (metadata).
486 */
487 static
488 int ctf_open_file_stream_read(struct ctf_trace *td, const char *path, int flags)
489 {
490 int ret;
491 struct ctf_file_stream *file_stream;
492
493 ret = openat(td->dirfd, path, flags);
494 if (ret < 0)
495 goto error;
496 file_stream = g_new0(struct ctf_file_stream, 1);
497 ctf_init_pos(&file_stream->pos, ret, flags);
498 ret = create_stream_packet_index(td, file_stream);
499 if (ret)
500 goto error_index;
501 /* Add stream file to stream class */
502 g_ptr_array_add(file_stream->stream->files, file_stream);
503 return 0;
504
505 error_index:
506 ctf_fini_pos(&file_stream->pos);
507 close(file_stream->pos.fd);
508 g_free(file_stream);
509 error:
510 return ret;
511 }
512
513 static
514 int ctf_open_trace_read(struct ctf_trace *td, const char *path, int flags)
515 {
516 int ret;
517 struct dirent *dirent;
518 struct dirent *diriter;
519 size_t dirent_len;
520
521 td->flags = flags;
522
523 /* Open trace directory */
524 td->dir = opendir(path);
525 if (!td->dir) {
526 fprintf(stdout, "[error] Unable to open trace directory.\n");
527 ret = -ENOENT;
528 goto error;
529 }
530
531 td->dirfd = open(path, 0);
532 if (td->dirfd < 0) {
533 fprintf(stdout, "[error] Unable to open trace directory file descriptor.\n");
534 ret = -ENOENT;
535 goto error_dirfd;
536 }
537
538 td->streams = g_ptr_array_new();
539
540 /*
541 * Keep the metadata file separate.
542 */
543
544 ret = ctf_open_trace_metadata_read(td);
545 if (ret) {
546 goto error_metadata;
547 }
548
549 /*
550 * Open each stream: for each file, try to open, check magic
551 * number, and get the stream ID to add to the right location in
552 * the stream array.
553 */
554
555 dirent_len = offsetof(struct dirent, d_name) +
556 fpathconf(td->dirfd, _PC_NAME_MAX) + 1;
557
558 dirent = malloc(dirent_len);
559
560 for (;;) {
561 ret = readdir_r(td->dir, dirent, &diriter);
562 if (ret) {
563 fprintf(stdout, "[error] Readdir error.\n");
564 goto readdir_error;
565 }
566 if (!diriter)
567 break;
568 /* Ignore hidden files, ., .. and metadata. */
569 if (!strncmp(diriter->d_name, ".", 1)
570 || !strcmp(diriter->d_name, "..")
571 || !strcmp(diriter->d_name, "metadata"))
572 continue;
573 ret = ctf_open_file_stream_read(td, diriter->d_name, flags);
574 if (ret) {
575 fprintf(stdout, "[error] Open file stream error.\n");
576 goto readdir_error;
577 }
578 }
579
580 free(dirent);
581 return 0;
582
583 readdir_error:
584 free(dirent);
585 error_metadata:
586 g_ptr_array_free(td->streams, TRUE);
587 close(td->dirfd);
588 error_dirfd:
589 closedir(td->dir);
590 error:
591 return ret;
592 }
593
594 struct trace_descriptor *ctf_open_trace(const char *path, int flags)
595 {
596 struct ctf_trace *td;
597 int ret;
598
599 td = g_new0(struct ctf_trace, 1);
600
601 switch (flags & O_ACCMODE) {
602 case O_RDONLY:
603 if (!path) {
604 fprintf(stdout, "[error] Path missing for input CTF trace.\n");
605 goto error;
606 }
607 ret = ctf_open_trace_read(td, path, flags);
608 if (ret)
609 goto error;
610 break;
611 case O_RDWR:
612 fprintf(stdout, "[error] Opening CTF traces for output is not supported yet.\n");
613 goto error;
614 default:
615 fprintf(stdout, "[error] Incorrect open flags.\n");
616 goto error;
617 }
618
619 return &td->parent;
620 error:
621 g_free(td);
622 return NULL;
623 }
624
625 static
626 void ctf_close_file_stream(struct ctf_file_stream *file_stream)
627 {
628 ctf_fini_pos(&file_stream->pos);
629 close(file_stream->pos.fd);
630 }
631
632 void ctf_close_trace(struct trace_descriptor *tdp)
633 {
634 struct ctf_trace *td = container_of(tdp, struct ctf_trace, parent);
635 int i;
636
637 if (td->streams) {
638 for (i = 0; i < td->streams->len; i++) {
639 struct ctf_stream *stream;
640 int j;
641 stream = g_ptr_array_index(td->streams, i);
642 for (j = 0; j < stream->files->len; j++) {
643 struct ctf_file_stream *file_stream;
644 file_stream = g_ptr_array_index(stream->files, j);
645 ctf_close_file_stream(file_stream);
646 }
647
648 }
649 g_ptr_array_free(td->streams, TRUE);
650 }
651 closedir(td->dir);
652 g_free(td);
653 }
654
655 void __attribute__((constructor)) ctf_init(void)
656 {
657 int ret;
658
659 ctf_format.name = g_quark_from_static_string("ctf");
660 ret = bt_register_format(&ctf_format);
661 assert(!ret);
662 }
663
664 /* TODO: finalize */
This page took 0.046029 seconds and 4 git commands to generate.