From: Mathieu Desnoyers Date: Wed, 19 Feb 2014 00:19:26 +0000 (-0500) Subject: CTF: Support incremental metadata append X-Git-Tag: v1.2.0-rc2~30 X-Git-Url: http://git.efficios.com/?p=babeltrace.git;a=commitdiff_plain;h=0c880b0af719e4d58ccebe61acdd0e77a4479bfd CTF: Support incremental metadata append Signed-off-by: Mathieu Desnoyers --- diff --git a/formats/ctf/ctf.c b/formats/ctf/ctf.c index 5ddf2f78..2ff68bbf 100644 --- a/formats/ctf/ctf.c +++ b/formats/ctf/ctf.c @@ -1049,7 +1049,7 @@ warning: } static -int ctf_open_trace_metadata_packet_read(struct ctf_trace *td, FILE *in, +int ctf_trace_metadata_packet_read(struct ctf_trace *td, FILE *in, FILE *out) { struct metadata_packet_header header; @@ -1139,7 +1139,7 @@ read_padding: } static -int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp, +int ctf_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp, char **buf) { FILE *in, *out; @@ -1158,7 +1158,7 @@ int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp, return -errno; } for (;;) { - ret = ctf_open_trace_metadata_packet_read(td, in, out); + ret = ctf_trace_metadata_packet_read(td, in, out); if (ret) { break; } @@ -1199,10 +1199,8 @@ int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp, } static -int ctf_open_trace_metadata_read(struct ctf_trace *td, - void (*packet_seek)(struct bt_stream_pos *pos, size_t index, - int whence), FILE *metadata_fp, - struct ctf_scanner *scanner) +int ctf_trace_metadata_read(struct ctf_trace *td, FILE *metadata_fp, + struct ctf_scanner *scanner, int append) { struct ctf_file_stream *metadata_stream; FILE *fp; @@ -1212,14 +1210,6 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td, metadata_stream = g_new0(struct ctf_file_stream, 1); metadata_stream->pos.last_offset = LAST_OFFSET_POISON; - if (packet_seek) { - metadata_stream->pos.packet_seek = packet_seek; - } else { - fprintf(stderr, "[error] packet_seek function undefined.\n"); - ret = -1; - goto end_free; - } - if (metadata_fp) { fp = metadata_fp; metadata_stream->pos.fd = -1; @@ -1246,7 +1236,7 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td, yydebug = 1; if (packet_metadata(td, fp)) { - ret = ctf_open_trace_metadata_stream_read(td, &fp, &buf); + ret = ctf_trace_metadata_stream_read(td, &fp, &buf); if (ret) { /* Warn about empty metadata */ fprintf(stderr, "[warning] Empty metadata.\n"); @@ -1255,20 +1245,22 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td, td->metadata_string = buf; td->metadata_packetized = 1; } else { - unsigned int major, minor; - ssize_t nr_items; - - td->byte_order = BYTE_ORDER; - - /* Check text-only metadata header and version */ - nr_items = fscanf(fp, "/* CTF %10u.%10u", &major, &minor); - if (nr_items < 2) - fprintf(stderr, "[warning] Ill-shapen or missing \"/* CTF x.y\" header for text-only metadata.\n"); - if (check_version(major, minor) < 0) { - ret = -EINVAL; - goto end; + if (!append) { + unsigned int major, minor; + ssize_t nr_items; + + td->byte_order = BYTE_ORDER; + + /* Check text-only metadata header and version */ + nr_items = fscanf(fp, "/* CTF %10u.%10u", &major, &minor); + if (nr_items < 2) + fprintf(stderr, "[warning] Ill-shapen or missing \"/* CTF x.y\" header for text-only metadata.\n"); + if (check_version(major, minor) < 0) { + ret = -EINVAL; + goto end; + } + rewind(fp); } - rewind(fp); } ret = ctf_scanner_append_ast(scanner, fp); @@ -1358,6 +1350,36 @@ error: return NULL; } +static +int copy_event_declarations_stream_class_to_stream(struct ctf_trace *td, + struct ctf_stream_declaration *stream_class, + struct ctf_stream_definition *stream) +{ + size_t def_size, class_size, i; + int ret = 0; + + def_size = stream->events_by_id->len; + class_size = stream_class->events_by_id->len; + + g_ptr_array_set_size(stream->events_by_id, class_size); + for (i = def_size; i < class_size; i++) { + struct ctf_event_declaration *event = + g_ptr_array_index(stream_class->events_by_id, i); + struct ctf_event_definition *stream_event; + + if (!event) + continue; + stream_event = create_event_definitions(td, stream, event); + if (!stream_event) { + ret = -EINVAL; + goto error; + } + g_ptr_array_index(stream->events_by_id, i) = stream_event; + } +error: + return ret; +} + static int create_stream_definitions(struct ctf_trace *td, struct ctf_stream_definition *stream) { @@ -1407,20 +1429,10 @@ int create_stream_definitions(struct ctf_trace *td, struct ctf_stream_definition stream->parent_def_scope = stream->stream_event_context->p.scope; } stream->events_by_id = g_ptr_array_new(); - g_ptr_array_set_size(stream->events_by_id, stream_class->events_by_id->len); - for (i = 0; i < stream->events_by_id->len; i++) { - struct ctf_event_declaration *event = g_ptr_array_index(stream_class->events_by_id, i); - struct ctf_event_definition *stream_event; - - if (!event) - continue; - stream_event = create_event_definitions(td, stream, event); - if (!stream_event) { - ret = -EINVAL; - goto error_event; - } - g_ptr_array_index(stream->events_by_id, i) = stream_event; - } + ret = copy_event_declarations_stream_class_to_stream(td, + stream_class, stream); + if (ret) + goto error_event; return 0; error_event: @@ -2065,6 +2077,8 @@ int ctf_open_trace_read(struct ctf_trace *td, /* * Keep the metadata file separate. + * Keep scanner object local to the open. We don't support + * incremental metadata append for on-disk traces. */ scanner = ctf_scanner_alloc(); if (!scanner) { @@ -2072,8 +2086,7 @@ int ctf_open_trace_read(struct ctf_trace *td, ret = -ENOMEM; goto error_metadata; } - ret = ctf_open_trace_metadata_read(td, packet_seek, metadata_fp, - scanner); + ret = ctf_trace_metadata_read(td, metadata_fp, scanner, 0); ctf_scanner_free(scanner); if (ret) { fprintf(stderr, "[warning] Unable to open trace metadata for path \"%s\".\n", path); @@ -2284,16 +2297,14 @@ int ctf_open_mmap_trace_read(struct ctf_trace *td, { int ret; struct bt_mmap_stream *mmap_info; - struct ctf_scanner *scanner; - scanner = ctf_scanner_alloc(); - if (!scanner) { + td->scanner = ctf_scanner_alloc(); + if (!td->scanner) { fprintf(stderr, "[error] Error allocating scanner\n"); ret = -ENOMEM; - goto error_scanner_alloc; + goto error; } - ret = ctf_open_trace_metadata_read(td, ctf_packet_seek, metadata_fp, - scanner); + ret = ctf_trace_metadata_read(td, metadata_fp, td->scanner, 0); if (ret) { goto error; } @@ -2309,12 +2320,10 @@ int ctf_open_mmap_trace_read(struct ctf_trace *td, goto error; } } - ctf_scanner_free(scanner); return 0; error: - ctf_scanner_free(scanner); -error_scanner_alloc: + ctf_scanner_free(td->scanner); return ret; } @@ -2351,6 +2360,41 @@ error: return NULL; } +int ctf_append_trace_metadata(struct bt_trace_descriptor *tdp, + FILE *metadata_fp) +{ + struct ctf_trace *td = container_of(tdp, struct ctf_trace, parent); + int i, j; + int ret; + + if (!td->scanner) + return -EINVAL; + ret = ctf_trace_metadata_read(td, metadata_fp, td->scanner, 1); + if (ret) + return ret; + /* for each stream_class */ + for (i = 0; i < td->streams->len; i++) { + struct ctf_stream_declaration *stream_class; + + stream_class = g_ptr_array_index(td->streams, i); + if (!stream_class) + continue; + /* for each stream */ + for (j = 0; j < stream_class->streams->len; j++) { + struct ctf_stream_definition *stream; + + stream = g_ptr_array_index(stream_class->streams, j); + if (!stream) + continue; + ret = copy_event_declarations_stream_class_to_stream(td, + stream_class, stream); + if (ret) + return ret; + } + } + return 0; +} + static int ctf_convert_index_timestamp(struct bt_trace_descriptor *tdp) { @@ -2443,6 +2487,7 @@ int ctf_close_trace(struct bt_trace_descriptor *tdp) } } ctf_destroy_metadata(td); + ctf_scanner_free(td->scanner); if (td->dirfd >= 0) { ret = close(td->dirfd); if (ret) { diff --git a/formats/ctf/metadata/ctf-ast.h b/formats/ctf/metadata/ctf-ast.h index 93bce7d8..2ba6e860 100644 --- a/formats/ctf/metadata/ctf-ast.h +++ b/formats/ctf/metadata/ctf-ast.h @@ -73,6 +73,14 @@ struct ctf_node { struct bt_list_head siblings; struct bt_list_head tmp_head; unsigned int lineno; + /* + * We mark nodes visited in the generate-io-struct phase (last + * phase). We only mark the 1-depth level nodes as visited + * (never the root node, and not their sub-nodes). This allows + * skipping already visited nodes when doing incremental + * metadata append. + */ + int visited; enum node_type type; union { diff --git a/formats/ctf/metadata/ctf-parser.y b/formats/ctf/metadata/ctf-parser.y index 61ab4f58..68fde931 100644 --- a/formats/ctf/metadata/ctf-parser.y +++ b/formats/ctf/metadata/ctf-parser.y @@ -1025,6 +1025,8 @@ void ctf_scanner_free(struct ctf_scanner *scanner) { int ret; + if (!scanner) + return; finalize_scope(&scanner->root_scope); objstack_destroy(scanner->objstack); ret = yylex_destroy(scanner->scanner); diff --git a/formats/ctf/metadata/ctf-visitor-generate-io-struct.c b/formats/ctf/metadata/ctf-visitor-generate-io-struct.c index 98169457..9acda567 100644 --- a/formats/ctf/metadata/ctf-visitor-generate-io-struct.c +++ b/formats/ctf/metadata/ctf-visitor-generate-io-struct.c @@ -1854,6 +1854,10 @@ int ctf_event_visit(FILE *fd, int depth, struct ctf_node *node, struct ctf_event_declaration *event; struct bt_ctf_event_decl *event_decl; + if (node->visited) + return 0; + node->visited = 1; + event_decl = g_new0(struct bt_ctf_event_decl, 1); event = &event_decl->parent; event->declaration_scope = bt_new_declaration_scope(parent_declaration_scope); @@ -2051,6 +2055,12 @@ int ctf_stream_visit(FILE *fd, int depth, struct ctf_node *node, struct ctf_node *iter; struct ctf_stream_declaration *stream; + if (node) { + if (node->visited) + return 0; + node->visited = 1; + } + stream = g_new0(struct ctf_stream_declaration, 1); stream->declaration_scope = bt_new_declaration_scope(parent_declaration_scope); stream->events_by_id = g_ptr_array_new(); @@ -2250,8 +2260,13 @@ int ctf_trace_visit(FILE *fd, int depth, struct ctf_node *node, struct ctf_trace int ret = 0; struct ctf_node *iter; + if (!trace->restart_root_decl && node->visited) + return 0; + node->visited = 1; + if (trace->declaration_scope) return -EEXIST; + trace->declaration_scope = bt_new_declaration_scope(trace->root_declaration_scope); trace->streams = g_ptr_array_new(); trace->event_declarations = g_ptr_array_new(); @@ -2443,6 +2458,10 @@ int ctf_clock_visit(FILE *fd, int depth, struct ctf_node *node, struct ctf_trace struct ctf_node *iter; struct ctf_clock *clock; + if (node->visited) + return 0; + node->visited = 1; + clock = g_new0(struct ctf_clock, 1); /* Default clock frequency is set to 1000000000 */ clock->freq = 1000000000ULL; @@ -2633,6 +2652,10 @@ int ctf_callsite_visit(FILE *fd, int depth, struct ctf_node *node, struct ctf_tr struct ctf_callsite *callsite; struct ctf_callsite_dups *cs_dups; + if (node->visited) + return 0; + node->visited = 1; + callsite = g_new0(struct ctf_callsite, 1); bt_list_for_each_entry(iter, &node->u.callsite.declaration_list, siblings) { ret = ctf_callsite_declaration_visit(fd, depth + 1, iter, callsite, trace); @@ -2867,6 +2890,10 @@ int ctf_env_visit(FILE *fd, int depth, struct ctf_node *node, struct ctf_trace * int ret = 0; struct ctf_node *iter; + if (node->visited) + return 0; + node->visited = 1; + trace->env.vpid = -1; trace->env.procname[0] = '\0'; trace->env.hostname[0] = '\0'; @@ -2888,6 +2915,10 @@ int ctf_root_declaration_visit(FILE *fd, int depth, struct ctf_node *node, struc { int ret = 0; + if (!trace->restart_root_decl && node->visited) + return 0; + node->visited = 1; + switch (node->type) { case NODE_TYPEDEF: ret = ctf_typedef_visit(fd, depth + 1, @@ -2933,7 +2964,6 @@ int ctf_visitor_construct_metadata(FILE *fd, int depth, struct ctf_node *node, { int ret = 0; struct ctf_node *iter; - int env_clock_done = 0; printf_verbose("CTF visitor: metadata construction...\n"); trace->byte_order = byte_order; @@ -2947,24 +2977,21 @@ retry: switch (node->type) { case NODE_ROOT: - if (!env_clock_done) { - /* - * declarations need to query clock hash table, - * so clock need to be treated first. - */ - if (bt_list_empty(&node->u.root.clock)) { - ctf_clock_default(fd, depth + 1, trace); - } else { - bt_list_for_each_entry(iter, &node->u.root.clock, siblings) { - ret = ctf_clock_visit(fd, depth + 1, iter, - trace); - if (ret) { - fprintf(fd, "[error] %s: clock declaration error\n", __func__); - goto error; - } + /* + * declarations need to query clock hash table, + * so clock need to be treated first. + */ + if (bt_list_empty(&node->u.root.clock)) { + ctf_clock_default(fd, depth + 1, trace); + } else { + bt_list_for_each_entry(iter, &node->u.root.clock, siblings) { + ret = ctf_clock_visit(fd, depth + 1, iter, + trace); + if (ret) { + fprintf(fd, "[error] %s: clock declaration error\n", __func__); + goto error; } } - env_clock_done = 1; } bt_list_for_each_entry(iter, &node->u.root.declaration_list, siblings) { @@ -2977,6 +3004,7 @@ retry: bt_list_for_each_entry(iter, &node->u.root.trace, siblings) { ret = ctf_trace_visit(fd, depth + 1, iter, trace); if (ret == -EINTR) { + trace->restart_root_decl = 1; bt_free_declaration_scope(trace->root_declaration_scope); /* * Need to restart creation of type @@ -2990,6 +3018,7 @@ retry: goto error; } } + trace->restart_root_decl = 0; bt_list_for_each_entry(iter, &node->u.root.callsite, siblings) { ret = ctf_callsite_visit(fd, depth + 1, iter, trace); diff --git a/formats/ctf/metadata/ctf-visitor-parent-links.c b/formats/ctf/metadata/ctf-visitor-parent-links.c index 104afb71..037496af 100644 --- a/formats/ctf/metadata/ctf-visitor-parent-links.c +++ b/formats/ctf/metadata/ctf-visitor-parent-links.c @@ -180,6 +180,9 @@ int ctf_visitor_parent_links(FILE *fd, int depth, struct ctf_node *node) int ret = 0; struct ctf_node *iter; + if (node->visited) + return 0; + switch (node->type) { case NODE_ROOT: bt_list_for_each_entry(iter, &node->u.root.declaration_list, siblings) { diff --git a/formats/ctf/metadata/ctf-visitor-semantic-validator.c b/formats/ctf/metadata/ctf-visitor-semantic-validator.c index 13b2fc55..96776a95 100644 --- a/formats/ctf/metadata/ctf-visitor-semantic-validator.c +++ b/formats/ctf/metadata/ctf-visitor-semantic-validator.c @@ -434,6 +434,9 @@ int _ctf_visitor_semantic_check(FILE *fd, int depth, struct ctf_node *node) int ret = 0; struct ctf_node *iter; + if (node->visited) + return 0; + switch (node->type) { case NODE_ROOT: bt_list_for_each_entry(iter, &node->u.root.declaration_list, siblings) { diff --git a/formats/ctf/metadata/ctf-visitor-xml.c b/formats/ctf/metadata/ctf-visitor-xml.c index 3c1d37e7..fca77046 100644 --- a/formats/ctf/metadata/ctf-visitor-xml.c +++ b/formats/ctf/metadata/ctf-visitor-xml.c @@ -357,6 +357,9 @@ int ctf_visitor_print_xml(FILE *fd, int depth, struct ctf_node *node) int ret = 0; struct ctf_node *iter; + if (node->visited) + return 0; + switch (node->type) { case NODE_ROOT: print_tabs(fd, depth); diff --git a/include/babeltrace/ctf-ir/metadata.h b/include/babeltrace/ctf-ir/metadata.h index 5e92984d..93c7a586 100644 --- a/include/babeltrace/ctf-ir/metadata.h +++ b/include/babeltrace/ctf-ir/metadata.h @@ -42,6 +42,7 @@ struct ctf_stream_declaration; struct ctf_event_declaration; struct ctf_clock; struct ctf_callsite; +struct ctf_scanner; struct ctf_stream_definition { struct ctf_stream_declaration *stream_class; @@ -192,6 +193,8 @@ struct ctf_trace { GPtrArray *event_declarations; /* Array of all the struct bt_ctf_event_decl */ struct declaration_struct *packet_header_decl; + struct ctf_scanner *scanner; + int restart_root_decl; uint64_t major; uint64_t minor; diff --git a/include/babeltrace/ctf/types.h b/include/babeltrace/ctf/types.h index f268e13a..67b24ee4 100644 --- a/include/babeltrace/ctf/types.h +++ b/include/babeltrace/ctf/types.h @@ -242,5 +242,7 @@ void ctf_pos_get_event(struct ctf_stream_pos *pos) void ctf_print_timestamp(FILE *fp, struct ctf_stream_definition *stream, uint64_t timestamp); +int ctf_append_trace_metadata(struct bt_trace_descriptor *tdp, + FILE *metadata_fp); #endif /* _BABELTRACE_CTF_TYPES_H */