X-Git-Url: http://git.efficios.com/?p=babeltrace.git;a=blobdiff_plain;f=formats%2Fctf%2Fctf.c;h=e31808c3843d27380a63b0d173a949aa027e336e;hp=eec3350f3ba73b2c40c27c479943f3d80b4c093c;hb=2654fe9bab8f0eaeb17264ef7abadfd14e245b23;hpb=e69dd258fa9c5f460a9071f4c6a9fcc98aaa010e

diff --git a/formats/ctf/ctf.c b/formats/ctf/ctf.c
index eec3350f..e31808c3 100644
--- a/formats/ctf/ctf.c
+++ b/formats/ctf/ctf.c
@@ -84,6 +84,13 @@ uint64_t opt_clock_offset_ns;
 
 extern int yydebug;
 
+/*
+ * TODO: babeltrace_ctf_console_output ensures that we only print
+ * discarded events when ctf-text plugin is used. Should be cleaned up
+ * with the plugin system redesign.
+ */
+int babeltrace_ctf_console_output;
+
 static
 struct bt_trace_descriptor *ctf_open_trace(const char *path, int flags,
 		void (*packet_seek)(struct bt_stream_pos *pos, size_t index,
@@ -181,26 +188,24 @@ uint64_t ctf_timestamp_begin(struct bt_trace_descriptor *descriptor,
 					parent);
 			stream_pos = &cfs->pos;
 
-			if (!stream_pos->packet_real_index)
+			if (!stream_pos->packet_index)
 				goto error;
 
-			if (stream_pos->packet_real_index->len <= 0)
+			if (stream_pos->packet_index->len <= 0)
 				continue;
 
+			index = &g_array_index(stream_pos->packet_index,
+					struct packet_index,
+					stream_pos->packet_index->len - 1);
 			if (type == BT_CLOCK_REAL) {
-				index = &g_array_index(stream_pos->packet_real_index,
-						struct packet_index,
-						stream_pos->packet_real_index->len - 1);
+				if (index->ts_real.timestamp_begin < begin)
+					begin = index->ts_real.timestamp_begin;
 			} else if (type == BT_CLOCK_CYCLES) {
-				index = &g_array_index(stream_pos->packet_cycles_index,
-						struct packet_index,
-						stream_pos->packet_real_index->len - 1);
-
+				if (index->ts_cycles.timestamp_begin < begin)
+					begin = index->ts_cycles.timestamp_begin;
 			} else {
 				goto error;
 			}
-			if (index->timestamp_begin < begin)
-				begin = index->timestamp_begin;
 		}
 	}
 
@@ -242,26 +247,24 @@ uint64_t ctf_timestamp_end(struct bt_trace_descriptor *descriptor,
 					parent);
 			stream_pos = &cfs->pos;
 
-			if (!stream_pos->packet_real_index)
+			if (!stream_pos->packet_index)
 				goto error;
 
-			if (stream_pos->packet_real_index->len <= 0)
+			if (stream_pos->packet_index->len <= 0)
 				continue;
 
+			index = &g_array_index(stream_pos->packet_index,
+					struct packet_index,
+					stream_pos->packet_index->len - 1);
 			if (type == BT_CLOCK_REAL) {
-				index = &g_array_index(stream_pos->packet_real_index,
-						struct packet_index,
-						stream_pos->packet_real_index->len - 1);
+				if (index->ts_real.timestamp_end > end)
+					end = index->ts_real.timestamp_end;
 			} else if (type == BT_CLOCK_CYCLES) {
-				index = &g_array_index(stream_pos->packet_cycles_index,
-						struct packet_index,
-						stream_pos->packet_real_index->len - 1);
-
+				if (index->ts_cycles.timestamp_end > end)
+					end = index->ts_cycles.timestamp_end;
 			} else {
 				goto error;
 			}
-			if (index->timestamp_end > end)
-				end = index->timestamp_end;
 		}
 	}
 
@@ -283,10 +286,7 @@ void ctf_update_timestamp(struct ctf_stream_definition *stream,
 	uint64_t oldval, newval, updateval;
 
 	if (unlikely(integer_declaration->len == 64)) {
-		stream->prev_cycles_timestamp = stream->cycles_timestamp;
 		stream->cycles_timestamp = integer_definition->value._unsigned;
-		stream->prev_real_timestamp = ctf_get_real_timestamp(stream,
-				stream->prev_cycles_timestamp);
 		stream->real_timestamp = ctf_get_real_timestamp(stream,
 				stream->cycles_timestamp);
 		return;
@@ -302,12 +302,9 @@ void ctf_update_timestamp(struct ctf_stream_definition *stream,
 	updateval = stream->cycles_timestamp;
 	updateval &= ~((1ULL << integer_declaration->len) - 1);
 	updateval += newval;
-	stream->prev_cycles_timestamp = stream->cycles_timestamp;
 	stream->cycles_timestamp = updateval;
 
 	/* convert to real timestamp */
-	stream->prev_real_timestamp = ctf_get_real_timestamp(stream,
-			stream->prev_cycles_timestamp);
 	stream->real_timestamp = ctf_get_real_timestamp(stream,
 			stream->cycles_timestamp);
 }
@@ -412,24 +409,39 @@ void print_uuid(FILE *fp, unsigned char *uuid)
 		fprintf(fp, "%x", (unsigned int) uuid[i]);
 }
 
-void ctf_print_discarded(FILE *fp, struct ctf_stream_definition *stream,
-		int end_stream)
+/*
+ * Discarded events can be either:
+ * - discarded after end of previous buffer due to buffer full:
+ *     happened within range: [ prev_timestamp_end, timestamp_begin ]
+ * - discarded within current buffer due to either event too large or
+ *   nested wrap-around:
+ *     happened within range: [ timestamp_begin, timestamp_end ]
+ *
+ * Given we have discarded counters of those two types merged into the
+ * events_discarded counter, we need to use the union of those ranges:
+ *   [ prev_timestamp_end, timestamp_end ]
+ */
+static
+void ctf_print_discarded(FILE *fp, struct ctf_stream_definition *stream)
 {
-	fprintf(fp, "[warning] Tracer discarded %" PRIu64 " events %sbetween [",
-		stream->events_discarded,
-		end_stream ? "at end of stream " : "");
+	if (!stream->events_discarded || !babeltrace_ctf_console_output) {
+		return;
+	}
+	fflush(stdout);
+	fprintf(fp, "[warning] Tracer discarded %" PRIu64 " events between [",
+		stream->events_discarded);
 	if (opt_clock_cycles) {
 		ctf_print_timestamp(fp, stream,
-				stream->prev_cycles_timestamp);
+				stream->prev.cycles.end);
 		fprintf(fp, "] and [");
 		ctf_print_timestamp(fp, stream,
-				stream->prev_cycles_timestamp_end);
+				stream->current.cycles.end);
 	} else {
 		ctf_print_timestamp(fp, stream,
-				stream->prev_real_timestamp);
+				stream->prev.real.end);
 		fprintf(fp, "] and [");
 		ctf_print_timestamp(fp, stream,
-				stream->prev_real_timestamp_end);
+				stream->current.real.end);
 	}
 	fprintf(fp, "] in trace UUID ");
 	print_uuid(fp, stream->stream_class->trace->uuid);
@@ -472,12 +484,16 @@ int ctf_read_event(struct bt_stream_pos *ppos, struct ctf_stream_definition *str
 	if (unlikely(pos->offset == EOF))
 		return EOF;
 
-	if (pos->content_size == 0) {
-		/* Stream is inactive for now (live reading). */
+	/* Stream is inactive for now (live reading). */
+	if (unlikely(pos->content_size == 0))
 		return EAGAIN;
-	}
-	/* Packet only contains headers */
-	if (pos->offset == pos->content_size)
+
+	/*
+	 * Packet seeked to by ctf_pos_get_event() only contains
+	 * headers, no event. Consider stream as inactive (live
+	 * reading).
+	 */
+	if (unlikely(pos->data_offset == pos->content_size))
 		return EAGAIN;
 
 	assert(pos->offset < pos->content_size);
@@ -733,13 +749,10 @@ int ctf_init_pos(struct ctf_stream_pos *pos, struct bt_trace_descriptor *trace,
 {
 	pos->fd = fd;
 	if (fd >= 0) {
-		pos->packet_cycles_index = g_array_new(FALSE, TRUE,
-						sizeof(struct packet_index));
-		pos->packet_real_index = g_array_new(FALSE, TRUE,
+		pos->packet_index = g_array_new(FALSE, TRUE,
 				sizeof(struct packet_index));
 	} else {
-		pos->packet_cycles_index = NULL;
-		pos->packet_real_index = NULL;
+		pos->packet_index = NULL;
 	}
 	switch (open_flags & O_ACCMODE) {
 	case O_RDONLY:
@@ -779,13 +792,65 @@ int ctf_fini_pos(struct ctf_stream_pos *pos)
 			return -1;
 		}
 	}
-	if (pos->packet_cycles_index)
-		(void) g_array_free(pos->packet_cycles_index, TRUE);
-	if (pos->packet_real_index)
-		(void) g_array_free(pos->packet_real_index, TRUE);
+	if (pos->packet_index)
+		(void) g_array_free(pos->packet_index, TRUE);
 	return 0;
 }
 
+void ctf_update_current_packet_index(struct ctf_stream_definition *stream,
+		struct packet_index *prev_index,
+		struct packet_index *cur_index)
+{
+	uint64_t events_discarded_diff;
+
+	/* Update packet index time information */
+
+	/* Current packet begin/end */
+	stream->current.real.begin =
+		cur_index->ts_real.timestamp_begin;
+	stream->current.cycles.begin =
+		cur_index->ts_cycles.timestamp_begin;
+	stream->current.real.end =
+		cur_index->ts_real.timestamp_end;
+	stream->current.cycles.end =
+		cur_index->ts_cycles.timestamp_end;
+
+	/* Update packet index discarded event information */
+	events_discarded_diff = cur_index->events_discarded;
+	if (prev_index) {
+		/* Previous packet begin/end */
+		stream->prev.cycles.begin =
+			prev_index->ts_cycles.timestamp_begin;
+		stream->prev.real.begin =
+			prev_index->ts_real.timestamp_begin;
+		stream->prev.cycles.end =
+			prev_index->ts_cycles.timestamp_end;
+		stream->prev.real.end =
+			prev_index->ts_real.timestamp_end;
+
+		events_discarded_diff -= prev_index->events_discarded;
+		/*
+		 * Deal with 32-bit wrap-around if the tracer provided a
+		 * 32-bit field.
+		 */
+		if (prev_index->events_discarded_len == 32) {
+			events_discarded_diff = (uint32_t) events_discarded_diff;
+		}
+	} else {
+		/*
+		 * First packet: use current packet info as limits for
+		 * previous packet.
+		 */
+		stream->prev.cycles.begin =
+			stream->prev.cycles.end =
+				stream->current.cycles.begin;
+		stream->prev.real.begin =
+			stream->prev.real.end =
+				stream->current.real.begin;
+	}
+	stream->events_discarded = events_discarded_diff;
+}
+
 /*
  * for SEEK_CUR: go to next packet.
  * for SEEK_SET: go to packet numer (index).
@@ -798,7 +863,7 @@ void ctf_packet_seek(struct bt_stream_pos *stream_pos, size_t index, int whence)
 		container_of(pos, struct ctf_file_stream, pos);
 	int ret;
 	off_t off;
-	struct packet_index *packet_index;
+	struct packet_index *packet_index, *prev_index;
 
 	switch (whence) {
 	case SEEK_CUR:
@@ -850,102 +915,58 @@ void ctf_packet_seek(struct bt_stream_pos *stream_pos, size_t index, int whence)
 		switch (whence) {
 		case SEEK_CUR:
 		{
-			uint64_t events_discarded_diff;
-
 			if (pos->offset == EOF) {
 				return;
 			}
-			assert(pos->cur_index < pos->packet_cycles_index->len);
-			assert(pos->cur_index < pos->packet_real_index->len);
-
-			/* For printing discarded event count */
-			packet_index = &g_array_index(pos->packet_cycles_index,
-					struct packet_index, pos->cur_index);
-			file_stream->parent.prev_cycles_timestamp_end =
-				packet_index->timestamp_end;
-			file_stream->parent.prev_cycles_timestamp =
-				packet_index->timestamp_begin;
-
-			packet_index = &g_array_index(pos->packet_real_index,
-					struct packet_index, pos->cur_index);
-			file_stream->parent.prev_real_timestamp_end =
-						packet_index->timestamp_end;
-			file_stream->parent.prev_real_timestamp =
-				packet_index->timestamp_begin;
-
-			events_discarded_diff = packet_index->events_discarded;
-			if (pos->cur_index > 0) {
-				packet_index = &g_array_index(pos->packet_real_index,
-						struct packet_index,
-						pos->cur_index - 1);
-				events_discarded_diff -= packet_index->events_discarded;
-				/*
-				 * Deal with 32-bit wrap-around if the
-				 * tracer provided a 32-bit field.
-				 */
-				if (packet_index->events_discarded_len == 32) {
-					events_discarded_diff = (uint32_t) events_discarded_diff;
-				}
-			}
-			file_stream->parent.events_discarded = events_discarded_diff;
-			file_stream->parent.prev_real_timestamp = file_stream->parent.real_timestamp;
-			file_stream->parent.prev_cycles_timestamp = file_stream->parent.cycles_timestamp;
+			assert(pos->cur_index < pos->packet_index->len);
 			/* The reader will expect us to skip padding */
 			++pos->cur_index;
 			break;
 		}
 		case SEEK_SET:
-			if (index >= pos->packet_cycles_index->len) {
+			if (index >= pos->packet_index->len) {
 				pos->offset = EOF;
 				return;
 			}
-			packet_index = &g_array_index(pos->packet_cycles_index,
-					struct packet_index, index);
-			pos->last_events_discarded = packet_index->events_discarded;
 			pos->cur_index = index;
-			file_stream->parent.prev_real_timestamp = 0;
-			file_stream->parent.prev_real_timestamp_end = 0;
-			file_stream->parent.prev_cycles_timestamp = 0;
-			file_stream->parent.prev_cycles_timestamp_end = 0;
 			break;
 		default:
 			assert(0);
 		}
-		if (pos->cur_index >= pos->packet_real_index->len) {
-			/*
-			 * We need to check if we are in trace read or
-			 * called from packet indexing.  In this last
-			 * case, the collection is not there, so we
-			 * cannot print the timestamps.
-			 */
-			if ((&file_stream->parent)->stream_class->trace->parent.collection) {
-				/*
-				 * When a stream reaches the end of the
-				 * file, we need to show the number of
-				 * events discarded ourselves, because
-				 * there is no next event scheduled to
-				 * be printed in the output.
-				 */
-				if (file_stream->parent.events_discarded) {
-					fflush(stdout);
-					ctf_print_discarded(stderr,
-						&file_stream->parent,
-						1);
-					file_stream->parent.events_discarded = 0;
-				}
-			}
+
+		packet_index = &g_array_index(pos->packet_index,
+				struct packet_index, pos->cur_index);
+		if (pos->cur_index > 0) {
+			prev_index = &g_array_index(pos->packet_index,
+					struct packet_index,
+					pos->cur_index - 1);
+		} else {
+			prev_index = NULL;
+		}
+		ctf_update_current_packet_index(&file_stream->parent,
+				prev_index, packet_index);
+
+		if (pos->cur_index >= pos->packet_index->len) {
 			pos->offset = EOF;
 			return;
 		}
-		packet_index = &g_array_index(pos->packet_cycles_index,
-				struct packet_index,
-				pos->cur_index);
-		file_stream->parent.cycles_timestamp = packet_index->timestamp_begin;
 
-		packet_index = &g_array_index(pos->packet_real_index,
+		/*
+		 * We need to check if we are in trace read or called
+		 * from packet indexing.  In this last case, the
+		 * collection is not there, so we cannot print the
+		 * timestamps.
+		 */
+		if ((&file_stream->parent)->stream_class->trace->parent.collection) {
+			ctf_print_discarded(stderr, &file_stream->parent);
+		}
+
+		packet_index = &g_array_index(pos->packet_index,
 				struct packet_index,
 				pos->cur_index);
-		file_stream->parent.real_timestamp = packet_index->timestamp_begin;
+		file_stream->parent.cycles_timestamp = packet_index->ts_cycles.timestamp_begin;
+
+		file_stream->parent.real_timestamp = packet_index->ts_real.timestamp_begin;
 
 		/* Lookup context/packet size in index */
 		if (packet_index->data_offset == -1) {
@@ -957,9 +978,10 @@ void ctf_packet_seek(struct bt_stream_pos *stream_pos, size_t index, int whence)
 		pos->content_size = packet_index->content_size;
 		pos->packet_size = packet_index->packet_size;
 		pos->mmap_offset = packet_index->offset;
-		if (packet_index->data_offset < packet_index->content_size) {
+		pos->data_offset = packet_index->data_offset;
+		if (pos->data_offset < packet_index->content_size) {
 			pos->offset = 0;	/* will read headers */
-		} else if (packet_index->data_offset == packet_index->content_size) {
+		} else if (pos->data_offset == packet_index->content_size) {
 			/* empty packet */
 			pos->offset = packet_index->data_offset;
 			whence = SEEK_CUR;
@@ -1044,7 +1066,7 @@ warning:
 }
 
 static
-int ctf_open_trace_metadata_packet_read(struct ctf_trace *td, FILE *in,
+int ctf_trace_metadata_packet_read(struct ctf_trace *td, FILE *in,
 					FILE *out)
 {
 	struct metadata_packet_header header;
@@ -1134,7 +1156,7 @@ read_padding:
 }
 
 static
-int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp,
+int ctf_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp,
 					char **buf)
 {
 	FILE *in, *out;
@@ -1153,7 +1175,7 @@ int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp,
 		return -errno;
 	}
 	for (;;) {
-		ret = ctf_open_trace_metadata_packet_read(td, in, out);
+		ret = ctf_trace_metadata_packet_read(td, in, out);
 		if (ret) {
 			break;
 		}
@@ -1194,11 +1216,9 @@ int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp,
 }
 
 static
-int ctf_open_trace_metadata_read(struct ctf_trace *td,
-		void (*packet_seek)(struct bt_stream_pos *pos, size_t index,
-			int whence), FILE *metadata_fp)
+int ctf_trace_metadata_read(struct ctf_trace *td, FILE *metadata_fp,
+		struct ctf_scanner *scanner, int append)
 {
-	struct ctf_scanner *scanner;
 	struct ctf_file_stream *metadata_stream;
 	FILE *fp;
 	char *buf = NULL;
@@ -1207,14 +1227,6 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td,
 	metadata_stream = g_new0(struct ctf_file_stream, 1);
 	metadata_stream->pos.last_offset = LAST_OFFSET_POISON;
 
-	if (packet_seek) {
-		metadata_stream->pos.packet_seek = packet_seek;
-	} else {
-		fprintf(stderr, "[error] packet_seek function undefined.\n");
-		ret = -1;
-		goto end_free;
-	}
-
 	if (metadata_fp) {
 		fp = metadata_fp;
 		metadata_stream->pos.fd = -1;
@@ -1241,38 +1253,32 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td,
 		yydebug = 1;
 
 	if (packet_metadata(td, fp)) {
-		ret = ctf_open_trace_metadata_stream_read(td, &fp, &buf);
+		ret = ctf_trace_metadata_stream_read(td, &fp, &buf);
 		if (ret) {
-			/* Warn about empty metadata */
-			fprintf(stderr, "[warning] Empty metadata.\n");
-			goto end_packet_read;
+			goto end;
 		}
 		td->metadata_string = buf;
 		td->metadata_packetized = 1;
 	} else {
-		unsigned int major, minor;
-		ssize_t nr_items;
-
-		td->byte_order = BYTE_ORDER;
-
-		/* Check text-only metadata header and version */
-		nr_items = fscanf(fp, "/* CTF %u.%u", &major, &minor);
-		if (nr_items < 2)
-			fprintf(stderr, "[warning] Ill-shapen or missing \"/* CTF x.y\" header for text-only metadata.\n");
-		if (check_version(major, minor) < 0) {
-			ret = -EINVAL;
-			goto end_packet_read;
+		if (!append) {
+			unsigned int major, minor;
+			ssize_t nr_items;
+
+			td->byte_order = BYTE_ORDER;
+
+			/* Check text-only metadata header and version */
+			nr_items = fscanf(fp, "/* CTF %10u.%10u", &major, &minor);
+			if (nr_items < 2)
+				fprintf(stderr, "[warning] Ill-shapen or missing \"/* CTF x.y\" header for text-only metadata.\n");
+			if (check_version(major, minor) < 0) {
+				ret = -EINVAL;
+				goto end;
+			}
+			rewind(fp);
 		}
-		rewind(fp);
 	}
 
-	scanner = ctf_scanner_alloc(fp);
-	if (!scanner) {
-		fprintf(stderr, "[error] Error allocating scanner\n");
-		ret = -ENOMEM;
-		goto end_scanner_alloc;
-	}
-	ret = ctf_scanner_append_ast(scanner);
+	ret = ctf_scanner_append_ast(scanner, fp);
 	if (ret) {
 		fprintf(stderr, "[error] Error creating AST\n");
 		goto end;
@@ -1298,9 +1304,6 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td,
 		goto end;
 	}
 end:
-	ctf_scanner_free(scanner);
-end_scanner_alloc:
-end_packet_read:
 	if (fp) {
 		closeret = fclose(fp);
 		if (closeret) {
@@ -1362,6 +1365,36 @@ error:
 	return NULL;
 }
 
+static
+int copy_event_declarations_stream_class_to_stream(struct ctf_trace *td,
+		struct ctf_stream_declaration *stream_class,
+		struct ctf_stream_definition *stream)
+{
+	size_t def_size, class_size, i;
+	int ret = 0;
+
+	def_size = stream->events_by_id->len;
+	class_size = stream_class->events_by_id->len;
+
+	g_ptr_array_set_size(stream->events_by_id, class_size);
+	for (i = def_size; i < class_size; i++) {
+		struct ctf_event_declaration *event =
+			g_ptr_array_index(stream_class->events_by_id, i);
+		struct ctf_event_definition *stream_event;
+
+		if (!event)
+			continue;
+		stream_event = create_event_definitions(td, stream, event);
+		if (!stream_event) {
+			ret = -EINVAL;
+			goto error;
+		}
+		g_ptr_array_index(stream->events_by_id, i) = stream_event;
+	}
+error:
+	return ret;
+}
+
 static
 int create_stream_definitions(struct ctf_trace *td, struct ctf_stream_definition *stream)
 {
@@ -1411,20 +1444,10 @@ int create_stream_definitions(struct ctf_trace *td, struct ctf_stream_definition
 		stream->parent_def_scope = stream->stream_event_context->p.scope;
 	}
 	stream->events_by_id = g_ptr_array_new();
-	g_ptr_array_set_size(stream->events_by_id, stream_class->events_by_id->len);
-	for (i = 0; i < stream->events_by_id->len; i++) {
-		struct ctf_event_declaration *event = g_ptr_array_index(stream_class->events_by_id, i);
-		struct ctf_event_definition *stream_event;
-
-		if (!event)
-			continue;
-		stream_event = create_event_definitions(td, stream, event);
-		if (!stream_event) {
-			ret = -EINVAL;
-			goto error_event;
-		}
-		g_ptr_array_index(stream->events_by_id, i) = stream_event;
-	}
+	ret = copy_event_declarations_stream_class_to_stream(td,
+			stream_class, stream);
+	if (ret)
+		goto error_event;
 	return 0;
 
 error_event:
@@ -1518,8 +1541,10 @@ begin:
 	packet_index.offset = pos->mmap_offset;
 	packet_index.content_size = 0;
 	packet_index.packet_size = 0;
-	packet_index.timestamp_begin = 0;
-	packet_index.timestamp_end = 0;
+	packet_index.ts_real.timestamp_begin = 0;
+	packet_index.ts_real.timestamp_end = 0;
+	packet_index.ts_cycles.timestamp_begin = 0;
+	packet_index.ts_cycles.timestamp_end = 0;
 	packet_index.events_discarded = 0;
 	packet_index.events_discarded_len = 0;
 
@@ -1543,7 +1568,7 @@ begin:
 			if (magic != CTF_MAGIC) {
 				fprintf(stderr, "[error] Invalid magic number 0x%" PRIX64 " at packet %u (file offset %zd).\n",
 						magic,
-						file_stream->pos.packet_cycles_index->len,
+						file_stream->pos.packet_index->len,
 						(ssize_t) pos->mmap_offset);
 				return -EINVAL;
 			}
@@ -1635,12 +1660,12 @@ begin:
 			struct bt_definition *field;
 
 			field = bt_struct_definition_get_field_from_index(file_stream->parent.stream_packet_context, len_index);
-			packet_index.timestamp_begin = bt_get_unsigned_int(field);
+			packet_index.ts_cycles.timestamp_begin = bt_get_unsigned_int(field);
 			if (file_stream->parent.stream_class->trace->parent.collection) {
-				packet_index.timestamp_begin =
+				packet_index.ts_real.timestamp_begin =
 					ctf_get_real_timestamp(
 						&file_stream->parent,
-						packet_index.timestamp_begin);
+						packet_index.ts_cycles.timestamp_begin);
 			}
 		}
 
@@ -1650,12 +1675,12 @@ begin:
 			struct bt_definition *field;
 
 			field = bt_struct_definition_get_field_from_index(file_stream->parent.stream_packet_context, len_index);
-			packet_index.timestamp_end = bt_get_unsigned_int(field);
+			packet_index.ts_cycles.timestamp_end = bt_get_unsigned_int(field);
 			if (file_stream->parent.stream_class->trace->parent.collection) {
-				packet_index.timestamp_end =
+				packet_index.ts_real.timestamp_end =
 					ctf_get_real_timestamp(
 						&file_stream->parent,
-						packet_index.timestamp_end);
+						packet_index.ts_cycles.timestamp_end);
 			}
 		}
 
@@ -1702,7 +1727,7 @@ begin:
 	packet_index.data_offset = pos->offset;
 
 	/* add index to packet array */
-	g_array_append_val(file_stream->pos.packet_cycles_index, packet_index);
+	g_array_append_val(file_stream->pos.packet_index, packet_index);
 
 	pos->mmap_offset += packet_index.packet_size >> LOG2_CHAR_BIT;
 
@@ -1804,12 +1829,11 @@ static
 int import_stream_packet_index(struct ctf_trace *td,
 		struct ctf_file_stream *file_stream)
 {
-	struct ctf_stream_declaration *stream;
 	struct ctf_stream_pos *pos;
-	struct ctf_packet_index ctf_index;
+	struct ctf_packet_index *ctf_index = NULL;
 	struct ctf_packet_index_file_hdr index_hdr;
 	struct packet_index index;
-	int index_read;
+	uint32_t packet_index_len;
 	int ret = 0;
 	int first_packet = 1;
 	size_t len;
@@ -1837,35 +1861,43 @@ int import_stream_packet_index(struct ctf_trace *td,
 		ret = -1;
 		goto error;
 	}
-	if (index_hdr.packet_index_len == 0) {
+	packet_index_len = be32toh(index_hdr.packet_index_len);
+	if (packet_index_len == 0) {
 		fprintf(stderr, "[error] Packet index length cannot be 0.\n");
 		ret = -1;
 		goto error;
 	}
-
-	while ((index_read = fread(&ctf_index, index_hdr.packet_index_len, 1,
-					pos->index_fp)) == 1) {
+	/*
+	 * Allocate the index length found in header, not internal
+	 * representation.
+	 */
+	ctf_index = g_malloc0(packet_index_len);
+	while (fread(ctf_index, packet_index_len, 1,
+			pos->index_fp) == 1) {
 		uint64_t stream_id;
+		struct ctf_stream_declaration *stream = NULL;
 
 		memset(&index, 0, sizeof(index));
-		index.offset = be64toh(ctf_index.offset);
-		index.packet_size = be64toh(ctf_index.packet_size);
-		index.content_size = be64toh(ctf_index.content_size);
-		index.timestamp_begin = be64toh(ctf_index.timestamp_begin);
-		index.timestamp_end = be64toh(ctf_index.timestamp_end);
-		index.events_discarded = be64toh(ctf_index.events_discarded);
+		index.offset = be64toh(ctf_index->offset);
+		index.packet_size = be64toh(ctf_index->packet_size);
+		index.content_size = be64toh(ctf_index->content_size);
+		index.ts_cycles.timestamp_begin = be64toh(ctf_index->timestamp_begin);
+		index.ts_cycles.timestamp_end = be64toh(ctf_index->timestamp_end);
+		index.events_discarded = be64toh(ctf_index->events_discarded);
 		index.events_discarded_len = 64;
 		index.data_offset = -1;
-		stream_id = be64toh(ctf_index.stream_id);
+		stream_id = be64toh(ctf_index->stream_id);
 
 		if (!first_packet) {
 			/* add index to packet array */
-			g_array_append_val(file_stream->pos.packet_cycles_index, index);
+			g_array_append_val(file_stream->pos.packet_index, index);
 			continue;
 		}
 
 		file_stream->parent.stream_id = stream_id;
-		stream = g_ptr_array_index(td->streams, stream_id);
+		if (stream_id < td->streams->len) {
+			stream = g_ptr_array_index(td->streams, stream_id);
+		}
 		if (!stream) {
 			fprintf(stderr, "[error] Stream %" PRIu64
 					" is not declared in metadata.\n",
@@ -1879,12 +1911,19 @@ int import_stream_packet_index(struct ctf_trace *td,
 			goto error;
 		first_packet = 0;
 		/* add index to packet array */
-		g_array_append_val(file_stream->pos.packet_cycles_index, index);
+		g_array_append_val(file_stream->pos.packet_index, index);
+	}
+
+	/* Index containing only the header. */
+	if (!file_stream->parent.stream_class) {
+		ret = -1;
+		goto error;
 	}
 
 	ret = 0;
 
 error:
+	g_free(ctf_index);
 	return ret;
 }
 
@@ -2031,6 +2070,7 @@ int ctf_open_trace_read(struct ctf_trace *td,
 		void (*packet_seek)(struct bt_stream_pos *pos, size_t index,
 			int whence), FILE *metadata_fp)
 {
+	struct ctf_scanner *scanner;
 	int ret, closeret;
 	struct dirent *dirent;
 	struct dirent *diriter;
@@ -2059,10 +2099,21 @@ int ctf_open_trace_read(struct ctf_trace *td,
 
 	/*
 	 * Keep the metadata file separate.
+	 * Keep scanner object local to the open. We don't support
+	 * incremental metadata append for on-disk traces.
 	 */
-
-	ret = ctf_open_trace_metadata_read(td, packet_seek, metadata_fp);
+	scanner = ctf_scanner_alloc();
+	if (!scanner) {
+		fprintf(stderr, "[error] Error allocating scanner\n");
+		ret = -ENOMEM;
+		goto error_metadata;
+	}
+	ret = ctf_trace_metadata_read(td, metadata_fp, scanner, 0);
+	ctf_scanner_free(scanner);
 	if (ret) {
+		if (ret == -ENOENT) {
+			fprintf(stderr, "[warning] Empty metadata.\n");
+		}
 		fprintf(stderr, "[warning] Unable to open trace metadata for path \"%s\".\n", path);
 		goto error_metadata;
 	}
@@ -2183,12 +2234,13 @@ void ctf_init_mmap_pos(struct ctf_stream_pos *pos,
 	pos->offset = 0;
 	pos->dummy = false;
 	pos->cur_index = 0;
-	pos->packet_cycles_index = NULL;
-	pos->packet_real_index = NULL;
 	pos->prot = PROT_READ;
 	pos->flags = MAP_PRIVATE;
 	pos->parent.rw_table = read_dispatch_table;
 	pos->parent.event_cb = ctf_read_event;
+	pos->priv = mmap_info->priv;
+	pos->packet_index = g_array_new(FALSE, TRUE,
+			sizeof(struct packet_index));
 }
 
 static
@@ -2271,8 +2323,17 @@ int ctf_open_mmap_trace_read(struct ctf_trace *td,
 	int ret;
 	struct bt_mmap_stream *mmap_info;
 
-	ret = ctf_open_trace_metadata_read(td, ctf_packet_seek, metadata_fp);
+	td->scanner = ctf_scanner_alloc();
+	if (!td->scanner) {
+		fprintf(stderr, "[error] Error allocating scanner\n");
+		ret = -ENOMEM;
+		goto error;
+	}
+	ret = ctf_trace_metadata_read(td, metadata_fp, td->scanner, 0);
 	if (ret) {
+		if (ret == -ENOENT) {
+			fprintf(stderr, "[warning] Empty metadata.\n");
+		}
 		goto error;
 	}
 
@@ -2287,10 +2348,10 @@ int ctf_open_mmap_trace_read(struct ctf_trace *td,
 			goto error;
 		}
 	}
-
 	return 0;
 
 error:
+	ctf_scanner_free(td->scanner);
 	return ret;
 }
 
@@ -2314,6 +2375,7 @@ struct bt_trace_descriptor *ctf_open_mmap_trace(
 		goto error;
 	}
 	td = g_new0(struct ctf_trace, 1);
+	td->dirfd = -1;
 	ret = ctf_open_mmap_trace_read(td, mmap_list, packet_seek, metadata_fp);
 	if (ret)
 		goto error_free;
@@ -2326,6 +2388,41 @@ error:
 	return NULL;
 }
 
+int ctf_append_trace_metadata(struct bt_trace_descriptor *tdp,
+		FILE *metadata_fp)
+{
+	struct ctf_trace *td = container_of(tdp, struct ctf_trace, parent);
+	int i, j;
+	int ret;
+
+	if (!td->scanner)
+		return -EINVAL;
+	ret = ctf_trace_metadata_read(td, metadata_fp, td->scanner, 1);
+	if (ret)
+		return ret;
+	/* for each stream_class */
+	for (i = 0; i < td->streams->len; i++) {
+		struct ctf_stream_declaration *stream_class;
+
+		stream_class = g_ptr_array_index(td->streams, i);
+		if (!stream_class)
+			continue;
+		/* for each stream */
+		for (j = 0; j < stream_class->streams->len; j++) {
+			struct ctf_stream_definition *stream;
+
+			stream = g_ptr_array_index(stream_class->streams, j);
+			if (!stream)
+				continue;
+			ret = copy_event_declarations_stream_class_to_stream(td,
+				stream_class, stream);
+			if (ret)
+				return ret;
+		}
+	}
+	return 0;
+}
+
 static
 int ctf_convert_index_timestamp(struct bt_trace_descriptor *tdp)
 {
@@ -2351,25 +2448,20 @@ int ctf_convert_index_timestamp(struct bt_trace_descriptor *tdp)
 			cfs = container_of(stream, struct ctf_file_stream,
 					parent);
 			stream_pos = &cfs->pos;
-			if (!stream_pos->packet_cycles_index)
+			if (!stream_pos->packet_index)
 				continue;
 
-			for (k = 0; k < stream_pos->packet_cycles_index->len; k++) {
+			for (k = 0; k < stream_pos->packet_index->len; k++) {
 				struct packet_index *index;
-				struct packet_index new_index;
 
-				index = &g_array_index(stream_pos->packet_cycles_index,
+				index = &g_array_index(stream_pos->packet_index,
 						struct packet_index, k);
-				memcpy(&new_index, index,
-						sizeof(struct packet_index));
-				new_index.timestamp_begin =
+				index->ts_real.timestamp_begin =
 					ctf_get_real_timestamp(stream,
-							index->timestamp_begin);
-				new_index.timestamp_end =
+							index->ts_cycles.timestamp_begin);
+				index->ts_real.timestamp_end =
 					ctf_get_real_timestamp(stream,
-							index->timestamp_end);
-				g_array_append_val(stream_pos->packet_real_index,
-						new_index);
+							index->ts_cycles.timestamp_end);
 			}
 		}
 	}
@@ -2386,10 +2478,12 @@ int ctf_close_file_stream(struct ctf_file_stream *file_stream)
 		fprintf(stderr, "Error on ctf_fini_pos\n");
 		return -1;
 	}
-	ret = close(file_stream->pos.fd);
-	if (ret) {
-		perror("Error closing file fd");
-		return -1;
+	if (file_stream->pos.fd >= 0) {
+		ret = close(file_stream->pos.fd);
+		if (ret) {
+			perror("Error closing file fd");
+			return -1;
+		}
 	}
 	return 0;
 }
@@ -2421,15 +2515,20 @@ int ctf_close_trace(struct bt_trace_descriptor *tdp)
 		}
 	}
 	ctf_destroy_metadata(td);
-	ret = close(td->dirfd);
-	if (ret) {
-		perror("Error closing dirfd");
-		return ret;
+	ctf_scanner_free(td->scanner);
+	if (td->dirfd >= 0) {
+		ret = close(td->dirfd);
+		if (ret) {
+			perror("Error closing dirfd");
+			return ret;
+		}
 	}
-	ret = closedir(td->dir);
-	if (ret) {
-		perror("Error closedir");
-		return ret;
+	if (td->dir) {
+		ret = closedir(td->dir);
+		if (ret) {
+			perror("Error closedir");
+			return ret;
+		}
 	}
 	free(td->metadata_string);
 	g_free(td);