X-Git-Url: http://git.efficios.com/?p=babeltrace.git;a=blobdiff_plain;f=formats%2Fctf%2Fctf.c;h=e31808c3843d27380a63b0d173a949aa027e336e;hp=9047b7c22eccb38fea4a4e917e37fab9c3905f2c;hb=2654fe9bab8f0eaeb17264ef7abadfd14e245b23;hpb=28f35f0aad3aa2741b7ca22a097d0b4f4fcec4ed

diff --git a/formats/ctf/ctf.c b/formats/ctf/ctf.c
index 9047b7c2..e31808c3 100644
--- a/formats/ctf/ctf.c
+++ b/formats/ctf/ctf.c
@@ -35,6 +35,7 @@
 #include <babeltrace/context-internal.h>
 #include <babeltrace/compat/uuid.h>
 #include <babeltrace/endian.h>
+#include <babeltrace/ctf/ctf-index.h>
 #include <inttypes.h>
 #include <stdio.h>
 #include <sys/mman.h>
@@ -71,6 +72,8 @@
 
 #define NSEC_PER_SEC 1000000000ULL
 
+#define INDEX_PATH "./index/%s.idx"
+
 int opt_clock_cycles,
 	opt_clock_seconds,
 	opt_clock_date,
@@ -81,6 +84,13 @@ uint64_t opt_clock_offset_ns;
 
 extern int yydebug;
 
+/*
+ * TODO: babeltrace_ctf_console_output ensures that we only print
+ * discarded events when ctf-text plugin is used. Should be cleaned up
+ * with the plugin system redesign.
+ */
+int babeltrace_ctf_console_output;
+
 static
 struct bt_trace_descriptor *ctf_open_trace(const char *path, int flags,
 		void (*packet_seek)(struct bt_stream_pos *pos, size_t index,
@@ -178,26 +188,24 @@ uint64_t ctf_timestamp_begin(struct bt_trace_descriptor *descriptor,
 					parent);
 			stream_pos = &cfs->pos;
 
-			if (!stream_pos->packet_real_index)
+			if (!stream_pos->packet_index)
 				goto error;
 
-			if (stream_pos->packet_real_index->len <= 0)
+			if (stream_pos->packet_index->len <= 0)
 				continue;
 
+			index = &g_array_index(stream_pos->packet_index,
+					struct packet_index,
+					stream_pos->packet_index->len - 1);
 			if (type == BT_CLOCK_REAL) {
-				index = &g_array_index(stream_pos->packet_real_index,
-						struct packet_index,
-						stream_pos->packet_real_index->len - 1);
+				if (index->ts_real.timestamp_begin < begin)
+					begin = index->ts_real.timestamp_begin;
 			} else if (type == BT_CLOCK_CYCLES) {
-				index = &g_array_index(stream_pos->packet_cycles_index,
-						struct packet_index,
-						stream_pos->packet_real_index->len - 1);
-
+				if (index->ts_cycles.timestamp_begin < begin)
+					begin = index->ts_cycles.timestamp_begin;
 			} else {
 				goto error;
 			}
-			if (index->timestamp_begin < begin)
-				begin = index->timestamp_begin;
 		}
 	}
 
@@ -239,26 +247,24 @@ uint64_t ctf_timestamp_end(struct bt_trace_descriptor *descriptor,
 					parent);
 			stream_pos = &cfs->pos;
 
-			if (!stream_pos->packet_real_index)
+			if (!stream_pos->packet_index)
 				goto error;
 
-			if (stream_pos->packet_real_index->len <= 0)
+			if (stream_pos->packet_index->len <= 0)
 				continue;
 
+			index = &g_array_index(stream_pos->packet_index,
+					struct packet_index,
+					stream_pos->packet_index->len - 1);
 			if (type == BT_CLOCK_REAL) {
-				index = &g_array_index(stream_pos->packet_real_index,
-						struct packet_index,
-						stream_pos->packet_real_index->len - 1);
+				if (index->ts_real.timestamp_end > end)
+					end = index->ts_real.timestamp_end;
 			} else if (type == BT_CLOCK_CYCLES) {
-				index = &g_array_index(stream_pos->packet_cycles_index,
-						struct packet_index,
-						stream_pos->packet_real_index->len - 1);
-
+				if (index->ts_cycles.timestamp_end > end)
+					end = index->ts_cycles.timestamp_end;
 			} else {
 				goto error;
 			}
-			if (index->timestamp_end > end)
-				end = index->timestamp_end;
 		}
 	}
 
@@ -280,10 +286,7 @@ void ctf_update_timestamp(struct ctf_stream_definition *stream,
 	uint64_t oldval, newval, updateval;
 
 	if (unlikely(integer_declaration->len == 64)) {
-		stream->prev_cycles_timestamp = stream->cycles_timestamp;
 		stream->cycles_timestamp = integer_definition->value._unsigned;
-		stream->prev_real_timestamp = ctf_get_real_timestamp(stream,
-				stream->prev_cycles_timestamp);
 		stream->real_timestamp = ctf_get_real_timestamp(stream,
 				stream->cycles_timestamp);
 		return;
@@ -299,12 +302,9 @@ void ctf_update_timestamp(struct ctf_stream_definition *stream,
 	updateval = stream->cycles_timestamp;
 	updateval &= ~((1ULL << integer_declaration->len) - 1);
 	updateval += newval;
-	stream->prev_cycles_timestamp = stream->cycles_timestamp;
 	stream->cycles_timestamp = updateval;
 
 	/* convert to real timestamp */
-	stream->prev_real_timestamp = ctf_get_real_timestamp(stream,
-			stream->prev_cycles_timestamp);
 	stream->real_timestamp = ctf_get_real_timestamp(stream,
 			stream->cycles_timestamp);
 }
@@ -409,24 +409,39 @@ void print_uuid(FILE *fp, unsigned char *uuid)
 		fprintf(fp, "%x", (unsigned int) uuid[i]);
 }
 
-void ctf_print_discarded(FILE *fp, struct ctf_stream_definition *stream,
-		int end_stream)
+/*
+ * Discarded events can be either:
+ * - discarded after end of previous buffer due to buffer full:
+ *     happened within range: [ prev_timestamp_end, timestamp_begin ]
+ * - discarded within current buffer due to either event too large or
+ *   nested wrap-around:
+ *     happened within range: [ timestamp_begin, timestamp_end ]
+ *
+ * Given we have discarded counters of those two types merged into the
+ * events_discarded counter, we need to use the union of those ranges:
+ *   [ prev_timestamp_end, timestamp_end ]
+ */
+static
+void ctf_print_discarded(FILE *fp, struct ctf_stream_definition *stream)
 {
-	fprintf(fp, "[warning] Tracer discarded %" PRIu64 " events %sbetween [",
-		stream->events_discarded,
-		end_stream ? "at end of stream " : "");
+	if (!stream->events_discarded || !babeltrace_ctf_console_output) {
+		return;
+	}
+	fflush(stdout);
+	fprintf(fp, "[warning] Tracer discarded %" PRIu64 " events between [",
+		stream->events_discarded);
 	if (opt_clock_cycles) {
 		ctf_print_timestamp(fp, stream,
-				stream->prev_cycles_timestamp);
+				stream->prev.cycles.end);
 		fprintf(fp, "] and [");
 		ctf_print_timestamp(fp, stream,
-				stream->prev_cycles_timestamp_end);
+				stream->current.cycles.end);
 	} else {
 		ctf_print_timestamp(fp, stream,
-				stream->prev_real_timestamp);
+				stream->prev.real.end);
 		fprintf(fp, "] and [");
 		ctf_print_timestamp(fp, stream,
-				stream->prev_real_timestamp_end);
+				stream->current.real.end);
 	}
 	fprintf(fp, "] in trace UUID ");
 	print_uuid(fp, stream->stream_class->trace->uuid);
@@ -468,6 +483,19 @@ int ctf_read_event(struct bt_stream_pos *ppos, struct ctf_stream_definition *str
 	 */
 	if (unlikely(pos->offset == EOF))
 		return EOF;
+
+	/* Stream is inactive for now (live reading). */
+	if (unlikely(pos->content_size == 0))
+		return EAGAIN;
+
+	/*
+	 * Packet seeked to by ctf_pos_get_event() only contains
+	 * headers, no event. Consider stream as inactive (live
+	 * reading).
+	 */
+	if (unlikely(pos->data_offset == pos->content_size))
+		return EAGAIN;
+
 	assert(pos->offset < pos->content_size);
 
 	/* Read event header */
@@ -556,7 +584,7 @@ int ctf_read_event(struct bt_stream_pos *ppos, struct ctf_stream_definition *str
 	return 0;
 
 error:
-	fprintf(stderr, "[error] Unexpected end of stream. Either the trace data stream is corrupted or metadata description does not match data layout.\n");
+	fprintf(stderr, "[error] Unexpected end of packet. Either the trace data stream is corrupted or metadata description does not match data layout.\n");
 	return ret;
 }
 
@@ -615,18 +643,116 @@ error:
 	return ret;
 }
 
+/*
+ * One side-effect of this function is to unmap pos mmap base if one is
+ * mapped.
+ */
+static
+int find_data_offset(struct ctf_stream_pos *pos,
+		struct ctf_file_stream *file_stream,
+		struct packet_index *packet_index)
+{
+	uint64_t packet_map_len = DEFAULT_HEADER_LEN, tmp_map_len;
+	struct stat filestats;
+	size_t filesize;
+	int ret;
+
+	pos = &file_stream->pos;
+
+	ret = fstat(pos->fd, &filestats);
+	if (ret < 0)
+		return ret;
+	filesize = filestats.st_size;
+
+	/* Deal with empty files */
+	if (!filesize) {
+		return 0;
+	}
+
+begin:
+	if (filesize - pos->mmap_offset < (packet_map_len >> LOG2_CHAR_BIT)) {
+		packet_map_len = (filesize - pos->mmap_offset) << LOG2_CHAR_BIT;
+	}
+
+	if (pos->base_mma) {
+		/* unmap old base */
+		ret = munmap_align(pos->base_mma);
+		if (ret) {
+			fprintf(stderr, "[error] Unable to unmap old base: %s.\n",
+					strerror(errno));
+			return ret;
+		}
+		pos->base_mma = NULL;
+	}
+	/* map new base. Need mapping length from header. */
+	pos->base_mma = mmap_align(packet_map_len >> LOG2_CHAR_BIT, PROT_READ,
+			MAP_PRIVATE, pos->fd, pos->mmap_offset);
+	assert(pos->base_mma != MAP_FAILED);
+
+	pos->content_size = packet_map_len;
+	pos->packet_size = packet_map_len;
+	pos->offset = 0;	/* Position of the packet header */
+
+	/* update trace_packet_header and stream_packet_context */
+	if (pos->prot == PROT_READ && file_stream->parent.trace_packet_header) {
+		/* Read packet header */
+		ret = generic_rw(&pos->parent, &file_stream->parent.trace_packet_header->p);
+		if (ret) {
+			if (ret == -EFAULT)
+				goto retry;
+		}
+	}
+	if (pos->prot == PROT_READ && file_stream->parent.stream_packet_context) {
+		/* Read packet context */
+		ret = generic_rw(&pos->parent, &file_stream->parent.stream_packet_context->p);
+		if (ret) {
+			if (ret == -EFAULT)
+				goto retry;
+		}
+	}
+	packet_index->data_offset = pos->offset;
+
+	/* unmap old base */
+	ret = munmap_align(pos->base_mma);
+	if (ret) {
+		fprintf(stderr, "[error] Unable to unmap old base: %s.\n",
+				strerror(errno));
+		return ret;
+	}
+	pos->base_mma = NULL;
+
+	return 0;
+
+	/* Retry with larger mapping */
+retry:
+	if (packet_map_len == ((filesize - pos->mmap_offset) << LOG2_CHAR_BIT)) {
+		/*
+		 * Reached EOF, but still expecting header/context data.
+		 */
+		fprintf(stderr, "[error] Reached end of file, but still expecting header or context fields.\n");
+		return -EFAULT;
+	}
+	/* Double the mapping len, and retry */
+	tmp_map_len = packet_map_len << 1;
+	if (tmp_map_len >> 1 != packet_map_len) {
+		/* Overflow */
+		fprintf(stderr, "[error] Packet mapping length overflow\n");
+		return -EFAULT;
+	}
+	packet_map_len = tmp_map_len;
+	goto begin;
+}
+
+
 int ctf_init_pos(struct ctf_stream_pos *pos, struct bt_trace_descriptor *trace,
 		int fd, int open_flags)
 {
 	pos->fd = fd;
 	if (fd >= 0) {
-		pos->packet_cycles_index = g_array_new(FALSE, TRUE,
-						sizeof(struct packet_index));
-		pos->packet_real_index = g_array_new(FALSE, TRUE,
+		pos->packet_index = g_array_new(FALSE, TRUE,
 				sizeof(struct packet_index));
 	} else {
-		pos->packet_cycles_index = NULL;
-		pos->packet_real_index = NULL;
+		pos->packet_index = NULL;
 	}
 	switch (open_flags & O_ACCMODE) {
 	case O_RDONLY:
@@ -666,13 +792,65 @@ int ctf_fini_pos(struct ctf_stream_pos *pos)
 			return -1;
 		}
 	}
-	if (pos->packet_cycles_index)
-		(void) g_array_free(pos->packet_cycles_index, TRUE);
-	if (pos->packet_real_index)
-		(void) g_array_free(pos->packet_real_index, TRUE);
+	if (pos->packet_index)
+		(void) g_array_free(pos->packet_index, TRUE);
 	return 0;
 }
 
+void ctf_update_current_packet_index(struct ctf_stream_definition *stream,
+		struct packet_index *prev_index,
+		struct packet_index *cur_index)
+{
+	uint64_t events_discarded_diff;
+
+	/* Update packet index time information */
+
+	/* Current packet begin/end */
+	stream->current.real.begin =
+		cur_index->ts_real.timestamp_begin;
+	stream->current.cycles.begin =
+		cur_index->ts_cycles.timestamp_begin;
+	stream->current.real.end =
+		cur_index->ts_real.timestamp_end;
+	stream->current.cycles.end =
+		cur_index->ts_cycles.timestamp_end;
+
+	/* Update packet index discarded event information */
+	events_discarded_diff = cur_index->events_discarded;
+	if (prev_index) {
+		/* Previous packet begin/end */
+		stream->prev.cycles.begin =
+			prev_index->ts_cycles.timestamp_begin;
+		stream->prev.real.begin =
+			prev_index->ts_real.timestamp_begin;
+		stream->prev.cycles.end =
+			prev_index->ts_cycles.timestamp_end;
+		stream->prev.real.end =
+			prev_index->ts_real.timestamp_end;
+
+		events_discarded_diff -= prev_index->events_discarded;
+		/*
+		 * Deal with 32-bit wrap-around if the tracer provided a
+		 * 32-bit field.
+		 */
+		if (prev_index->events_discarded_len == 32) {
+			events_discarded_diff = (uint32_t) events_discarded_diff;
+		}
+	} else {
+		/*
+		 * First packet: use current packet info as limits for
+		 * previous packet.
+		 */
+		stream->prev.cycles.begin =
+			stream->prev.cycles.end =
+				stream->current.cycles.begin;
+		stream->prev.real.begin =
+			stream->prev.real.end =
+				stream->current.real.begin;
+	}
+	stream->events_discarded = events_discarded_diff;
+}
+
 /*
  * for SEEK_CUR: go to next packet.
  * for SEEK_SET: go to packet numer (index).
@@ -685,7 +863,7 @@ void ctf_packet_seek(struct bt_stream_pos *stream_pos, size_t index, int whence)
 		container_of(pos, struct ctf_file_stream, pos);
 	int ret;
 	off_t off;
-	struct packet_index *packet_index;
+	struct packet_index *packet_index, *prev_index;
 
 	switch (whence) {
 	case SEEK_CUR:
@@ -737,110 +915,73 @@ void ctf_packet_seek(struct bt_stream_pos *stream_pos, size_t index, int whence)
 		switch (whence) {
 		case SEEK_CUR:
 		{
-			uint64_t events_discarded_diff;
-
 			if (pos->offset == EOF) {
 				return;
 			}
-			assert(pos->cur_index < pos->packet_cycles_index->len);
-			assert(pos->cur_index < pos->packet_real_index->len);
-
-			/* For printing discarded event count */
-			packet_index = &g_array_index(pos->packet_cycles_index,
-					struct packet_index, pos->cur_index);
-			file_stream->parent.prev_cycles_timestamp_end =
-				packet_index->timestamp_end;
-			file_stream->parent.prev_cycles_timestamp =
-				packet_index->timestamp_begin;
-
-			packet_index = &g_array_index(pos->packet_real_index,
-					struct packet_index, pos->cur_index);
-			file_stream->parent.prev_real_timestamp_end =
-						packet_index->timestamp_end;
-			file_stream->parent.prev_real_timestamp =
-				packet_index->timestamp_begin;
-
-			events_discarded_diff = packet_index->events_discarded;
-			if (pos->cur_index > 0) {
-				packet_index = &g_array_index(pos->packet_real_index,
-						struct packet_index,
-						pos->cur_index - 1);
-				events_discarded_diff -= packet_index->events_discarded;
-				/*
-				 * Deal with 32-bit wrap-around if the
-				 * tracer provided a 32-bit field.
-				 */
-				if (packet_index->events_discarded_len == 32) {
-					events_discarded_diff = (uint32_t) events_discarded_diff;
-				}
-			}
-			file_stream->parent.events_discarded = events_discarded_diff;
-			file_stream->parent.prev_real_timestamp = file_stream->parent.real_timestamp;
-			file_stream->parent.prev_cycles_timestamp = file_stream->parent.cycles_timestamp;
+			assert(pos->cur_index < pos->packet_index->len);
 			/* The reader will expect us to skip padding */
 			++pos->cur_index;
 			break;
 		}
 		case SEEK_SET:
-			if (index >= pos->packet_cycles_index->len) {
+			if (index >= pos->packet_index->len) {
 				pos->offset = EOF;
 				return;
 			}
-			packet_index = &g_array_index(pos->packet_cycles_index,
-					struct packet_index, index);
-			pos->last_events_discarded = packet_index->events_discarded;
 			pos->cur_index = index;
-			file_stream->parent.prev_real_timestamp = 0;
-			file_stream->parent.prev_real_timestamp_end = 0;
-			file_stream->parent.prev_cycles_timestamp = 0;
-			file_stream->parent.prev_cycles_timestamp_end = 0;
 			break;
 		default:
 			assert(0);
 		}
-		if (pos->cur_index >= pos->packet_real_index->len) {
-			/*
-			 * We need to check if we are in trace read or
-			 * called from packet indexing.  In this last
-			 * case, the collection is not there, so we
-			 * cannot print the timestamps.
-			 */
-			if ((&file_stream->parent)->stream_class->trace->parent.collection) {
-				/*
-				 * When a stream reaches the end of the
-				 * file, we need to show the number of
-				 * events discarded ourselves, because
-				 * there is no next event scheduled to
-				 * be printed in the output.
-				 */
-				if (file_stream->parent.events_discarded) {
-					fflush(stdout);
-					ctf_print_discarded(stderr,
-						&file_stream->parent,
-						1);
-					file_stream->parent.events_discarded = 0;
-				}
-			}
+
+		packet_index = &g_array_index(pos->packet_index,
+				struct packet_index, pos->cur_index);
+		if (pos->cur_index > 0) {
+			prev_index = &g_array_index(pos->packet_index,
+					struct packet_index,
+					pos->cur_index - 1);
+		} else {
+			prev_index = NULL;
+		}
+		ctf_update_current_packet_index(&file_stream->parent,
+				prev_index, packet_index);
+
+		if (pos->cur_index >= pos->packet_index->len) {
 			pos->offset = EOF;
 			return;
 		}
-		packet_index = &g_array_index(pos->packet_cycles_index,
-				struct packet_index,
-				pos->cur_index);
-		file_stream->parent.cycles_timestamp = packet_index->timestamp_begin;
 
-		packet_index = &g_array_index(pos->packet_real_index,
+		/*
+		 * We need to check if we are in trace read or called
+		 * from packet indexing.  In this last case, the
+		 * collection is not there, so we cannot print the
+		 * timestamps.
+		 */
+		if ((&file_stream->parent)->stream_class->trace->parent.collection) {
+			ctf_print_discarded(stderr, &file_stream->parent);
+		}
+
+		packet_index = &g_array_index(pos->packet_index,
 				struct packet_index,
 				pos->cur_index);
-		file_stream->parent.real_timestamp = packet_index->timestamp_begin;
-		pos->mmap_offset = packet_index->offset;
+		file_stream->parent.cycles_timestamp = packet_index->ts_cycles.timestamp_begin;
+
+		file_stream->parent.real_timestamp = packet_index->ts_real.timestamp_begin;
 
 		/* Lookup context/packet size in index */
+		if (packet_index->data_offset == -1) {
+			ret = find_data_offset(pos, file_stream, packet_index);
+			if (ret < 0) {
+				return;
+			}
+		}
 		pos->content_size = packet_index->content_size;
 		pos->packet_size = packet_index->packet_size;
-		if (packet_index->data_offset < packet_index->content_size) {
+		pos->mmap_offset = packet_index->offset;
+		pos->data_offset = packet_index->data_offset;
+		if (pos->data_offset < packet_index->content_size) {
 			pos->offset = 0;	/* will read headers */
-		} else if (packet_index->data_offset == packet_index->content_size) {
+		} else if (pos->data_offset == packet_index->content_size) {
 			/* empty packet */
 			pos->offset = packet_index->data_offset;
 			whence = SEEK_CUR;
@@ -925,7 +1066,7 @@ warning:
 }
 
 static
-int ctf_open_trace_metadata_packet_read(struct ctf_trace *td, FILE *in,
+int ctf_trace_metadata_packet_read(struct ctf_trace *td, FILE *in,
 					FILE *out)
 {
 	struct metadata_packet_header header;
@@ -1015,7 +1156,7 @@ read_padding:
 }
 
 static
-int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp,
+int ctf_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp,
 					char **buf)
 {
 	FILE *in, *out;
@@ -1034,7 +1175,7 @@ int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp,
 		return -errno;
 	}
 	for (;;) {
-		ret = ctf_open_trace_metadata_packet_read(td, in, out);
+		ret = ctf_trace_metadata_packet_read(td, in, out);
 		if (ret) {
 			break;
 		}
@@ -1075,11 +1216,9 @@ int ctf_open_trace_metadata_stream_read(struct ctf_trace *td, FILE **fp,
 }
 
 static
-int ctf_open_trace_metadata_read(struct ctf_trace *td,
-		void (*packet_seek)(struct bt_stream_pos *pos, size_t index,
-			int whence), FILE *metadata_fp)
+int ctf_trace_metadata_read(struct ctf_trace *td, FILE *metadata_fp,
+		struct ctf_scanner *scanner, int append)
 {
-	struct ctf_scanner *scanner;
 	struct ctf_file_stream *metadata_stream;
 	FILE *fp;
 	char *buf = NULL;
@@ -1088,14 +1227,6 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td,
 	metadata_stream = g_new0(struct ctf_file_stream, 1);
 	metadata_stream->pos.last_offset = LAST_OFFSET_POISON;
 
-	if (packet_seek) {
-		metadata_stream->pos.packet_seek = packet_seek;
-	} else {
-		fprintf(stderr, "[error] packet_seek function undefined.\n");
-		ret = -1;
-		goto end_free;
-	}
-
 	if (metadata_fp) {
 		fp = metadata_fp;
 		metadata_stream->pos.fd = -1;
@@ -1122,38 +1253,32 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td,
 		yydebug = 1;
 
 	if (packet_metadata(td, fp)) {
-		ret = ctf_open_trace_metadata_stream_read(td, &fp, &buf);
+		ret = ctf_trace_metadata_stream_read(td, &fp, &buf);
 		if (ret) {
-			/* Warn about empty metadata */
-			fprintf(stderr, "[warning] Empty metadata.\n");
-			goto end_packet_read;
+			goto end;
 		}
 		td->metadata_string = buf;
 		td->metadata_packetized = 1;
 	} else {
-		unsigned int major, minor;
-		ssize_t nr_items;
-
-		td->byte_order = BYTE_ORDER;
-
-		/* Check text-only metadata header and version */
-		nr_items = fscanf(fp, "/* CTF %u.%u", &major, &minor);
-		if (nr_items < 2)
-			fprintf(stderr, "[warning] Ill-shapen or missing \"/* CTF x.y\" header for text-only metadata.\n");
-		if (check_version(major, minor) < 0) {
-			ret = -EINVAL;
-			goto end_packet_read;
+		if (!append) {
+			unsigned int major, minor;
+			ssize_t nr_items;
+
+			td->byte_order = BYTE_ORDER;
+
+			/* Check text-only metadata header and version */
+			nr_items = fscanf(fp, "/* CTF %10u.%10u", &major, &minor);
+			if (nr_items < 2)
+				fprintf(stderr, "[warning] Ill-shapen or missing \"/* CTF x.y\" header for text-only metadata.\n");
+			if (check_version(major, minor) < 0) {
+				ret = -EINVAL;
+				goto end;
+			}
+			rewind(fp);
 		}
-		rewind(fp);
 	}
 
-	scanner = ctf_scanner_alloc(fp);
-	if (!scanner) {
-		fprintf(stderr, "[error] Error allocating scanner\n");
-		ret = -ENOMEM;
-		goto end_scanner_alloc;
-	}
-	ret = ctf_scanner_append_ast(scanner);
+	ret = ctf_scanner_append_ast(scanner, fp);
 	if (ret) {
 		fprintf(stderr, "[error] Error creating AST\n");
 		goto end;
@@ -1179,9 +1304,6 @@ int ctf_open_trace_metadata_read(struct ctf_trace *td,
 		goto end;
 	}
 end:
-	ctf_scanner_free(scanner);
-end_scanner_alloc:
-end_packet_read:
 	if (fp) {
 		closeret = fclose(fp);
 		if (closeret) {
@@ -1243,6 +1365,36 @@ error:
 	return NULL;
 }
 
+static
+int copy_event_declarations_stream_class_to_stream(struct ctf_trace *td,
+		struct ctf_stream_declaration *stream_class,
+		struct ctf_stream_definition *stream)
+{
+	size_t def_size, class_size, i;
+	int ret = 0;
+
+	def_size = stream->events_by_id->len;
+	class_size = stream_class->events_by_id->len;
+
+	g_ptr_array_set_size(stream->events_by_id, class_size);
+	for (i = def_size; i < class_size; i++) {
+		struct ctf_event_declaration *event =
+			g_ptr_array_index(stream_class->events_by_id, i);
+		struct ctf_event_definition *stream_event;
+
+		if (!event)
+			continue;
+		stream_event = create_event_definitions(td, stream, event);
+		if (!stream_event) {
+			ret = -EINVAL;
+			goto error;
+		}
+		g_ptr_array_index(stream->events_by_id, i) = stream_event;
+	}
+error:
+	return ret;
+}
+
 static
 int create_stream_definitions(struct ctf_trace *td, struct ctf_stream_definition *stream)
 {
@@ -1292,20 +1444,10 @@ int create_stream_definitions(struct ctf_trace *td, struct ctf_stream_definition
 		stream->parent_def_scope = stream->stream_event_context->p.scope;
 	}
 	stream->events_by_id = g_ptr_array_new();
-	g_ptr_array_set_size(stream->events_by_id, stream_class->events_by_id->len);
-	for (i = 0; i < stream->events_by_id->len; i++) {
-		struct ctf_event_declaration *event = g_ptr_array_index(stream_class->events_by_id, i);
-		struct ctf_event_definition *stream_event;
-
-		if (!event)
-			continue;
-		stream_event = create_event_definitions(td, stream, event);
-		if (!stream_event) {
-			ret = -EINVAL;
-			goto error_event;
-		}
-		g_ptr_array_index(stream->events_by_id, i) = stream_event;
-	}
+	ret = copy_event_declarations_stream_class_to_stream(td,
+			stream_class, stream);
+	if (ret)
+		goto error_event;
 	return 0;
 
 error_event:
@@ -1399,8 +1541,10 @@ begin:
 	packet_index.offset = pos->mmap_offset;
 	packet_index.content_size = 0;
 	packet_index.packet_size = 0;
-	packet_index.timestamp_begin = 0;
-	packet_index.timestamp_end = 0;
+	packet_index.ts_real.timestamp_begin = 0;
+	packet_index.ts_real.timestamp_end = 0;
+	packet_index.ts_cycles.timestamp_begin = 0;
+	packet_index.ts_cycles.timestamp_end = 0;
 	packet_index.events_discarded = 0;
 	packet_index.events_discarded_len = 0;
 
@@ -1424,7 +1568,7 @@ begin:
 			if (magic != CTF_MAGIC) {
 				fprintf(stderr, "[error] Invalid magic number 0x%" PRIX64 " at packet %u (file offset %zd).\n",
 						magic,
-						file_stream->pos.packet_cycles_index->len,
+						file_stream->pos.packet_index->len,
 						(ssize_t) pos->mmap_offset);
 				return -EINVAL;
 			}
@@ -1486,28 +1630,28 @@ begin:
 			fprintf(stderr, "[error] Unable to read packet context: %s\n", strerror(-ret));
 			return ret;
 		}
-		/* read content size from header */
-		len_index = bt_struct_declaration_lookup_field_index(file_stream->parent.stream_packet_context->declaration, g_quark_from_static_string("content_size"));
+		/* read packet size from header */
+		len_index = bt_struct_declaration_lookup_field_index(file_stream->parent.stream_packet_context->declaration, g_quark_from_static_string("packet_size"));
 		if (len_index >= 0) {
 			struct bt_definition *field;
 
 			field = bt_struct_definition_get_field_from_index(file_stream->parent.stream_packet_context, len_index);
-			packet_index.content_size = bt_get_unsigned_int(field);
+			packet_index.packet_size = bt_get_unsigned_int(field);
 		} else {
 			/* Use file size for packet size */
-			packet_index.content_size = filesize * CHAR_BIT;
+			packet_index.packet_size = filesize * CHAR_BIT;
 		}
 
-		/* read packet size from header */
-		len_index = bt_struct_declaration_lookup_field_index(file_stream->parent.stream_packet_context->declaration, g_quark_from_static_string("packet_size"));
+		/* read content size from header */
+		len_index = bt_struct_declaration_lookup_field_index(file_stream->parent.stream_packet_context->declaration, g_quark_from_static_string("content_size"));
 		if (len_index >= 0) {
 			struct bt_definition *field;
 
 			field = bt_struct_definition_get_field_from_index(file_stream->parent.stream_packet_context, len_index);
-			packet_index.packet_size = bt_get_unsigned_int(field);
+			packet_index.content_size = bt_get_unsigned_int(field);
 		} else {
-			/* Use content size if non-zero, else file size */
-			packet_index.packet_size = packet_index.content_size ? : filesize * CHAR_BIT;
+			/* Use packet size if non-zero, else file size */
+			packet_index.content_size = packet_index.packet_size ? : filesize * CHAR_BIT;
 		}
 
 		/* read timestamp begin from header */
@@ -1516,12 +1660,12 @@ begin:
 			struct bt_definition *field;
 
 			field = bt_struct_definition_get_field_from_index(file_stream->parent.stream_packet_context, len_index);
-			packet_index.timestamp_begin = bt_get_unsigned_int(field);
+			packet_index.ts_cycles.timestamp_begin = bt_get_unsigned_int(field);
 			if (file_stream->parent.stream_class->trace->parent.collection) {
-				packet_index.timestamp_begin =
+				packet_index.ts_real.timestamp_begin =
 					ctf_get_real_timestamp(
 						&file_stream->parent,
-						packet_index.timestamp_begin);
+						packet_index.ts_cycles.timestamp_begin);
 			}
 		}
 
@@ -1531,12 +1675,12 @@ begin:
 			struct bt_definition *field;
 
 			field = bt_struct_definition_get_field_from_index(file_stream->parent.stream_packet_context, len_index);
-			packet_index.timestamp_end = bt_get_unsigned_int(field);
+			packet_index.ts_cycles.timestamp_end = bt_get_unsigned_int(field);
 			if (file_stream->parent.stream_class->trace->parent.collection) {
-				packet_index.timestamp_end =
+				packet_index.ts_real.timestamp_end =
 					ctf_get_real_timestamp(
 						&file_stream->parent,
-						packet_index.timestamp_end);
+						packet_index.ts_cycles.timestamp_end);
 			}
 		}
 
@@ -1551,9 +1695,9 @@ begin:
 		}
 	} else {
 		/* Use file size for packet size */
-		packet_index.content_size = filesize * CHAR_BIT;
-		/* Use content size if non-zero, else file size */
-		packet_index.packet_size = packet_index.content_size ? : filesize * CHAR_BIT;
+		packet_index.packet_size = filesize * CHAR_BIT;
+		/* Use packet size if non-zero, else file size */
+		packet_index.content_size = packet_index.packet_size ? : filesize * CHAR_BIT;
 	}
 
 	/* Validate content size and packet size values */
@@ -1569,11 +1713,21 @@ begin:
 		return -EINVAL;
 	}
 
+	if (packet_index.content_size < pos->offset) {
+		fprintf(stderr, "[error] Invalid CTF stream: content size is smaller than packet headers.\n");
+		return -EINVAL;
+	}
+
+	if ((packet_index.packet_size >> LOG2_CHAR_BIT) == 0) {
+		fprintf(stderr, "[error] Invalid CTF stream: packet size needs to be at least one byte\n");
+		return -EINVAL;
+	}
+
 	/* Save position after header and context */
 	packet_index.data_offset = pos->offset;
 
 	/* add index to packet array */
-	g_array_append_val(file_stream->pos.packet_cycles_index, packet_index);
+	g_array_append_val(file_stream->pos.packet_index, packet_index);
 
 	pos->mmap_offset += packet_index.packet_size >> LOG2_CHAR_BIT;
 
@@ -1671,6 +1825,108 @@ error:
 	return ret;
 }
 
+static
+int import_stream_packet_index(struct ctf_trace *td,
+		struct ctf_file_stream *file_stream)
+{
+	struct ctf_stream_pos *pos;
+	struct ctf_packet_index *ctf_index = NULL;
+	struct ctf_packet_index_file_hdr index_hdr;
+	struct packet_index index;
+	uint32_t packet_index_len;
+	int ret = 0;
+	int first_packet = 1;
+	size_t len;
+
+	pos = &file_stream->pos;
+
+	len = fread(&index_hdr, sizeof(index_hdr), 1, pos->index_fp);
+	if (len != 1) {
+		perror("read index file header");
+		goto error;
+	}
+
+	/* Check the index header */
+	if (be32toh(index_hdr.magic) != CTF_INDEX_MAGIC) {
+		fprintf(stderr, "[error] wrong index magic\n");
+		ret = -1;
+		goto error;
+	}
+	if (be32toh(index_hdr.index_major) != CTF_INDEX_MAJOR) {
+		fprintf(stderr, "[error] Incompatible index file %" PRIu64
+				".%" PRIu64 ", supported %d.%d\n",
+				be64toh(index_hdr.index_major),
+				be64toh(index_hdr.index_minor), CTF_INDEX_MAJOR,
+				CTF_INDEX_MINOR);
+		ret = -1;
+		goto error;
+	}
+	packet_index_len = be32toh(index_hdr.packet_index_len);
+	if (packet_index_len == 0) {
+		fprintf(stderr, "[error] Packet index length cannot be 0.\n");
+		ret = -1;
+		goto error;
+	}
+	/*
+	 * Allocate the index length found in header, not internal
+	 * representation.
+	 */
+	ctf_index = g_malloc0(packet_index_len);
+	while (fread(ctf_index, packet_index_len, 1,
+			pos->index_fp) == 1) {
+		uint64_t stream_id;
+		struct ctf_stream_declaration *stream = NULL;
+
+		memset(&index, 0, sizeof(index));
+		index.offset = be64toh(ctf_index->offset);
+		index.packet_size = be64toh(ctf_index->packet_size);
+		index.content_size = be64toh(ctf_index->content_size);
+		index.ts_cycles.timestamp_begin = be64toh(ctf_index->timestamp_begin);
+		index.ts_cycles.timestamp_end = be64toh(ctf_index->timestamp_end);
+		index.events_discarded = be64toh(ctf_index->events_discarded);
+		index.events_discarded_len = 64;
+		index.data_offset = -1;
+		stream_id = be64toh(ctf_index->stream_id);
+
+		if (!first_packet) {
+			/* add index to packet array */
+			g_array_append_val(file_stream->pos.packet_index, index);
+			continue;
+		}
+
+		file_stream->parent.stream_id = stream_id;
+		if (stream_id < td->streams->len) {
+			stream = g_ptr_array_index(td->streams, stream_id);
+		}
+		if (!stream) {
+			fprintf(stderr, "[error] Stream %" PRIu64
+					" is not declared in metadata.\n",
+					stream_id);
+			ret = -EINVAL;
+			goto error;
+		}
+		file_stream->parent.stream_class = stream;
+		ret = create_stream_definitions(td, &file_stream->parent);
+		if (ret)
+			goto error;
+		first_packet = 0;
+		/* add index to packet array */
+		g_array_append_val(file_stream->pos.packet_index, index);
+	}
+
+	/* Index containing only the header. */
+	if (!file_stream->parent.stream_class) {
+		ret = -1;
+		goto error;
+	}
+
+	ret = 0;
+
+error:
+	g_free(ctf_index);
+	return ret;
+}
+
 /*
  * Note: many file streams can inherit from the same stream class
  * description (metadata).
@@ -1683,6 +1939,7 @@ int ctf_open_file_stream_read(struct ctf_trace *td, const char *path, int flags,
 	int ret, fd, closeret;
 	struct ctf_file_stream *file_stream;
 	struct stat statbuf;
+	char *index_name;
 
 	fd = openat(td->dirfd, path, flags);
 	if (fd < 0) {
@@ -1698,13 +1955,18 @@ int ctf_open_file_stream_read(struct ctf_trace *td, const char *path, int flags,
 		goto fstat_error;
 	}
 	if (S_ISDIR(statbuf.st_mode)) {
-		fprintf(stderr, "[warning] Skipping directory '%s' found in trace\n", path);
+		if (strncmp(path, "index", 5) != 0) {
+			fprintf(stderr, "[warning] Skipping directory '%s' "
+					"found in trace\n", path);
+		}
 		ret = 0;
 		goto fd_is_dir_ok;
 	}
 
 	file_stream = g_new0(struct ctf_file_stream, 1);
 	file_stream->pos.last_offset = LAST_OFFSET_POISON;
+	file_stream->pos.fd = -1;
+	file_stream->pos.index_fp = NULL;
 
 	strncpy(file_stream->parent.path, path, PATH_MAX);
 	file_stream->parent.path[PATH_MAX - 1] = '\0';
@@ -1727,19 +1989,65 @@ int ctf_open_file_stream_read(struct ctf_trace *td, const char *path, int flags,
 	 * For now, only a single clock per trace is supported.
 	 */
 	file_stream->parent.current_clock = td->parent.single_clock;
-	ret = create_stream_packet_index(td, file_stream);
-	if (ret) {
-		fprintf(stderr, "[error] Stream index creation error.\n");
-		goto error_index;
+
+	/*
+	 * Allocate the index name for this stream and try to open it.
+	 */
+	index_name = malloc((strlen(path) + sizeof(INDEX_PATH)) * sizeof(char));
+	if (!index_name) {
+		fprintf(stderr, "[error] Cannot allocate index filename\n");
+		goto error_def;
+	}
+	snprintf(index_name, strlen(path) + sizeof(INDEX_PATH),
+			INDEX_PATH, path);
+
+	if (faccessat(td->dirfd, index_name, O_RDONLY, flags) < 0) {
+		ret = create_stream_packet_index(td, file_stream);
+		if (ret) {
+			fprintf(stderr, "[error] Stream index creation error.\n");
+			goto error_index;
+		}
+	} else {
+		ret = openat(td->dirfd, index_name, flags);
+		if (ret < 0) {
+			perror("Index file openat()");
+			ret = -1;
+			goto error_free;
+		}
+		file_stream->pos.index_fp = fdopen(ret, "r");
+		if (!file_stream->pos.index_fp) {
+			perror("fdopen() error");
+			goto error_free;
+		}
+		ret = import_stream_packet_index(td, file_stream);
+		if (ret) {
+			ret = -1;
+			goto error_index;
+		}
+		ret = fclose(file_stream->pos.index_fp);
+		if (ret < 0) {
+			perror("close index");
+			goto error_free;
+		}
 	}
+	free(index_name);
+
 	/* Add stream file to stream class */
 	g_ptr_array_add(file_stream->parent.stream_class->streams,
 			&file_stream->parent);
 	return 0;
 
 error_index:
+	if (file_stream->pos.index_fp) {
+		ret = fclose(file_stream->pos.index_fp);
+		if (ret < 0) {
+			perror("close index");
+		}
+	}
 	if (file_stream->parent.trace_packet_header)
 		bt_definition_unref(&file_stream->parent.trace_packet_header->p);
+error_free:
+	free(index_name);
 error_def:
 	closeret = ctf_fini_pos(&file_stream->pos);
 	if (closeret) {
@@ -1762,10 +2070,12 @@ int ctf_open_trace_read(struct ctf_trace *td,
 		void (*packet_seek)(struct bt_stream_pos *pos, size_t index,
 			int whence), FILE *metadata_fp)
 {
+	struct ctf_scanner *scanner;
 	int ret, closeret;
 	struct dirent *dirent;
 	struct dirent *diriter;
 	size_t dirent_len;
+	char *ext;
 
 	td->flags = flags;
 
@@ -1789,10 +2099,21 @@ int ctf_open_trace_read(struct ctf_trace *td,
 
 	/*
 	 * Keep the metadata file separate.
+	 * Keep scanner object local to the open. We don't support
+	 * incremental metadata append for on-disk traces.
 	 */
-
-	ret = ctf_open_trace_metadata_read(td, packet_seek, metadata_fp);
+	scanner = ctf_scanner_alloc();
+	if (!scanner) {
+		fprintf(stderr, "[error] Error allocating scanner\n");
+		ret = -ENOMEM;
+		goto error_metadata;
+	}
+	ret = ctf_trace_metadata_read(td, metadata_fp, scanner, 0);
+	ctf_scanner_free(scanner);
 	if (ret) {
+		if (ret == -ENOENT) {
+			fprintf(stderr, "[warning] Empty metadata.\n");
+		}
 		fprintf(stderr, "[warning] Unable to open trace metadata for path \"%s\".\n", path);
 		goto error_metadata;
 	}
@@ -1821,6 +2142,13 @@ int ctf_open_trace_read(struct ctf_trace *td,
 				|| !strcmp(diriter->d_name, "..")
 				|| !strcmp(diriter->d_name, "metadata"))
 			continue;
+
+		/* Ignore index files : *.idx */
+		ext = strrchr(diriter->d_name, '.');
+		if (ext && (!strcmp(ext, ".idx"))) {
+			continue;
+		}
+
 		ret = ctf_open_file_stream_read(td, diriter->d_name,
 					flags, packet_seek);
 		if (ret) {
@@ -1906,12 +2234,13 @@ void ctf_init_mmap_pos(struct ctf_stream_pos *pos,
 	pos->offset = 0;
 	pos->dummy = false;
 	pos->cur_index = 0;
-	pos->packet_cycles_index = NULL;
-	pos->packet_real_index = NULL;
 	pos->prot = PROT_READ;
 	pos->flags = MAP_PRIVATE;
 	pos->parent.rw_table = read_dispatch_table;
 	pos->parent.event_cb = ctf_read_event;
+	pos->priv = mmap_info->priv;
+	pos->packet_index = g_array_new(FALSE, TRUE,
+			sizeof(struct packet_index));
 }
 
 static
@@ -1994,8 +2323,17 @@ int ctf_open_mmap_trace_read(struct ctf_trace *td,
 	int ret;
 	struct bt_mmap_stream *mmap_info;
 
-	ret = ctf_open_trace_metadata_read(td, ctf_packet_seek, metadata_fp);
+	td->scanner = ctf_scanner_alloc();
+	if (!td->scanner) {
+		fprintf(stderr, "[error] Error allocating scanner\n");
+		ret = -ENOMEM;
+		goto error;
+	}
+	ret = ctf_trace_metadata_read(td, metadata_fp, td->scanner, 0);
 	if (ret) {
+		if (ret == -ENOENT) {
+			fprintf(stderr, "[warning] Empty metadata.\n");
+		}
 		goto error;
 	}
 
@@ -2010,10 +2348,10 @@ int ctf_open_mmap_trace_read(struct ctf_trace *td,
 			goto error;
 		}
 	}
-
 	return 0;
 
 error:
+	ctf_scanner_free(td->scanner);
 	return ret;
 }
 
@@ -2037,6 +2375,7 @@ struct bt_trace_descriptor *ctf_open_mmap_trace(
 		goto error;
 	}
 	td = g_new0(struct ctf_trace, 1);
+	td->dirfd = -1;
 	ret = ctf_open_mmap_trace_read(td, mmap_list, packet_seek, metadata_fp);
 	if (ret)
 		goto error_free;
@@ -2049,6 +2388,41 @@ error:
 	return NULL;
 }
 
+int ctf_append_trace_metadata(struct bt_trace_descriptor *tdp,
+		FILE *metadata_fp)
+{
+	struct ctf_trace *td = container_of(tdp, struct ctf_trace, parent);
+	int i, j;
+	int ret;
+
+	if (!td->scanner)
+		return -EINVAL;
+	ret = ctf_trace_metadata_read(td, metadata_fp, td->scanner, 1);
+	if (ret)
+		return ret;
+	/* for each stream_class */
+	for (i = 0; i < td->streams->len; i++) {
+		struct ctf_stream_declaration *stream_class;
+
+		stream_class = g_ptr_array_index(td->streams, i);
+		if (!stream_class)
+			continue;
+		/* for each stream */
+		for (j = 0; j < stream_class->streams->len; j++) {
+			struct ctf_stream_definition *stream;
+
+			stream = g_ptr_array_index(stream_class->streams, j);
+			if (!stream)
+				continue;
+			ret = copy_event_declarations_stream_class_to_stream(td,
+				stream_class, stream);
+			if (ret)
+				return ret;
+		}
+	}
+	return 0;
+}
+
 static
 int ctf_convert_index_timestamp(struct bt_trace_descriptor *tdp)
 {
@@ -2074,25 +2448,20 @@ int ctf_convert_index_timestamp(struct bt_trace_descriptor *tdp)
 			cfs = container_of(stream, struct ctf_file_stream,
 					parent);
 			stream_pos = &cfs->pos;
-			if (!stream_pos->packet_cycles_index)
+			if (!stream_pos->packet_index)
 				continue;
 
-			for (k = 0; k < stream_pos->packet_cycles_index->len; k++) {
+			for (k = 0; k < stream_pos->packet_index->len; k++) {
 				struct packet_index *index;
-				struct packet_index new_index;
 
-				index = &g_array_index(stream_pos->packet_cycles_index,
+				index = &g_array_index(stream_pos->packet_index,
 						struct packet_index, k);
-				memcpy(&new_index, index,
-						sizeof(struct packet_index));
-				new_index.timestamp_begin =
+				index->ts_real.timestamp_begin =
 					ctf_get_real_timestamp(stream,
-							index->timestamp_begin);
-				new_index.timestamp_end =
+							index->ts_cycles.timestamp_begin);
+				index->ts_real.timestamp_end =
 					ctf_get_real_timestamp(stream,
-							index->timestamp_end);
-				g_array_append_val(stream_pos->packet_real_index,
-						new_index);
+							index->ts_cycles.timestamp_end);
 			}
 		}
 	}
@@ -2109,10 +2478,12 @@ int ctf_close_file_stream(struct ctf_file_stream *file_stream)
 		fprintf(stderr, "Error on ctf_fini_pos\n");
 		return -1;
 	}
-	ret = close(file_stream->pos.fd);
-	if (ret) {
-		perror("Error closing file fd");
-		return -1;
+	if (file_stream->pos.fd >= 0) {
+		ret = close(file_stream->pos.fd);
+		if (ret) {
+			perror("Error closing file fd");
+			return -1;
+		}
 	}
 	return 0;
 }
@@ -2144,15 +2515,20 @@ int ctf_close_trace(struct bt_trace_descriptor *tdp)
 		}
 	}
 	ctf_destroy_metadata(td);
-	ret = close(td->dirfd);
-	if (ret) {
-		perror("Error closing dirfd");
-		return ret;
+	ctf_scanner_free(td->scanner);
+	if (td->dirfd >= 0) {
+		ret = close(td->dirfd);
+		if (ret) {
+			perror("Error closing dirfd");
+			return ret;
+		}
 	}
-	ret = closedir(td->dir);
-	if (ret) {
-		perror("Error closedir");
-		return ret;
+	if (td->dir) {
+		ret = closedir(td->dir);
+		if (ret) {
+			perror("Error closedir");
+			return ret;
+		}
 	}
 	free(td->metadata_string);
 	g_free(td);