960fe8e46439a6447c17198f0732c08b507bee4a
[lttng-tools.git] / src / bin / lttng-sessiond / consumer.c
1 /*
2 * Copyright (C) 2012 - David Goulet <dgoulet@efficios.com>
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License, version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 51
15 * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
16 */
17
18 #define _LGPL_SOURCE
19 #include <assert.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <sys/stat.h>
24 #include <sys/types.h>
25 #include <unistd.h>
26 #include <inttypes.h>
27
28 #include <common/common.h>
29 #include <common/defaults.h>
30 #include <common/uri.h>
31 #include <common/relayd/relayd.h>
32
33 #include "consumer.h"
34 #include "health-sessiond.h"
35 #include "ust-app.h"
36 #include "utils.h"
37
38 /*
39 * Send a data payload using a given consumer socket of size len.
40 *
41 * The consumer socket lock MUST be acquired before calling this since this
42 * function can change the fd value.
43 *
44 * Return 0 on success else a negative value on error.
45 */
46 int consumer_socket_send(struct consumer_socket *socket, void *msg, size_t len)
47 {
48 int fd;
49 ssize_t size;
50
51 assert(socket);
52 assert(socket->fd_ptr);
53 assert(msg);
54
55 /* Consumer socket is invalid. Stopping. */
56 fd = *socket->fd_ptr;
57 if (fd < 0) {
58 goto error;
59 }
60
61 size = lttcomm_send_unix_sock(fd, msg, len);
62 if (size < 0) {
63 /* The above call will print a PERROR on error. */
64 DBG("Error when sending data to consumer on sock %d", fd);
65 /*
66 * At this point, the socket is not usable anymore thus closing it and
67 * setting the file descriptor to -1 so it is not reused.
68 */
69
70 /* This call will PERROR on error. */
71 (void) lttcomm_close_unix_sock(fd);
72 *socket->fd_ptr = -1;
73 goto error;
74 }
75
76 return 0;
77
78 error:
79 return -1;
80 }
81
82 /*
83 * Receive a data payload using a given consumer socket of size len.
84 *
85 * The consumer socket lock MUST be acquired before calling this since this
86 * function can change the fd value.
87 *
88 * Return 0 on success else a negative value on error.
89 */
90 int consumer_socket_recv(struct consumer_socket *socket, void *msg, size_t len)
91 {
92 int fd;
93 ssize_t size;
94
95 assert(socket);
96 assert(socket->fd_ptr);
97 assert(msg);
98
99 /* Consumer socket is invalid. Stopping. */
100 fd = *socket->fd_ptr;
101 if (fd < 0) {
102 goto error;
103 }
104
105 size = lttcomm_recv_unix_sock(fd, msg, len);
106 if (size <= 0) {
107 /* The above call will print a PERROR on error. */
108 DBG("Error when receiving data from the consumer socket %d", fd);
109 /*
110 * At this point, the socket is not usable anymore thus closing it and
111 * setting the file descriptor to -1 so it is not reused.
112 */
113
114 /* This call will PERROR on error. */
115 (void) lttcomm_close_unix_sock(fd);
116 *socket->fd_ptr = -1;
117 goto error;
118 }
119
120 return 0;
121
122 error:
123 return -1;
124 }
125
126 /*
127 * Receive a reply command status message from the consumer. Consumer socket
128 * lock MUST be acquired before calling this function.
129 *
130 * Return 0 on success, -1 on recv error or a negative lttng error code which
131 * was possibly returned by the consumer.
132 */
133 int consumer_recv_status_reply(struct consumer_socket *sock)
134 {
135 int ret;
136 struct lttcomm_consumer_status_msg reply;
137
138 assert(sock);
139
140 ret = consumer_socket_recv(sock, &reply, sizeof(reply));
141 if (ret < 0) {
142 goto end;
143 }
144
145 if (reply.ret_code == LTTCOMM_CONSUMERD_SUCCESS) {
146 /* All good. */
147 ret = 0;
148 } else {
149 ret = -reply.ret_code;
150 DBG("Consumer ret code %d", ret);
151 }
152
153 end:
154 return ret;
155 }
156
157 /*
158 * Once the ASK_CHANNEL command is sent to the consumer, the channel
159 * information are sent back. This call receives that data and populates key
160 * and stream_count.
161 *
162 * On success return 0 and both key and stream_count are set. On error, a
163 * negative value is sent back and both parameters are untouched.
164 */
165 int consumer_recv_status_channel(struct consumer_socket *sock,
166 uint64_t *key, unsigned int *stream_count)
167 {
168 int ret;
169 struct lttcomm_consumer_status_channel reply;
170
171 assert(sock);
172 assert(stream_count);
173 assert(key);
174
175 ret = consumer_socket_recv(sock, &reply, sizeof(reply));
176 if (ret < 0) {
177 goto end;
178 }
179
180 /* An error is possible so don't touch the key and stream_count. */
181 if (reply.ret_code != LTTCOMM_CONSUMERD_SUCCESS) {
182 ret = -1;
183 goto end;
184 }
185
186 *key = reply.key;
187 *stream_count = reply.stream_count;
188 ret = 0;
189
190 end:
191 return ret;
192 }
193
194 /*
195 * Send destroy relayd command to consumer.
196 *
197 * On success return positive value. On error, negative value.
198 */
199 int consumer_send_destroy_relayd(struct consumer_socket *sock,
200 struct consumer_output *consumer)
201 {
202 int ret;
203 struct lttcomm_consumer_msg msg;
204
205 assert(consumer);
206 assert(sock);
207
208 DBG2("Sending destroy relayd command to consumer sock %d", *sock->fd_ptr);
209
210 memset(&msg, 0, sizeof(msg));
211 msg.cmd_type = LTTNG_CONSUMER_DESTROY_RELAYD;
212 msg.u.destroy_relayd.net_seq_idx = consumer->net_seq_index;
213
214 pthread_mutex_lock(sock->lock);
215 ret = consumer_socket_send(sock, &msg, sizeof(msg));
216 if (ret < 0) {
217 goto error;
218 }
219
220 /* Don't check the return value. The caller will do it. */
221 ret = consumer_recv_status_reply(sock);
222
223 DBG2("Consumer send destroy relayd command done");
224
225 error:
226 pthread_mutex_unlock(sock->lock);
227 return ret;
228 }
229
230 /*
231 * For each consumer socket in the consumer output object, send a destroy
232 * relayd command.
233 */
234 void consumer_output_send_destroy_relayd(struct consumer_output *consumer)
235 {
236 struct lttng_ht_iter iter;
237 struct consumer_socket *socket;
238
239 assert(consumer);
240
241 /* Destroy any relayd connection */
242 if (consumer->type == CONSUMER_DST_NET) {
243 rcu_read_lock();
244 cds_lfht_for_each_entry(consumer->socks->ht, &iter.iter, socket,
245 node.node) {
246 int ret;
247
248 /* Send destroy relayd command */
249 ret = consumer_send_destroy_relayd(socket, consumer);
250 if (ret < 0) {
251 DBG("Unable to send destroy relayd command to consumer");
252 /* Continue since we MUST delete everything at this point. */
253 }
254 }
255 rcu_read_unlock();
256 }
257 }
258
259 /*
260 * From a consumer_data structure, allocate and add a consumer socket to the
261 * consumer output.
262 *
263 * Return 0 on success, else negative value on error
264 */
265 int consumer_create_socket(struct consumer_data *data,
266 struct consumer_output *output)
267 {
268 int ret = 0;
269 struct consumer_socket *socket;
270
271 assert(data);
272
273 if (output == NULL || data->cmd_sock < 0) {
274 /*
275 * Not an error. Possible there is simply not spawned consumer or it's
276 * disabled for the tracing session asking the socket.
277 */
278 goto error;
279 }
280
281 rcu_read_lock();
282 socket = consumer_find_socket(data->cmd_sock, output);
283 rcu_read_unlock();
284 if (socket == NULL) {
285 socket = consumer_allocate_socket(&data->cmd_sock);
286 if (socket == NULL) {
287 ret = -1;
288 goto error;
289 }
290
291 socket->registered = 0;
292 socket->lock = &data->lock;
293 rcu_read_lock();
294 consumer_add_socket(socket, output);
295 rcu_read_unlock();
296 }
297
298 socket->type = data->type;
299
300 DBG3("Consumer socket created (fd: %d) and added to output",
301 data->cmd_sock);
302
303 error:
304 return ret;
305 }
306
307 /*
308 * Return the consumer socket from the given consumer output with the right
309 * bitness. On error, returns NULL.
310 *
311 * The caller MUST acquire a rcu read side lock and keep it until the socket
312 * object reference is not needed anymore.
313 */
314 struct consumer_socket *consumer_find_socket_by_bitness(int bits,
315 struct consumer_output *consumer)
316 {
317 int consumer_fd;
318 struct consumer_socket *socket = NULL;
319
320 switch (bits) {
321 case 64:
322 consumer_fd = uatomic_read(&ust_consumerd64_fd);
323 break;
324 case 32:
325 consumer_fd = uatomic_read(&ust_consumerd32_fd);
326 break;
327 default:
328 assert(0);
329 goto end;
330 }
331
332 socket = consumer_find_socket(consumer_fd, consumer);
333 if (!socket) {
334 ERR("Consumer socket fd %d not found in consumer obj %p",
335 consumer_fd, consumer);
336 }
337
338 end:
339 return socket;
340 }
341
342 /*
343 * Find a consumer_socket in a consumer_output hashtable. Read side lock must
344 * be acquired before calling this function and across use of the
345 * returned consumer_socket.
346 */
347 struct consumer_socket *consumer_find_socket(int key,
348 struct consumer_output *consumer)
349 {
350 struct lttng_ht_iter iter;
351 struct lttng_ht_node_ulong *node;
352 struct consumer_socket *socket = NULL;
353
354 /* Negative keys are lookup failures */
355 if (key < 0 || consumer == NULL) {
356 return NULL;
357 }
358
359 lttng_ht_lookup(consumer->socks, (void *)((unsigned long) key),
360 &iter);
361 node = lttng_ht_iter_get_node_ulong(&iter);
362 if (node != NULL) {
363 socket = caa_container_of(node, struct consumer_socket, node);
364 }
365
366 return socket;
367 }
368
369 /*
370 * Allocate a new consumer_socket and return the pointer.
371 */
372 struct consumer_socket *consumer_allocate_socket(int *fd)
373 {
374 struct consumer_socket *socket = NULL;
375
376 assert(fd);
377
378 socket = zmalloc(sizeof(struct consumer_socket));
379 if (socket == NULL) {
380 PERROR("zmalloc consumer socket");
381 goto error;
382 }
383
384 socket->fd_ptr = fd;
385 lttng_ht_node_init_ulong(&socket->node, *fd);
386
387 error:
388 return socket;
389 }
390
391 /*
392 * Add consumer socket to consumer output object. Read side lock must be
393 * acquired before calling this function.
394 */
395 void consumer_add_socket(struct consumer_socket *sock,
396 struct consumer_output *consumer)
397 {
398 assert(sock);
399 assert(consumer);
400
401 lttng_ht_add_unique_ulong(consumer->socks, &sock->node);
402 }
403
404 /*
405 * Delte consumer socket to consumer output object. Read side lock must be
406 * acquired before calling this function.
407 */
408 void consumer_del_socket(struct consumer_socket *sock,
409 struct consumer_output *consumer)
410 {
411 int ret;
412 struct lttng_ht_iter iter;
413
414 assert(sock);
415 assert(consumer);
416
417 iter.iter.node = &sock->node.node;
418 ret = lttng_ht_del(consumer->socks, &iter);
419 assert(!ret);
420 }
421
422 /*
423 * RCU destroy call function.
424 */
425 static void destroy_socket_rcu(struct rcu_head *head)
426 {
427 struct lttng_ht_node_ulong *node =
428 caa_container_of(head, struct lttng_ht_node_ulong, head);
429 struct consumer_socket *socket =
430 caa_container_of(node, struct consumer_socket, node);
431
432 free(socket);
433 }
434
435 /*
436 * Destroy and free socket pointer in a call RCU. Read side lock must be
437 * acquired before calling this function.
438 */
439 void consumer_destroy_socket(struct consumer_socket *sock)
440 {
441 assert(sock);
442
443 /*
444 * We DO NOT close the file descriptor here since it is global to the
445 * session daemon and is closed only if the consumer dies or a custom
446 * consumer was registered,
447 */
448 if (sock->registered) {
449 DBG3("Consumer socket was registered. Closing fd %d", *sock->fd_ptr);
450 lttcomm_close_unix_sock(*sock->fd_ptr);
451 }
452
453 call_rcu(&sock->node.head, destroy_socket_rcu);
454 }
455
456 /*
457 * Allocate and assign data to a consumer_output object.
458 *
459 * Return pointer to structure.
460 */
461 struct consumer_output *consumer_create_output(enum consumer_dst_type type)
462 {
463 struct consumer_output *output = NULL;
464
465 output = zmalloc(sizeof(struct consumer_output));
466 if (output == NULL) {
467 PERROR("zmalloc consumer_output");
468 goto error;
469 }
470
471 /* By default, consumer output is enabled */
472 output->enabled = 1;
473 output->type = type;
474 output->net_seq_index = (uint64_t) -1ULL;
475 urcu_ref_init(&output->ref);
476
477 output->socks = lttng_ht_new(0, LTTNG_HT_TYPE_ULONG);
478
479 error:
480 return output;
481 }
482
483 /*
484 * Iterate over the consumer output socket hash table and destroy them. The
485 * socket file descriptor are only closed if the consumer output was
486 * registered meaning it's an external consumer.
487 */
488 void consumer_destroy_output_sockets(struct consumer_output *obj)
489 {
490 struct lttng_ht_iter iter;
491 struct consumer_socket *socket;
492
493 if (!obj->socks) {
494 return;
495 }
496
497 rcu_read_lock();
498 cds_lfht_for_each_entry(obj->socks->ht, &iter.iter, socket, node.node) {
499 consumer_del_socket(socket, obj);
500 consumer_destroy_socket(socket);
501 }
502 rcu_read_unlock();
503 }
504
505 /*
506 * Delete the consumer_output object from the list and free the ptr.
507 *
508 * Should *NOT* be called with RCU read-side lock held.
509 */
510 static void consumer_release_output(struct urcu_ref *ref)
511 {
512 struct consumer_output *obj =
513 caa_container_of(ref, struct consumer_output, ref);
514
515 consumer_destroy_output_sockets(obj);
516
517 if (obj->socks) {
518 /* Finally destroy HT */
519 ht_cleanup_push(obj->socks);
520 }
521
522 free(obj);
523 }
524
525 /*
526 * Get the consumer_output object.
527 */
528 void consumer_output_get(struct consumer_output *obj)
529 {
530 urcu_ref_get(&obj->ref);
531 }
532
533 /*
534 * Put the consumer_output object.
535 *
536 * Should *NOT* be called with RCU read-side lock held.
537 */
538 void consumer_output_put(struct consumer_output *obj)
539 {
540 if (!obj) {
541 return;
542 }
543 urcu_ref_put(&obj->ref, consumer_release_output);
544 }
545
546 /*
547 * Copy consumer output and returned the newly allocated copy.
548 *
549 * Should *NOT* be called with RCU read-side lock held.
550 */
551 struct consumer_output *consumer_copy_output(struct consumer_output *obj)
552 {
553 int ret;
554 struct consumer_output *output;
555
556 assert(obj);
557
558 output = consumer_create_output(obj->type);
559 if (output == NULL) {
560 goto end;
561 }
562 output->enabled = obj->enabled;
563 output->net_seq_index = obj->net_seq_index;
564 memcpy(output->subdir, obj->subdir, PATH_MAX);
565 output->snapshot = obj->snapshot;
566 output->relay_major_version = obj->relay_major_version;
567 output->relay_minor_version = obj->relay_minor_version;
568 memcpy(&output->dst, &obj->dst, sizeof(output->dst));
569 ret = consumer_copy_sockets(output, obj);
570 if (ret < 0) {
571 goto error_put;
572 }
573 end:
574 return output;
575
576 error_put:
577 consumer_output_put(output);
578 return NULL;
579 }
580
581 /*
582 * Copy consumer sockets from src to dst.
583 *
584 * Return 0 on success or else a negative value.
585 */
586 int consumer_copy_sockets(struct consumer_output *dst,
587 struct consumer_output *src)
588 {
589 int ret = 0;
590 struct lttng_ht_iter iter;
591 struct consumer_socket *socket, *copy_sock;
592
593 assert(dst);
594 assert(src);
595
596 rcu_read_lock();
597 cds_lfht_for_each_entry(src->socks->ht, &iter.iter, socket, node.node) {
598 /* Ignore socket that are already there. */
599 copy_sock = consumer_find_socket(*socket->fd_ptr, dst);
600 if (copy_sock) {
601 continue;
602 }
603
604 /* Create new socket object. */
605 copy_sock = consumer_allocate_socket(socket->fd_ptr);
606 if (copy_sock == NULL) {
607 rcu_read_unlock();
608 ret = -ENOMEM;
609 goto error;
610 }
611
612 copy_sock->registered = socket->registered;
613 /*
614 * This is valid because this lock is shared accross all consumer
615 * object being the global lock of the consumer data structure of the
616 * session daemon.
617 */
618 copy_sock->lock = socket->lock;
619 consumer_add_socket(copy_sock, dst);
620 }
621 rcu_read_unlock();
622
623 error:
624 return ret;
625 }
626
627 /*
628 * Set network URI to the consumer output object.
629 *
630 * Return 0 on success. Return 1 if the URI were equal. Else, negative value on
631 * error.
632 */
633 int consumer_set_network_uri(struct consumer_output *obj,
634 struct lttng_uri *uri)
635 {
636 int ret;
637 char tmp_path[PATH_MAX];
638 char hostname[HOST_NAME_MAX];
639 struct lttng_uri *dst_uri = NULL;
640
641 /* Code flow error safety net. */
642 assert(obj);
643 assert(uri);
644
645 switch (uri->stype) {
646 case LTTNG_STREAM_CONTROL:
647 dst_uri = &obj->dst.net.control;
648 obj->dst.net.control_isset = 1;
649 if (uri->port == 0) {
650 /* Assign default port. */
651 uri->port = DEFAULT_NETWORK_CONTROL_PORT;
652 } else {
653 if (obj->dst.net.data_isset && uri->port ==
654 obj->dst.net.data.port) {
655 ret = -LTTNG_ERR_INVALID;
656 goto error;
657 }
658 }
659 DBG3("Consumer control URI set with port %d", uri->port);
660 break;
661 case LTTNG_STREAM_DATA:
662 dst_uri = &obj->dst.net.data;
663 obj->dst.net.data_isset = 1;
664 if (uri->port == 0) {
665 /* Assign default port. */
666 uri->port = DEFAULT_NETWORK_DATA_PORT;
667 } else {
668 if (obj->dst.net.control_isset && uri->port ==
669 obj->dst.net.control.port) {
670 ret = -LTTNG_ERR_INVALID;
671 goto error;
672 }
673 }
674 DBG3("Consumer data URI set with port %d", uri->port);
675 break;
676 default:
677 ERR("Set network uri type unknown %d", uri->stype);
678 ret = -LTTNG_ERR_INVALID;
679 goto error;
680 }
681
682 ret = uri_compare(dst_uri, uri);
683 if (!ret) {
684 /* Same URI, don't touch it and return success. */
685 DBG3("URI network compare are the same");
686 goto equal;
687 }
688
689 /* URIs were not equal, replacing it. */
690 memset(dst_uri, 0, sizeof(struct lttng_uri));
691 memcpy(dst_uri, uri, sizeof(struct lttng_uri));
692 obj->type = CONSUMER_DST_NET;
693
694 /* Handle subdir and add hostname in front. */
695 if (dst_uri->stype == LTTNG_STREAM_CONTROL) {
696 /* Get hostname to append it in the pathname */
697 ret = gethostname(hostname, sizeof(hostname));
698 if (ret < 0) {
699 PERROR("gethostname. Fallback on default localhost");
700 strncpy(hostname, "localhost", sizeof(hostname));
701 }
702 hostname[sizeof(hostname) - 1] = '\0';
703
704 /* Setup consumer subdir if none present in the control URI */
705 if (strlen(dst_uri->subdir) == 0) {
706 ret = snprintf(tmp_path, sizeof(tmp_path), "%s/%s",
707 hostname, obj->subdir);
708 } else {
709 ret = snprintf(tmp_path, sizeof(tmp_path), "%s/%s",
710 hostname, dst_uri->subdir);
711 }
712 if (ret < 0) {
713 PERROR("snprintf set consumer uri subdir");
714 ret = -LTTNG_ERR_NOMEM;
715 goto error;
716 }
717
718 if (lttng_strncpy(obj->subdir, tmp_path, sizeof(obj->subdir))) {
719 ret = -LTTNG_ERR_INVALID;
720 goto error;
721 }
722 DBG3("Consumer set network uri subdir path %s", tmp_path);
723 }
724
725 return 0;
726 equal:
727 return 1;
728 error:
729 return ret;
730 }
731
732 /*
733 * Send file descriptor to consumer via sock.
734 *
735 * The consumer socket lock must be held by the caller.
736 */
737 int consumer_send_fds(struct consumer_socket *sock, int *fds, size_t nb_fd)
738 {
739 int ret;
740
741 assert(fds);
742 assert(sock);
743 assert(nb_fd > 0);
744 assert(pthread_mutex_trylock(sock->lock) == EBUSY);
745
746 ret = lttcomm_send_fds_unix_sock(*sock->fd_ptr, fds, nb_fd);
747 if (ret < 0) {
748 /* The above call will print a PERROR on error. */
749 DBG("Error when sending consumer fds on sock %d", *sock->fd_ptr);
750 goto error;
751 }
752
753 ret = consumer_recv_status_reply(sock);
754
755 error:
756 return ret;
757 }
758
759 /*
760 * Consumer send communication message structure to consumer.
761 *
762 * The consumer socket lock must be held by the caller.
763 */
764 int consumer_send_msg(struct consumer_socket *sock,
765 struct lttcomm_consumer_msg *msg)
766 {
767 int ret;
768
769 assert(msg);
770 assert(sock);
771 assert(pthread_mutex_trylock(sock->lock) == EBUSY);
772
773 ret = consumer_socket_send(sock, msg, sizeof(struct lttcomm_consumer_msg));
774 if (ret < 0) {
775 goto error;
776 }
777
778 ret = consumer_recv_status_reply(sock);
779
780 error:
781 return ret;
782 }
783
784 /*
785 * Consumer send channel communication message structure to consumer.
786 *
787 * The consumer socket lock must be held by the caller.
788 */
789 int consumer_send_channel(struct consumer_socket *sock,
790 struct lttcomm_consumer_msg *msg)
791 {
792 int ret;
793
794 assert(msg);
795 assert(sock);
796
797 ret = consumer_send_msg(sock, msg);
798 if (ret < 0) {
799 goto error;
800 }
801
802 error:
803 return ret;
804 }
805
806 /*
807 * Populate the given consumer msg structure with the ask_channel command
808 * information.
809 */
810 void consumer_init_ask_channel_comm_msg(struct lttcomm_consumer_msg *msg,
811 uint64_t subbuf_size,
812 uint64_t num_subbuf,
813 int overwrite,
814 unsigned int switch_timer_interval,
815 unsigned int read_timer_interval,
816 unsigned int live_timer_interval,
817 int output,
818 int type,
819 uint64_t session_id,
820 const char *pathname,
821 const char *name,
822 uid_t uid,
823 gid_t gid,
824 uint64_t relayd_id,
825 uint64_t key,
826 unsigned char *uuid,
827 uint32_t chan_id,
828 uint64_t tracefile_size,
829 uint64_t tracefile_count,
830 uint64_t session_id_per_pid,
831 unsigned int monitor,
832 uint32_t ust_app_uid,
833 const char *root_shm_path,
834 const char *shm_path)
835 {
836 assert(msg);
837
838 /* Zeroed structure */
839 memset(msg, 0, sizeof(struct lttcomm_consumer_msg));
840
841 msg->cmd_type = LTTNG_CONSUMER_ASK_CHANNEL_CREATION;
842 msg->u.ask_channel.subbuf_size = subbuf_size;
843 msg->u.ask_channel.num_subbuf = num_subbuf ;
844 msg->u.ask_channel.overwrite = overwrite;
845 msg->u.ask_channel.switch_timer_interval = switch_timer_interval;
846 msg->u.ask_channel.read_timer_interval = read_timer_interval;
847 msg->u.ask_channel.live_timer_interval = live_timer_interval;
848 msg->u.ask_channel.output = output;
849 msg->u.ask_channel.type = type;
850 msg->u.ask_channel.session_id = session_id;
851 msg->u.ask_channel.session_id_per_pid = session_id_per_pid;
852 msg->u.ask_channel.uid = uid;
853 msg->u.ask_channel.gid = gid;
854 msg->u.ask_channel.relayd_id = relayd_id;
855 msg->u.ask_channel.key = key;
856 msg->u.ask_channel.chan_id = chan_id;
857 msg->u.ask_channel.tracefile_size = tracefile_size;
858 msg->u.ask_channel.tracefile_count = tracefile_count;
859 msg->u.ask_channel.monitor = monitor;
860 msg->u.ask_channel.ust_app_uid = ust_app_uid;
861
862 memcpy(msg->u.ask_channel.uuid, uuid, sizeof(msg->u.ask_channel.uuid));
863
864 if (pathname) {
865 strncpy(msg->u.ask_channel.pathname, pathname,
866 sizeof(msg->u.ask_channel.pathname));
867 msg->u.ask_channel.pathname[sizeof(msg->u.ask_channel.pathname)-1] = '\0';
868 }
869
870 strncpy(msg->u.ask_channel.name, name, sizeof(msg->u.ask_channel.name));
871 msg->u.ask_channel.name[sizeof(msg->u.ask_channel.name) - 1] = '\0';
872
873 if (root_shm_path) {
874 strncpy(msg->u.ask_channel.root_shm_path, root_shm_path,
875 sizeof(msg->u.ask_channel.root_shm_path));
876 msg->u.ask_channel.root_shm_path[sizeof(msg->u.ask_channel.root_shm_path) - 1] = '\0';
877 }
878 if (shm_path) {
879 strncpy(msg->u.ask_channel.shm_path, shm_path,
880 sizeof(msg->u.ask_channel.shm_path));
881 msg->u.ask_channel.shm_path[sizeof(msg->u.ask_channel.shm_path) - 1] = '\0';
882 }
883 }
884
885 /*
886 * Init channel communication message structure.
887 */
888 void consumer_init_channel_comm_msg(struct lttcomm_consumer_msg *msg,
889 enum lttng_consumer_command cmd,
890 uint64_t channel_key,
891 uint64_t session_id,
892 const char *pathname,
893 uid_t uid,
894 gid_t gid,
895 uint64_t relayd_id,
896 const char *name,
897 unsigned int nb_init_streams,
898 enum lttng_event_output output,
899 int type,
900 uint64_t tracefile_size,
901 uint64_t tracefile_count,
902 unsigned int monitor,
903 unsigned int live_timer_interval)
904 {
905 assert(msg);
906
907 /* Zeroed structure */
908 memset(msg, 0, sizeof(struct lttcomm_consumer_msg));
909
910 /* Send channel */
911 msg->cmd_type = cmd;
912 msg->u.channel.channel_key = channel_key;
913 msg->u.channel.session_id = session_id;
914 msg->u.channel.uid = uid;
915 msg->u.channel.gid = gid;
916 msg->u.channel.relayd_id = relayd_id;
917 msg->u.channel.nb_init_streams = nb_init_streams;
918 msg->u.channel.output = output;
919 msg->u.channel.type = type;
920 msg->u.channel.tracefile_size = tracefile_size;
921 msg->u.channel.tracefile_count = tracefile_count;
922 msg->u.channel.monitor = monitor;
923 msg->u.channel.live_timer_interval = live_timer_interval;
924
925 strncpy(msg->u.channel.pathname, pathname,
926 sizeof(msg->u.channel.pathname));
927 msg->u.channel.pathname[sizeof(msg->u.channel.pathname) - 1] = '\0';
928
929 strncpy(msg->u.channel.name, name, sizeof(msg->u.channel.name));
930 msg->u.channel.name[sizeof(msg->u.channel.name) - 1] = '\0';
931 }
932
933 /*
934 * Init stream communication message structure.
935 */
936 void consumer_init_stream_comm_msg(struct lttcomm_consumer_msg *msg,
937 enum lttng_consumer_command cmd,
938 uint64_t channel_key,
939 uint64_t stream_key,
940 int cpu)
941 {
942 assert(msg);
943
944 memset(msg, 0, sizeof(struct lttcomm_consumer_msg));
945
946 msg->cmd_type = cmd;
947 msg->u.stream.channel_key = channel_key;
948 msg->u.stream.stream_key = stream_key;
949 msg->u.stream.cpu = cpu;
950 }
951
952 void consumer_init_streams_sent_comm_msg(struct lttcomm_consumer_msg *msg,
953 enum lttng_consumer_command cmd,
954 uint64_t channel_key, uint64_t net_seq_idx)
955 {
956 assert(msg);
957
958 memset(msg, 0, sizeof(struct lttcomm_consumer_msg));
959
960 msg->cmd_type = cmd;
961 msg->u.sent_streams.channel_key = channel_key;
962 msg->u.sent_streams.net_seq_idx = net_seq_idx;
963 }
964
965 /*
966 * Send stream communication structure to the consumer.
967 */
968 int consumer_send_stream(struct consumer_socket *sock,
969 struct consumer_output *dst, struct lttcomm_consumer_msg *msg,
970 int *fds, size_t nb_fd)
971 {
972 int ret;
973
974 assert(msg);
975 assert(dst);
976 assert(sock);
977 assert(fds);
978
979 ret = consumer_send_msg(sock, msg);
980 if (ret < 0) {
981 goto error;
982 }
983
984 ret = consumer_send_fds(sock, fds, nb_fd);
985 if (ret < 0) {
986 goto error;
987 }
988
989 error:
990 return ret;
991 }
992
993 /*
994 * Send relayd socket to consumer associated with a session name.
995 *
996 * The consumer socket lock must be held by the caller.
997 *
998 * On success return positive value. On error, negative value.
999 */
1000 int consumer_send_relayd_socket(struct consumer_socket *consumer_sock,
1001 struct lttcomm_relayd_sock *rsock, struct consumer_output *consumer,
1002 enum lttng_stream_type type, uint64_t session_id,
1003 char *session_name, char *hostname, int session_live_timer)
1004 {
1005 int ret;
1006 struct lttcomm_consumer_msg msg;
1007
1008 /* Code flow error. Safety net. */
1009 assert(rsock);
1010 assert(consumer);
1011 assert(consumer_sock);
1012
1013 memset(&msg, 0, sizeof(msg));
1014 /* Bail out if consumer is disabled */
1015 if (!consumer->enabled) {
1016 ret = LTTNG_OK;
1017 goto error;
1018 }
1019
1020 if (type == LTTNG_STREAM_CONTROL) {
1021 ret = relayd_create_session(rsock,
1022 &msg.u.relayd_sock.relayd_session_id,
1023 session_name, hostname, session_live_timer,
1024 consumer->snapshot);
1025 if (ret < 0) {
1026 /* Close the control socket. */
1027 (void) relayd_close(rsock);
1028 goto error;
1029 }
1030 }
1031
1032 msg.cmd_type = LTTNG_CONSUMER_ADD_RELAYD_SOCKET;
1033 /*
1034 * Assign network consumer output index using the temporary consumer since
1035 * this call should only be made from within a set_consumer_uri() function
1036 * call in the session daemon.
1037 */
1038 msg.u.relayd_sock.net_index = consumer->net_seq_index;
1039 msg.u.relayd_sock.type = type;
1040 msg.u.relayd_sock.session_id = session_id;
1041 memcpy(&msg.u.relayd_sock.sock, rsock, sizeof(msg.u.relayd_sock.sock));
1042
1043 DBG3("Sending relayd sock info to consumer on %d", *consumer_sock->fd_ptr);
1044 ret = consumer_send_msg(consumer_sock, &msg);
1045 if (ret < 0) {
1046 goto error;
1047 }
1048
1049 DBG3("Sending relayd socket file descriptor to consumer");
1050 ret = consumer_send_fds(consumer_sock, &rsock->sock.fd, 1);
1051 if (ret < 0) {
1052 goto error;
1053 }
1054
1055 DBG2("Consumer relayd socket sent");
1056
1057 error:
1058 return ret;
1059 }
1060
1061 /*
1062 * Set consumer subdirectory using the session name and a generated datetime if
1063 * needed. This is appended to the current subdirectory.
1064 */
1065 int consumer_set_subdir(struct consumer_output *consumer,
1066 const char *session_name)
1067 {
1068 int ret = 0;
1069 unsigned int have_default_name = 0;
1070 char datetime[16], tmp_path[PATH_MAX];
1071 time_t rawtime;
1072 struct tm *timeinfo;
1073
1074 assert(consumer);
1075 assert(session_name);
1076
1077 memset(tmp_path, 0, sizeof(tmp_path));
1078
1079 /* Flag if we have a default session. */
1080 if (strncmp(session_name, DEFAULT_SESSION_NAME "-",
1081 strlen(DEFAULT_SESSION_NAME) + 1) == 0) {
1082 have_default_name = 1;
1083 } else {
1084 /* Get date and time for session path */
1085 time(&rawtime);
1086 timeinfo = localtime(&rawtime);
1087 strftime(datetime, sizeof(datetime), "%Y%m%d-%H%M%S", timeinfo);
1088 }
1089
1090 if (have_default_name) {
1091 ret = snprintf(tmp_path, sizeof(tmp_path),
1092 "%s/%s", consumer->subdir, session_name);
1093 } else {
1094 ret = snprintf(tmp_path, sizeof(tmp_path),
1095 "%s/%s-%s/", consumer->subdir, session_name, datetime);
1096 }
1097 if (ret < 0) {
1098 PERROR("snprintf session name date");
1099 goto error;
1100 }
1101
1102 if (lttng_strncpy(consumer->subdir, tmp_path,
1103 sizeof(consumer->subdir))) {
1104 ret = -EINVAL;
1105 goto error;
1106 }
1107 DBG2("Consumer subdir set to %s", consumer->subdir);
1108
1109 error:
1110 return ret;
1111 }
1112
1113 /*
1114 * Ask the consumer if the data is pending for the specific session id.
1115 * Returns 1 if data is pending, 0 otherwise, or < 0 on error.
1116 */
1117 int consumer_is_data_pending(uint64_t session_id,
1118 struct consumer_output *consumer)
1119 {
1120 int ret;
1121 int32_t ret_code = 0; /* Default is that the data is NOT pending */
1122 struct consumer_socket *socket;
1123 struct lttng_ht_iter iter;
1124 struct lttcomm_consumer_msg msg;
1125
1126 assert(consumer);
1127
1128 DBG3("Consumer data pending for id %" PRIu64, session_id);
1129
1130 memset(&msg, 0, sizeof(msg));
1131 msg.cmd_type = LTTNG_CONSUMER_DATA_PENDING;
1132 msg.u.data_pending.session_id = session_id;
1133
1134 /* Send command for each consumer */
1135 rcu_read_lock();
1136 cds_lfht_for_each_entry(consumer->socks->ht, &iter.iter, socket,
1137 node.node) {
1138 pthread_mutex_lock(socket->lock);
1139 ret = consumer_socket_send(socket, &msg, sizeof(msg));
1140 if (ret < 0) {
1141 pthread_mutex_unlock(socket->lock);
1142 goto error_unlock;
1143 }
1144
1145 /*
1146 * No need for a recv reply status because the answer to the command is
1147 * the reply status message.
1148 */
1149
1150 ret = consumer_socket_recv(socket, &ret_code, sizeof(ret_code));
1151 if (ret < 0) {
1152 pthread_mutex_unlock(socket->lock);
1153 goto error_unlock;
1154 }
1155 pthread_mutex_unlock(socket->lock);
1156
1157 if (ret_code == 1) {
1158 break;
1159 }
1160 }
1161 rcu_read_unlock();
1162
1163 DBG("Consumer data is %s pending for session id %" PRIu64,
1164 ret_code == 1 ? "" : "NOT", session_id);
1165 return ret_code;
1166
1167 error_unlock:
1168 rcu_read_unlock();
1169 return -1;
1170 }
1171
1172 /*
1173 * Send a flush command to consumer using the given channel key.
1174 *
1175 * Return 0 on success else a negative value.
1176 */
1177 int consumer_flush_channel(struct consumer_socket *socket, uint64_t key)
1178 {
1179 int ret;
1180 struct lttcomm_consumer_msg msg;
1181
1182 assert(socket);
1183
1184 DBG2("Consumer flush channel key %" PRIu64, key);
1185
1186 memset(&msg, 0, sizeof(msg));
1187 msg.cmd_type = LTTNG_CONSUMER_FLUSH_CHANNEL;
1188 msg.u.flush_channel.key = key;
1189
1190 pthread_mutex_lock(socket->lock);
1191 health_code_update();
1192
1193 ret = consumer_send_msg(socket, &msg);
1194 if (ret < 0) {
1195 goto end;
1196 }
1197
1198 end:
1199 health_code_update();
1200 pthread_mutex_unlock(socket->lock);
1201 return ret;
1202 }
1203
1204 /*
1205 * Send a clear quiescent command to consumer using the given channel key.
1206 *
1207 * Return 0 on success else a negative value.
1208 */
1209 int consumer_clear_quiescent_channel(struct consumer_socket *socket, uint64_t key)
1210 {
1211 int ret;
1212 struct lttcomm_consumer_msg msg;
1213
1214 assert(socket);
1215
1216 DBG2("Consumer clear quiescent channel key %" PRIu64, key);
1217
1218 memset(&msg, 0, sizeof(msg));
1219 msg.cmd_type = LTTNG_CONSUMER_CLEAR_QUIESCENT_CHANNEL;
1220 msg.u.clear_quiescent_channel.key = key;
1221
1222 pthread_mutex_lock(socket->lock);
1223 health_code_update();
1224
1225 ret = consumer_send_msg(socket, &msg);
1226 if (ret < 0) {
1227 goto end;
1228 }
1229
1230 end:
1231 health_code_update();
1232 pthread_mutex_unlock(socket->lock);
1233 return ret;
1234 }
1235
1236 /*
1237 * Send a close metadata command to consumer using the given channel key.
1238 * Called with registry lock held.
1239 *
1240 * Return 0 on success else a negative value.
1241 */
1242 int consumer_close_metadata(struct consumer_socket *socket,
1243 uint64_t metadata_key)
1244 {
1245 int ret;
1246 struct lttcomm_consumer_msg msg;
1247
1248 assert(socket);
1249
1250 DBG2("Consumer close metadata channel key %" PRIu64, metadata_key);
1251
1252 memset(&msg, 0, sizeof(msg));
1253 msg.cmd_type = LTTNG_CONSUMER_CLOSE_METADATA;
1254 msg.u.close_metadata.key = metadata_key;
1255
1256 pthread_mutex_lock(socket->lock);
1257 health_code_update();
1258
1259 ret = consumer_send_msg(socket, &msg);
1260 if (ret < 0) {
1261 goto end;
1262 }
1263
1264 end:
1265 health_code_update();
1266 pthread_mutex_unlock(socket->lock);
1267 return ret;
1268 }
1269
1270 /*
1271 * Send a setup metdata command to consumer using the given channel key.
1272 *
1273 * Return 0 on success else a negative value.
1274 */
1275 int consumer_setup_metadata(struct consumer_socket *socket,
1276 uint64_t metadata_key)
1277 {
1278 int ret;
1279 struct lttcomm_consumer_msg msg;
1280
1281 assert(socket);
1282
1283 DBG2("Consumer setup metadata channel key %" PRIu64, metadata_key);
1284
1285 memset(&msg, 0, sizeof(msg));
1286 msg.cmd_type = LTTNG_CONSUMER_SETUP_METADATA;
1287 msg.u.setup_metadata.key = metadata_key;
1288
1289 pthread_mutex_lock(socket->lock);
1290 health_code_update();
1291
1292 ret = consumer_send_msg(socket, &msg);
1293 if (ret < 0) {
1294 goto end;
1295 }
1296
1297 end:
1298 health_code_update();
1299 pthread_mutex_unlock(socket->lock);
1300 return ret;
1301 }
1302
1303 /*
1304 * Send metadata string to consumer.
1305 * RCU read-side lock must be held to guarantee existence of socket.
1306 *
1307 * Return 0 on success else a negative value.
1308 */
1309 int consumer_push_metadata(struct consumer_socket *socket,
1310 uint64_t metadata_key, char *metadata_str, size_t len,
1311 size_t target_offset, uint64_t version)
1312 {
1313 int ret;
1314 struct lttcomm_consumer_msg msg;
1315
1316 assert(socket);
1317
1318 DBG2("Consumer push metadata to consumer socket %d", *socket->fd_ptr);
1319
1320 pthread_mutex_lock(socket->lock);
1321
1322 memset(&msg, 0, sizeof(msg));
1323 msg.cmd_type = LTTNG_CONSUMER_PUSH_METADATA;
1324 msg.u.push_metadata.key = metadata_key;
1325 msg.u.push_metadata.target_offset = target_offset;
1326 msg.u.push_metadata.len = len;
1327 msg.u.push_metadata.version = version;
1328
1329 health_code_update();
1330 ret = consumer_send_msg(socket, &msg);
1331 if (ret < 0 || len == 0) {
1332 goto end;
1333 }
1334
1335 DBG3("Consumer pushing metadata on sock %d of len %zu", *socket->fd_ptr,
1336 len);
1337
1338 ret = consumer_socket_send(socket, metadata_str, len);
1339 if (ret < 0) {
1340 goto end;
1341 }
1342
1343 health_code_update();
1344 ret = consumer_recv_status_reply(socket);
1345 if (ret < 0) {
1346 goto end;
1347 }
1348
1349 end:
1350 pthread_mutex_unlock(socket->lock);
1351 health_code_update();
1352 return ret;
1353 }
1354
1355 /*
1356 * Ask the consumer to snapshot a specific channel using the key.
1357 *
1358 * Return 0 on success or else a negative error.
1359 */
1360 int consumer_snapshot_channel(struct consumer_socket *socket, uint64_t key,
1361 struct snapshot_output *output, int metadata, uid_t uid, gid_t gid,
1362 const char *session_path, int wait, uint64_t nb_packets_per_stream)
1363 {
1364 int ret;
1365 struct lttcomm_consumer_msg msg;
1366
1367 assert(socket);
1368 assert(output);
1369 assert(output->consumer);
1370
1371 DBG("Consumer snapshot channel key %" PRIu64, key);
1372
1373 memset(&msg, 0, sizeof(msg));
1374 msg.cmd_type = LTTNG_CONSUMER_SNAPSHOT_CHANNEL;
1375 msg.u.snapshot_channel.key = key;
1376 msg.u.snapshot_channel.nb_packets_per_stream = nb_packets_per_stream;
1377 msg.u.snapshot_channel.metadata = metadata;
1378
1379 if (output->consumer->type == CONSUMER_DST_NET) {
1380 msg.u.snapshot_channel.relayd_id = output->consumer->net_seq_index;
1381 msg.u.snapshot_channel.use_relayd = 1;
1382 ret = snprintf(msg.u.snapshot_channel.pathname,
1383 sizeof(msg.u.snapshot_channel.pathname),
1384 "%s/%s-%s-%" PRIu64 "%s", output->consumer->subdir,
1385 output->name, output->datetime, output->nb_snapshot,
1386 session_path);
1387 if (ret < 0) {
1388 ret = -LTTNG_ERR_NOMEM;
1389 goto error;
1390 }
1391 } else {
1392 ret = snprintf(msg.u.snapshot_channel.pathname,
1393 sizeof(msg.u.snapshot_channel.pathname),
1394 "%s/%s-%s-%" PRIu64 "%s", output->consumer->dst.trace_path,
1395 output->name, output->datetime, output->nb_snapshot,
1396 session_path);
1397 if (ret < 0) {
1398 ret = -LTTNG_ERR_NOMEM;
1399 goto error;
1400 }
1401 msg.u.snapshot_channel.relayd_id = (uint64_t) -1ULL;
1402
1403 /* Create directory. Ignore if exist. */
1404 ret = run_as_mkdir_recursive(msg.u.snapshot_channel.pathname,
1405 S_IRWXU | S_IRWXG, uid, gid);
1406 if (ret < 0) {
1407 if (errno != EEXIST) {
1408 ERR("Trace directory creation error");
1409 goto error;
1410 }
1411 }
1412 }
1413
1414 health_code_update();
1415 pthread_mutex_lock(socket->lock);
1416 ret = consumer_send_msg(socket, &msg);
1417 pthread_mutex_unlock(socket->lock);
1418 if (ret < 0) {
1419 goto error;
1420 }
1421
1422 error:
1423 health_code_update();
1424 return ret;
1425 }
1426
1427 /*
1428 * Ask the consumer the number of discarded events for a channel.
1429 */
1430 int consumer_get_discarded_events(uint64_t session_id, uint64_t channel_key,
1431 struct consumer_output *consumer, uint64_t *discarded)
1432 {
1433 int ret;
1434 struct consumer_socket *socket;
1435 struct lttng_ht_iter iter;
1436 struct lttcomm_consumer_msg msg;
1437
1438 assert(consumer);
1439
1440 DBG3("Consumer discarded events id %" PRIu64, session_id);
1441
1442 memset(&msg, 0, sizeof(msg));
1443 msg.cmd_type = LTTNG_CONSUMER_DISCARDED_EVENTS;
1444 msg.u.discarded_events.session_id = session_id;
1445 msg.u.discarded_events.channel_key = channel_key;
1446
1447 *discarded = 0;
1448
1449 /* Send command for each consumer */
1450 rcu_read_lock();
1451 cds_lfht_for_each_entry(consumer->socks->ht, &iter.iter, socket,
1452 node.node) {
1453 uint64_t consumer_discarded = 0;
1454 pthread_mutex_lock(socket->lock);
1455 ret = consumer_socket_send(socket, &msg, sizeof(msg));
1456 if (ret < 0) {
1457 pthread_mutex_unlock(socket->lock);
1458 goto end;
1459 }
1460
1461 /*
1462 * No need for a recv reply status because the answer to the
1463 * command is the reply status message.
1464 */
1465 ret = consumer_socket_recv(socket, &consumer_discarded,
1466 sizeof(consumer_discarded));
1467 if (ret < 0) {
1468 ERR("get discarded events");
1469 pthread_mutex_unlock(socket->lock);
1470 goto end;
1471 }
1472 pthread_mutex_unlock(socket->lock);
1473 *discarded += consumer_discarded;
1474 }
1475 ret = 0;
1476 DBG("Consumer discarded %" PRIu64 " events in session id %" PRIu64,
1477 *discarded, session_id);
1478
1479 end:
1480 rcu_read_unlock();
1481 return ret;
1482 }
1483
1484 /*
1485 * Ask the consumer the number of lost packets for a channel.
1486 */
1487 int consumer_get_lost_packets(uint64_t session_id, uint64_t channel_key,
1488 struct consumer_output *consumer, uint64_t *lost)
1489 {
1490 int ret;
1491 struct consumer_socket *socket;
1492 struct lttng_ht_iter iter;
1493 struct lttcomm_consumer_msg msg;
1494
1495 assert(consumer);
1496
1497 DBG3("Consumer lost packets id %" PRIu64, session_id);
1498
1499 memset(&msg, 0, sizeof(msg));
1500 msg.cmd_type = LTTNG_CONSUMER_LOST_PACKETS;
1501 msg.u.lost_packets.session_id = session_id;
1502 msg.u.lost_packets.channel_key = channel_key;
1503
1504 *lost = 0;
1505
1506 /* Send command for each consumer */
1507 rcu_read_lock();
1508 cds_lfht_for_each_entry(consumer->socks->ht, &iter.iter, socket,
1509 node.node) {
1510 uint64_t consumer_lost = 0;
1511 pthread_mutex_lock(socket->lock);
1512 ret = consumer_socket_send(socket, &msg, sizeof(msg));
1513 if (ret < 0) {
1514 pthread_mutex_unlock(socket->lock);
1515 goto end;
1516 }
1517
1518 /*
1519 * No need for a recv reply status because the answer to the
1520 * command is the reply status message.
1521 */
1522 ret = consumer_socket_recv(socket, &consumer_lost,
1523 sizeof(consumer_lost));
1524 if (ret < 0) {
1525 ERR("get lost packets");
1526 pthread_mutex_unlock(socket->lock);
1527 goto end;
1528 }
1529 pthread_mutex_unlock(socket->lock);
1530 *lost += consumer_lost;
1531 }
1532 ret = 0;
1533 DBG("Consumer lost %" PRIu64 " packets in session id %" PRIu64,
1534 *lost, session_id);
1535
1536 end:
1537 rcu_read_unlock();
1538 return ret;
1539 }
This page took 0.075173 seconds and 4 git commands to generate.