Fix: Use 2.12 custom mutex nest getter

Eliminate the following deadlock by calling libc close directly when
called from within the 2.12 fd tracker locking. The 2.13 locks always
need to be taken outside of the 2.12 locks. In the scenario below,
Thread 3 (lttng-ust 2.12 listener thread) takes the locks in the wrong
order.

This fix depends on custom commit "Add ust fd mutex nesting getter" in
LTTng-UST 2.12.

                Thread 3 (Thread 0x7f7679247700 (LWP 621683) "hello-ust"):
                #0  __lll_lock_wait (futex=futex@entry=0x7f767a392b60 <ust_safe_guard_fd_mutex>, private=0) at lowlevellock.c:52
                #1  0x00007f767a39d843 in __GI___pthread_mutex_lock (mutex=0x7f767a392b60 <ust_safe_guard_fd_mutex>) at ../nptl/pthread_mutex_lock.c:80
-> 2.13 locks fd tracker
2.13            #2  0x00007f767a37ff82 in lttng_ust_lock_fd_tracker_orig () at fd-tracker.c:163
2.13            #3  0x00007f767a380b66 in lttng_ust_safe_close_fd_orig (fd=3, close_cb=0x7f767a56b070 <__GI___close>) at fd-tracker.c:385
2.13            #4  0x00007f767a6e5557 in close (fd=3) at lttng-ust-fd.c:101
2.12            #5  0x00007f767a297e7d in ustcomm_connect_unix_sock (pathname=0x7f767a32bc64 <local_apps+36> "/home/compudj/.lttng/lttng-ust-sock-8", timeout=3000) at lttng-ust-comm.c:157
-> 2.12 listener thread locks fd tracker
-> 2.12 listener thread locks ust_lock
2.12            #6  0x00007f767a2a1f77 in ust_listener_thread (arg=0x7f767a32bc40 <local_apps>) at lttng-ust-comm.c:1591
                #7  0x00007f767a39aea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
                #8  0x00007f767a57aaef in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

                Thread 2 (Thread 0x7f7678a46700 (LWP 621682) "hello-ust"):
                #0  __lll_lock_wait (futex=futex@entry=0x7f767a32f780 <ust_mutex>, private=0) at lowlevellock.c:52
                #1  0x00007f767a39d843 in __GI___pthread_mutex_lock (mutex=0x7f767a32f780 <ust_mutex>) at ../nptl/pthread_mutex_lock.c:80
2.12            #2  0x00007f767a29da59 in ust_lock () at lttng-ust-comm.c:167
2.12            #3  0x00007f767a2a1d95 in ust_listener_thread (arg=0x7f767a329be0 <global_apps>) at lttng-ust-comm.c:1558
                #4  0x00007f767a39aea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
                #5  0x00007f767a57aaef in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

                Thread 1 (Thread 0x7f767a33a040 (LWP 621681) "hello"):
                #0  __lll_lock_wait (futex=futex@entry=0x7f767a32f720 <ust_safe_guard_fd_mutex>, private=0) at lowlevellock.c:52
                #1  0x00007f767a39d843 in __GI___pthread_mutex_lock (mutex=0x7f767a32f720 <ust_safe_guard_fd_mutex>) at ../nptl/pthread_mutex_lock.c:80
-> 2.12 lock fd tracker
2.12            #2  0x00007f767a29c37c in lttng_ust_lock_fd_tracker () at lttng-ust-fd-tracker.c:153
2.12            #3  0x00007f767a29ceba in lttng_ust_safe_close_fd (fd=4, close_cb=0x7f767a56b070 <__GI___close>) at lttng-ust-fd-tracker.c:341
2.13            #4  0x00007f767a380b06 in lttng_ust_safe_close_fd_chain (fd=4, close_cb=0x7f767a56b070 <__GI___close>) at fd-tracker.c:348
2.13            #5  0x00007f767a380b5c in lttng_ust_safe_close_fd_orig (fd=4, close_cb=0x7f767a56b070 <__GI___close>) at fd-tracker.c:379
2.13            #6  0x00007f767a6e5557 in close (fd=4) at lttng-ust-fd.c:101
-> 2.13 lock fd tracker
2.13            #7  0x00007f767a43b5e6 in lttng_ust_elf_destroy (elf=0x55b870f044c0) at elf.c:352
2.13            #8  0x00007f767a3f2797 in get_elf_info (bin_data=0x7ffd5481ef70) at lttng-ust-statedump.c:296
2.13            #9  0x00007f767a3f283f in extract_baddr (bin_data=0x7ffd5481ef70) at lttng-ust-statedump.c:319
2.13            #10 0x00007f767a3f3173 in extract_bin_info_events (info=0x7ffd5481fff0, size=64, _data=0x7ffd54820098) at lttng-ust-statedump.c:518
                #11 0x00007f767a5b3ad5 in __GI___dl_iterate_phdr (callback=0x7f767a3f2f5a <extract_bin_info_events>, data=0x7ffd54820098) at dl-iteratephdr.c:75
2.13            #12 0x00007f767a3f32c5 in lttng_ust_dl_update_orig (ip=0x7f767a3cb599 <lttng_ust_ctor+683>) at lttng-ust-statedump.c:574
2.13            #13 0x00007f767a3f33f7 in lttng_ust_statedump_init () at lttng-ust-statedump.c:638
2.13            #14 0x00007f767a3cb599 in lttng_ust_ctor () at lttng-ust-comm.c:2246
2.13            #15 0x00007f767a3ccaad in lttng_ust_after_fork_child (restore_sigset=0x7ffd548207e0) at lttng-ust-comm.c:2577
2.13            #16 0x00007f767a6f687f in fork () at ustfork.c:271
                #17 0x000055b8705f1034 in make_child ()
                #18 0x000055b8705daacc in execute_command_internal ()
                #19 0x000055b8705ddcf5 in execute_command ()
                #20 0x000055b8705df951 in ?? ()
                #21 0x000055b8705daf79 in execute_command_internal ()
                #22 0x000055b8705ddcf5 in execute_command ()
                #23 0x000055b8705c49db in reader_loop ()
                #24 0x000055b8705c3668 in main ()

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: Ife72c3532dde1357c6c7937aa4c5d8135fd52e84

diff --git a/src/lib/lttng-ust-common/fd-tracker.c b/src/lib/lttng-ust-common/fd-tracker.c
index c47dc1fc527eae5e9d2f0c95f22686189cb98abe..4a527ee56d002d4bf840227a36778e89d92c60bc 100644 (file)
--- a/src/lib/lttng-ust-common/fd-tracker.c
+++ b/src/lib/lttng-ust-common/fd-tracker.c
@@ -374,6 +374,38 @@ static void lttng_ust_delete_fd_from_tracker_orig(int fd)
        DEL_FD_FROM_SET(fd, lttng_fd_set);
 }
 
+#if !defined(LTTNG_UST_CUSTOM_UPGRADE_CONFLICTING_SYMBOLS)
+/* lttng-ust 2.12 (custom branch) getter */
+static int (*__lttng_ust_get_fd_mutex_nest)(void) = NULL;
+
+void *lttng_ust_get_fd_mutex_nest_init(void)
+{
+       if (__lttng_ust_get_fd_mutex_nest == NULL) {
+               __lttng_ust_get_fd_mutex_nest = dlsym(RTLD_DEFAULT, "lttng_ust_get_fd_mutex_nest");
+
+               if (__lttng_ust_get_fd_mutex_nest == NULL) {
+                       __lttng_ust_get_fd_mutex_nest = (void *) LTTNG_UST_DLSYM_FAILED_PTR;
+                       fprintf(stderr, "%s\n", dlerror());
+               }
+       }
+
+       return __lttng_ust_get_fd_mutex_nest;
+}
+
+static int lttng_ust_get_fd_mutex_nest_chain(void)
+{
+       assert(__lttng_ust_get_fd_mutex_nest != NULL);
+       if (__lttng_ust_get_fd_mutex_nest != (void *) LTTNG_UST_DLSYM_FAILED_PTR)
+               return __lttng_ust_get_fd_mutex_nest();
+       return 0;
+}
+#else
+static int lttng_ust_get_fd_mutex_nest_chain(void)
+{
+       return 0;
+}
+#endif
+
 /*
  * Interface allowing applications to close arbitrary file descriptors.
  * We check if it is owned by lttng-ust, and return -1, errno=EBADF
@@ -395,13 +427,8 @@ static int lttng_ust_safe_close_fd_orig(int fd, int (*close_cb)(int fd))
         * If called from lttng-ust, we directly call close without
         * validating whether the FD is part of the tracked set.
         */
-       if (URCU_TLS(ust_fd_mutex_nest)) {
-#if !defined(LTTNG_UST_CUSTOM_UPGRADE_CONFLICTING_SYMBOLS)
-               return lttng_ust_safe_close_fd_chain(fd, close_cb);
-#else
+       if (URCU_TLS(ust_fd_mutex_nest) || lttng_ust_get_fd_mutex_nest_chain())
                return close_cb(fd);
-#endif
-       }
 
        lttng_ust_lock_fd_tracker();
        if (IS_FD_VALID(fd) && IS_FD_SET(fd, lttng_fd_set)) {
@@ -436,13 +463,8 @@ static int lttng_ust_safe_fclose_stream_orig(FILE *stream, int (*fclose_cb)(FILE
         * If called from lttng-ust, we directly call fclose without
         * validating whether the FD is part of the tracked set.
         */
-       if (URCU_TLS(ust_fd_mutex_nest)) {
-#if !defined(LTTNG_UST_CUSTOM_UPGRADE_CONFLICTING_SYMBOLS)
-               return lttng_ust_safe_fclose_stream_chain(stream, fclose_cb);
-#else
+       if (URCU_TLS(ust_fd_mutex_nest) || lttng_ust_get_fd_mutex_nest_chain())
                return fclose_cb(stream);
-#endif
-       }
 
        fd = fileno(stream);
 
@@ -504,7 +526,7 @@ static int lttng_ust_safe_closefrom_fd_orig(int lowfd, int (*close_cb)(int fd))
         * If called from lttng-ust, we directly call close without
         * validating whether the FD is part of the tracked set.
         */
-       if (URCU_TLS(ust_fd_mutex_nest)) {
+       if (URCU_TLS(ust_fd_mutex_nest) || lttng_ust_get_fd_mutex_nest_chain()) {
                for (i = lowfd; i < lttng_ust_max_fd; i++) {
                        if (close_cb(i) < 0) {
                                switch (errno) {

diff --git a/src/lib/lttng-ust-common/fd-tracker.h b/src/lib/lttng-ust-common/fd-tracker.h
index 657d99eb3ff26441e3ca72d0b334a729e16e17f4..4186f305bc7443a8eaa4ff08e18916a730bc77d4 100644 (file)
--- a/src/lib/lttng-ust-common/fd-tracker.h
+++ b/src/lib/lttng-ust-common/fd-tracker.h
@@ -16,6 +16,7 @@ void lttng_ust_fd_tracker_alloc_tls(void)
 #if !defined(LTTNG_UST_CUSTOM_UPGRADE_CONFLICTING_SYMBOLS)
 void *lttng_ust_safe_close_fd_init(void);
 void *lttng_ust_safe_fclose_stream_init(void);
+void *lttng_ust_get_fd_mutex_nest_init(void);
 #endif
 
 #endif

diff --git a/src/lib/lttng-ust-fd/lttng-ust-fd.c b/src/lib/lttng-ust-fd/lttng-ust-fd.c
index af208a9852dbf03e9442140d6089c9f18472d110..5071f659321984158a662ead2dc4f2fbadf017f0 100644 (file)
--- a/src/lib/lttng-ust-fd/lttng-ust-fd.c
+++ b/src/lib/lttng-ust-fd/lttng-ust-fd.c
@@ -85,6 +85,7 @@ void lttng_ust_init_close_chaining(void)
         */
        (void) lttng_ust_safe_close_fd_init();
        (void) lttng_ust_safe_fclose_stream_init();
+       (void) lttng_ust_get_fd_mutex_nest_init();
 }
 #else
 static