Fix: perform the initialization memory barrier out of loop body

[lttng-tools.git] / src / bin / lttng-sessiond / main.c

diff --git a/src/bin/lttng-sessiond/main.c b/src/bin/lttng-sessiond/main.c
index 37c0a9325960f84940ae268039b3c4e8bea212ce..3d2cad9361464534128a44f5917d01f1e58899fe 100644 (file)
--- a/src/bin/lttng-sessiond/main.c
+++ b/src/bin/lttng-sessiond/main.c
@@ -299,13 +299,41 @@ struct rotation_thread_handle *rotation_thread_handle;
 struct lttng_ht *agent_apps_ht_by_sock = NULL;
 
 /*
- * Whether sessiond is ready for commands/notification channel/health check
+ * The initialization of the session daemon is done in multiple phases.
+ *
+ * While all threads are launched near-simultaneously, only some of them
+ * are needed to ensure the session daemon can start to respond to client
  * requests.
- * NR_LTTNG_SESSIOND_READY must match the number of calls to
- * sessiond_notify_ready().
+ *
+ * There are two important guarantees that we wish to offer with respect
+ * to the initialisation of the session daemon:
+ *   - When the daemonize/background launcher process exits, the sessiond
+ *     is fully able to respond to client requests,
+ *   - Auto-loaded sessions are visible to clients.
+ *
+ * In order to achieve this, a number of support threads have to be launched
+ * to allow the "client" thread to function properly. Moreover, since the
+ * "load session" thread needs the client thread, we must provide a way
+ * for the "load session" thread to know that the "client" thread is up
+ * and running.
+ *
+ * Hence, the support threads decrement the lttng_sessiond_ready counter
+ * while the "client" threads waits for it to reach 0. Once the "client" thread
+ * unblocks, it posts the message_thread_ready semaphore which allows the
+ * "load session" thread to progress.
+ *
+ * This implies that the "load session" thread is the last to be initialized
+ * and will explicitly call sessiond_signal_parents(), which signals the parents
+ * that the session daemon is fully initialized.
+ *
+ * The four (4) support threads are:
+ *  - agent_thread
+ *  - notification_thread
+ *  - rotation_thread
+ *  - health_thread
  */
-#define NR_LTTNG_SESSIOND_READY                6
-int lttng_sessiond_ready = NR_LTTNG_SESSIOND_READY;
+#define NR_LTTNG_SESSIOND_SUPPORT_THREADS 4
+int lttng_sessiond_ready = NR_LTTNG_SESSIOND_SUPPORT_THREADS;
 
 int sessiond_check_thread_quit_pipe(int fd, uint32_t events)
 {
@@ -314,28 +342,47 @@ int sessiond_check_thread_quit_pipe(int fd, uint32_t events)
 
 /* Notify parents that we are ready for cmd and health check */
 LTTNG_HIDDEN
-void sessiond_notify_ready(void)
+void sessiond_signal_parents(void)
 {
-       if (uatomic_sub_return(&lttng_sessiond_ready, 1) == 0) {
-               /*
-                * Notify parent pid that we are ready to accept command
-                * for client side.  This ppid is the one from the
-                * external process that spawned us.
-                */
-               if (config.sig_parent) {
-                       kill(ppid, SIGUSR1);
-               }
+       /*
+        * Notify parent pid that we are ready to accept command
+        * for client side.  This ppid is the one from the
+        * external process that spawned us.
+        */
+       if (config.sig_parent) {
+               kill(ppid, SIGUSR1);
+       }
 
-               /*
-                * Notify the parent of the fork() process that we are
-                * ready.
-                */
-               if (config.daemonize || config.background) {
-                       kill(child_ppid, SIGUSR1);
-               }
+       /*
+        * Notify the parent of the fork() process that we are
+        * ready.
+        */
+       if (config.daemonize || config.background) {
+               kill(child_ppid, SIGUSR1);
        }
 }
 
+LTTNG_HIDDEN
+void sessiond_notify_ready(void)
+{
+       /*
+        * This memory barrier is paired with the one performed by
+        * the client thread after it has seen that 'lttng_sessiond_ready' is 0.
+        *
+        * The purpose of these memory barriers is to ensure that all
+        * initialization operations of the various threads that call this
+        * function to signal that they are ready are commited/published
+        * before the client thread can see the 'lttng_sessiond_ready' counter
+        * reach 0.
+        *
+        * Note that this could be a 'write' memory barrier, but a full barrier
+        * is used in case the code using this utility changes. The performance
+        * implications of this choice are minimal since this is a slow path.
+        */
+       cmm_smp_mb();
+       uatomic_sub(&lttng_sessiond_ready, 1);
+}
+
 static
 int __sessiond_set_thread_pollset(struct lttng_poll_event *events, size_t size,
                int *a_pipe)
@@ -4495,8 +4542,6 @@ static void *thread_manage_clients(void *data)
                goto error;
        }
 
-       sessiond_notify_ready();
-
        ret = sem_post(&load_info->message_thread_ready);
        if (ret) {
                PERROR("sem_post message_thread_ready");
@@ -4530,6 +4575,16 @@ static void *thread_manage_clients(void *data)
                        goto exit;
                }
        }
+       /*
+        * This barrier is paired with the one in sessiond_notify_ready() to
+        * ensure that loads accessing data initialized by the other threads,
+        * on which this thread was waiting, are not performed before this point.
+        *
+        * Note that this could be a 'read' memory barrier, but a full barrier
+        * is used in case the code changes. The performance implications of
+        * this choice are minimal since this is a slow path.
+        */
+       cmm_smp_mb();
 
        /* This testpoint is after we signal readiness to the parent. */
        if (testpoint(sessiond_thread_manage_clients)) {