tools/testing/selftests/rseq/rseq.c

   1 /*
   2  * rseq.c
   3  *
   4  * Copyright (C) 2016 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; only
   9  * version 2.1 of the License.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14  * Lesser General Public License for more details.
  15  */
  16
  17 #define _GNU_SOURCE
  18 #include <errno.h>
  19 #include <sched.h>
  20 #include <stdio.h>
  21 #include <stdlib.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <syscall.h>
  25 #include <assert.h>
  26 #include <signal.h>
  27 #include <linux/membarrier.h>
  28
  29 #include <rseq.h>
  30
  31 #ifdef __NR_membarrier
  32 # define membarrier(...)                syscall(__NR_membarrier, __VA_ARGS__)
  33 #else
  34 # define membarrier(...)                -ENOSYS
  35 #endif
  36
  37 struct rseq_thread_state {
  38         uint32_t fallback_wait_cnt;
  39         uint32_t fallback_cnt;
  40         sigset_t sigmask_saved;
  41 };
  42
  43 __attribute__((weak)) __thread volatile struct rseq __rseq_abi = {
  44         .u.e.cpu_id = -1,
  45 };
  46
  47 static __thread volatile struct rseq_thread_state rseq_thread_state;
  48
  49 int rseq_has_sys_membarrier;
  50
  51 static int sys_rseq(volatile struct rseq *rseq_abi, int flags)
  52 {
  53         return syscall(__NR_rseq, rseq_abi, flags);
  54 }
  55
  56 int rseq_register_current_thread(void)
  57 {
  58         int rc;
  59
  60         rc = sys_rseq(&__rseq_abi, 0);
  61         if (rc) {
  62                 fprintf(stderr, "Error: sys_rseq(...) failed(%d): %s\n",
  63                         errno, strerror(errno));
  64                 return -1;
  65         }
  66         assert(rseq_current_cpu() >= 0);
  67         return 0;
  68 }
  69
  70 int rseq_unregister_current_thread(void)
  71 {
  72         int rc;
  73
  74         rc = sys_rseq(NULL, 0);
  75         if (rc) {
  76                 fprintf(stderr, "Error: sys_rseq(...) failed(%d): %s\n",
  77                         errno, strerror(errno));
  78                 return -1;
  79         }
  80         return 0;
  81 }
  82
  83 int rseq_init_lock(struct rseq_lock *rlock)
  84 {
  85         int ret;
  86
  87         ret = pthread_mutex_init(&rlock->lock, NULL);
  88         if (ret) {
  89                 errno = ret;
  90                 return -1;
  91         }
  92         rlock->state = RSEQ_LOCK_STATE_RESTART;
  93         return 0;
  94 }
  95
  96 int rseq_destroy_lock(struct rseq_lock *rlock)
  97 {
  98         int ret;
  99
 100         ret = pthread_mutex_destroy(&rlock->lock);
 101         if (ret) {
 102                 errno = ret;
 103                 return -1;
 104         }
 105         return 0;
 106 }
 107
 108 static void signal_off_save(sigset_t *oldset)
 109 {
 110         sigset_t set;
 111         int ret;
 112
 113         sigfillset(&set);
 114         ret = pthread_sigmask(SIG_BLOCK, &set, oldset);
 115         if (ret)
 116                 abort();
 117 }
 118
 119 static void signal_restore(sigset_t oldset)
 120 {
 121         int ret;
 122
 123         ret = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
 124         if (ret)
 125                 abort();
 126 }
 127
 128 static void rseq_fallback_lock(struct rseq_lock *rlock)
 129 {
 130         signal_off_save((sigset_t *)&rseq_thread_state.sigmask_saved);
 131         pthread_mutex_lock(&rlock->lock);
 132         rseq_thread_state.fallback_cnt++;
 133         /*
 134          * For concurrent threads arriving before we set LOCK:
 135          * reading cpu_id after setting the state to LOCK
 136          * ensures they restart.
 137          */
 138         ACCESS_ONCE(rlock->state) = RSEQ_LOCK_STATE_LOCK;
 139         /*
 140          * For concurrent threads arriving after we set LOCK:
 141          * those will grab the lock, so we are protected by
 142          * mutual exclusion.
 143          */
 144 }
 145
 146 void rseq_fallback_wait(struct rseq_lock *rlock)
 147 {
 148         signal_off_save((sigset_t *)&rseq_thread_state.sigmask_saved);
 149         pthread_mutex_lock(&rlock->lock);
 150         rseq_thread_state.fallback_wait_cnt++;
 151         pthread_mutex_unlock(&rlock->lock);
 152         signal_restore(rseq_thread_state.sigmask_saved);
 153 }
 154
 155 static void rseq_fallback_unlock(struct rseq_lock *rlock, int cpu_at_start)
 156 {
 157         /*
 158          * Concurrent rseq arriving before we set state back to RESTART
 159          * grab the lock. Those arriving after we set state back to
 160          * RESTART will perform restartable critical sections. The next
 161          * owner of the lock will take take of making sure it prevents
 162          * concurrent restartable sequences from completing.  We may be
 163          * writing from another CPU, so update the state with a store
 164          * release semantic to ensure restartable sections will see our
 165          * side effect (writing to *p) before they enter their
 166          * restartable critical section.
 167          *
 168          * In cases where we observe that we are on the right CPU after the
 169          * critical section, program order ensures that following restartable
 170          * critical sections will see our stores, so we don't have to use
 171          * store-release or membarrier.
 172          *
 173          * Use sys_membarrier when available to remove the memory barrier
 174          * implied by smp_load_acquire().
 175          */
 176         barrier();
 177         if (likely(rseq_current_cpu() == cpu_at_start)) {
 178                 ACCESS_ONCE(rlock->state) = RSEQ_LOCK_STATE_RESTART;
 179         } else {
 180                 if (!has_fast_acquire_release() && rseq_has_sys_membarrier) {
 181                         if (membarrier(MEMBARRIER_CMD_SHARED, 0))
 182                                 abort();
 183                         ACCESS_ONCE(rlock->state) = RSEQ_LOCK_STATE_RESTART;
 184                 } else {
 185                         /*
 186                          * Store with release semantic to ensure
 187                          * restartable sections will see our side effect
 188                          * (writing to *p) before they enter their
 189                          * restartable critical section. Matches
 190                          * smp_load_acquire() in rseq_start().
 191                          */
 192                         smp_store_release(&rlock->state,
 193                                 RSEQ_LOCK_STATE_RESTART);
 194                 }
 195         }
 196         pthread_mutex_unlock(&rlock->lock);
 197         signal_restore(rseq_thread_state.sigmask_saved);
 198 }
 199
 200 int rseq_fallback_current_cpu(void)
 201 {
 202         int cpu;
 203
 204         cpu = sched_getcpu();
 205         if (cpu < 0) {
 206                 perror("sched_getcpu()");
 207                 abort();
 208         }
 209         return cpu;
 210 }
 211
 212 int rseq_fallback_begin(struct rseq_lock *rlock)
 213 {
 214         rseq_fallback_lock(rlock);
 215         return rseq_fallback_current_cpu();
 216 }
 217
 218 void rseq_fallback_end(struct rseq_lock *rlock, int cpu)
 219 {
 220         rseq_fallback_unlock(rlock, cpu);
 221 }
 222
 223 /* Handle non-initialized rseq for this thread. */
 224 void rseq_fallback_noinit(struct rseq_state *rseq_state)
 225 {
 226         rseq_state->lock_state = RSEQ_LOCK_STATE_FAIL;
 227         rseq_state->cpu_id = 0;
 228 }
 229
 230 uint32_t rseq_get_fallback_wait_cnt(void)
 231 {
 232         return rseq_thread_state.fallback_wait_cnt;
 233 }
 234
 235 uint32_t rseq_get_fallback_cnt(void)
 236 {
 237         return rseq_thread_state.fallback_cnt;
 238 }
 239
 240 void __attribute__((constructor)) rseq_init(void)
 241 {
 242         int ret;
 243
 244         ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
 245         if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED))
 246                 rseq_has_sys_membarrier = 1;
 247 }