| 1 | /* |
| 2 | * rseq.c |
| 3 | * |
| 4 | * Copyright (C) 2016 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
| 5 | * |
| 6 | * This library is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU Lesser General Public |
| 8 | * License as published by the Free Software Foundation; only |
| 9 | * version 2.1 of the License. |
| 10 | * |
| 11 | * This library is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | * Lesser General Public License for more details. |
| 15 | */ |
| 16 | |
| 17 | #define _GNU_SOURCE |
| 18 | #include <errno.h> |
| 19 | #include <sched.h> |
| 20 | #include <stdio.h> |
| 21 | #include <stdlib.h> |
| 22 | #include <string.h> |
| 23 | #include <unistd.h> |
| 24 | #include <syscall.h> |
| 25 | #include <assert.h> |
| 26 | #include <signal.h> |
| 27 | #include <linux/membarrier.h> |
| 28 | |
| 29 | #include <rseq.h> |
| 30 | |
| 31 | #ifdef __NR_membarrier |
| 32 | # define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__) |
| 33 | #else |
| 34 | # define membarrier(...) -ENOSYS |
| 35 | #endif |
| 36 | |
| 37 | struct rseq_thread_state { |
| 38 | uint32_t fallback_wait_cnt; |
| 39 | uint32_t fallback_cnt; |
| 40 | sigset_t sigmask_saved; |
| 41 | }; |
| 42 | |
| 43 | __attribute__((weak)) __thread volatile struct rseq __rseq_abi = { |
| 44 | .u.e.cpu_id = -1, |
| 45 | }; |
| 46 | |
| 47 | static __thread volatile struct rseq_thread_state rseq_thread_state; |
| 48 | |
| 49 | int rseq_has_sys_membarrier; |
| 50 | |
| 51 | static int sys_rseq(volatile struct rseq *rseq_abi, int flags) |
| 52 | { |
| 53 | return syscall(__NR_rseq, rseq_abi, flags); |
| 54 | } |
| 55 | |
| 56 | int rseq_register_current_thread(void) |
| 57 | { |
| 58 | int rc; |
| 59 | |
| 60 | rc = sys_rseq(&__rseq_abi, 0); |
| 61 | if (rc) { |
| 62 | fprintf(stderr, "Error: sys_rseq(...) failed(%d): %s\n", |
| 63 | errno, strerror(errno)); |
| 64 | return -1; |
| 65 | } |
| 66 | assert(rseq_current_cpu() >= 0); |
| 67 | return 0; |
| 68 | } |
| 69 | |
| 70 | int rseq_unregister_current_thread(void) |
| 71 | { |
| 72 | int rc; |
| 73 | |
| 74 | rc = sys_rseq(NULL, 0); |
| 75 | if (rc) { |
| 76 | fprintf(stderr, "Error: sys_rseq(...) failed(%d): %s\n", |
| 77 | errno, strerror(errno)); |
| 78 | return -1; |
| 79 | } |
| 80 | return 0; |
| 81 | } |
| 82 | |
| 83 | int rseq_init_lock(struct rseq_lock *rlock) |
| 84 | { |
| 85 | int ret; |
| 86 | |
| 87 | ret = pthread_mutex_init(&rlock->lock, NULL); |
| 88 | if (ret) { |
| 89 | errno = ret; |
| 90 | return -1; |
| 91 | } |
| 92 | rlock->state = RSEQ_LOCK_STATE_RESTART; |
| 93 | return 0; |
| 94 | } |
| 95 | |
| 96 | int rseq_destroy_lock(struct rseq_lock *rlock) |
| 97 | { |
| 98 | int ret; |
| 99 | |
| 100 | ret = pthread_mutex_destroy(&rlock->lock); |
| 101 | if (ret) { |
| 102 | errno = ret; |
| 103 | return -1; |
| 104 | } |
| 105 | return 0; |
| 106 | } |
| 107 | |
| 108 | static void signal_off_save(sigset_t *oldset) |
| 109 | { |
| 110 | sigset_t set; |
| 111 | int ret; |
| 112 | |
| 113 | sigfillset(&set); |
| 114 | ret = pthread_sigmask(SIG_BLOCK, &set, oldset); |
| 115 | if (ret) |
| 116 | abort(); |
| 117 | } |
| 118 | |
| 119 | static void signal_restore(sigset_t oldset) |
| 120 | { |
| 121 | int ret; |
| 122 | |
| 123 | ret = pthread_sigmask(SIG_SETMASK, &oldset, NULL); |
| 124 | if (ret) |
| 125 | abort(); |
| 126 | } |
| 127 | |
| 128 | static void rseq_fallback_lock(struct rseq_lock *rlock) |
| 129 | { |
| 130 | signal_off_save((sigset_t *)&rseq_thread_state.sigmask_saved); |
| 131 | pthread_mutex_lock(&rlock->lock); |
| 132 | rseq_thread_state.fallback_cnt++; |
| 133 | /* |
| 134 | * For concurrent threads arriving before we set LOCK: |
| 135 | * reading cpu_id after setting the state to LOCK |
| 136 | * ensures they restart. |
| 137 | */ |
| 138 | ACCESS_ONCE(rlock->state) = RSEQ_LOCK_STATE_LOCK; |
| 139 | /* |
| 140 | * For concurrent threads arriving after we set LOCK: |
| 141 | * those will grab the lock, so we are protected by |
| 142 | * mutual exclusion. |
| 143 | */ |
| 144 | } |
| 145 | |
| 146 | void rseq_fallback_wait(struct rseq_lock *rlock) |
| 147 | { |
| 148 | signal_off_save((sigset_t *)&rseq_thread_state.sigmask_saved); |
| 149 | pthread_mutex_lock(&rlock->lock); |
| 150 | rseq_thread_state.fallback_wait_cnt++; |
| 151 | pthread_mutex_unlock(&rlock->lock); |
| 152 | signal_restore(rseq_thread_state.sigmask_saved); |
| 153 | } |
| 154 | |
| 155 | static void rseq_fallback_unlock(struct rseq_lock *rlock, int cpu_at_start) |
| 156 | { |
| 157 | /* |
| 158 | * Concurrent rseq arriving before we set state back to RESTART |
| 159 | * grab the lock. Those arriving after we set state back to |
| 160 | * RESTART will perform restartable critical sections. The next |
| 161 | * owner of the lock will take take of making sure it prevents |
| 162 | * concurrent restartable sequences from completing. We may be |
| 163 | * writing from another CPU, so update the state with a store |
| 164 | * release semantic to ensure restartable sections will see our |
| 165 | * side effect (writing to *p) before they enter their |
| 166 | * restartable critical section. |
| 167 | * |
| 168 | * In cases where we observe that we are on the right CPU after the |
| 169 | * critical section, program order ensures that following restartable |
| 170 | * critical sections will see our stores, so we don't have to use |
| 171 | * store-release or membarrier. |
| 172 | * |
| 173 | * Use sys_membarrier when available to remove the memory barrier |
| 174 | * implied by smp_load_acquire(). |
| 175 | */ |
| 176 | barrier(); |
| 177 | if (likely(rseq_current_cpu() == cpu_at_start)) { |
| 178 | ACCESS_ONCE(rlock->state) = RSEQ_LOCK_STATE_RESTART; |
| 179 | } else { |
| 180 | if (!has_fast_acquire_release() && rseq_has_sys_membarrier) { |
| 181 | if (membarrier(MEMBARRIER_CMD_SHARED, 0)) |
| 182 | abort(); |
| 183 | ACCESS_ONCE(rlock->state) = RSEQ_LOCK_STATE_RESTART; |
| 184 | } else { |
| 185 | /* |
| 186 | * Store with release semantic to ensure |
| 187 | * restartable sections will see our side effect |
| 188 | * (writing to *p) before they enter their |
| 189 | * restartable critical section. Matches |
| 190 | * smp_load_acquire() in rseq_start(). |
| 191 | */ |
| 192 | smp_store_release(&rlock->state, |
| 193 | RSEQ_LOCK_STATE_RESTART); |
| 194 | } |
| 195 | } |
| 196 | pthread_mutex_unlock(&rlock->lock); |
| 197 | signal_restore(rseq_thread_state.sigmask_saved); |
| 198 | } |
| 199 | |
| 200 | int rseq_fallback_current_cpu(void) |
| 201 | { |
| 202 | int cpu; |
| 203 | |
| 204 | cpu = sched_getcpu(); |
| 205 | if (cpu < 0) { |
| 206 | perror("sched_getcpu()"); |
| 207 | abort(); |
| 208 | } |
| 209 | return cpu; |
| 210 | } |
| 211 | |
| 212 | int rseq_fallback_begin(struct rseq_lock *rlock) |
| 213 | { |
| 214 | rseq_fallback_lock(rlock); |
| 215 | return rseq_fallback_current_cpu(); |
| 216 | } |
| 217 | |
| 218 | void rseq_fallback_end(struct rseq_lock *rlock, int cpu) |
| 219 | { |
| 220 | rseq_fallback_unlock(rlock, cpu); |
| 221 | } |
| 222 | |
| 223 | /* Handle non-initialized rseq for this thread. */ |
| 224 | void rseq_fallback_noinit(struct rseq_state *rseq_state) |
| 225 | { |
| 226 | rseq_state->lock_state = RSEQ_LOCK_STATE_FAIL; |
| 227 | rseq_state->cpu_id = 0; |
| 228 | } |
| 229 | |
| 230 | uint32_t rseq_get_fallback_wait_cnt(void) |
| 231 | { |
| 232 | return rseq_thread_state.fallback_wait_cnt; |
| 233 | } |
| 234 | |
| 235 | uint32_t rseq_get_fallback_cnt(void) |
| 236 | { |
| 237 | return rseq_thread_state.fallback_cnt; |
| 238 | } |
| 239 | |
| 240 | void __attribute__((constructor)) rseq_init(void) |
| 241 | { |
| 242 | int ret; |
| 243 | |
| 244 | ret = membarrier(MEMBARRIER_CMD_QUERY, 0); |
| 245 | if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) |
| 246 | rseq_has_sys_membarrier = 1; |
| 247 | } |