[deliverable/linux.git] / tools / testing / selftests / rseq / rseq.h

/*
 * rseq.h
 *
 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifndef RSEQ_H
#define RSEQ_H

#include <stdint.h>
#include <stdbool.h>
#include <pthread.h>
#include <signal.h>
#include <sched.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <sched.h>
#include <linux/rseq.h>

/*
 * Empty code injection macros, override when testing.
 * It is important to consider that the ASM injection macros need to be
 * fully reentrant (e.g. do not modify the stack).
 */
#ifndef RSEQ_INJECT_ASM
#define RSEQ_INJECT_ASM(n)
#endif

#ifndef RSEQ_INJECT_C
#define RSEQ_INJECT_C(n)
#endif

#ifndef RSEQ_INJECT_INPUT
#define RSEQ_INJECT_INPUT
#endif

#ifndef RSEQ_INJECT_CLOBBER
#define RSEQ_INJECT_CLOBBER
#endif

#ifndef RSEQ_INJECT_FAILED
#define RSEQ_INJECT_FAILED
#endif

#ifndef RSEQ_FALLBACK_CNT
#define RSEQ_FALLBACK_CNT	3
#endif

uint32_t rseq_get_fallback_wait_cnt(void);
uint32_t rseq_get_fallback_cnt(void);

extern __thread volatile struct rseq __rseq_abi;
extern int rseq_has_sys_membarrier;

#define likely(x)		__builtin_expect(!!(x), 1)
#define unlikely(x)		__builtin_expect(!!(x), 0)
#define barrier()		__asm__ __volatile__("" : : : "memory")

#define ACCESS_ONCE(x)		(*(__volatile__  __typeof__(x) *)&(x))
#define WRITE_ONCE(x, v)	__extension__ ({ ACCESS_ONCE(x) = (v); })
#define READ_ONCE(x)		ACCESS_ONCE(x)

#if defined(__x86_64__) || defined(__i386__)
#include <rseq-x86.h>
#elif defined(__ARMEL__)
#include <rseq-arm.h>
#elif defined(__PPC__)
#include <rseq-ppc.h>
#else
#error unsupported target
#endif

enum rseq_lock_state {
	RSEQ_LOCK_STATE_RESTART = 0,
	RSEQ_LOCK_STATE_LOCK = 1,
	RSEQ_LOCK_STATE_FAIL = 2,
};

struct rseq_lock {
	pthread_mutex_t lock;
	int32_t state;		/* enum rseq_lock_state */
};

/* State returned by rseq_start, passed as argument to rseq_finish. */
struct rseq_state {
	volatile struct rseq *rseqp;
	int32_t cpu_id;		/* cpu_id at start. */
	uint32_t event_counter;	/* event_counter at start. */
	int32_t lock_state;	/* Lock state at start. */
};

/*
 * Register rseq for the current thread. This needs to be called once
 * by any thread which uses restartable sequences, before they start
 * using restartable sequences. If initialization is not invoked, or if
 * it fails, the restartable critical sections will fall-back on locking
 * (rseq_lock).
 */
int rseq_register_current_thread(void);

/*
 * Unregister rseq for current thread.
 */
int rseq_unregister_current_thread(void);

/*
 * The fallback lock should be initialized before being used by any
 * thread, and destroyed after all threads are done using it. This lock
 * should be used by all rseq calls associated with shared data, either
 * between threads, or between processes in a shared memory.
 *
 * There may be many rseq_lock per process, e.g. one per protected data
 * structure.
 */
int rseq_init_lock(struct rseq_lock *rlock);
int rseq_destroy_lock(struct rseq_lock *rlock);

/*
 * Restartable sequence fallback prototypes. Fallback on locking when
 * rseq is not initialized, not available on the system, or during
 * single-stepping to ensure forward progress.
 */
int rseq_fallback_begin(struct rseq_lock *rlock);
void rseq_fallback_end(struct rseq_lock *rlock, int cpu);
void rseq_fallback_wait(struct rseq_lock *rlock);
void rseq_fallback_noinit(struct rseq_state *rseq_state);

/*
 * Restartable sequence fallback for reading the current CPU number.
 */
int rseq_fallback_current_cpu(void);

static inline int32_t rseq_cpu_at_start(struct rseq_state start_value)
{
	return start_value.cpu_id;
}

static inline int32_t rseq_current_cpu_raw(void)
{
	return ACCESS_ONCE(__rseq_abi.u.e.cpu_id);
}

static inline int32_t rseq_current_cpu(void)
{
	int32_t cpu;

	cpu = rseq_current_cpu_raw();
	if (unlikely(cpu < 0))
		cpu = rseq_fallback_current_cpu();
	return cpu;
}

static inline __attribute__((always_inline))
struct rseq_state rseq_start(void)
{
	struct rseq_state result;

	result.rseqp = &__rseq_abi;
	if (has_single_copy_load_64()) {
		union rseq_cpu_event u;

		u.v = ACCESS_ONCE(result.rseqp->u.v);
		result.event_counter = u.e.event_counter;
		result.cpu_id = u.e.cpu_id;
	} else {
		result.event_counter =
			ACCESS_ONCE(result.rseqp->u.e.event_counter);
		/* load event_counter before cpu_id. */
		RSEQ_INJECT_C(6)
		result.cpu_id = ACCESS_ONCE(result.rseqp->u.e.cpu_id);
	}
	RSEQ_INJECT_C(7)
	/*
	 * Ensure the compiler does not re-order loads of protected
	 * values before we load the event counter.
	 */
	barrier();
	return result;
}

static inline __attribute__((always_inline))
struct rseq_state rseq_start_rlock(struct rseq_lock *rlock)
{
	struct rseq_state result;

	result = rseq_start();
	/*
	 * Read event counter before lock state and cpu_id. This ensures
	 * that when the state changes from RESTART to LOCK, if we have
	 * some threads that have already seen the RESTART still in
	 * flight, they will necessarily be preempted/signalled before a
	 * thread can see the LOCK state for that same CPU. That
	 * preemption/signalling will cause them to restart, so they
	 * don't interfere with the lock.
	 */

	if (!has_fast_acquire_release() && likely(rseq_has_sys_membarrier)) {
		result.lock_state = ACCESS_ONCE(rlock->state);
		barrier();
	} else {
		/*
		 * Load lock state with acquire semantic. Matches
		 * smp_store_release() in rseq_fallback_end().
		 */
		result.lock_state = smp_load_acquire(&rlock->state);
	}
	if (unlikely(result.cpu_id < 0))
		rseq_fallback_noinit(&result);
	return result;
}

enum rseq_finish_type {
	RSEQ_FINISH_SINGLE,
	RSEQ_FINISH_TWO,
	RSEQ_FINISH_MEMCPY,
};

/*
 * p_spec and to_write_spec are used for a speculative write attempted
 * near the end of the restartable sequence. A rseq_finish2 may fail
 * even after this write takes place.
 *
 * p_final and to_write_final are used for the final write. If this
 * write takes place, the rseq_finish2 is guaranteed to succeed.
 */
static inline __attribute__((always_inline))
bool __rseq_finish(intptr_t *p_spec, intptr_t to_write_spec,
		void *p_memcpy, void *to_write_memcpy, size_t len_memcpy,
		intptr_t *p_final, intptr_t to_write_final,
		struct rseq_state start_value,
		enum rseq_finish_type type, bool release)
{
	RSEQ_INJECT_C(9)

	switch (type) {
	case RSEQ_FINISH_SINGLE:
		RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
			/* no speculative write */, /* no speculative write */,
			RSEQ_FINISH_FINAL_STORE_ASM(),
			RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
			/* no extra clobber */, /* no arg */, /* no arg */,
			/* no arg */
		);
		break;
	case RSEQ_FINISH_TWO:
		if (release) {
			RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
				RSEQ_FINISH_SPECULATIVE_STORE_ASM(),
				RSEQ_FINISH_SPECULATIVE_STORE_INPUT(p_spec, to_write_spec),
				RSEQ_FINISH_FINAL_STORE_RELEASE_ASM(),
				RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
				/* no extra clobber */, /* no arg */, /* no arg */,
				/* no arg */
			);
		} else {
			RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
				RSEQ_FINISH_SPECULATIVE_STORE_ASM(),
				RSEQ_FINISH_SPECULATIVE_STORE_INPUT(p_spec, to_write_spec),
				RSEQ_FINISH_FINAL_STORE_ASM(),
				RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
				/* no extra clobber */, /* no arg */, /* no arg */,
				/* no arg */
			);
		}
		break;
	case RSEQ_FINISH_MEMCPY:
		if (release) {
			RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
				RSEQ_FINISH_MEMCPY_STORE_ASM(),
				RSEQ_FINISH_MEMCPY_STORE_INPUT(p_memcpy, to_write_memcpy, len_memcpy),
				RSEQ_FINISH_FINAL_STORE_RELEASE_ASM(),
				RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
				RSEQ_FINISH_MEMCPY_CLOBBER(),
				RSEQ_FINISH_MEMCPY_SETUP(),
				RSEQ_FINISH_MEMCPY_TEARDOWN(),
				RSEQ_FINISH_MEMCPY_SCRATCH()
			);
		} else {
			RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
				RSEQ_FINISH_MEMCPY_STORE_ASM(),
				RSEQ_FINISH_MEMCPY_STORE_INPUT(p_memcpy, to_write_memcpy, len_memcpy),
				RSEQ_FINISH_FINAL_STORE_ASM(),
				RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
				RSEQ_FINISH_MEMCPY_CLOBBER(),
				RSEQ_FINISH_MEMCPY_SETUP(),
				RSEQ_FINISH_MEMCPY_TEARDOWN(),
				RSEQ_FINISH_MEMCPY_SCRATCH()
			);
		}
		break;
	}
	return true;
failure:
	RSEQ_INJECT_FAILED
	return false;
}

static inline __attribute__((always_inline))
bool rseq_finish_rlock(struct rseq_lock *rlock,
		intptr_t *p_spec, intptr_t to_write_spec,
		void *p_memcpy, void *to_write_memcpy, size_t len_memcpy,
		intptr_t *p_final, intptr_t to_write_final,
		struct rseq_state start_value,
		enum rseq_finish_type type, bool release)
{
	if (unlikely(start_value.lock_state != RSEQ_LOCK_STATE_RESTART)) {
		if (start_value.lock_state == RSEQ_LOCK_STATE_LOCK)
			rseq_fallback_wait(rlock);
		return false;
	}
	return __rseq_finish(p_spec, to_write_spec, p_memcpy,
		to_write_memcpy, len_memcpy,
		p_final, to_write_final,
		start_value, type, release);
}

static inline __attribute__((always_inline))
bool rseq_finish(intptr_t *p, intptr_t to_write,
		struct rseq_state start_value)
{
	return __rseq_finish(NULL, 0,
			NULL, NULL, 0,
			p, to_write, start_value,
			RSEQ_FINISH_SINGLE, false);
}

static inline __attribute__((always_inline))
bool rseq_finish2(intptr_t *p_spec, intptr_t to_write_spec,
		intptr_t *p_final, intptr_t to_write_final,
		struct rseq_state start_value)
{
	return __rseq_finish(p_spec, to_write_spec,
			NULL, NULL, 0,
			p_final, to_write_final, start_value,
			RSEQ_FINISH_TWO, false);
}

static inline __attribute__((always_inline))
bool rseq_finish2_release(intptr_t *p_spec, intptr_t to_write_spec,
		intptr_t *p_final, intptr_t to_write_final,
		struct rseq_state start_value)
{
	return __rseq_finish(p_spec, to_write_spec,
			NULL, NULL, 0,
			p_final, to_write_final, start_value,
			RSEQ_FINISH_TWO, true);
}

static inline __attribute__((always_inline))
bool rseq_finish_memcpy(void *p_memcpy, void *to_write_memcpy,
		size_t len_memcpy, intptr_t *p_final, intptr_t to_write_final,
		struct rseq_state start_value)
{
	return __rseq_finish(NULL, 0,
			p_memcpy, to_write_memcpy, len_memcpy,
			p_final, to_write_final, start_value,
			RSEQ_FINISH_MEMCPY, false);
}

static inline __attribute__((always_inline))
bool rseq_finish_memcpy_release(void *p_memcpy, void *to_write_memcpy,
		size_t len_memcpy, intptr_t *p_final, intptr_t to_write_final,
		struct rseq_state start_value)
{
	return __rseq_finish(NULL, 0,
			p_memcpy, to_write_memcpy, len_memcpy,
			p_final, to_write_final, start_value,
			RSEQ_FINISH_MEMCPY, true);
}

#define __rseq_store_RSEQ_FINISH_SINGLE(_targetptr_spec, _newval_spec,	\
		_dest_memcpy, _src_memcpy, _len_memcpy,			\
		_targetptr_final, _newval_final)			\
	do {								\
		*(_targetptr_final) = (_newval_final);			\
	} while (0)

#define __rseq_store_RSEQ_FINISH_TWO(_targetptr_spec, _newval_spec,	\
		_dest_memcpy, _src_memcpy, _len_memcpy,			\
		_targetptr_final, _newval_final)			\
	do {								\
		*(_targetptr_spec) = (_newval_spec);			\
		*(_targetptr_final) = (_newval_final);			\
	} while (0)

#define __rseq_store_RSEQ_FINISH_MEMCPY(_targetptr_spec,		\
		_newval_spec, _dest_memcpy, _src_memcpy, _len_memcpy,	\
		_targetptr_final, _newval_final)			\
	do {								\
		memcpy(_dest_memcpy, _src_memcpy, _len_memcpy);		\
		*(_targetptr_final) = (_newval_final);			\
	} while (0)

/*
 * Helper macro doing two restartable critical section attempts, and if
 * they fail, fallback on locking.
 */
#define __do_rseq(_type, _lock, _rseq_state, _cpu, _result,		\
		_targetptr_spec, _newval_spec,				\
		_dest_memcpy, _src_memcpy, _len_memcpy,			\
		_targetptr_final, _newval_final, _code, _release)	\
	do {								\
		_rseq_state = rseq_start_rlock(_lock);			\
		_cpu = rseq_cpu_at_start(_rseq_state);			\
		_result = true;						\
		_code							\
		if (unlikely(!_result))					\
			break;						\
		if (likely(rseq_finish_rlock(_lock,			\
				_targetptr_spec, _newval_spec,		\
				_dest_memcpy, _src_memcpy, _len_memcpy,	\
				_targetptr_final, _newval_final,	\
				_rseq_state, _type, _release)))		\
			break;						\
		_rseq_state = rseq_start_rlock(_lock);			\
		_cpu = rseq_cpu_at_start(_rseq_state);			\
		_result = true;						\
		_code							\
		if (unlikely(!_result))					\
			break;						\
		if (likely(rseq_finish_rlock(_lock,			\
				_targetptr_spec, _newval_spec,		\
				_dest_memcpy, _src_memcpy, _len_memcpy,	\
				_targetptr_final, _newval_final,	\
				_rseq_state, _type, _release)))		\
			break;						\
		_cpu = rseq_fallback_begin(_lock);			\
		_result = true;						\
		_code							\
		if (likely(_result))					\
			__rseq_store_##_type(_targetptr_spec,		\
				 _newval_spec, _dest_memcpy,		\
				_src_memcpy, _len_memcpy,		\
				_targetptr_final, _newval_final);	\
		rseq_fallback_end(_lock, _cpu);				\
	} while (0)

#define do_rseq(_lock, _rseq_state, _cpu, _result, _targetptr, _newval,	\
		_code)							\
	__do_rseq(RSEQ_FINISH_SINGLE, _lock, _rseq_state, _cpu, _result,\
		NULL, 0, NULL, NULL, 0, _targetptr, _newval, _code, false)

#define do_rseq2(_lock, _rseq_state, _cpu, _result,			\
		_targetptr_spec, _newval_spec,				\
		_targetptr_final, _newval_final, _code)			\
	__do_rseq(RSEQ_FINISH_TWO, _lock, _rseq_state, _cpu, _result,	\
		_targetptr_spec, _newval_spec,				\
		NULL, NULL, 0,						\
		_targetptr_final, _newval_final, _code, false)

#define do_rseq2_release(_lock, _rseq_state, _cpu, _result,		\
		_targetptr_spec, _newval_spec,				\
		_targetptr_final, _newval_final, _code)			\
	__do_rseq(RSEQ_FINISH_TWO, _lock, _rseq_state, _cpu, _result,	\
		_targetptr_spec, _newval_spec,				\
		NULL, NULL, 0,						\
		_targetptr_final, _newval_final, _code, true)

#define do_rseq_memcpy(_lock, _rseq_state, _cpu, _result,		\
		_dest_memcpy, _src_memcpy, _len_memcpy,			\
		_targetptr_final, _newval_final, _code)			\
	__do_rseq(RSEQ_FINISH_MEMCPY, _lock, _rseq_state, _cpu, _result,\
		NULL, 0,						\
		_dest_memcpy, _src_memcpy, _len_memcpy,			\
		_targetptr_final, _newval_final, _code, false)

#define do_rseq_memcpy_release(_lock, _rseq_state, _cpu, _result,	\
		_dest_memcpy, _src_memcpy, _len_memcpy,			\
		_targetptr_final, _newval_final, _code)			\
	__do_rseq(RSEQ_FINISH_MEMCPY, _lock, _rseq_state, _cpu, _result,\
		NULL, 0,						\
		_dest_memcpy, _src_memcpy, _len_memcpy,			\
		_targetptr_final, _newval_final, _code, true)

#endif  /* RSEQ_H_ */
Commit	Line	Data
b54c5158 MD	1	/*
	2	* rseq.h
	3	*
	4	* (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	5	*
	6	* Permission is hereby granted, free of charge, to any person obtaining a copy
	7	* of this software and associated documentation files (the "Software"), to deal
	8	* in the Software without restriction, including without limitation the rights
	9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	10	* copies of the Software, and to permit persons to whom the Software is
	11	* furnished to do so, subject to the following conditions:
	12	*
	13	* The above copyright notice and this permission notice shall be included in
	14	* all copies or substantial portions of the Software.
	15	*
	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	19	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	22	* SOFTWARE.
	23	*/
	24
	25	#ifndef RSEQ_H
	26	#define RSEQ_H
	27
	28	#include <stdint.h>
	29	#include <stdbool.h>
	30	#include <pthread.h>
	31	#include <signal.h>
	32	#include <sched.h>
	33	#include <errno.h>
	34	#include <stdio.h>
	35	#include <stdlib.h>
	36	#include <sched.h>
	37	#include <linux/rseq.h>
	38
	39	/*
	40	* Empty code injection macros, override when testing.
	41	* It is important to consider that the ASM injection macros need to be
	42	* fully reentrant (e.g. do not modify the stack).
	43	*/
	44	#ifndef RSEQ_INJECT_ASM
	45	#define RSEQ_INJECT_ASM(n)
	46	#endif
	47
	48	#ifndef RSEQ_INJECT_C
	49	#define RSEQ_INJECT_C(n)
	50	#endif
	51
	52	#ifndef RSEQ_INJECT_INPUT
	53	#define RSEQ_INJECT_INPUT
	54	#endif
	55
	56	#ifndef RSEQ_INJECT_CLOBBER
	57	#define RSEQ_INJECT_CLOBBER
	58	#endif
	59
	60	#ifndef RSEQ_INJECT_FAILED
	61	#define RSEQ_INJECT_FAILED
	62	#endif
	63
	64	#ifndef RSEQ_FALLBACK_CNT
65	#define RSEQ_FALLBACK_CNT 3
66	#endif
67
68	uint32_t rseq_get_fallback_wait_cnt(void);
69	uint32_t rseq_get_fallback_cnt(void);
70
71	extern __thread volatile struct rseq __rseq_abi;
72	extern int rseq_has_sys_membarrier;
73
74	#define likely(x) __builtin_expect(!!(x), 1)
75	#define unlikely(x) __builtin_expect(!!(x), 0)
76	#define barrier() __asm__ __volatile__("" : : : "memory")
77
78	#define ACCESS_ONCE(x) ((__volatile__ __typeof__(x) )&(x))
79	#define WRITE_ONCE(x, v) __extension__ ({ ACCESS_ONCE(x) = (v); })
80	#define READ_ONCE(x) ACCESS_ONCE(x)
81
82	#if defined(__x86_64__) \|\| defined(__i386__)
83	#include <rseq-x86.h>
84	#elif defined(__ARMEL__)
85	#include <rseq-arm.h>
86	#elif defined(__PPC__)
87	#include <rseq-ppc.h>
88	#else
89	#error unsupported target
90	#endif
91
92	enum rseq_lock_state {
93	RSEQ_LOCK_STATE_RESTART = 0,
94	RSEQ_LOCK_STATE_LOCK = 1,
95	RSEQ_LOCK_STATE_FAIL = 2,
96	};
97
98	struct rseq_lock {
99	pthread_mutex_t lock;
100	int32_t state; /* enum rseq_lock_state */
101	};
102
103	/* State returned by rseq_start, passed as argument to rseq_finish. */
104	struct rseq_state {
105	volatile struct rseq *rseqp;
106	int32_t cpu_id; /* cpu_id at start. */
107	uint32_t event_counter; /* event_counter at start. */
108	int32_t lock_state; /* Lock state at start. */
109	};
110
111	/*
112	* Register rseq for the current thread. This needs to be called once
113	* by any thread which uses restartable sequences, before they start
114	* using restartable sequences. If initialization is not invoked, or if
115	* it fails, the restartable critical sections will fall-back on locking
116	* (rseq_lock).
117	*/
118	int rseq_register_current_thread(void);
119
120	/*
121	* Unregister rseq for current thread.
122	*/
123	int rseq_unregister_current_thread(void);
124
125	/*
126	* The fallback lock should be initialized before being used by any
127	* thread, and destroyed after all threads are done using it. This lock
128	* should be used by all rseq calls associated with shared data, either
129	* between threads, or between processes in a shared memory.
130	*
131	* There may be many rseq_lock per process, e.g. one per protected data
132	* structure.
133	*/
134	int rseq_init_lock(struct rseq_lock *rlock);
135	int rseq_destroy_lock(struct rseq_lock *rlock);
136
137	/*
138	* Restartable sequence fallback prototypes. Fallback on locking when
139	* rseq is not initialized, not available on the system, or during
140	* single-stepping to ensure forward progress.
141	*/
142	int rseq_fallback_begin(struct rseq_lock *rlock);
143	void rseq_fallback_end(struct rseq_lock *rlock, int cpu);
144	void rseq_fallback_wait(struct rseq_lock *rlock);
145	void rseq_fallback_noinit(struct rseq_state *rseq_state);
146
147	/*
148	* Restartable sequence fallback for reading the current CPU number.
149	*/
150	int rseq_fallback_current_cpu(void);
151
152	static inline int32_t rseq_cpu_at_start(struct rseq_state start_value)
153	{
154	return start_value.cpu_id;
155	}
156
157	static inline int32_t rseq_current_cpu_raw(void)
158	{
159	return ACCESS_ONCE(__rseq_abi.u.e.cpu_id);
160	}
161
162	static inline int32_t rseq_current_cpu(void)
163	{
164	int32_t cpu;
165
166	cpu = rseq_current_cpu_raw();
167	if (unlikely(cpu < 0))
168	cpu = rseq_fallback_current_cpu();
169	return cpu;
170	}
171
172	static inline __attribute__((always_inline))
a2ac5f39	173	struct rseq_state rseq_start(void)
b54c5158 MD	174	{
	175	struct rseq_state result;
	176
	177	result.rseqp = &__rseq_abi;
	178	if (has_single_copy_load_64()) {
	179	union rseq_cpu_event u;
	180
	181	u.v = ACCESS_ONCE(result.rseqp->u.v);
	182	result.event_counter = u.e.event_counter;
	183	result.cpu_id = u.e.cpu_id;
	184	} else {
	185	result.event_counter =
	186	ACCESS_ONCE(result.rseqp->u.e.event_counter);
	187	/* load event_counter before cpu_id. */
	188	RSEQ_INJECT_C(6)
	189	result.cpu_id = ACCESS_ONCE(result.rseqp->u.e.cpu_id);
	190	}
a2ac5f39 MD	191	RSEQ_INJECT_C(7)
	192	/*
	193	* Ensure the compiler does not re-order loads of protected
	194	* values before we load the event counter.
	195	*/
	196	barrier();
	197	return result;
	198	}
	199
	200	static inline __attribute__((always_inline))
	201	struct rseq_state rseq_start_rlock(struct rseq_lock *rlock)
	202	{
	203	struct rseq_state result;
	204
	205	result = rseq_start();
b54c5158 MD	206	/*
	207	* Read event counter before lock state and cpu_id. This ensures
	208	* that when the state changes from RESTART to LOCK, if we have
	209	* some threads that have already seen the RESTART still in
	210	* flight, they will necessarily be preempted/signalled before a
	211	* thread can see the LOCK state for that same CPU. That
	212	* preemption/signalling will cause them to restart, so they
	213	* don't interfere with the lock.
	214	*/
b54c5158 MD	215
	216	if (!has_fast_acquire_release() && likely(rseq_has_sys_membarrier)) {
	217	result.lock_state = ACCESS_ONCE(rlock->state);
	218	barrier();
	219	} else {
	220	/*
	221	* Load lock state with acquire semantic. Matches
	222	* smp_store_release() in rseq_fallback_end().
	223	*/
	224	result.lock_state = smp_load_acquire(&rlock->state);
	225	}
	226	if (unlikely(result.cpu_id < 0))
	227	rseq_fallback_noinit(&result);
b54c5158 MD	228	return result;
	229	}
	230
	231	enum rseq_finish_type {
	232	RSEQ_FINISH_SINGLE,
	233	RSEQ_FINISH_TWO,
	234	RSEQ_FINISH_MEMCPY,
	235	};
	236
	237	/*
	238	* p_spec and to_write_spec are used for a speculative write attempted
	239	* near the end of the restartable sequence. A rseq_finish2 may fail
	240	* even after this write takes place.
	241	*
	242	* p_final and to_write_final are used for the final write. If this
	243	* write takes place, the rseq_finish2 is guaranteed to succeed.
	244	*/
	245	static inline __attribute__((always_inline))
a2ac5f39	246	bool __rseq_finish(intptr_t *p_spec, intptr_t to_write_spec,
b54c5158 MD	247	void p_memcpy, void to_write_memcpy, size_t len_memcpy,
	248	intptr_t *p_final, intptr_t to_write_final,
	249	struct rseq_state start_value,
	250	enum rseq_finish_type type, bool release)
	251	{
	252	RSEQ_INJECT_C(9)
	253
b54c5158 MD	254	switch (type) {
	255	case RSEQ_FINISH_SINGLE:
	256	RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
	257	/* no speculative write /, / no speculative write */,
	258	RSEQ_FINISH_FINAL_STORE_ASM(),
	259	RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
	260	/* no extra clobber /, / no arg /, / no arg */,
	261	/* no arg */
	262	);
	263	break;
	264	case RSEQ_FINISH_TWO:
	265	if (release) {
	266	RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
	267	RSEQ_FINISH_SPECULATIVE_STORE_ASM(),
	268	RSEQ_FINISH_SPECULATIVE_STORE_INPUT(p_spec, to_write_spec),
	269	RSEQ_FINISH_FINAL_STORE_RELEASE_ASM(),
	270	RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
	271	/* no extra clobber /, / no arg /, / no arg */,
	272	/* no arg */
	273	);
	274	} else {
	275	RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
	276	RSEQ_FINISH_SPECULATIVE_STORE_ASM(),
	277	RSEQ_FINISH_SPECULATIVE_STORE_INPUT(p_spec, to_write_spec),
	278	RSEQ_FINISH_FINAL_STORE_ASM(),
	279	RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
	280	/* no extra clobber /, / no arg /, / no arg */,
	281	/* no arg */
	282	);
	283	}
	284	break;
	285	case RSEQ_FINISH_MEMCPY:
	286	if (release) {
	287	RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
	288	RSEQ_FINISH_MEMCPY_STORE_ASM(),
	289	RSEQ_FINISH_MEMCPY_STORE_INPUT(p_memcpy, to_write_memcpy, len_memcpy),
	290	RSEQ_FINISH_FINAL_STORE_RELEASE_ASM(),
	291	RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
	292	RSEQ_FINISH_MEMCPY_CLOBBER(),
	293	RSEQ_FINISH_MEMCPY_SETUP(),
	294	RSEQ_FINISH_MEMCPY_TEARDOWN(),
	295	RSEQ_FINISH_MEMCPY_SCRATCH()
	296	);
	297	} else {
	298	RSEQ_FINISH_ASM(p_final, to_write_final, start_value, failure,
	299	RSEQ_FINISH_MEMCPY_STORE_ASM(),
	300	RSEQ_FINISH_MEMCPY_STORE_INPUT(p_memcpy, to_write_memcpy, len_memcpy),
	301	RSEQ_FINISH_FINAL_STORE_ASM(),
	302	RSEQ_FINISH_FINAL_STORE_INPUT(p_final, to_write_final),
	303	RSEQ_FINISH_MEMCPY_CLOBBER(),
	304	RSEQ_FINISH_MEMCPY_SETUP(),
	305	RSEQ_FINISH_MEMCPY_TEARDOWN(),
	306	RSEQ_FINISH_MEMCPY_SCRATCH()
	307	);
	308	}
	309	break;
	310	}
	311	return true;
	312	failure:
	313	RSEQ_INJECT_FAILED
	314	return false;
	315	}
	316
	317	static inline __attribute__((always_inline))
a2ac5f39 MD	318	bool rseq_finish_rlock(struct rseq_lock *rlock,
	319	intptr_t *p_spec, intptr_t to_write_spec,
	320	void p_memcpy, void to_write_memcpy, size_t len_memcpy,
	321	intptr_t *p_final, intptr_t to_write_final,
	322	struct rseq_state start_value,
	323	enum rseq_finish_type type, bool release)
	324	{
	325	if (unlikely(start_value.lock_state != RSEQ_LOCK_STATE_RESTART)) {
	326	if (start_value.lock_state == RSEQ_LOCK_STATE_LOCK)
	327	rseq_fallback_wait(rlock);
	328	return false;
	329	}
	330	return __rseq_finish(p_spec, to_write_spec, p_memcpy,
	331	to_write_memcpy, len_memcpy,
	332	p_final, to_write_final,
	333	start_value, type, release);
	334	}
	335
	336	static inline __attribute__((always_inline))
	337	bool rseq_finish(intptr_t *p, intptr_t to_write,
b54c5158 MD	338	struct rseq_state start_value)
b54c5158 MD	339	{
a2ac5f39	340	return __rseq_finish(NULL, 0,
b54c5158 MD	341	NULL, NULL, 0,
	342	p, to_write, start_value,
	343	RSEQ_FINISH_SINGLE, false);
	344	}
	345
	346	static inline __attribute__((always_inline))
a2ac5f39	347	bool rseq_finish2(intptr_t *p_spec, intptr_t to_write_spec,
b54c5158 MD	348	intptr_t *p_final, intptr_t to_write_final,
	349	struct rseq_state start_value)
	350	{
a2ac5f39	351	return __rseq_finish(p_spec, to_write_spec,
b54c5158 MD	352	NULL, NULL, 0,
	353	p_final, to_write_final, start_value,
	354	RSEQ_FINISH_TWO, false);
	355	}
	356
	357	static inline __attribute__((always_inline))
a2ac5f39	358	bool rseq_finish2_release(intptr_t *p_spec, intptr_t to_write_spec,
b54c5158 MD	359	intptr_t *p_final, intptr_t to_write_final,
	360	struct rseq_state start_value)
	361	{
a2ac5f39	362	return __rseq_finish(p_spec, to_write_spec,
b54c5158 MD	363	NULL, NULL, 0,
	364	p_final, to_write_final, start_value,
	365	RSEQ_FINISH_TWO, true);
	366	}
	367
	368	static inline __attribute__((always_inline))
a2ac5f39 MD	369	bool rseq_finish_memcpy(void p_memcpy, void to_write_memcpy,
a2ac5f39 MD	370	size_t len_memcpy, intptr_t *p_final, intptr_t to_write_final,
b54c5158 MD	371	struct rseq_state start_value)
b54c5158 MD	372	{
a2ac5f39	373	return __rseq_finish(NULL, 0,
b54c5158 MD	374	p_memcpy, to_write_memcpy, len_memcpy,
	375	p_final, to_write_final, start_value,
	376	RSEQ_FINISH_MEMCPY, false);
	377	}
	378
	379	static inline __attribute__((always_inline))
a2ac5f39 MD	380	bool rseq_finish_memcpy_release(void p_memcpy, void to_write_memcpy,
a2ac5f39 MD	381	size_t len_memcpy, intptr_t *p_final, intptr_t to_write_final,
b54c5158 MD	382	struct rseq_state start_value)
b54c5158 MD	383	{
a2ac5f39	384	return __rseq_finish(NULL, 0,
b54c5158 MD	385	p_memcpy, to_write_memcpy, len_memcpy,
	386	p_final, to_write_final, start_value,
	387	RSEQ_FINISH_MEMCPY, true);
	388	}
	389
	390	#define __rseq_store_RSEQ_FINISH_SINGLE(_targetptr_spec, _newval_spec, \
	391	_dest_memcpy, _src_memcpy, _len_memcpy, \
	392	_targetptr_final, _newval_final) \
	393	do { \
	394	*(_targetptr_final) = (_newval_final); \
	395	} while (0)
	396
	397	#define __rseq_store_RSEQ_FINISH_TWO(_targetptr_spec, _newval_spec, \
	398	_dest_memcpy, _src_memcpy, _len_memcpy, \
	399	_targetptr_final, _newval_final) \
	400	do { \
	401	*(_targetptr_spec) = (_newval_spec); \
	402	*(_targetptr_final) = (_newval_final); \
	403	} while (0)
	404
	405	#define __rseq_store_RSEQ_FINISH_MEMCPY(_targetptr_spec, \
	406	_newval_spec, _dest_memcpy, _src_memcpy, _len_memcpy, \
	407	_targetptr_final, _newval_final) \
	408	do { \
	409	memcpy(_dest_memcpy, _src_memcpy, _len_memcpy); \
	410	*(_targetptr_final) = (_newval_final); \
	411	} while (0)
	412
	413	/*
	414	* Helper macro doing two restartable critical section attempts, and if
	415	* they fail, fallback on locking.
	416	*/
	417	#define __do_rseq(_type, _lock, _rseq_state, _cpu, _result, \
	418	_targetptr_spec, _newval_spec, \
	419	_dest_memcpy, _src_memcpy, _len_memcpy, \
	420	_targetptr_final, _newval_final, _code, _release) \
	421	do { \
a2ac5f39	422	_rseq_state = rseq_start_rlock(_lock); \
b54c5158 MD	423	_cpu = rseq_cpu_at_start(_rseq_state); \
	424	_result = true; \
	425	_code \
	426	if (unlikely(!_result)) \
	427	break; \
a2ac5f39	428	if (likely(rseq_finish_rlock(_lock, \
b54c5158 MD	429	_targetptr_spec, _newval_spec, \
	430	_dest_memcpy, _src_memcpy, _len_memcpy, \
	431	_targetptr_final, _newval_final, \
	432	_rseq_state, _type, _release))) \
	433	break; \
a2ac5f39	434	_rseq_state = rseq_start_rlock(_lock); \
b54c5158 MD	435	_cpu = rseq_cpu_at_start(_rseq_state); \
	436	_result = true; \
	437	_code \
	438	if (unlikely(!_result)) \
	439	break; \
a2ac5f39	440	if (likely(rseq_finish_rlock(_lock, \
b54c5158 MD	441	_targetptr_spec, _newval_spec, \
	442	_dest_memcpy, _src_memcpy, _len_memcpy, \
	443	_targetptr_final, _newval_final, \
	444	_rseq_state, _type, _release))) \
	445	break; \
	446	_cpu = rseq_fallback_begin(_lock); \
	447	_result = true; \
	448	_code \
	449	if (likely(_result)) \
	450	__rseq_store_##_type(_targetptr_spec, \
	451	_newval_spec, _dest_memcpy, \
	452	_src_memcpy, _len_memcpy, \
	453	_targetptr_final, _newval_final); \
	454	rseq_fallback_end(_lock, _cpu); \
	455	} while (0)
	456
	457	#define do_rseq(_lock, _rseq_state, _cpu, _result, _targetptr, _newval, \
	458	_code) \
	459	__do_rseq(RSEQ_FINISH_SINGLE, _lock, _rseq_state, _cpu, _result,\
	460	NULL, 0, NULL, NULL, 0, _targetptr, _newval, _code, false)
	461
	462	#define do_rseq2(_lock, _rseq_state, _cpu, _result, \
	463	_targetptr_spec, _newval_spec, \
	464	_targetptr_final, _newval_final, _code) \
	465	__do_rseq(RSEQ_FINISH_TWO, _lock, _rseq_state, _cpu, _result, \
	466	_targetptr_spec, _newval_spec, \
	467	NULL, NULL, 0, \
	468	_targetptr_final, _newval_final, _code, false)
	469
	470	#define do_rseq2_release(_lock, _rseq_state, _cpu, _result, \
	471	_targetptr_spec, _newval_spec, \
	472	_targetptr_final, _newval_final, _code) \
	473	__do_rseq(RSEQ_FINISH_TWO, _lock, _rseq_state, _cpu, _result, \
	474	_targetptr_spec, _newval_spec, \
	475	NULL, NULL, 0, \
	476	_targetptr_final, _newval_final, _code, true)
	477
	478	#define do_rseq_memcpy(_lock, _rseq_state, _cpu, _result, \
	479	_dest_memcpy, _src_memcpy, _len_memcpy, \
	480	_targetptr_final, _newval_final, _code) \
	481	__do_rseq(RSEQ_FINISH_MEMCPY, _lock, _rseq_state, _cpu, _result,\
	482	NULL, 0, \
	483	_dest_memcpy, _src_memcpy, _len_memcpy, \
	484	_targetptr_final, _newval_final, _code, false)
	485
	486	#define do_rseq_memcpy_release(_lock, _rseq_state, _cpu, _result, \
	487	_dest_memcpy, _src_memcpy, _len_memcpy, \
	488	_targetptr_final, _newval_final, _code) \
	489	__do_rseq(RSEQ_FINISH_MEMCPY, _lock, _rseq_state, _cpu, _result,\
	490	NULL, 0, \
	491	_dest_memcpy, _src_memcpy, _len_memcpy, \
	492	_targetptr_final, _newval_final, _code, true)
	493
	494	#endif /* RSEQ_H_ */