[lttng-ust.git] / libringbuffer / rseq-ppc.h

/*
 * rseq-ppc.h
 *
 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * (C) Copyright 2016 - Boqun Feng <boqun.feng@gmail.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#define has_fast_acquire_release()	0

#ifdef __PPC64__
#define has_single_copy_load_64()	1
#else
#define has_single_copy_load_64()	0
#endif

/*
 * The __rseq_table section can be used by debuggers to better handle
 * single-stepping through the restartable critical sections.
 */

#ifdef __PPC64__

#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
		_failure, _spec_store, _spec_input, \
		_final_store, _final_input, _extra_clobber, \
		_setup, _teardown, _scratch) \
	__asm__ __volatile__ goto ( \
		".pushsection __rseq_table, \"aw\"\n\t" \
		".balign 32\n\t" \
		"3:\n\t" \
		".quad 1f, 2f, 4f, 0x0\n\t" \
		".popsection\n\t" \
		"1:\n\t" \
		_setup \
		RSEQ_INJECT_ASM(1) \
		"lis %%r17, (3b)@highest\n\t" \
		"ori %%r17, %%r17, (3b)@higher\n\t" \
		"rldicr %%r17, %%r17, 32, 31\n\t" \
		"oris %%r17, %%r17, (3b)@h\n\t" \
		"ori %%r17, %%r17, (3b)@l\n\t" \
		"std %%r17, 0(%[rseq_cs])\n\t" \
		RSEQ_INJECT_ASM(2) \
		"lwz %%r17, %[current_event_counter]\n\t" \
		"cmpw cr7, %[start_event_counter], %%r17\n\t" \
		"bne- cr7, 4f\n\t" \
		RSEQ_INJECT_ASM(3) \
		_spec_store \
		_final_store \
		"2:\n\t" \
		RSEQ_INJECT_ASM(5) \
		"li %%r17, 0\n\t" \
		"std %%r17, 0(%[rseq_cs])\n\t" \
		_teardown \
		"b 5f\n\t" \
		"4:\n\t" \
		"li %%r17, 0\n\t" \
		"std %%r17, 0(%[rseq_cs])\n\t" \
		_teardown \
		"b %l[failure]\n\t" \
		"5:\n\t" \
		: /* no outputs */ \
		: [start_event_counter]"r"((_start_value).event_counter), \
		  [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
		  [rseq_cs]"b"(&(_start_value).rseqp->rseq_cs) \
		  _spec_input \
		  _final_input \
		  RSEQ_INJECT_INPUT \
		: "r17", "memory", "cc" \
		  _extra_clobber \
		  RSEQ_INJECT_CLOBBER \
		: _failure \
	)

#define RSEQ_FINISH_FINAL_STORE_ASM() \
		"std %[to_write_final], 0(%[target_final])\n\t"

#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
		"lwsync\n\t" \
		RSEQ_FINISH_FINAL_STORE_ASM()

#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
		, [to_write_final]"r"(_to_write_final), \
		[target_final]"b"(_target_final)

#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
		"std %[to_write_spec], 0(%[target_spec])\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
		, [to_write_spec]"r"(_to_write_spec), \
		[target_spec]"b"(_target_spec)

/* TODO: implement a faster memcpy. */
#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
		"cmpdi %%r19, 0\n\t" \
		"beq 333f\n\t" \
		"addi %%r20, %%r20, -1\n\t" \
		"addi %%r21, %%r21, -1\n\t" \
		"222:\n\t" \
		"lbzu %%r18, 1(%%r20)\n\t" \
		"stbu %%r18, 1(%%r21)\n\t" \
		"addi %%r19, %%r19, -1\n\t" \
		"cmpdi %%r19, 0\n\t" \
		"bne 222b\n\t" \
		"333:\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
		, [to_write_memcpy]"r"(_to_write_memcpy), \
		[target_memcpy]"r"(_target_memcpy), \
		[len_memcpy]"r"(_len_memcpy)

#define RSEQ_FINISH_MEMCPY_CLOBBER() \
		, "r18", "r19", "r20", "r21"

#define RSEQ_FINISH_MEMCPY_SCRATCH()

/*
 * We use extra registers to hold the input registers, and we don't need to
 * save and restore the input registers.
 */
#define RSEQ_FINISH_MEMCPY_SETUP() \
		"mr %%r19, %[len_memcpy]\n\t" \
		"mr %%r20, %[to_write_memcpy]\n\t" \
		"mr %%r21, %[target_memcpy]\n\t" \

#define RSEQ_FINISH_MEMCPY_TEARDOWN()

#else	/* #ifdef __PPC64__ */

#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
		_failure, _spec_store, _spec_input, \
		_final_store, _final_input, _extra_clobber, \
		_setup, _teardown, _scratch) \
	__asm__ __volatile__ goto ( \
		".pushsection __rseq_table, \"aw\"\n\t" \
		".balign 32\n\t" \
		"3:\n\t" \
		/* 32-bit only supported on BE */ \
		".long 0x0, 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0\n\t" \
		".popsection\n\t" \
		"1:\n\t" \
		_setup \
		RSEQ_INJECT_ASM(1) \
		"lis %%r17, (3b)@ha\n\t" \
		"addi %%r17, %%r17, (3b)@l\n\t" \
		"stw %%r17, 0(%[rseq_cs])\n\t" \
		RSEQ_INJECT_ASM(2) \
		"lwz %%r17, %[current_event_counter]\n\t" \
		"cmpw cr7, %[start_event_counter], %%r17\n\t" \
		"bne- cr7, 4f\n\t" \
		RSEQ_INJECT_ASM(3) \
		_spec_store \
		_final_store \
		"2:\n\t" \
		RSEQ_INJECT_ASM(5) \
		"li %%r17, 0\n\t" \
		"stw %%r17, 0(%[rseq_cs])\n\t" \
		_teardown \
		"b 5f\n\t" \
		"4:\n\t" \
		"li %%r17, 0\n\t" \
		"std %%r17, 0(%[rseq_cs])\n\t" \
		_teardown \
		"b %l[failure]\n\t" \
		"5:\n\t" \
		: /* no outputs */ \
		: [start_event_counter]"r"((_start_value).event_counter), \
		  [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
		  [rseq_cs]"b"(&(_start_value).rseqp->rseq_cs) \
		  _spec_input \
		  _final_input \
		  RSEQ_INJECT_INPUT \
		: "r17", "memory", "cc" \
		  _extra_clobber \
		  RSEQ_INJECT_CLOBBER \
		: _failure \
	)

#define RSEQ_FINISH_FINAL_STORE_ASM() \
		"stw %[to_write_final], 0(%[target_final])\n\t"

#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
		"lwsync\n\t" \
		RSEQ_FINISH_FINAL_STORE_ASM()

#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
		, [to_write_final]"r"(_to_write_final), \
		[target_final]"b"(_target_final)

#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
		"stw %[to_write_spec], 0(%[target_spec])\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
		, [to_write_spec]"r"(_to_write_spec), \
		[target_spec]"b"(_target_spec)

/* TODO: implement a faster memcpy. */
#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
		"cmpwi %%r19, 0\n\t" \
		"beq 333f\n\t" \
		"addi %%r20, %%r20, -1\n\t" \
		"addi %%r21, %%r21, -1\n\t" \
		"222:\n\t" \
		"lbzu %%r18, 1(%%r20)\n\t" \
		"stbu %%r18, 1(%%r21)\n\t" \
		"addi %%r19, %%r19, -1\n\t" \
		"cmpwi %%r19, 0\n\t" \
		"bne 222b\n\t" \
		"333:\n\t" \
		RSEQ_INJECT_ASM(4)

#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
		, [to_write_memcpy]"r"(_to_write_memcpy), \
		[target_memcpy]"r"(_target_memcpy), \
		[len_memcpy]"r"(_len_memcpy)

#define RSEQ_FINISH_MEMCPY_CLOBBER() \
		, "r18", "r19", "r20", "r21"

#define RSEQ_FINISH_MEMCPY_SCRATCH()

/*
 * We use extra registers to hold the input registers, and we don't need to
 * save and restore the input registers.
 */
#define RSEQ_FINISH_MEMCPY_SETUP() \
		"mr %%r19, %[len_memcpy]\n\t" \
		"mr %%r20, %[to_write_memcpy]\n\t" \
		"mr %%r21, %[target_memcpy]\n\t" \

#define RSEQ_FINISH_MEMCPY_TEARDOWN()

#endif	/* #else #ifdef __PPC64__ */
Commit	Line	Data
b76e5200 MD	1	/*
	2	* rseq-ppc.h
	3	*
	4	* (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	5	* (C) Copyright 2016 - Boqun Feng <boqun.feng@gmail.com>
	6	*
	7	* Permission is hereby granted, free of charge, to any person obtaining a copy
	8	* of this software and associated documentation files (the "Software"), to deal
	9	* in the Software without restriction, including without limitation the rights
	10	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	11	* copies of the Software, and to permit persons to whom the Software is
	12	* furnished to do so, subject to the following conditions:
	13	*
	14	* The above copyright notice and this permission notice shall be included in
	15	* all copies or substantial portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	18	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	19	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	20	* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	21	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	22	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	23	* SOFTWARE.
	24	*/
	25
b76e5200 MD	26	#define has_fast_acquire_release() 0
	27
	28	#ifdef __PPC64__
	29	#define has_single_copy_load_64() 1
	30	#else
	31	#define has_single_copy_load_64() 0
	32	#endif
	33
	34	/*
	35	* The __rseq_table section can be used by debuggers to better handle
	36	* single-stepping through the restartable critical sections.
	37	*/
	38
	39	#ifdef __PPC64__
	40
	41	#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
	42	_failure, _spec_store, _spec_input, \
	43	_final_store, _final_input, _extra_clobber, \
	44	_setup, _teardown, _scratch) \
	45	__asm__ __volatile__ goto ( \
	46	".pushsection __rseq_table, \"aw\"\n\t" \
	47	".balign 32\n\t" \
	48	"3:\n\t" \
	49	".quad 1f, 2f, 4f, 0x0\n\t" \
	50	".popsection\n\t" \
	51	"1:\n\t" \
	52	_setup \
	53	RSEQ_INJECT_ASM(1) \
	54	"lis %%r17, (3b)@highest\n\t" \
	55	"ori %%r17, %%r17, (3b)@higher\n\t" \
	56	"rldicr %%r17, %%r17, 32, 31\n\t" \
	57	"oris %%r17, %%r17, (3b)@h\n\t" \
	58	"ori %%r17, %%r17, (3b)@l\n\t" \
	59	"std %%r17, 0(%[rseq_cs])\n\t" \
	60	RSEQ_INJECT_ASM(2) \
	61	"lwz %%r17, %[current_event_counter]\n\t" \
	62	"cmpw cr7, %[start_event_counter], %%r17\n\t" \
	63	"bne- cr7, 4f\n\t" \
	64	RSEQ_INJECT_ASM(3) \
	65	_spec_store \
	66	_final_store \
	67	"2:\n\t" \
	68	RSEQ_INJECT_ASM(5) \
	69	"li %%r17, 0\n\t" \
	70	"std %%r17, 0(%[rseq_cs])\n\t" \
	71	_teardown \
	72	"b 5f\n\t" \
	73	"4:\n\t" \
	74	"li %%r17, 0\n\t" \
	75	"std %%r17, 0(%[rseq_cs])\n\t" \
	76	_teardown \
	77	"b %l[failure]\n\t" \
	78	"5:\n\t" \
	79	: /* no outputs */ \
	80	: [start_event_counter]"r"((_start_value).event_counter), \
	81	[current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
	82	[rseq_cs]"b"(&(_start_value).rseqp->rseq_cs) \
	83	_spec_input \
	84	_final_input \
	85	RSEQ_INJECT_INPUT \
	86	: "r17", "memory", "cc" \
	87	_extra_clobber \
	88	RSEQ_INJECT_CLOBBER \
	89	: _failure \
90	)
91
92	#define RSEQ_FINISH_FINAL_STORE_ASM() \
93	"std %[to_write_final], 0(%[target_final])\n\t"
94
95	#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
96	"lwsync\n\t" \
97	RSEQ_FINISH_FINAL_STORE_ASM()
98
99	#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
100	, [to_write_final]"r"(_to_write_final), \
101	[target_final]"b"(_target_final)
102
103	#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
104	"std %[to_write_spec], 0(%[target_spec])\n\t" \
105	RSEQ_INJECT_ASM(4)
106
107	#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
108	, [to_write_spec]"r"(_to_write_spec), \
109	[target_spec]"b"(_target_spec)
110
111	/* TODO: implement a faster memcpy. */
112	#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
113	"cmpdi %%r19, 0\n\t" \
114	"beq 333f\n\t" \
115	"addi %%r20, %%r20, -1\n\t" \
116	"addi %%r21, %%r21, -1\n\t" \
117	"222:\n\t" \
118	"lbzu %%r18, 1(%%r20)\n\t" \
119	"stbu %%r18, 1(%%r21)\n\t" \
120	"addi %%r19, %%r19, -1\n\t" \
121	"cmpdi %%r19, 0\n\t" \
122	"bne 222b\n\t" \
123	"333:\n\t" \
124	RSEQ_INJECT_ASM(4)
125
126	#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
127	, [to_write_memcpy]"r"(_to_write_memcpy), \
128	[target_memcpy]"r"(_target_memcpy), \
129	[len_memcpy]"r"(_len_memcpy)
130
131	#define RSEQ_FINISH_MEMCPY_CLOBBER() \
132	, "r18", "r19", "r20", "r21"
133
134	#define RSEQ_FINISH_MEMCPY_SCRATCH()
135
136	/*
137	* We use extra registers to hold the input registers, and we don't need to
138	* save and restore the input registers.
139	*/
140	#define RSEQ_FINISH_MEMCPY_SETUP() \
141	"mr %%r19, %[len_memcpy]\n\t" \
142	"mr %%r20, %[to_write_memcpy]\n\t" \
143	"mr %%r21, %[target_memcpy]\n\t" \
144
145	#define RSEQ_FINISH_MEMCPY_TEARDOWN()
146
147	#else /* #ifdef __PPC64__ */
148
149	#define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
150	_failure, _spec_store, _spec_input, \
151	_final_store, _final_input, _extra_clobber, \
152	_setup, _teardown, _scratch) \
153	__asm__ __volatile__ goto ( \
154	".pushsection __rseq_table, \"aw\"\n\t" \
155	".balign 32\n\t" \
156	"3:\n\t" \
157	/* 32-bit only supported on BE */ \
158	".long 0x0, 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0\n\t" \
159	".popsection\n\t" \
160	"1:\n\t" \
161	_setup \
162	RSEQ_INJECT_ASM(1) \
163	"lis %%r17, (3b)@ha\n\t" \
164	"addi %%r17, %%r17, (3b)@l\n\t" \
165	"stw %%r17, 0(%[rseq_cs])\n\t" \
166	RSEQ_INJECT_ASM(2) \
167	"lwz %%r17, %[current_event_counter]\n\t" \
168	"cmpw cr7, %[start_event_counter], %%r17\n\t" \
169	"bne- cr7, 4f\n\t" \
170	RSEQ_INJECT_ASM(3) \
171	_spec_store \
172	_final_store \
173	"2:\n\t" \
174	RSEQ_INJECT_ASM(5) \
175	"li %%r17, 0\n\t" \
176	"stw %%r17, 0(%[rseq_cs])\n\t" \
177	_teardown \
178	"b 5f\n\t" \
179	"4:\n\t" \
180	"li %%r17, 0\n\t" \
181	"std %%r17, 0(%[rseq_cs])\n\t" \
182	_teardown \
183	"b %l[failure]\n\t" \
184	"5:\n\t" \
185	: /* no outputs */ \
186	: [start_event_counter]"r"((_start_value).event_counter), \
187	[current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
188	[rseq_cs]"b"(&(_start_value).rseqp->rseq_cs) \
189	_spec_input \
190	_final_input \
191	RSEQ_INJECT_INPUT \
192	: "r17", "memory", "cc" \
193	_extra_clobber \
194	RSEQ_INJECT_CLOBBER \
195	: _failure \
196	)
197
198	#define RSEQ_FINISH_FINAL_STORE_ASM() \
199	"stw %[to_write_final], 0(%[target_final])\n\t"
200
201	#define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
202	"lwsync\n\t" \
203	RSEQ_FINISH_FINAL_STORE_ASM()
204
205	#define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
206	, [to_write_final]"r"(_to_write_final), \
207	[target_final]"b"(_target_final)
208
209	#define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
210	"stw %[to_write_spec], 0(%[target_spec])\n\t" \
211	RSEQ_INJECT_ASM(4)
212
213	#define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
214	, [to_write_spec]"r"(_to_write_spec), \
215	[target_spec]"b"(_target_spec)
216
217	/* TODO: implement a faster memcpy. */
218	#define RSEQ_FINISH_MEMCPY_STORE_ASM() \
219	"cmpwi %%r19, 0\n\t" \
220	"beq 333f\n\t" \
221	"addi %%r20, %%r20, -1\n\t" \
222	"addi %%r21, %%r21, -1\n\t" \
223	"222:\n\t" \
224	"lbzu %%r18, 1(%%r20)\n\t" \
225	"stbu %%r18, 1(%%r21)\n\t" \
226	"addi %%r19, %%r19, -1\n\t" \
227	"cmpwi %%r19, 0\n\t" \
228	"bne 222b\n\t" \
229	"333:\n\t" \
230	RSEQ_INJECT_ASM(4)
231
232	#define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
233	, [to_write_memcpy]"r"(_to_write_memcpy), \
234	[target_memcpy]"r"(_target_memcpy), \
235	[len_memcpy]"r"(_len_memcpy)
236
237	#define RSEQ_FINISH_MEMCPY_CLOBBER() \
238	, "r18", "r19", "r20", "r21"
239
240	#define RSEQ_FINISH_MEMCPY_SCRATCH()
241
242	/*
243	* We use extra registers to hold the input registers, and we don't need to
244	* save and restore the input registers.
245	*/
246	#define RSEQ_FINISH_MEMCPY_SETUP() \
247	"mr %%r19, %[len_memcpy]\n\t" \
248	"mr %%r20, %[to_write_memcpy]\n\t" \
249	"mr %%r21, %[target_memcpy]\n\t" \
250
251	#define RSEQ_FINISH_MEMCPY_TEARDOWN()
252
253	#endif /* #else #ifdef __PPC64__ */