Commit | Line | Data |
---|---|---|
c10302ef MS |
1 | /* |
2 | * BPF Jit compiler for s390. | |
3 | * | |
4 | * Copyright IBM Corp. 2012 | |
5 | * | |
6 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> | |
7 | */ | |
8 | #include <linux/moduleloader.h> | |
9 | #include <linux/netdevice.h> | |
10 | #include <linux/filter.h> | |
11 | #include <asm/cacheflush.h> | |
12 | #include <asm/processor.h> | |
68d9884d | 13 | #include <asm/facility.h> |
c10302ef MS |
14 | |
15 | /* | |
16 | * Conventions: | |
17 | * %r2 = skb pointer | |
18 | * %r3 = offset parameter | |
19 | * %r4 = scratch register / length parameter | |
20 | * %r5 = BPF A accumulator | |
21 | * %r8 = return address | |
22 | * %r9 = save register for skb pointer | |
23 | * %r10 = skb->data | |
24 | * %r11 = skb->len - skb->data_len (headlen) | |
25 | * %r12 = BPF X accumulator | |
26 | * %r13 = literal pool pointer | |
27 | * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS | |
28 | */ | |
29 | int bpf_jit_enable __read_mostly; | |
30 | ||
31 | /* | |
32 | * assembly code in arch/x86/net/bpf_jit.S | |
33 | */ | |
34 | extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; | |
35 | extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; | |
36 | ||
37 | struct bpf_jit { | |
38 | unsigned int seen; | |
39 | u8 *start; | |
40 | u8 *prg; | |
41 | u8 *mid; | |
42 | u8 *lit; | |
43 | u8 *end; | |
44 | u8 *base_ip; | |
45 | u8 *ret0_ip; | |
46 | u8 *exit_ip; | |
47 | unsigned int off_load_word; | |
48 | unsigned int off_load_half; | |
49 | unsigned int off_load_byte; | |
50 | unsigned int off_load_bmsh; | |
51 | unsigned int off_load_iword; | |
52 | unsigned int off_load_ihalf; | |
53 | unsigned int off_load_ibyte; | |
54 | }; | |
55 | ||
56 | #define BPF_SIZE_MAX 4096 /* Max size for program */ | |
57 | ||
58 | #define SEEN_DATAREF 1 /* might call external helpers */ | |
59 | #define SEEN_XREG 2 /* ebx is used */ | |
60 | #define SEEN_MEM 4 /* use mem[] for temporary storage */ | |
61 | #define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ | |
62 | #define SEEN_LITERAL 16 /* code uses literals */ | |
63 | #define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ | |
64 | #define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ | |
65 | #define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ | |
66 | #define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ | |
67 | #define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ | |
68 | #define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ | |
69 | #define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ | |
70 | ||
71 | #define EMIT2(op) \ | |
72 | ({ \ | |
73 | if (jit->prg + 2 <= jit->mid) \ | |
74 | *(u16 *) jit->prg = op; \ | |
75 | jit->prg += 2; \ | |
76 | }) | |
77 | ||
78 | #define EMIT4(op) \ | |
79 | ({ \ | |
80 | if (jit->prg + 4 <= jit->mid) \ | |
81 | *(u32 *) jit->prg = op; \ | |
82 | jit->prg += 4; \ | |
83 | }) | |
84 | ||
85 | #define EMIT4_DISP(op, disp) \ | |
86 | ({ \ | |
87 | unsigned int __disp = (disp) & 0xfff; \ | |
88 | EMIT4(op | __disp); \ | |
89 | }) | |
90 | ||
91 | #define EMIT4_IMM(op, imm) \ | |
92 | ({ \ | |
93 | unsigned int __imm = (imm) & 0xffff; \ | |
94 | EMIT4(op | __imm); \ | |
95 | }) | |
96 | ||
97 | #define EMIT4_PCREL(op, pcrel) \ | |
98 | ({ \ | |
99 | long __pcrel = ((pcrel) >> 1) & 0xffff; \ | |
100 | EMIT4(op | __pcrel); \ | |
101 | }) | |
102 | ||
103 | #define EMIT6(op1, op2) \ | |
104 | ({ \ | |
105 | if (jit->prg + 6 <= jit->mid) { \ | |
106 | *(u32 *) jit->prg = op1; \ | |
107 | *(u16 *) (jit->prg + 4) = op2; \ | |
108 | } \ | |
109 | jit->prg += 6; \ | |
110 | }) | |
111 | ||
112 | #define EMIT6_DISP(op1, op2, disp) \ | |
113 | ({ \ | |
114 | unsigned int __disp = (disp) & 0xfff; \ | |
115 | EMIT6(op1 | __disp, op2); \ | |
116 | }) | |
117 | ||
68d9884d HC |
118 | #define EMIT6_IMM(op, imm) \ |
119 | ({ \ | |
120 | unsigned int __imm = (imm); \ | |
121 | EMIT6(op | (__imm >> 16), __imm & 0xffff); \ | |
122 | }) | |
123 | ||
c10302ef MS |
124 | #define EMIT_CONST(val) \ |
125 | ({ \ | |
126 | unsigned int ret; \ | |
127 | ret = (unsigned int) (jit->lit - jit->base_ip); \ | |
128 | jit->seen |= SEEN_LITERAL; \ | |
129 | if (jit->lit + 4 <= jit->end) \ | |
130 | *(u32 *) jit->lit = val; \ | |
131 | jit->lit += 4; \ | |
132 | ret; \ | |
133 | }) | |
134 | ||
135 | #define EMIT_FN_CONST(bit, fn) \ | |
136 | ({ \ | |
137 | unsigned int ret; \ | |
138 | ret = (unsigned int) (jit->lit - jit->base_ip); \ | |
139 | if (jit->seen & bit) { \ | |
140 | jit->seen |= SEEN_LITERAL; \ | |
141 | if (jit->lit + 8 <= jit->end) \ | |
142 | *(void **) jit->lit = fn; \ | |
143 | jit->lit += 8; \ | |
144 | } \ | |
145 | ret; \ | |
146 | }) | |
147 | ||
148 | static void bpf_jit_prologue(struct bpf_jit *jit) | |
149 | { | |
150 | /* Save registers and create stack frame if necessary */ | |
151 | if (jit->seen & SEEN_DATAREF) { | |
152 | /* stmg %r8,%r15,88(%r15) */ | |
153 | EMIT6(0xeb8ff058, 0x0024); | |
154 | /* lgr %r14,%r15 */ | |
155 | EMIT4(0xb90400ef); | |
156 | /* ahi %r15,<offset> */ | |
157 | EMIT4_IMM(0xa7fa0000, (jit->seen & SEEN_MEM) ? -112 : -80); | |
158 | /* stg %r14,152(%r15) */ | |
159 | EMIT6(0xe3e0f098, 0x0024); | |
160 | } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) | |
161 | /* stmg %r12,%r13,120(%r15) */ | |
162 | EMIT6(0xebcdf078, 0x0024); | |
163 | else if (jit->seen & SEEN_XREG) | |
164 | /* stg %r12,120(%r15) */ | |
165 | EMIT6(0xe3c0f078, 0x0024); | |
166 | else if (jit->seen & SEEN_LITERAL) | |
167 | /* stg %r13,128(%r15) */ | |
168 | EMIT6(0xe3d0f080, 0x0024); | |
169 | ||
170 | /* Setup literal pool */ | |
171 | if (jit->seen & SEEN_LITERAL) { | |
172 | /* basr %r13,0 */ | |
173 | EMIT2(0x0dd0); | |
174 | jit->base_ip = jit->prg; | |
175 | } | |
176 | jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); | |
177 | jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); | |
178 | jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); | |
179 | jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); | |
180 | jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); | |
181 | jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); | |
182 | jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); | |
183 | ||
184 | /* Filter needs to access skb data */ | |
185 | if (jit->seen & SEEN_DATAREF) { | |
186 | /* l %r11,<len>(%r2) */ | |
187 | EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); | |
188 | /* s %r11,<data_len>(%r2) */ | |
189 | EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); | |
190 | /* lg %r10,<data>(%r2) */ | |
191 | EMIT6_DISP(0xe3a02000, 0x0004, | |
192 | offsetof(struct sk_buff, data)); | |
193 | } | |
194 | } | |
195 | ||
196 | static void bpf_jit_epilogue(struct bpf_jit *jit) | |
197 | { | |
198 | /* Return 0 */ | |
199 | if (jit->seen & SEEN_RET0) { | |
200 | jit->ret0_ip = jit->prg; | |
201 | /* lghi %r2,0 */ | |
202 | EMIT4(0xa7290000); | |
203 | } | |
204 | jit->exit_ip = jit->prg; | |
205 | /* Restore registers */ | |
206 | if (jit->seen & SEEN_DATAREF) | |
207 | /* lmg %r8,%r15,<offset>(%r15) */ | |
208 | EMIT6_DISP(0xeb8ff000, 0x0004, | |
209 | (jit->seen & SEEN_MEM) ? 200 : 168); | |
210 | else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) | |
211 | /* lmg %r12,%r13,120(%r15) */ | |
212 | EMIT6(0xebcdf078, 0x0004); | |
213 | else if (jit->seen & SEEN_XREG) | |
214 | /* lg %r12,120(%r15) */ | |
215 | EMIT6(0xe3c0f078, 0x0004); | |
216 | else if (jit->seen & SEEN_LITERAL) | |
217 | /* lg %r13,128(%r15) */ | |
218 | EMIT6(0xe3d0f080, 0x0004); | |
219 | /* br %r14 */ | |
220 | EMIT2(0x07fe); | |
221 | } | |
222 | ||
223 | /* | |
224 | * make sure we dont leak kernel information to user | |
225 | */ | |
226 | static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) | |
227 | { | |
228 | /* Clear temporary memory if (seen & SEEN_MEM) */ | |
229 | if (jit->seen & SEEN_MEM) | |
230 | /* xc 0(64,%r15),0(%r15) */ | |
231 | EMIT6(0xd73ff000, 0xf000); | |
232 | /* Clear X if (seen & SEEN_XREG) */ | |
233 | if (jit->seen & SEEN_XREG) | |
234 | /* lhi %r12,0 */ | |
235 | EMIT4(0xa7c80000); | |
236 | /* Clear A if the first register does not set it. */ | |
237 | switch (filter[0].code) { | |
238 | case BPF_S_LD_W_ABS: | |
239 | case BPF_S_LD_H_ABS: | |
240 | case BPF_S_LD_B_ABS: | |
241 | case BPF_S_LD_W_LEN: | |
242 | case BPF_S_LD_W_IND: | |
243 | case BPF_S_LD_H_IND: | |
244 | case BPF_S_LD_B_IND: | |
245 | case BPF_S_LDX_B_MSH: | |
246 | case BPF_S_LD_IMM: | |
247 | case BPF_S_LD_MEM: | |
248 | case BPF_S_MISC_TXA: | |
249 | case BPF_S_ANC_PROTOCOL: | |
250 | case BPF_S_ANC_PKTTYPE: | |
251 | case BPF_S_ANC_IFINDEX: | |
252 | case BPF_S_ANC_MARK: | |
253 | case BPF_S_ANC_QUEUE: | |
254 | case BPF_S_ANC_HATYPE: | |
255 | case BPF_S_ANC_RXHASH: | |
256 | case BPF_S_ANC_CPU: | |
257 | case BPF_S_RET_K: | |
258 | /* first instruction sets A register */ | |
259 | break; | |
260 | default: /* A = 0 */ | |
261 | /* lhi %r5,0 */ | |
262 | EMIT4(0xa7580000); | |
263 | } | |
264 | } | |
265 | ||
266 | static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, | |
267 | unsigned int *addrs, int i, int last) | |
268 | { | |
269 | unsigned int K; | |
270 | int offset; | |
271 | unsigned int mask; | |
272 | ||
273 | K = filter->k; | |
274 | switch (filter->code) { | |
275 | case BPF_S_ALU_ADD_X: /* A += X */ | |
276 | jit->seen |= SEEN_XREG; | |
277 | /* ar %r5,%r12 */ | |
278 | EMIT2(0x1a5c); | |
279 | break; | |
280 | case BPF_S_ALU_ADD_K: /* A += K */ | |
281 | if (!K) | |
282 | break; | |
283 | if (K <= 16383) | |
284 | /* ahi %r5,<K> */ | |
285 | EMIT4_IMM(0xa75a0000, K); | |
68d9884d HC |
286 | else if (test_facility(21)) |
287 | /* alfi %r5,<K> */ | |
288 | EMIT6_IMM(0xc25b0000, K); | |
c10302ef MS |
289 | else |
290 | /* a %r5,<d(K)>(%r13) */ | |
291 | EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); | |
292 | break; | |
293 | case BPF_S_ALU_SUB_X: /* A -= X */ | |
294 | jit->seen |= SEEN_XREG; | |
295 | /* sr %r5,%r12 */ | |
296 | EMIT2(0x1b5c); | |
297 | break; | |
298 | case BPF_S_ALU_SUB_K: /* A -= K */ | |
299 | if (!K) | |
300 | break; | |
301 | if (K <= 16384) | |
302 | /* ahi %r5,-K */ | |
303 | EMIT4_IMM(0xa75a0000, -K); | |
68d9884d HC |
304 | else if (test_facility(21)) |
305 | /* alfi %r5,-K */ | |
306 | EMIT6_IMM(0xc25b0000, -K); | |
c10302ef MS |
307 | else |
308 | /* s %r5,<d(K)>(%r13) */ | |
309 | EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); | |
310 | break; | |
311 | case BPF_S_ALU_MUL_X: /* A *= X */ | |
312 | jit->seen |= SEEN_XREG; | |
313 | /* msr %r5,%r12 */ | |
314 | EMIT4(0xb252005c); | |
315 | break; | |
316 | case BPF_S_ALU_MUL_K: /* A *= K */ | |
317 | if (K <= 16383) | |
318 | /* mhi %r5,K */ | |
319 | EMIT4_IMM(0xa75c0000, K); | |
68d9884d HC |
320 | else if (test_facility(34)) |
321 | /* msfi %r5,<K> */ | |
322 | EMIT6_IMM(0xc2510000, K); | |
c10302ef MS |
323 | else |
324 | /* ms %r5,<d(K)>(%r13) */ | |
325 | EMIT4_DISP(0x7150d000, EMIT_CONST(K)); | |
326 | break; | |
327 | case BPF_S_ALU_DIV_X: /* A /= X */ | |
328 | jit->seen |= SEEN_XREG | SEEN_RET0; | |
329 | /* ltr %r12,%r12 */ | |
330 | EMIT2(0x12cc); | |
331 | /* jz <ret0> */ | |
332 | EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); | |
333 | /* lhi %r4,0 */ | |
334 | EMIT4(0xa7480000); | |
335 | /* dr %r4,%r12 */ | |
336 | EMIT2(0x1d4c); | |
337 | break; | |
338 | case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K) */ | |
339 | /* m %r4,<d(K)>(%r13) */ | |
340 | EMIT4_DISP(0x5c40d000, EMIT_CONST(K)); | |
32472745 HC |
341 | /* lr %r5,%r4 */ |
342 | EMIT2(0x1854); | |
343 | break; | |
344 | case BPF_S_ALU_MOD_X: /* A %= X */ | |
345 | jit->seen |= SEEN_XREG | SEEN_RET0; | |
346 | /* ltr %r12,%r12 */ | |
347 | EMIT2(0x12cc); | |
348 | /* jz <ret0> */ | |
349 | EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); | |
350 | /* lhi %r4,0 */ | |
351 | EMIT4(0xa7480000); | |
352 | /* dr %r4,%r12 */ | |
353 | EMIT2(0x1d4c); | |
354 | /* lr %r5,%r4 */ | |
355 | EMIT2(0x1854); | |
356 | break; | |
357 | case BPF_S_ALU_MOD_K: /* A %= K */ | |
358 | /* lhi %r4,0 */ | |
359 | EMIT4(0xa7480000); | |
360 | /* d %r4,<d(K)>(%r13) */ | |
361 | EMIT4_DISP(0x5d40d000, EMIT_CONST(K)); | |
c10302ef MS |
362 | /* lr %r5,%r4 */ |
363 | EMIT2(0x1854); | |
364 | break; | |
365 | case BPF_S_ALU_AND_X: /* A &= X */ | |
366 | jit->seen |= SEEN_XREG; | |
367 | /* nr %r5,%r12 */ | |
368 | EMIT2(0x145c); | |
369 | break; | |
370 | case BPF_S_ALU_AND_K: /* A &= K */ | |
68d9884d HC |
371 | if (test_facility(21)) |
372 | /* nilf %r5,<K> */ | |
373 | EMIT6_IMM(0xc05b0000, K); | |
374 | else | |
375 | /* n %r5,<d(K)>(%r13) */ | |
376 | EMIT4_DISP(0x5450d000, EMIT_CONST(K)); | |
c10302ef MS |
377 | break; |
378 | case BPF_S_ALU_OR_X: /* A |= X */ | |
379 | jit->seen |= SEEN_XREG; | |
380 | /* or %r5,%r12 */ | |
381 | EMIT2(0x165c); | |
382 | break; | |
383 | case BPF_S_ALU_OR_K: /* A |= K */ | |
68d9884d HC |
384 | if (test_facility(21)) |
385 | /* oilf %r5,<K> */ | |
386 | EMIT6_IMM(0xc05d0000, K); | |
387 | else | |
388 | /* o %r5,<d(K)>(%r13) */ | |
389 | EMIT4_DISP(0x5650d000, EMIT_CONST(K)); | |
c10302ef | 390 | break; |
c59eed11 HC |
391 | case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */ |
392 | jit->seen |= SEEN_XREG; | |
393 | /* xr %r5,%r12 */ | |
394 | EMIT2(0x175c); | |
395 | break; | |
c10302ef MS |
396 | case BPF_S_ALU_LSH_X: /* A <<= X; */ |
397 | jit->seen |= SEEN_XREG; | |
398 | /* sll %r5,0(%r12) */ | |
399 | EMIT4(0x8950c000); | |
400 | break; | |
401 | case BPF_S_ALU_LSH_K: /* A <<= K */ | |
402 | if (K == 0) | |
403 | break; | |
404 | /* sll %r5,K */ | |
405 | EMIT4_DISP(0x89500000, K); | |
406 | break; | |
407 | case BPF_S_ALU_RSH_X: /* A >>= X; */ | |
408 | jit->seen |= SEEN_XREG; | |
409 | /* srl %r5,0(%r12) */ | |
410 | EMIT4(0x8850c000); | |
411 | break; | |
412 | case BPF_S_ALU_RSH_K: /* A >>= K; */ | |
413 | if (K == 0) | |
414 | break; | |
415 | /* srl %r5,K */ | |
416 | EMIT4_DISP(0x88500000, K); | |
417 | break; | |
418 | case BPF_S_ALU_NEG: /* A = -A */ | |
419 | /* lnr %r5,%r5 */ | |
420 | EMIT2(0x1155); | |
421 | break; | |
422 | case BPF_S_JMP_JA: /* ip += K */ | |
423 | offset = addrs[i + K] + jit->start - jit->prg; | |
424 | EMIT4_PCREL(0xa7f40000, offset); | |
425 | break; | |
426 | case BPF_S_JMP_JGT_K: /* ip += (A > K) ? jt : jf */ | |
427 | mask = 0x200000; /* jh */ | |
428 | goto kbranch; | |
429 | case BPF_S_JMP_JGE_K: /* ip += (A >= K) ? jt : jf */ | |
430 | mask = 0xa00000; /* jhe */ | |
431 | goto kbranch; | |
432 | case BPF_S_JMP_JEQ_K: /* ip += (A == K) ? jt : jf */ | |
433 | mask = 0x800000; /* je */ | |
434 | kbranch: /* Emit compare if the branch targets are different */ | |
435 | if (filter->jt != filter->jf) { | |
436 | if (K <= 16383) | |
437 | /* chi %r5,<K> */ | |
438 | EMIT4_IMM(0xa75e0000, K); | |
68d9884d HC |
439 | else if (test_facility(21)) |
440 | /* clfi %r5,<K> */ | |
441 | EMIT6_IMM(0xc25f0000, K); | |
c10302ef MS |
442 | else |
443 | /* c %r5,<d(K)>(%r13) */ | |
444 | EMIT4_DISP(0x5950d000, EMIT_CONST(K)); | |
445 | } | |
446 | branch: if (filter->jt == filter->jf) { | |
447 | if (filter->jt == 0) | |
448 | break; | |
449 | /* j <jt> */ | |
450 | offset = addrs[i + filter->jt] + jit->start - jit->prg; | |
451 | EMIT4_PCREL(0xa7f40000, offset); | |
452 | break; | |
453 | } | |
454 | if (filter->jt != 0) { | |
455 | /* brc <mask>,<jt> */ | |
456 | offset = addrs[i + filter->jt] + jit->start - jit->prg; | |
457 | EMIT4_PCREL(0xa7040000 | mask, offset); | |
458 | } | |
459 | if (filter->jf != 0) { | |
460 | /* brc <mask^15>,<jf> */ | |
461 | offset = addrs[i + filter->jf] + jit->start - jit->prg; | |
462 | EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset); | |
463 | } | |
464 | break; | |
465 | case BPF_S_JMP_JSET_K: /* ip += (A & K) ? jt : jf */ | |
466 | mask = 0x700000; /* jnz */ | |
467 | /* Emit test if the branch targets are different */ | |
468 | if (filter->jt != filter->jf) { | |
469 | if (K > 65535) { | |
470 | /* lr %r4,%r5 */ | |
471 | EMIT2(0x1845); | |
472 | /* n %r4,<d(K)>(%r13) */ | |
473 | EMIT4_DISP(0x5440d000, EMIT_CONST(K)); | |
474 | } else | |
475 | /* tmll %r5,K */ | |
476 | EMIT4_IMM(0xa7510000, K); | |
477 | } | |
478 | goto branch; | |
479 | case BPF_S_JMP_JGT_X: /* ip += (A > X) ? jt : jf */ | |
480 | mask = 0x200000; /* jh */ | |
481 | goto xbranch; | |
482 | case BPF_S_JMP_JGE_X: /* ip += (A >= X) ? jt : jf */ | |
483 | mask = 0xa00000; /* jhe */ | |
484 | goto xbranch; | |
485 | case BPF_S_JMP_JEQ_X: /* ip += (A == X) ? jt : jf */ | |
486 | mask = 0x800000; /* je */ | |
487 | xbranch: /* Emit compare if the branch targets are different */ | |
488 | if (filter->jt != filter->jf) { | |
489 | jit->seen |= SEEN_XREG; | |
490 | /* cr %r5,%r12 */ | |
491 | EMIT2(0x195c); | |
492 | } | |
493 | goto branch; | |
494 | case BPF_S_JMP_JSET_X: /* ip += (A & X) ? jt : jf */ | |
495 | mask = 0x700000; /* jnz */ | |
496 | /* Emit test if the branch targets are different */ | |
497 | if (filter->jt != filter->jf) { | |
498 | jit->seen |= SEEN_XREG; | |
499 | /* lr %r4,%r5 */ | |
500 | EMIT2(0x1845); | |
501 | /* nr %r4,%r12 */ | |
502 | EMIT2(0x144c); | |
503 | } | |
504 | goto branch; | |
505 | case BPF_S_LD_W_ABS: /* A = *(u32 *) (skb->data+K) */ | |
506 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD; | |
507 | offset = jit->off_load_word; | |
508 | goto load_abs; | |
509 | case BPF_S_LD_H_ABS: /* A = *(u16 *) (skb->data+K) */ | |
510 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF; | |
511 | offset = jit->off_load_half; | |
512 | goto load_abs; | |
513 | case BPF_S_LD_B_ABS: /* A = *(u8 *) (skb->data+K) */ | |
514 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE; | |
515 | offset = jit->off_load_byte; | |
516 | load_abs: if ((int) K < 0) | |
517 | goto out; | |
518 | call_fn: /* lg %r1,<d(function)>(%r13) */ | |
519 | EMIT6_DISP(0xe310d000, 0x0004, offset); | |
520 | /* l %r3,<d(K)>(%r13) */ | |
521 | EMIT4_DISP(0x5830d000, EMIT_CONST(K)); | |
522 | /* basr %r8,%r1 */ | |
523 | EMIT2(0x0d81); | |
524 | /* jnz <ret0> */ | |
525 | EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg)); | |
526 | break; | |
527 | case BPF_S_LD_W_IND: /* A = *(u32 *) (skb->data+K+X) */ | |
528 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD; | |
529 | offset = jit->off_load_iword; | |
530 | goto call_fn; | |
531 | case BPF_S_LD_H_IND: /* A = *(u16 *) (skb->data+K+X) */ | |
532 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF; | |
533 | offset = jit->off_load_ihalf; | |
534 | goto call_fn; | |
535 | case BPF_S_LD_B_IND: /* A = *(u8 *) (skb->data+K+X) */ | |
536 | jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE; | |
537 | offset = jit->off_load_ibyte; | |
538 | goto call_fn; | |
539 | case BPF_S_LDX_B_MSH: | |
540 | /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ | |
541 | jit->seen |= SEEN_RET0; | |
542 | if ((int) K < 0) { | |
543 | /* j <ret0> */ | |
544 | EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg)); | |
545 | break; | |
546 | } | |
547 | jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH; | |
548 | offset = jit->off_load_bmsh; | |
549 | goto call_fn; | |
550 | case BPF_S_LD_W_LEN: /* A = skb->len; */ | |
551 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); | |
552 | /* l %r5,<d(len)>(%r2) */ | |
553 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len)); | |
554 | break; | |
555 | case BPF_S_LDX_W_LEN: /* X = skb->len; */ | |
556 | jit->seen |= SEEN_XREG; | |
557 | /* l %r12,<d(len)>(%r2) */ | |
558 | EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len)); | |
559 | break; | |
560 | case BPF_S_LD_IMM: /* A = K */ | |
561 | if (K <= 16383) | |
562 | /* lhi %r5,K */ | |
563 | EMIT4_IMM(0xa7580000, K); | |
68d9884d HC |
564 | else if (test_facility(21)) |
565 | /* llilf %r5,<K> */ | |
566 | EMIT6_IMM(0xc05f0000, K); | |
c10302ef MS |
567 | else |
568 | /* l %r5,<d(K)>(%r13) */ | |
569 | EMIT4_DISP(0x5850d000, EMIT_CONST(K)); | |
570 | break; | |
571 | case BPF_S_LDX_IMM: /* X = K */ | |
572 | jit->seen |= SEEN_XREG; | |
573 | if (K <= 16383) | |
574 | /* lhi %r12,<K> */ | |
575 | EMIT4_IMM(0xa7c80000, K); | |
68d9884d HC |
576 | else if (test_facility(21)) |
577 | /* llilf %r12,<K> */ | |
578 | EMIT6_IMM(0xc0cf0000, K); | |
c10302ef MS |
579 | else |
580 | /* l %r12,<d(K)>(%r13) */ | |
581 | EMIT4_DISP(0x58c0d000, EMIT_CONST(K)); | |
582 | break; | |
583 | case BPF_S_LD_MEM: /* A = mem[K] */ | |
584 | jit->seen |= SEEN_MEM; | |
585 | /* l %r5,<K>(%r15) */ | |
586 | EMIT4_DISP(0x5850f000, | |
587 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | |
588 | break; | |
589 | case BPF_S_LDX_MEM: /* X = mem[K] */ | |
590 | jit->seen |= SEEN_XREG | SEEN_MEM; | |
591 | /* l %r12,<K>(%r15) */ | |
592 | EMIT4_DISP(0x58c0f000, | |
593 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | |
594 | break; | |
595 | case BPF_S_MISC_TAX: /* X = A */ | |
596 | jit->seen |= SEEN_XREG; | |
597 | /* lr %r12,%r5 */ | |
598 | EMIT2(0x18c5); | |
599 | break; | |
600 | case BPF_S_MISC_TXA: /* A = X */ | |
601 | jit->seen |= SEEN_XREG; | |
602 | /* lr %r5,%r12 */ | |
603 | EMIT2(0x185c); | |
604 | break; | |
605 | case BPF_S_RET_K: | |
606 | if (K == 0) { | |
607 | jit->seen |= SEEN_RET0; | |
608 | if (last) | |
609 | break; | |
610 | /* j <ret0> */ | |
611 | EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg); | |
612 | } else { | |
613 | if (K <= 16383) | |
614 | /* lghi %r2,K */ | |
615 | EMIT4_IMM(0xa7290000, K); | |
616 | else | |
617 | /* llgf %r2,<K>(%r13) */ | |
618 | EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K)); | |
619 | /* j <exit> */ | |
620 | if (last && !(jit->seen & SEEN_RET0)) | |
621 | break; | |
622 | EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); | |
623 | } | |
624 | break; | |
625 | case BPF_S_RET_A: | |
626 | /* llgfr %r2,%r5 */ | |
627 | EMIT4(0xb9160025); | |
628 | /* j <exit> */ | |
629 | EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); | |
630 | break; | |
631 | case BPF_S_ST: /* mem[K] = A */ | |
632 | jit->seen |= SEEN_MEM; | |
633 | /* st %r5,<K>(%r15) */ | |
634 | EMIT4_DISP(0x5050f000, | |
635 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | |
636 | break; | |
637 | case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ | |
638 | jit->seen |= SEEN_XREG | SEEN_MEM; | |
639 | /* st %r12,<K>(%r15) */ | |
640 | EMIT4_DISP(0x50c0f000, | |
641 | (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); | |
642 | break; | |
643 | case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */ | |
644 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); | |
645 | /* lhi %r5,0 */ | |
646 | EMIT4(0xa7580000); | |
647 | /* icm %r5,3,<d(protocol)>(%r2) */ | |
648 | EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol)); | |
649 | break; | |
650 | case BPF_S_ANC_IFINDEX: /* if (!skb->dev) return 0; | |
651 | * A = skb->dev->ifindex */ | |
652 | BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); | |
653 | jit->seen |= SEEN_RET0; | |
654 | /* lg %r1,<d(dev)>(%r2) */ | |
655 | EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); | |
656 | /* ltgr %r1,%r1 */ | |
657 | EMIT4(0xb9020011); | |
658 | /* jz <ret0> */ | |
659 | EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); | |
660 | /* l %r5,<d(ifindex)>(%r1) */ | |
661 | EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex)); | |
662 | break; | |
663 | case BPF_S_ANC_MARK: /* A = skb->mark */ | |
664 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); | |
665 | /* l %r5,<d(mark)>(%r2) */ | |
666 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark)); | |
667 | break; | |
668 | case BPF_S_ANC_QUEUE: /* A = skb->queue_mapping */ | |
669 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); | |
670 | /* lhi %r5,0 */ | |
671 | EMIT4(0xa7580000); | |
672 | /* icm %r5,3,<d(queue_mapping)>(%r2) */ | |
673 | EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping)); | |
674 | break; | |
675 | case BPF_S_ANC_HATYPE: /* if (!skb->dev) return 0; | |
676 | * A = skb->dev->type */ | |
677 | BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); | |
678 | jit->seen |= SEEN_RET0; | |
679 | /* lg %r1,<d(dev)>(%r2) */ | |
680 | EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); | |
681 | /* ltgr %r1,%r1 */ | |
682 | EMIT4(0xb9020011); | |
683 | /* jz <ret0> */ | |
684 | EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); | |
685 | /* lhi %r5,0 */ | |
686 | EMIT4(0xa7580000); | |
687 | /* icm %r5,3,<d(type)>(%r1) */ | |
688 | EMIT4_DISP(0xbf531000, offsetof(struct net_device, type)); | |
689 | break; | |
690 | case BPF_S_ANC_RXHASH: /* A = skb->rxhash */ | |
691 | BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4); | |
692 | /* l %r5,<d(rxhash)>(%r2) */ | |
693 | EMIT4_DISP(0x58502000, offsetof(struct sk_buff, rxhash)); | |
694 | break; | |
695 | case BPF_S_ANC_CPU: /* A = smp_processor_id() */ | |
696 | #ifdef CONFIG_SMP | |
697 | /* l %r5,<d(cpu_nr)> */ | |
698 | EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr)); | |
699 | #else | |
700 | /* lhi %r5,0 */ | |
701 | EMIT4(0xa7580000); | |
702 | #endif | |
703 | break; | |
704 | default: /* too complex, give up */ | |
705 | goto out; | |
706 | } | |
707 | addrs[i] = jit->prg - jit->start; | |
708 | return 0; | |
709 | out: | |
710 | return -1; | |
711 | } | |
712 | ||
713 | void bpf_jit_compile(struct sk_filter *fp) | |
714 | { | |
715 | unsigned long size, prg_len, lit_len; | |
716 | struct bpf_jit jit, cjit; | |
717 | unsigned int *addrs; | |
718 | int pass, i; | |
719 | ||
720 | if (!bpf_jit_enable) | |
721 | return; | |
722 | addrs = kmalloc(fp->len * sizeof(*addrs), GFP_KERNEL); | |
723 | if (addrs == NULL) | |
724 | return; | |
725 | memset(addrs, 0, fp->len * sizeof(*addrs)); | |
726 | memset(&jit, 0, sizeof(cjit)); | |
727 | memset(&cjit, 0, sizeof(cjit)); | |
728 | ||
729 | for (pass = 0; pass < 10; pass++) { | |
730 | jit.prg = jit.start; | |
731 | jit.lit = jit.mid; | |
732 | ||
733 | bpf_jit_prologue(&jit); | |
734 | bpf_jit_noleaks(&jit, fp->insns); | |
735 | for (i = 0; i < fp->len; i++) { | |
736 | if (bpf_jit_insn(&jit, fp->insns + i, addrs, i, | |
737 | i == fp->len - 1)) | |
738 | goto out; | |
739 | } | |
740 | bpf_jit_epilogue(&jit); | |
741 | if (jit.start) { | |
742 | WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit); | |
743 | if (memcmp(&jit, &cjit, sizeof(jit)) == 0) | |
744 | break; | |
745 | } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) { | |
746 | prg_len = jit.prg - jit.start; | |
747 | lit_len = jit.lit - jit.mid; | |
748 | size = max_t(unsigned long, prg_len + lit_len, | |
749 | sizeof(struct work_struct)); | |
750 | if (size >= BPF_SIZE_MAX) | |
751 | goto out; | |
752 | jit.start = module_alloc(size); | |
753 | if (!jit.start) | |
754 | goto out; | |
755 | jit.prg = jit.mid = jit.start + prg_len; | |
756 | jit.lit = jit.end = jit.start + prg_len + lit_len; | |
757 | jit.base_ip += (unsigned long) jit.start; | |
758 | jit.exit_ip += (unsigned long) jit.start; | |
759 | jit.ret0_ip += (unsigned long) jit.start; | |
760 | } | |
761 | cjit = jit; | |
762 | } | |
763 | if (bpf_jit_enable > 1) { | |
764 | pr_err("flen=%d proglen=%lu pass=%d image=%p\n", | |
765 | fp->len, jit.end - jit.start, pass, jit.start); | |
766 | if (jit.start) { | |
767 | printk(KERN_ERR "JIT code:\n"); | |
768 | print_fn_code(jit.start, jit.mid - jit.start); | |
769 | print_hex_dump(KERN_ERR, "JIT literals:\n", | |
770 | DUMP_PREFIX_ADDRESS, 16, 1, | |
771 | jit.mid, jit.end - jit.mid, false); | |
772 | } | |
773 | } | |
774 | if (jit.start) | |
775 | fp->bpf_func = (void *) jit.start; | |
776 | out: | |
777 | kfree(addrs); | |
778 | } | |
779 | ||
780 | static void jit_free_defer(struct work_struct *arg) | |
781 | { | |
782 | module_free(NULL, arg); | |
783 | } | |
784 | ||
785 | /* run from softirq, we must use a work_struct to call | |
786 | * module_free() from process context | |
787 | */ | |
788 | void bpf_jit_free(struct sk_filter *fp) | |
789 | { | |
790 | struct work_struct *work; | |
791 | ||
792 | if (fp->bpf_func == sk_run_filter) | |
793 | return; | |
794 | work = (struct work_struct *)fp->bpf_func; | |
795 | INIT_WORK(work, jit_free_defer); | |
796 | schedule_work(work); | |
797 | } |