Commit | Line | Data |
---|---|---|
0a14842f ED |
1 | /* bpf_jit.S : BPF JIT helper functions |
2 | * | |
3 | * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com) | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or | |
6 | * modify it under the terms of the GNU General Public License | |
7 | * as published by the Free Software Foundation; version 2 | |
8 | * of the License. | |
9 | */ | |
10 | #include <linux/linkage.h> | |
d21001cc | 11 | #include <asm/frame.h> |
0a14842f ED |
12 | |
13 | /* | |
14 | * Calling convention : | |
62258278 | 15 | * rbx : skb pointer (callee saved) |
0a14842f | 16 | * esi : offset of byte(s) to fetch in skb (can be scratched) |
62258278 | 17 | * r10 : copy of skb->data |
0a14842f ED |
18 | * r9d : hlen = skb->len - skb->data_len |
19 | */ | |
62258278 | 20 | #define SKBDATA %r10 |
a998d434 | 21 | #define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */ |
62258278 AS |
22 | #define MAX_BPF_STACK (512 /* from filter.h */ + \ |
23 | 32 /* space for rbx,r13,r14,r15 */ + \ | |
24 | 8 /* space for skb_copy_bits */) | |
0a14842f | 25 | |
2d8fe90a JP |
26 | #define FUNC(name) \ |
27 | .globl name; \ | |
28 | .type name, @function; \ | |
29 | name: | |
0a14842f | 30 | |
2d8fe90a | 31 | FUNC(sk_load_word) |
a998d434 JS |
32 | test %esi,%esi |
33 | js bpf_slow_path_word_neg | |
34 | ||
2d8fe90a | 35 | FUNC(sk_load_word_positive_offset) |
0a14842f ED |
36 | mov %r9d,%eax # hlen |
37 | sub %esi,%eax # hlen - offset | |
38 | cmp $3,%eax | |
39 | jle bpf_slow_path_word | |
40 | mov (SKBDATA,%rsi),%eax | |
41 | bswap %eax /* ntohl() */ | |
42 | ret | |
43 | ||
2d8fe90a | 44 | FUNC(sk_load_half) |
a998d434 JS |
45 | test %esi,%esi |
46 | js bpf_slow_path_half_neg | |
47 | ||
2d8fe90a | 48 | FUNC(sk_load_half_positive_offset) |
0a14842f ED |
49 | mov %r9d,%eax |
50 | sub %esi,%eax # hlen - offset | |
51 | cmp $1,%eax | |
52 | jle bpf_slow_path_half | |
53 | movzwl (SKBDATA,%rsi),%eax | |
54 | rol $8,%ax # ntohs() | |
55 | ret | |
56 | ||
2d8fe90a | 57 | FUNC(sk_load_byte) |
a998d434 JS |
58 | test %esi,%esi |
59 | js bpf_slow_path_byte_neg | |
60 | ||
2d8fe90a | 61 | FUNC(sk_load_byte_positive_offset) |
0a14842f ED |
62 | cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */ |
63 | jle bpf_slow_path_byte | |
64 | movzbl (SKBDATA,%rsi),%eax | |
65 | ret | |
66 | ||
0a14842f ED |
67 | /* rsi contains offset and can be scratched */ |
68 | #define bpf_slow_path_common(LEN) \ | |
d21001cc JP |
69 | lea -MAX_BPF_STACK + 32(%rbp), %rdx;\ |
70 | FRAME_BEGIN; \ | |
62258278 | 71 | mov %rbx, %rdi; /* arg1 == skb */ \ |
0a14842f ED |
72 | push %r9; \ |
73 | push SKBDATA; \ | |
74 | /* rsi already has offset */ \ | |
75 | mov $LEN,%ecx; /* len */ \ | |
0a14842f ED |
76 | call skb_copy_bits; \ |
77 | test %eax,%eax; \ | |
78 | pop SKBDATA; \ | |
d21001cc JP |
79 | pop %r9; \ |
80 | FRAME_END | |
0a14842f ED |
81 | |
82 | ||
83 | bpf_slow_path_word: | |
84 | bpf_slow_path_common(4) | |
85 | js bpf_error | |
62258278 | 86 | mov - MAX_BPF_STACK + 32(%rbp),%eax |
0a14842f ED |
87 | bswap %eax |
88 | ret | |
89 | ||
90 | bpf_slow_path_half: | |
91 | bpf_slow_path_common(2) | |
92 | js bpf_error | |
62258278 | 93 | mov - MAX_BPF_STACK + 32(%rbp),%ax |
0a14842f ED |
94 | rol $8,%ax |
95 | movzwl %ax,%eax | |
96 | ret | |
97 | ||
98 | bpf_slow_path_byte: | |
99 | bpf_slow_path_common(1) | |
100 | js bpf_error | |
62258278 | 101 | movzbl - MAX_BPF_STACK + 32(%rbp),%eax |
0a14842f | 102 | ret |
a998d434 JS |
103 | |
104 | #define sk_negative_common(SIZE) \ | |
d21001cc | 105 | FRAME_BEGIN; \ |
62258278 | 106 | mov %rbx, %rdi; /* arg1 == skb */ \ |
a998d434 JS |
107 | push %r9; \ |
108 | push SKBDATA; \ | |
109 | /* rsi already has offset */ \ | |
fdfaf64e | 110 | mov $SIZE,%edx; /* size */ \ |
a998d434 JS |
111 | call bpf_internal_load_pointer_neg_helper; \ |
112 | test %rax,%rax; \ | |
113 | pop SKBDATA; \ | |
114 | pop %r9; \ | |
d21001cc | 115 | FRAME_END; \ |
a998d434 JS |
116 | jz bpf_error |
117 | ||
a998d434 JS |
118 | bpf_slow_path_word_neg: |
119 | cmp SKF_MAX_NEG_OFF, %esi /* test range */ | |
120 | jl bpf_error /* offset lower -> error */ | |
2d8fe90a JP |
121 | |
122 | FUNC(sk_load_word_negative_offset) | |
a998d434 JS |
123 | sk_negative_common(4) |
124 | mov (%rax), %eax | |
125 | bswap %eax | |
126 | ret | |
127 | ||
128 | bpf_slow_path_half_neg: | |
129 | cmp SKF_MAX_NEG_OFF, %esi | |
130 | jl bpf_error | |
2d8fe90a JP |
131 | |
132 | FUNC(sk_load_half_negative_offset) | |
a998d434 JS |
133 | sk_negative_common(2) |
134 | mov (%rax),%ax | |
135 | rol $8,%ax | |
136 | movzwl %ax,%eax | |
137 | ret | |
138 | ||
139 | bpf_slow_path_byte_neg: | |
140 | cmp SKF_MAX_NEG_OFF, %esi | |
141 | jl bpf_error | |
2d8fe90a JP |
142 | |
143 | FUNC(sk_load_byte_negative_offset) | |
a998d434 JS |
144 | sk_negative_common(1) |
145 | movzbl (%rax), %eax | |
146 | ret | |
147 | ||
a998d434 JS |
148 | bpf_error: |
149 | # force a return 0 from jit handler | |
62258278 AS |
150 | xor %eax,%eax |
151 | mov - MAX_BPF_STACK(%rbp),%rbx | |
152 | mov - MAX_BPF_STACK + 8(%rbp),%r13 | |
153 | mov - MAX_BPF_STACK + 16(%rbp),%r14 | |
154 | mov - MAX_BPF_STACK + 24(%rbp),%r15 | |
a998d434 JS |
155 | leaveq |
156 | ret |