Commit | Line | Data |
---|---|---|
5ead97c8 JF |
1 | /* |
2 | * Xen hypercall batching. | |
3 | * | |
4 | * Xen allows multiple hypercalls to be issued at once, using the | |
5 | * multicall interface. This allows the cost of trapping into the | |
6 | * hypervisor to be amortized over several calls. | |
7 | * | |
8 | * This file implements a simple interface for multicalls. There's a | |
9 | * per-cpu buffer of outstanding multicalls. When you want to queue a | |
10 | * multicall for issuing, you can allocate a multicall slot for the | |
11 | * call and its arguments, along with storage for space which is | |
12 | * pointed to by the arguments (for passing pointers to structures, | |
13 | * etc). When the multicall is actually issued, all the space for the | |
14 | * commands and allocated memory is freed for reuse. | |
15 | * | |
16 | * Multicalls are flushed whenever any of the buffers get full, or | |
17 | * when explicitly requested. There's no way to get per-multicall | |
18 | * return results back. It will BUG if any of the multicalls fail. | |
19 | * | |
20 | * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 | |
21 | */ | |
22 | #include <linux/percpu.h> | |
f120f13e | 23 | #include <linux/hardirq.h> |
994025ca | 24 | #include <linux/debugfs.h> |
5ead97c8 JF |
25 | |
26 | #include <asm/xen/hypercall.h> | |
27 | ||
28 | #include "multicalls.h" | |
994025ca JF |
29 | #include "debugfs.h" |
30 | ||
31 | #define MC_BATCH 32 | |
5ead97c8 | 32 | |
a122d623 JF |
33 | #define MC_DEBUG 1 |
34 | ||
400d3494 | 35 | #define MC_ARGS (MC_BATCH * 16) |
5ead97c8 | 36 | |
994025ca | 37 | |
5ead97c8 JF |
38 | struct mc_buffer { |
39 | struct multicall_entry entries[MC_BATCH]; | |
a122d623 JF |
40 | #if MC_DEBUG |
41 | struct multicall_entry debug[MC_BATCH]; | |
b93d51dc | 42 | void *caller[MC_BATCH]; |
a122d623 | 43 | #endif |
400d3494 | 44 | unsigned char args[MC_ARGS]; |
91e0c5f3 JF |
45 | struct callback { |
46 | void (*fn)(void *); | |
47 | void *data; | |
48 | } callbacks[MC_BATCH]; | |
49 | unsigned mcidx, argidx, cbidx; | |
5ead97c8 JF |
50 | }; |
51 | ||
52 | static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); | |
53 | DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); | |
54 | ||
994025ca JF |
55 | /* flush reasons 0- slots, 1- args, 2- callbacks */ |
56 | enum flush_reasons | |
57 | { | |
58 | FL_SLOTS, | |
59 | FL_ARGS, | |
60 | FL_CALLBACKS, | |
61 | ||
62 | FL_N_REASONS | |
63 | }; | |
64 | ||
65 | #ifdef CONFIG_XEN_DEBUG_FS | |
66 | #define NHYPERCALLS 40 /* not really */ | |
67 | ||
68 | static struct { | |
69 | unsigned histo[MC_BATCH+1]; | |
70 | ||
71 | unsigned issued; | |
72 | unsigned arg_total; | |
73 | unsigned hypercalls; | |
74 | unsigned histo_hypercalls[NHYPERCALLS]; | |
75 | ||
76 | unsigned flush[FL_N_REASONS]; | |
77 | } mc_stats; | |
78 | ||
79 | static u8 zero_stats; | |
80 | ||
81 | static inline void check_zero(void) | |
82 | { | |
83 | if (unlikely(zero_stats)) { | |
84 | memset(&mc_stats, 0, sizeof(mc_stats)); | |
85 | zero_stats = 0; | |
86 | } | |
87 | } | |
88 | ||
89 | static void mc_add_stats(const struct mc_buffer *mc) | |
90 | { | |
91 | int i; | |
92 | ||
93 | check_zero(); | |
94 | ||
95 | mc_stats.issued++; | |
96 | mc_stats.hypercalls += mc->mcidx; | |
97 | mc_stats.arg_total += mc->argidx; | |
98 | ||
99 | mc_stats.histo[mc->mcidx]++; | |
100 | for(i = 0; i < mc->mcidx; i++) { | |
101 | unsigned op = mc->entries[i].op; | |
102 | if (op < NHYPERCALLS) | |
103 | mc_stats.histo_hypercalls[op]++; | |
104 | } | |
105 | } | |
106 | ||
107 | static void mc_stats_flush(enum flush_reasons idx) | |
108 | { | |
109 | check_zero(); | |
110 | ||
111 | mc_stats.flush[idx]++; | |
112 | } | |
113 | ||
114 | #else /* !CONFIG_XEN_DEBUG_FS */ | |
115 | ||
116 | static inline void mc_add_stats(const struct mc_buffer *mc) | |
117 | { | |
118 | } | |
119 | ||
120 | static inline void mc_stats_flush(enum flush_reasons idx) | |
121 | { | |
122 | } | |
123 | #endif /* CONFIG_XEN_DEBUG_FS */ | |
124 | ||
5ead97c8 JF |
125 | void xen_mc_flush(void) |
126 | { | |
f120f13e | 127 | struct mc_buffer *b = &__get_cpu_var(mc_buffer); |
5ead97c8 JF |
128 | int ret = 0; |
129 | unsigned long flags; | |
91e0c5f3 | 130 | int i; |
5ead97c8 | 131 | |
f120f13e JF |
132 | BUG_ON(preemptible()); |
133 | ||
5ead97c8 JF |
134 | /* Disable interrupts in case someone comes in and queues |
135 | something in the middle */ | |
136 | local_irq_save(flags); | |
137 | ||
994025ca JF |
138 | mc_add_stats(b); |
139 | ||
5ead97c8 | 140 | if (b->mcidx) { |
a122d623 JF |
141 | #if MC_DEBUG |
142 | memcpy(b->debug, b->entries, | |
143 | b->mcidx * sizeof(struct multicall_entry)); | |
144 | #endif | |
145 | ||
5ead97c8 JF |
146 | if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0) |
147 | BUG(); | |
148 | for (i = 0; i < b->mcidx; i++) | |
149 | if (b->entries[i].result < 0) | |
150 | ret++; | |
a122d623 JF |
151 | |
152 | #if MC_DEBUG | |
153 | if (ret) { | |
154 | printk(KERN_ERR "%d multicall(s) failed: cpu %d\n", | |
155 | ret, smp_processor_id()); | |
8ba6c2b0 | 156 | dump_stack(); |
7ebed39f | 157 | for (i = 0; i < b->mcidx; i++) { |
b93d51dc | 158 | printk(KERN_DEBUG " call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n", |
a122d623 JF |
159 | i+1, b->mcidx, |
160 | b->debug[i].op, | |
161 | b->debug[i].args[0], | |
b93d51dc IC |
162 | b->entries[i].result, |
163 | b->caller[i]); | |
a122d623 JF |
164 | } |
165 | } | |
166 | #endif | |
167 | ||
5ead97c8 JF |
168 | b->mcidx = 0; |
169 | b->argidx = 0; | |
170 | } else | |
171 | BUG_ON(b->argidx != 0); | |
172 | ||
7ebed39f | 173 | for (i = 0; i < b->cbidx; i++) { |
91e0c5f3 JF |
174 | struct callback *cb = &b->callbacks[i]; |
175 | ||
176 | (*cb->fn)(cb->data); | |
177 | } | |
178 | b->cbidx = 0; | |
179 | ||
c9960863 JF |
180 | local_irq_restore(flags); |
181 | ||
3d39e9d0 | 182 | WARN_ON(ret); |
5ead97c8 JF |
183 | } |
184 | ||
185 | struct multicall_space __xen_mc_entry(size_t args) | |
186 | { | |
f120f13e | 187 | struct mc_buffer *b = &__get_cpu_var(mc_buffer); |
5ead97c8 | 188 | struct multicall_space ret; |
400d3494 | 189 | unsigned argidx = roundup(b->argidx, sizeof(u64)); |
5ead97c8 | 190 | |
f120f13e | 191 | BUG_ON(preemptible()); |
400d3494 | 192 | BUG_ON(b->argidx > MC_ARGS); |
5ead97c8 JF |
193 | |
194 | if (b->mcidx == MC_BATCH || | |
400d3494 | 195 | (argidx + args) > MC_ARGS) { |
994025ca | 196 | mc_stats_flush(b->mcidx == MC_BATCH ? FL_SLOTS : FL_ARGS); |
5ead97c8 | 197 | xen_mc_flush(); |
400d3494 JF |
198 | argidx = roundup(b->argidx, sizeof(u64)); |
199 | } | |
5ead97c8 JF |
200 | |
201 | ret.mc = &b->entries[b->mcidx]; | |
b93d51dc IC |
202 | #ifdef MC_DEBUG |
203 | b->caller[b->mcidx] = __builtin_return_address(0); | |
204 | #endif | |
5ead97c8 | 205 | b->mcidx++; |
400d3494 JF |
206 | ret.args = &b->args[argidx]; |
207 | b->argidx = argidx + args; | |
208 | ||
209 | BUG_ON(b->argidx > MC_ARGS); | |
210 | return ret; | |
211 | } | |
212 | ||
213 | struct multicall_space xen_mc_extend_args(unsigned long op, size_t size) | |
214 | { | |
215 | struct mc_buffer *b = &__get_cpu_var(mc_buffer); | |
216 | struct multicall_space ret = { NULL, NULL }; | |
217 | ||
218 | BUG_ON(preemptible()); | |
219 | BUG_ON(b->argidx > MC_ARGS); | |
220 | ||
221 | if (b->mcidx == 0) | |
222 | return ret; | |
223 | ||
224 | if (b->entries[b->mcidx - 1].op != op) | |
225 | return ret; | |
226 | ||
227 | if ((b->argidx + size) > MC_ARGS) | |
228 | return ret; | |
229 | ||
230 | ret.mc = &b->entries[b->mcidx - 1]; | |
5ead97c8 | 231 | ret.args = &b->args[b->argidx]; |
400d3494 | 232 | b->argidx += size; |
5ead97c8 | 233 | |
400d3494 | 234 | BUG_ON(b->argidx > MC_ARGS); |
5ead97c8 JF |
235 | return ret; |
236 | } | |
91e0c5f3 JF |
237 | |
238 | void xen_mc_callback(void (*fn)(void *), void *data) | |
239 | { | |
240 | struct mc_buffer *b = &__get_cpu_var(mc_buffer); | |
241 | struct callback *cb; | |
242 | ||
994025ca JF |
243 | if (b->cbidx == MC_BATCH) { |
244 | mc_stats_flush(FL_CALLBACKS); | |
91e0c5f3 | 245 | xen_mc_flush(); |
994025ca | 246 | } |
91e0c5f3 JF |
247 | |
248 | cb = &b->callbacks[b->cbidx++]; | |
249 | cb->fn = fn; | |
250 | cb->data = data; | |
251 | } | |
994025ca JF |
252 | |
253 | #ifdef CONFIG_XEN_DEBUG_FS | |
254 | ||
255 | static struct dentry *d_mc_debug; | |
256 | ||
257 | static int __init xen_mc_debugfs(void) | |
258 | { | |
259 | struct dentry *d_xen = xen_init_debugfs(); | |
260 | ||
261 | if (d_xen == NULL) | |
262 | return -ENOMEM; | |
263 | ||
264 | d_mc_debug = debugfs_create_dir("multicalls", d_xen); | |
265 | ||
266 | debugfs_create_u8("zero_stats", 0644, d_mc_debug, &zero_stats); | |
267 | ||
268 | debugfs_create_u32("batches", 0444, d_mc_debug, &mc_stats.issued); | |
269 | debugfs_create_u32("hypercalls", 0444, d_mc_debug, &mc_stats.hypercalls); | |
270 | debugfs_create_u32("arg_total", 0444, d_mc_debug, &mc_stats.arg_total); | |
271 | ||
272 | xen_debugfs_create_u32_array("batch_histo", 0444, d_mc_debug, | |
273 | mc_stats.histo, MC_BATCH); | |
274 | xen_debugfs_create_u32_array("hypercall_histo", 0444, d_mc_debug, | |
275 | mc_stats.histo_hypercalls, NHYPERCALLS); | |
276 | xen_debugfs_create_u32_array("flush_reasons", 0444, d_mc_debug, | |
277 | mc_stats.flush, FL_N_REASONS); | |
278 | ||
279 | return 0; | |
280 | } | |
281 | fs_initcall(xen_mc_debugfs); | |
282 | ||
283 | #endif /* CONFIG_XEN_DEBUG_FS */ |