Commit | Line | Data |
---|---|---|
eee3af4a MM |
1 | /* |
2 | * Debug Store support | |
3 | * | |
4 | * This provides a low-level interface to the hardware's Debug Store | |
93fa7636 | 5 | * feature that is used for branch trace store (BTS) and |
eee3af4a MM |
6 | * precise-event based sampling (PEBS). |
7 | * | |
93fa7636 MM |
8 | * It manages: |
9 | * - per-thread and per-cpu allocation of BTS and PEBS | |
10 | * - buffer memory allocation (optional) | |
11 | * - buffer overflow handling | |
12 | * - buffer access | |
eee3af4a | 13 | * |
93fa7636 MM |
14 | * It assumes: |
15 | * - get_task_struct on all parameter tasks | |
16 | * - current is allowed to trace parameter tasks | |
eee3af4a | 17 | * |
eee3af4a | 18 | * |
93fa7636 MM |
19 | * Copyright (C) 2007-2008 Intel Corporation. |
20 | * Markus Metzger <markus.t.metzger@intel.com>, 2007-2008 | |
eee3af4a MM |
21 | */ |
22 | ||
93fa7636 | 23 | |
eee3af4a MM |
24 | #include <asm/ds.h> |
25 | ||
26 | #include <linux/errno.h> | |
27 | #include <linux/string.h> | |
28 | #include <linux/slab.h> | |
93fa7636 | 29 | #include <linux/sched.h> |
3c933904 | 30 | #include <linux/mm.h> |
93fa7636 MM |
31 | |
32 | ||
33 | /* | |
34 | * The configuration for a particular DS hardware implementation. | |
35 | */ | |
36 | struct ds_configuration { | |
37 | /* the size of the DS structure in bytes */ | |
38 | unsigned char sizeof_ds; | |
39 | /* the size of one pointer-typed field in the DS structure in bytes; | |
40 | this covers the first 8 fields related to buffer management. */ | |
41 | unsigned char sizeof_field; | |
42 | /* the size of a BTS/PEBS record in bytes */ | |
43 | unsigned char sizeof_rec[2]; | |
44 | }; | |
45 | static struct ds_configuration ds_cfg; | |
eee3af4a MM |
46 | |
47 | ||
48 | /* | |
49 | * Debug Store (DS) save area configuration (see Intel64 and IA32 | |
50 | * Architectures Software Developer's Manual, section 18.5) | |
51 | * | |
52 | * The DS configuration consists of the following fields; different | |
53 | * architetures vary in the size of those fields. | |
54 | * - double-word aligned base linear address of the BTS buffer | |
55 | * - write pointer into the BTS buffer | |
56 | * - end linear address of the BTS buffer (one byte beyond the end of | |
57 | * the buffer) | |
58 | * - interrupt pointer into BTS buffer | |
59 | * (interrupt occurs when write pointer passes interrupt pointer) | |
60 | * - double-word aligned base linear address of the PEBS buffer | |
61 | * - write pointer into the PEBS buffer | |
62 | * - end linear address of the PEBS buffer (one byte beyond the end of | |
63 | * the buffer) | |
64 | * - interrupt pointer into PEBS buffer | |
65 | * (interrupt occurs when write pointer passes interrupt pointer) | |
66 | * - value to which counter is reset following counter overflow | |
67 | * | |
93fa7636 MM |
68 | * Later architectures use 64bit pointers throughout, whereas earlier |
69 | * architectures use 32bit pointers in 32bit mode. | |
eee3af4a | 70 | * |
eee3af4a | 71 | * |
93fa7636 MM |
72 | * We compute the base address for the first 8 fields based on: |
73 | * - the field size stored in the DS configuration | |
74 | * - the relative field position | |
75 | * - an offset giving the start of the respective region | |
eee3af4a | 76 | * |
93fa7636 MM |
77 | * This offset is further used to index various arrays holding |
78 | * information for BTS and PEBS at the respective index. | |
eee3af4a | 79 | * |
93fa7636 MM |
80 | * On later 32bit processors, we only access the lower 32bit of the |
81 | * 64bit pointer fields. The upper halves will be zeroed out. | |
eee3af4a MM |
82 | */ |
83 | ||
93fa7636 MM |
84 | enum ds_field { |
85 | ds_buffer_base = 0, | |
86 | ds_index, | |
87 | ds_absolute_maximum, | |
88 | ds_interrupt_threshold, | |
89 | }; | |
eee3af4a | 90 | |
93fa7636 MM |
91 | enum ds_qualifier { |
92 | ds_bts = 0, | |
93 | ds_pebs | |
eee3af4a MM |
94 | }; |
95 | ||
93fa7636 MM |
96 | static inline unsigned long ds_get(const unsigned char *base, |
97 | enum ds_qualifier qual, enum ds_field field) | |
98 | { | |
99 | base += (ds_cfg.sizeof_field * (field + (4 * qual))); | |
100 | return *(unsigned long *)base; | |
101 | } | |
102 | ||
103 | static inline void ds_set(unsigned char *base, enum ds_qualifier qual, | |
104 | enum ds_field field, unsigned long value) | |
105 | { | |
106 | base += (ds_cfg.sizeof_field * (field + (4 * qual))); | |
107 | (*(unsigned long *)base) = value; | |
108 | } | |
109 | ||
110 | ||
eee3af4a | 111 | /* |
93fa7636 MM |
112 | * Locking is done only for allocating BTS or PEBS resources and for |
113 | * guarding context and buffer memory allocation. | |
114 | * | |
115 | * Most functions require the current task to own the ds context part | |
116 | * they are going to access. All the locking is done when validating | |
117 | * access to the context. | |
eee3af4a | 118 | */ |
93fa7636 | 119 | static spinlock_t ds_lock = __SPIN_LOCK_UNLOCKED(ds_lock); |
eee3af4a MM |
120 | |
121 | /* | |
93fa7636 MM |
122 | * Validate that the current task is allowed to access the BTS/PEBS |
123 | * buffer of the parameter task. | |
124 | * | |
125 | * Returns 0, if access is granted; -Eerrno, otherwise. | |
eee3af4a | 126 | */ |
93fa7636 MM |
127 | static inline int ds_validate_access(struct ds_context *context, |
128 | enum ds_qualifier qual) | |
129 | { | |
130 | if (!context) | |
131 | return -EPERM; | |
132 | ||
133 | if (context->owner[qual] == current) | |
134 | return 0; | |
135 | ||
136 | return -EPERM; | |
137 | } | |
138 | ||
eee3af4a MM |
139 | |
140 | /* | |
93fa7636 MM |
141 | * We either support (system-wide) per-cpu or per-thread allocation. |
142 | * We distinguish the two based on the task_struct pointer, where a | |
143 | * NULL pointer indicates per-cpu allocation for the current cpu. | |
144 | * | |
145 | * Allocations are use-counted. As soon as resources are allocated, | |
146 | * further allocations must be of the same type (per-cpu or | |
147 | * per-thread). We model this by counting allocations (i.e. the number | |
148 | * of tracers of a certain type) for one type negatively: | |
149 | * =0 no tracers | |
150 | * >0 number of per-thread tracers | |
151 | * <0 number of per-cpu tracers | |
152 | * | |
153 | * The below functions to get and put tracers and to check the | |
154 | * allocation type require the ds_lock to be held by the caller. | |
155 | * | |
156 | * Tracers essentially gives the number of ds contexts for a certain | |
157 | * type of allocation. | |
eee3af4a | 158 | */ |
93fa7636 MM |
159 | static long tracers; |
160 | ||
161 | static inline void get_tracer(struct task_struct *task) | |
eee3af4a | 162 | { |
93fa7636 | 163 | tracers += (task ? 1 : -1); |
eee3af4a | 164 | } |
93fa7636 MM |
165 | |
166 | static inline void put_tracer(struct task_struct *task) | |
eee3af4a | 167 | { |
93fa7636 | 168 | tracers -= (task ? 1 : -1); |
eee3af4a | 169 | } |
93fa7636 MM |
170 | |
171 | static inline int check_tracer(struct task_struct *task) | |
eee3af4a | 172 | { |
93fa7636 | 173 | return (task ? (tracers >= 0) : (tracers <= 0)); |
eee3af4a | 174 | } |
93fa7636 MM |
175 | |
176 | ||
177 | /* | |
178 | * The DS context is either attached to a thread or to a cpu: | |
179 | * - in the former case, the thread_struct contains a pointer to the | |
180 | * attached context. | |
181 | * - in the latter case, we use a static array of per-cpu context | |
182 | * pointers. | |
183 | * | |
184 | * Contexts are use-counted. They are allocated on first access and | |
185 | * deallocated when the last user puts the context. | |
186 | * | |
187 | * We distinguish between an allocating and a non-allocating get of a | |
188 | * context: | |
189 | * - the allocating get is used for requesting BTS/PEBS resources. It | |
190 | * requires the caller to hold the global ds_lock. | |
191 | * - the non-allocating get is used for all other cases. A | |
192 | * non-existing context indicates an error. It acquires and releases | |
193 | * the ds_lock itself for obtaining the context. | |
194 | * | |
195 | * A context and its DS configuration are allocated and deallocated | |
196 | * together. A context always has a DS configuration of the | |
197 | * appropriate size. | |
198 | */ | |
199 | static DEFINE_PER_CPU(struct ds_context *, system_context); | |
200 | ||
201 | #define this_system_context per_cpu(system_context, smp_processor_id()) | |
202 | ||
203 | /* | |
204 | * Returns the pointer to the parameter task's context or to the | |
205 | * system-wide context, if task is NULL. | |
206 | * | |
207 | * Increases the use count of the returned context, if not NULL. | |
208 | */ | |
209 | static inline struct ds_context *ds_get_context(struct task_struct *task) | |
eee3af4a | 210 | { |
93fa7636 | 211 | struct ds_context *context; |
de90add3 | 212 | unsigned long irq; |
93fa7636 | 213 | |
de90add3 | 214 | spin_lock_irqsave(&ds_lock, irq); |
93fa7636 MM |
215 | |
216 | context = (task ? task->thread.ds_ctx : this_system_context); | |
217 | if (context) | |
218 | context->count++; | |
219 | ||
de90add3 | 220 | spin_unlock_irqrestore(&ds_lock, irq); |
93fa7636 MM |
221 | |
222 | return context; | |
eee3af4a | 223 | } |
93fa7636 MM |
224 | |
225 | /* | |
226 | * Same as ds_get_context, but allocates the context and it's DS | |
227 | * structure, if necessary; returns NULL; if out of memory. | |
93fa7636 MM |
228 | */ |
229 | static inline struct ds_context *ds_alloc_context(struct task_struct *task) | |
eee3af4a | 230 | { |
93fa7636 MM |
231 | struct ds_context **p_context = |
232 | (task ? &task->thread.ds_ctx : &this_system_context); | |
233 | struct ds_context *context = *p_context; | |
de90add3 | 234 | unsigned long irq; |
93fa7636 MM |
235 | |
236 | if (!context) { | |
237 | context = kzalloc(sizeof(*context), GFP_KERNEL); | |
de90add3 | 238 | if (!context) |
573da422 | 239 | return NULL; |
93fa7636 MM |
240 | |
241 | context->ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL); | |
242 | if (!context->ds) { | |
243 | kfree(context); | |
573da422 | 244 | return NULL; |
93fa7636 MM |
245 | } |
246 | ||
de90add3 MM |
247 | spin_lock_irqsave(&ds_lock, irq); |
248 | ||
10db4ef7 IM |
249 | if (*p_context) { |
250 | kfree(context->ds); | |
251 | kfree(context); | |
93fa7636 | 252 | |
de90add3 MM |
253 | context = *p_context; |
254 | } else { | |
255 | *p_context = context; | |
93fa7636 | 256 | |
de90add3 MM |
257 | context->this = p_context; |
258 | context->task = task; | |
93fa7636 | 259 | |
de90add3 MM |
260 | if (task) |
261 | set_tsk_thread_flag(task, TIF_DS_AREA_MSR); | |
93fa7636 | 262 | |
de90add3 MM |
263 | if (!task || (task == current)) |
264 | wrmsrl(MSR_IA32_DS_AREA, | |
265 | (unsigned long)context->ds); | |
266 | } | |
267 | spin_unlock_irqrestore(&ds_lock, irq); | |
93fa7636 MM |
268 | } |
269 | ||
270 | context->count++; | |
271 | ||
272 | return context; | |
eee3af4a | 273 | } |
93fa7636 MM |
274 | |
275 | /* | |
276 | * Decreases the use count of the parameter context, if not NULL. | |
277 | * Deallocates the context, if the use count reaches zero. | |
278 | */ | |
279 | static inline void ds_put_context(struct ds_context *context) | |
eee3af4a | 280 | { |
de90add3 MM |
281 | unsigned long irq; |
282 | ||
93fa7636 MM |
283 | if (!context) |
284 | return; | |
285 | ||
de90add3 | 286 | spin_lock_irqsave(&ds_lock, irq); |
93fa7636 MM |
287 | |
288 | if (--context->count) | |
289 | goto out; | |
290 | ||
573da422 | 291 | *(context->this) = NULL; |
93fa7636 MM |
292 | |
293 | if (context->task) | |
294 | clear_tsk_thread_flag(context->task, TIF_DS_AREA_MSR); | |
295 | ||
296 | if (!context->task || (context->task == current)) | |
297 | wrmsrl(MSR_IA32_DS_AREA, 0); | |
298 | ||
299 | put_tracer(context->task); | |
300 | ||
301 | /* free any leftover buffers from tracers that did not | |
302 | * deallocate them properly. */ | |
303 | kfree(context->buffer[ds_bts]); | |
304 | kfree(context->buffer[ds_pebs]); | |
305 | kfree(context->ds); | |
306 | kfree(context); | |
307 | out: | |
de90add3 | 308 | spin_unlock_irqrestore(&ds_lock, irq); |
eee3af4a | 309 | } |
93fa7636 MM |
310 | |
311 | ||
312 | /* | |
313 | * Handle a buffer overflow | |
314 | * | |
315 | * task: the task whose buffers are overflowing; | |
316 | * NULL for a buffer overflow on the current cpu | |
317 | * context: the ds context | |
318 | * qual: the buffer type | |
319 | */ | |
320 | static void ds_overflow(struct task_struct *task, struct ds_context *context, | |
321 | enum ds_qualifier qual) | |
eee3af4a | 322 | { |
93fa7636 MM |
323 | if (!context) |
324 | return; | |
325 | ||
326 | if (context->callback[qual]) | |
327 | (*context->callback[qual])(task); | |
328 | ||
329 | /* todo: do some more overflow handling */ | |
eee3af4a | 330 | } |
93fa7636 MM |
331 | |
332 | ||
333 | /* | |
334 | * Allocate a non-pageable buffer of the parameter size. | |
335 | * Checks the memory and the locked memory rlimit. | |
336 | * | |
337 | * Returns the buffer, if successful; | |
338 | * NULL, if out of memory or rlimit exceeded. | |
339 | * | |
340 | * size: the requested buffer size in bytes | |
341 | * pages (out): if not NULL, contains the number of pages reserved | |
342 | */ | |
343 | static inline void *ds_allocate_buffer(size_t size, unsigned int *pages) | |
eee3af4a | 344 | { |
93fa7636 MM |
345 | unsigned long rlim, vm, pgsz; |
346 | void *buffer; | |
347 | ||
348 | pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT; | |
349 | ||
350 | rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT; | |
351 | vm = current->mm->total_vm + pgsz; | |
352 | if (rlim < vm) | |
573da422 | 353 | return NULL; |
93fa7636 MM |
354 | |
355 | rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT; | |
356 | vm = current->mm->locked_vm + pgsz; | |
357 | if (rlim < vm) | |
573da422 | 358 | return NULL; |
93fa7636 MM |
359 | |
360 | buffer = kzalloc(size, GFP_KERNEL); | |
361 | if (!buffer) | |
573da422 | 362 | return NULL; |
93fa7636 MM |
363 | |
364 | current->mm->total_vm += pgsz; | |
365 | current->mm->locked_vm += pgsz; | |
366 | ||
367 | if (pages) | |
368 | *pages = pgsz; | |
369 | ||
370 | return buffer; | |
eee3af4a | 371 | } |
93fa7636 MM |
372 | |
373 | static int ds_request(struct task_struct *task, void *base, size_t size, | |
374 | ds_ovfl_callback_t ovfl, enum ds_qualifier qual) | |
eee3af4a | 375 | { |
93fa7636 MM |
376 | struct ds_context *context; |
377 | unsigned long buffer, adj; | |
378 | const unsigned long alignment = (1 << 3); | |
de90add3 | 379 | unsigned long irq; |
93fa7636 MM |
380 | int error = 0; |
381 | ||
382 | if (!ds_cfg.sizeof_ds) | |
383 | return -EOPNOTSUPP; | |
384 | ||
385 | /* we require some space to do alignment adjustments below */ | |
386 | if (size < (alignment + ds_cfg.sizeof_rec[qual])) | |
387 | return -EINVAL; | |
388 | ||
389 | /* buffer overflow notification is not yet implemented */ | |
390 | if (ovfl) | |
391 | return -EOPNOTSUPP; | |
392 | ||
393 | ||
93fa7636 MM |
394 | context = ds_alloc_context(task); |
395 | if (!context) | |
de90add3 MM |
396 | return -ENOMEM; |
397 | ||
398 | spin_lock_irqsave(&ds_lock, irq); | |
93fa7636 | 399 | |
10db4ef7 IM |
400 | error = -EPERM; |
401 | if (!check_tracer(task)) | |
402 | goto out_unlock; | |
403 | ||
de90add3 MM |
404 | get_tracer(task); |
405 | ||
93fa7636 MM |
406 | error = -EALREADY; |
407 | if (context->owner[qual] == current) | |
de90add3 | 408 | goto out_put_tracer; |
93fa7636 | 409 | error = -EPERM; |
573da422 | 410 | if (context->owner[qual] != NULL) |
de90add3 | 411 | goto out_put_tracer; |
93fa7636 MM |
412 | context->owner[qual] = current; |
413 | ||
de90add3 | 414 | spin_unlock_irqrestore(&ds_lock, irq); |
93fa7636 MM |
415 | |
416 | ||
417 | error = -ENOMEM; | |
418 | if (!base) { | |
419 | base = ds_allocate_buffer(size, &context->pages[qual]); | |
420 | if (!base) | |
421 | goto out_release; | |
422 | ||
423 | context->buffer[qual] = base; | |
424 | } | |
425 | error = 0; | |
426 | ||
427 | context->callback[qual] = ovfl; | |
428 | ||
429 | /* adjust the buffer address and size to meet alignment | |
430 | * constraints: | |
431 | * - buffer is double-word aligned | |
432 | * - size is multiple of record size | |
433 | * | |
434 | * We checked the size at the very beginning; we have enough | |
435 | * space to do the adjustment. | |
436 | */ | |
437 | buffer = (unsigned long)base; | |
438 | ||
439 | adj = ALIGN(buffer, alignment) - buffer; | |
440 | buffer += adj; | |
441 | size -= adj; | |
442 | ||
443 | size /= ds_cfg.sizeof_rec[qual]; | |
444 | size *= ds_cfg.sizeof_rec[qual]; | |
445 | ||
446 | ds_set(context->ds, qual, ds_buffer_base, buffer); | |
447 | ds_set(context->ds, qual, ds_index, buffer); | |
448 | ds_set(context->ds, qual, ds_absolute_maximum, buffer + size); | |
449 | ||
450 | if (ovfl) { | |
451 | /* todo: select a suitable interrupt threshold */ | |
452 | } else | |
453 | ds_set(context->ds, qual, | |
454 | ds_interrupt_threshold, buffer + size + 1); | |
455 | ||
456 | /* we keep the context until ds_release */ | |
457 | return error; | |
458 | ||
459 | out_release: | |
573da422 | 460 | context->owner[qual] = NULL; |
93fa7636 | 461 | ds_put_context(context); |
de90add3 MM |
462 | put_tracer(task); |
463 | return error; | |
464 | ||
465 | out_put_tracer: | |
466 | spin_unlock_irqrestore(&ds_lock, irq); | |
467 | ds_put_context(context); | |
468 | put_tracer(task); | |
93fa7636 MM |
469 | return error; |
470 | ||
471 | out_unlock: | |
de90add3 | 472 | spin_unlock_irqrestore(&ds_lock, irq); |
93fa7636 MM |
473 | ds_put_context(context); |
474 | return error; | |
eee3af4a | 475 | } |
93fa7636 MM |
476 | |
477 | int ds_request_bts(struct task_struct *task, void *base, size_t size, | |
478 | ds_ovfl_callback_t ovfl) | |
eee3af4a | 479 | { |
93fa7636 | 480 | return ds_request(task, base, size, ovfl, ds_bts); |
eee3af4a | 481 | } |
93fa7636 MM |
482 | |
483 | int ds_request_pebs(struct task_struct *task, void *base, size_t size, | |
484 | ds_ovfl_callback_t ovfl) | |
eee3af4a | 485 | { |
93fa7636 | 486 | return ds_request(task, base, size, ovfl, ds_pebs); |
eee3af4a | 487 | } |
93fa7636 MM |
488 | |
489 | static int ds_release(struct task_struct *task, enum ds_qualifier qual) | |
eee3af4a | 490 | { |
93fa7636 MM |
491 | struct ds_context *context; |
492 | int error; | |
493 | ||
494 | context = ds_get_context(task); | |
495 | error = ds_validate_access(context, qual); | |
496 | if (error < 0) | |
497 | goto out; | |
498 | ||
499 | kfree(context->buffer[qual]); | |
493cd912 | 500 | context->buffer[qual] = NULL; |
93fa7636 MM |
501 | |
502 | current->mm->total_vm -= context->pages[qual]; | |
503 | current->mm->locked_vm -= context->pages[qual]; | |
504 | context->pages[qual] = 0; | |
493cd912 | 505 | context->owner[qual] = NULL; |
93fa7636 MM |
506 | |
507 | /* | |
508 | * we put the context twice: | |
509 | * once for the ds_get_context | |
510 | * once for the corresponding ds_request | |
511 | */ | |
512 | ds_put_context(context); | |
513 | out: | |
514 | ds_put_context(context); | |
515 | return error; | |
eee3af4a | 516 | } |
93fa7636 MM |
517 | |
518 | int ds_release_bts(struct task_struct *task) | |
eee3af4a | 519 | { |
93fa7636 | 520 | return ds_release(task, ds_bts); |
eee3af4a | 521 | } |
93fa7636 MM |
522 | |
523 | int ds_release_pebs(struct task_struct *task) | |
eee3af4a | 524 | { |
93fa7636 | 525 | return ds_release(task, ds_pebs); |
eee3af4a | 526 | } |
93fa7636 MM |
527 | |
528 | static int ds_get_index(struct task_struct *task, size_t *pos, | |
529 | enum ds_qualifier qual) | |
eee3af4a | 530 | { |
93fa7636 MM |
531 | struct ds_context *context; |
532 | unsigned long base, index; | |
533 | int error; | |
534 | ||
535 | context = ds_get_context(task); | |
536 | error = ds_validate_access(context, qual); | |
537 | if (error < 0) | |
538 | goto out; | |
539 | ||
540 | base = ds_get(context->ds, qual, ds_buffer_base); | |
541 | index = ds_get(context->ds, qual, ds_index); | |
542 | ||
543 | error = ((index - base) / ds_cfg.sizeof_rec[qual]); | |
544 | if (pos) | |
545 | *pos = error; | |
546 | out: | |
547 | ds_put_context(context); | |
548 | return error; | |
eee3af4a | 549 | } |
93fa7636 MM |
550 | |
551 | int ds_get_bts_index(struct task_struct *task, size_t *pos) | |
eee3af4a | 552 | { |
93fa7636 | 553 | return ds_get_index(task, pos, ds_bts); |
eee3af4a MM |
554 | } |
555 | ||
93fa7636 MM |
556 | int ds_get_pebs_index(struct task_struct *task, size_t *pos) |
557 | { | |
558 | return ds_get_index(task, pos, ds_pebs); | |
559 | } | |
eee3af4a | 560 | |
93fa7636 MM |
561 | static int ds_get_end(struct task_struct *task, size_t *pos, |
562 | enum ds_qualifier qual) | |
eee3af4a | 563 | { |
93fa7636 MM |
564 | struct ds_context *context; |
565 | unsigned long base, end; | |
566 | int error; | |
567 | ||
568 | context = ds_get_context(task); | |
569 | error = ds_validate_access(context, qual); | |
570 | if (error < 0) | |
571 | goto out; | |
572 | ||
573 | base = ds_get(context->ds, qual, ds_buffer_base); | |
574 | end = ds_get(context->ds, qual, ds_absolute_maximum); | |
575 | ||
576 | error = ((end - base) / ds_cfg.sizeof_rec[qual]); | |
577 | if (pos) | |
578 | *pos = error; | |
579 | out: | |
580 | ds_put_context(context); | |
581 | return error; | |
582 | } | |
eee3af4a | 583 | |
93fa7636 MM |
584 | int ds_get_bts_end(struct task_struct *task, size_t *pos) |
585 | { | |
586 | return ds_get_end(task, pos, ds_bts); | |
587 | } | |
eee3af4a | 588 | |
93fa7636 MM |
589 | int ds_get_pebs_end(struct task_struct *task, size_t *pos) |
590 | { | |
591 | return ds_get_end(task, pos, ds_pebs); | |
592 | } | |
eee3af4a | 593 | |
93fa7636 MM |
594 | static int ds_access(struct task_struct *task, size_t index, |
595 | const void **record, enum ds_qualifier qual) | |
596 | { | |
597 | struct ds_context *context; | |
598 | unsigned long base, idx; | |
599 | int error; | |
eee3af4a | 600 | |
93fa7636 | 601 | if (!record) |
eee3af4a MM |
602 | return -EINVAL; |
603 | ||
93fa7636 MM |
604 | context = ds_get_context(task); |
605 | error = ds_validate_access(context, qual); | |
606 | if (error < 0) | |
607 | goto out; | |
eee3af4a | 608 | |
93fa7636 MM |
609 | base = ds_get(context->ds, qual, ds_buffer_base); |
610 | idx = base + (index * ds_cfg.sizeof_rec[qual]); | |
eee3af4a | 611 | |
93fa7636 MM |
612 | error = -EINVAL; |
613 | if (idx > ds_get(context->ds, qual, ds_absolute_maximum)) | |
614 | goto out; | |
eee3af4a | 615 | |
93fa7636 MM |
616 | *record = (const void *)idx; |
617 | error = ds_cfg.sizeof_rec[qual]; | |
618 | out: | |
619 | ds_put_context(context); | |
620 | return error; | |
eee3af4a MM |
621 | } |
622 | ||
93fa7636 | 623 | int ds_access_bts(struct task_struct *task, size_t index, const void **record) |
eee3af4a | 624 | { |
93fa7636 | 625 | return ds_access(task, index, record, ds_bts); |
eee3af4a MM |
626 | } |
627 | ||
93fa7636 | 628 | int ds_access_pebs(struct task_struct *task, size_t index, const void **record) |
eee3af4a | 629 | { |
93fa7636 | 630 | return ds_access(task, index, record, ds_pebs); |
a95d67f8 MM |
631 | } |
632 | ||
93fa7636 MM |
633 | static int ds_write(struct task_struct *task, const void *record, size_t size, |
634 | enum ds_qualifier qual, int force) | |
a95d67f8 | 635 | { |
93fa7636 MM |
636 | struct ds_context *context; |
637 | int error; | |
eee3af4a | 638 | |
93fa7636 MM |
639 | if (!record) |
640 | return -EINVAL; | |
eee3af4a | 641 | |
93fa7636 MM |
642 | error = -EPERM; |
643 | context = ds_get_context(task); | |
644 | if (!context) | |
645 | goto out; | |
eee3af4a | 646 | |
93fa7636 MM |
647 | if (!force) { |
648 | error = ds_validate_access(context, qual); | |
649 | if (error < 0) | |
650 | goto out; | |
651 | } | |
eee3af4a | 652 | |
93fa7636 MM |
653 | error = 0; |
654 | while (size) { | |
655 | unsigned long base, index, end, write_end, int_th; | |
656 | unsigned long write_size, adj_write_size; | |
657 | ||
658 | /* | |
659 | * write as much as possible without producing an | |
660 | * overflow interrupt. | |
661 | * | |
662 | * interrupt_threshold must either be | |
663 | * - bigger than absolute_maximum or | |
664 | * - point to a record between buffer_base and absolute_maximum | |
665 | * | |
666 | * index points to a valid record. | |
667 | */ | |
668 | base = ds_get(context->ds, qual, ds_buffer_base); | |
669 | index = ds_get(context->ds, qual, ds_index); | |
670 | end = ds_get(context->ds, qual, ds_absolute_maximum); | |
671 | int_th = ds_get(context->ds, qual, ds_interrupt_threshold); | |
672 | ||
673 | write_end = min(end, int_th); | |
674 | ||
675 | /* if we are already beyond the interrupt threshold, | |
676 | * we fill the entire buffer */ | |
677 | if (write_end <= index) | |
678 | write_end = end; | |
679 | ||
680 | if (write_end <= index) | |
681 | goto out; | |
682 | ||
683 | write_size = min((unsigned long) size, write_end - index); | |
684 | memcpy((void *)index, record, write_size); | |
685 | ||
686 | record = (const char *)record + write_size; | |
687 | size -= write_size; | |
688 | error += write_size; | |
689 | ||
690 | adj_write_size = write_size / ds_cfg.sizeof_rec[qual]; | |
691 | adj_write_size *= ds_cfg.sizeof_rec[qual]; | |
692 | ||
693 | /* zero out trailing bytes */ | |
694 | memset((char *)index + write_size, 0, | |
695 | adj_write_size - write_size); | |
696 | index += adj_write_size; | |
697 | ||
698 | if (index >= end) | |
699 | index = base; | |
700 | ds_set(context->ds, qual, ds_index, index); | |
701 | ||
702 | if (index >= int_th) | |
703 | ds_overflow(task, context, qual); | |
704 | } | |
eee3af4a | 705 | |
93fa7636 MM |
706 | out: |
707 | ds_put_context(context); | |
708 | return error; | |
eee3af4a MM |
709 | } |
710 | ||
93fa7636 | 711 | int ds_write_bts(struct task_struct *task, const void *record, size_t size) |
a95d67f8 | 712 | { |
93fa7636 | 713 | return ds_write(task, record, size, ds_bts, /* force = */ 0); |
a95d67f8 MM |
714 | } |
715 | ||
93fa7636 | 716 | int ds_write_pebs(struct task_struct *task, const void *record, size_t size) |
a95d67f8 | 717 | { |
93fa7636 | 718 | return ds_write(task, record, size, ds_pebs, /* force = */ 0); |
a95d67f8 MM |
719 | } |
720 | ||
93fa7636 MM |
721 | int ds_unchecked_write_bts(struct task_struct *task, |
722 | const void *record, size_t size) | |
a95d67f8 | 723 | { |
93fa7636 | 724 | return ds_write(task, record, size, ds_bts, /* force = */ 1); |
a95d67f8 MM |
725 | } |
726 | ||
93fa7636 MM |
727 | int ds_unchecked_write_pebs(struct task_struct *task, |
728 | const void *record, size_t size) | |
eee3af4a | 729 | { |
93fa7636 MM |
730 | return ds_write(task, record, size, ds_pebs, /* force = */ 1); |
731 | } | |
eee3af4a | 732 | |
93fa7636 MM |
733 | static int ds_reset_or_clear(struct task_struct *task, |
734 | enum ds_qualifier qual, int clear) | |
735 | { | |
736 | struct ds_context *context; | |
737 | unsigned long base, end; | |
738 | int error; | |
eee3af4a | 739 | |
93fa7636 MM |
740 | context = ds_get_context(task); |
741 | error = ds_validate_access(context, qual); | |
742 | if (error < 0) | |
743 | goto out; | |
eee3af4a | 744 | |
93fa7636 MM |
745 | base = ds_get(context->ds, qual, ds_buffer_base); |
746 | end = ds_get(context->ds, qual, ds_absolute_maximum); | |
eee3af4a | 747 | |
93fa7636 MM |
748 | if (clear) |
749 | memset((void *)base, 0, end - base); | |
eee3af4a | 750 | |
93fa7636 | 751 | ds_set(context->ds, qual, ds_index, base); |
eee3af4a | 752 | |
93fa7636 MM |
753 | error = 0; |
754 | out: | |
755 | ds_put_context(context); | |
756 | return error; | |
eee3af4a MM |
757 | } |
758 | ||
93fa7636 | 759 | int ds_reset_bts(struct task_struct *task) |
eee3af4a | 760 | { |
93fa7636 MM |
761 | return ds_reset_or_clear(task, ds_bts, /* clear = */ 0); |
762 | } | |
eee3af4a | 763 | |
93fa7636 MM |
764 | int ds_reset_pebs(struct task_struct *task) |
765 | { | |
766 | return ds_reset_or_clear(task, ds_pebs, /* clear = */ 0); | |
767 | } | |
eee3af4a | 768 | |
93fa7636 MM |
769 | int ds_clear_bts(struct task_struct *task) |
770 | { | |
771 | return ds_reset_or_clear(task, ds_bts, /* clear = */ 1); | |
772 | } | |
eee3af4a | 773 | |
93fa7636 MM |
774 | int ds_clear_pebs(struct task_struct *task) |
775 | { | |
776 | return ds_reset_or_clear(task, ds_pebs, /* clear = */ 1); | |
777 | } | |
eee3af4a | 778 | |
93fa7636 MM |
779 | int ds_get_pebs_reset(struct task_struct *task, u64 *value) |
780 | { | |
781 | struct ds_context *context; | |
782 | int error; | |
eee3af4a | 783 | |
93fa7636 | 784 | if (!value) |
eee3af4a | 785 | return -EINVAL; |
eee3af4a | 786 | |
93fa7636 MM |
787 | context = ds_get_context(task); |
788 | error = ds_validate_access(context, ds_pebs); | |
789 | if (error < 0) | |
790 | goto out; | |
eee3af4a | 791 | |
93fa7636 MM |
792 | *value = *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)); |
793 | ||
794 | error = 0; | |
795 | out: | |
796 | ds_put_context(context); | |
797 | return error; | |
eee3af4a MM |
798 | } |
799 | ||
93fa7636 | 800 | int ds_set_pebs_reset(struct task_struct *task, u64 value) |
eee3af4a | 801 | { |
93fa7636 MM |
802 | struct ds_context *context; |
803 | int error; | |
eee3af4a | 804 | |
93fa7636 MM |
805 | context = ds_get_context(task); |
806 | error = ds_validate_access(context, ds_pebs); | |
807 | if (error < 0) | |
808 | goto out; | |
eee3af4a | 809 | |
93fa7636 MM |
810 | *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)) = value; |
811 | ||
812 | error = 0; | |
813 | out: | |
814 | ds_put_context(context); | |
815 | return error; | |
816 | } | |
817 | ||
818 | static const struct ds_configuration ds_cfg_var = { | |
819 | .sizeof_ds = sizeof(long) * 12, | |
820 | .sizeof_field = sizeof(long), | |
821 | .sizeof_rec[ds_bts] = sizeof(long) * 3, | |
c4858ffc | 822 | #ifdef __i386__ |
93fa7636 | 823 | .sizeof_rec[ds_pebs] = sizeof(long) * 10 |
c4858ffc MM |
824 | #else |
825 | .sizeof_rec[ds_pebs] = sizeof(long) * 18 | |
826 | #endif | |
eee3af4a | 827 | }; |
93fa7636 MM |
828 | static const struct ds_configuration ds_cfg_64 = { |
829 | .sizeof_ds = 8 * 12, | |
830 | .sizeof_field = 8, | |
831 | .sizeof_rec[ds_bts] = 8 * 3, | |
c4858ffc | 832 | #ifdef __i386__ |
93fa7636 | 833 | .sizeof_rec[ds_pebs] = 8 * 10 |
c4858ffc MM |
834 | #else |
835 | .sizeof_rec[ds_pebs] = 8 * 18 | |
836 | #endif | |
eee3af4a MM |
837 | }; |
838 | ||
839 | static inline void | |
840 | ds_configure(const struct ds_configuration *cfg) | |
841 | { | |
842 | ds_cfg = *cfg; | |
843 | } | |
844 | ||
845 | void __cpuinit ds_init_intel(struct cpuinfo_x86 *c) | |
846 | { | |
847 | switch (c->x86) { | |
848 | case 0x6: | |
849 | switch (c->x86_model) { | |
eee3af4a MM |
850 | case 0xD: |
851 | case 0xE: /* Pentium M */ | |
93fa7636 | 852 | ds_configure(&ds_cfg_var); |
eee3af4a | 853 | break; |
eee3af4a | 854 | case 0xF: /* Core2 */ |
573da422 | 855 | case 0x1C: /* Atom */ |
93fa7636 | 856 | ds_configure(&ds_cfg_64); |
eee3af4a MM |
857 | break; |
858 | default: | |
859 | /* sorry, don't know about them */ | |
860 | break; | |
861 | } | |
862 | break; | |
863 | case 0xF: | |
864 | switch (c->x86_model) { | |
eee3af4a MM |
865 | case 0x0: |
866 | case 0x1: | |
867 | case 0x2: /* Netburst */ | |
93fa7636 | 868 | ds_configure(&ds_cfg_var); |
eee3af4a | 869 | break; |
eee3af4a MM |
870 | default: |
871 | /* sorry, don't know about them */ | |
872 | break; | |
873 | } | |
874 | break; | |
875 | default: | |
876 | /* sorry, don't know about them */ | |
877 | break; | |
878 | } | |
879 | } | |
93fa7636 MM |
880 | |
881 | void ds_free(struct ds_context *context) | |
882 | { | |
883 | /* This is called when the task owning the parameter context | |
884 | * is dying. There should not be any user of that context left | |
885 | * to disturb us, anymore. */ | |
886 | unsigned long leftovers = context->count; | |
887 | while (leftovers--) | |
888 | ds_put_context(context); | |
889 | } |