projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
[S390] oprofile: add missing irq stats counter
[deliverable/linux.git] / arch / s390 / oprofile / hwsampler.c
1 /**
2 * arch/s390/oprofile/hwsampler.c
3 *
4 * Copyright IBM Corp. 2010
5 * Author: Heinz Graalfs <graalfs@de.ibm.com>
6 */
7
8 #include <linux/kernel_stat.h>
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/smp.h>
12 #include <linux/errno.h>
13 #include <linux/workqueue.h>
14 #include <linux/interrupt.h>
15 #include <linux/notifier.h>
16 #include <linux/cpu.h>
17 #include <linux/semaphore.h>
18 #include <linux/oom.h>
19 #include <linux/oprofile.h>
20
21 #include <asm/lowcore.h>
22 #include <asm/s390_ext.h>
23
24 #include "hwsampler.h"
25
26 #define MAX_NUM_SDB 511
27 #define MIN_NUM_SDB 1
28
29 #define ALERT_REQ_MASK 0x4000000000000000ul
30 #define BUFFER_FULL_MASK 0x8000000000000000ul
31
32 #define EI_IEA (1 << 31) /* invalid entry address */
33 #define EI_ISE (1 << 30) /* incorrect SDBT entry */
34 #define EI_PRA (1 << 29) /* program request alert */
35 #define EI_SACA (1 << 23) /* sampler authorization change alert */
36 #define EI_LSDA (1 << 22) /* loss of sample data alert */
37
38 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
39
40 struct hws_execute_parms {
41 void *buffer;
42 signed int rc;
43 };
44
45 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
46 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
47
48 static DEFINE_MUTEX(hws_sem);
49 static DEFINE_MUTEX(hws_sem_oom);
50
51 static unsigned char hws_flush_all;
52 static unsigned int hws_oom;
53 static struct workqueue_struct *hws_wq;
54
55 static unsigned int hws_state;
56 enum {
57 HWS_INIT = 1,
58 HWS_DEALLOCATED,
59 HWS_STOPPED,
60 HWS_STARTED,
61 HWS_STOPPING };
62
63 /* set to 1 if called by kernel during memory allocation */
64 static unsigned char oom_killer_was_active;
65 /* size of SDBT and SDB as of allocate API */
66 static unsigned long num_sdbt = 100;
67 static unsigned long num_sdb = 511;
68 /* sampling interval (machine cycles) */
69 static unsigned long interval;
70
71 static unsigned long min_sampler_rate;
72 static unsigned long max_sampler_rate;
73
74 static int ssctl(void *buffer)
75 {
76 int cc;
77
78 /* set in order to detect a program check */
79 cc = 1;
80
81 asm volatile(
82 "0: .insn s,0xB2870000,0(%1)\n"
83 "1: ipm %0\n"
84 " srl %0,28\n"
85 "2:\n"
86 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
87 : "+d" (cc), "+a" (buffer)
88 : "m" (*((struct hws_ssctl_request_block *)buffer))
89 : "cc", "memory");
90
91 return cc ? -EINVAL : 0 ;
92 }
93
94 static int qsi(void *buffer)
95 {
96 int cc;
97 cc = 1;
98
99 asm volatile(
100 "0: .insn s,0xB2860000,0(%1)\n"
101 "1: lhi %0,0\n"
102 "2:\n"
103 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
104 : "=d" (cc), "+a" (buffer)
105 : "m" (*((struct hws_qsi_info_block *)buffer))
106 : "cc", "memory");
107
108 return cc ? -EINVAL : 0;
109 }
110
111 static void execute_qsi(void *parms)
112 {
113 struct hws_execute_parms *ep = parms;
114
115 ep->rc = qsi(ep->buffer);
116 }
117
118 static void execute_ssctl(void *parms)
119 {
120 struct hws_execute_parms *ep = parms;
121
122 ep->rc = ssctl(ep->buffer);
123 }
124
125 static int smp_ctl_ssctl_stop(int cpu)
126 {
127 int rc;
128 struct hws_execute_parms ep;
129 struct hws_cpu_buffer *cb;
130
131 cb = &per_cpu(sampler_cpu_buffer, cpu);
132
133 cb->ssctl.es = 0;
134 cb->ssctl.cs = 0;
135
136 ep.buffer = &cb->ssctl;
137 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
138 rc = ep.rc;
139 if (rc) {
140 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
141 dump_stack();
142 }
143
144 ep.buffer = &cb->qsi;
145 smp_call_function_single(cpu, execute_qsi, &ep, 1);
146
147 if (cb->qsi.es || cb->qsi.cs) {
148 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
149 dump_stack();
150 }
151
152 return rc;
153 }
154
155 static int smp_ctl_ssctl_deactivate(int cpu)
156 {
157 int rc;
158 struct hws_execute_parms ep;
159 struct hws_cpu_buffer *cb;
160
161 cb = &per_cpu(sampler_cpu_buffer, cpu);
162
163 cb->ssctl.es = 1;
164 cb->ssctl.cs = 0;
165
166 ep.buffer = &cb->ssctl;
167 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
168 rc = ep.rc;
169 if (rc)
170 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
171
172 ep.buffer = &cb->qsi;
173 smp_call_function_single(cpu, execute_qsi, &ep, 1);
174
175 if (cb->qsi.cs)
176 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
177
178 return rc;
179 }
180
181 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
182 {
183 int rc;
184 struct hws_execute_parms ep;
185 struct hws_cpu_buffer *cb;
186
187 cb = &per_cpu(sampler_cpu_buffer, cpu);
188
189 cb->ssctl.h = 1;
190 cb->ssctl.tear = cb->first_sdbt;
191 cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
192 cb->ssctl.interval = interval;
193 cb->ssctl.es = 1;
194 cb->ssctl.cs = 1;
195
196 ep.buffer = &cb->ssctl;
197 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
198 rc = ep.rc;
199 if (rc)
200 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
201
202 ep.buffer = &cb->qsi;
203 smp_call_function_single(cpu, execute_qsi, &ep, 1);
204 if (ep.rc)
205 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
206
207 return rc;
208 }
209
210 static int smp_ctl_qsi(int cpu)
211 {
212 struct hws_execute_parms ep;
213 struct hws_cpu_buffer *cb;
214
215 cb = &per_cpu(sampler_cpu_buffer, cpu);
216
217 ep.buffer = &cb->qsi;
218 smp_call_function_single(cpu, execute_qsi, &ep, 1);
219
220 return ep.rc;
221 }
222
223 static inline unsigned long *trailer_entry_ptr(unsigned long v)
224 {
225 void *ret;
226
227 ret = (void *)v;
228 ret += PAGE_SIZE;
229 ret -= sizeof(struct hws_trailer_entry);
230
231 return (unsigned long *) ret;
232 }
233
234 /* prototypes for external interrupt handler and worker */
235 static void hws_ext_handler(unsigned int ext_int_code,
236 unsigned int param32, unsigned long param64);
237
238 static void worker(struct work_struct *work);
239
240 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
241 unsigned long *dear);
242
243 static void init_all_cpu_buffers(void)
244 {
245 int cpu;
246 struct hws_cpu_buffer *cb;
247
248 for_each_online_cpu(cpu) {
249 cb = &per_cpu(sampler_cpu_buffer, cpu);
250 memset(cb, 0, sizeof(struct hws_cpu_buffer));
251 }
252 }
253
254 static int is_link_entry(unsigned long *s)
255 {
256 return *s & 0x1ul ? 1 : 0;
257 }
258
259 static unsigned long *get_next_sdbt(unsigned long *s)
260 {
261 return (unsigned long *) (*s & ~0x1ul);
262 }
263
264 static int prepare_cpu_buffers(void)
265 {
266 int cpu;
267 int rc;
268 struct hws_cpu_buffer *cb;
269
270 rc = 0;
271 for_each_online_cpu(cpu) {
272 cb = &per_cpu(sampler_cpu_buffer, cpu);
273 atomic_set(&cb->ext_params, 0);
274 cb->worker_entry = 0;
275 cb->sample_overflow = 0;
276 cb->req_alert = 0;
277 cb->incorrect_sdbt_entry = 0;
278 cb->invalid_entry_address = 0;
279 cb->loss_of_sample_data = 0;
280 cb->sample_auth_change_alert = 0;
281 cb->finish = 0;
282 cb->oom = 0;
283 cb->stop_mode = 0;
284 }
285
286 return rc;
287 }
288
289 /*
290 * allocate_sdbt() - allocate sampler memory
291 * @cpu: the cpu for which sampler memory is allocated
292 *
293 * A 4K page is allocated for each requested SDBT.
294 * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
295 * Set ALERT_REQ mask in each SDBs trailer.
296 * Returns zero if successful, <0 otherwise.
297 */
298 static int allocate_sdbt(int cpu)
299 {
300 int j, k, rc;
301 unsigned long *sdbt;
302 unsigned long sdb;
303 unsigned long *tail;
304 unsigned long *trailer;
305 struct hws_cpu_buffer *cb;
306
307 cb = &per_cpu(sampler_cpu_buffer, cpu);
308
309 if (cb->first_sdbt)
310 return -EINVAL;
311
312 sdbt = NULL;
313 tail = sdbt;
314
315 for (j = 0; j < num_sdbt; j++) {
316 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
317
318 mutex_lock(&hws_sem_oom);
319 /* OOM killer might have been activated */
320 barrier();
321 if (oom_killer_was_active || !sdbt) {
322 if (sdbt)
323 free_page((unsigned long)sdbt);
324
325 goto allocate_sdbt_error;
326 }
327 if (cb->first_sdbt == 0)
328 cb->first_sdbt = (unsigned long)sdbt;
329
330 /* link current page to tail of chain */
331 if (tail)
332 *tail = (unsigned long)(void *)sdbt + 1;
333
334 mutex_unlock(&hws_sem_oom);
335
336 for (k = 0; k < num_sdb; k++) {
337 /* get and set SDB page */
338 sdb = get_zeroed_page(GFP_KERNEL);
339
340 mutex_lock(&hws_sem_oom);
341 /* OOM killer might have been activated */
342 barrier();
343 if (oom_killer_was_active || !sdb) {
344 if (sdb)
345 free_page(sdb);
346
347 goto allocate_sdbt_error;
348 }
349 *sdbt = sdb;
350 trailer = trailer_entry_ptr(*sdbt);
351 *trailer = ALERT_REQ_MASK;
352 sdbt++;
353 mutex_unlock(&hws_sem_oom);
354 }
355 tail = sdbt;
356 }
357 mutex_lock(&hws_sem_oom);
358 if (oom_killer_was_active)
359 goto allocate_sdbt_error;
360
361 rc = 0;
362 if (tail)
363 *tail = (unsigned long)
364 ((void *)cb->first_sdbt) + 1;
365
366 allocate_sdbt_exit:
367 mutex_unlock(&hws_sem_oom);
368 return rc;
369
370 allocate_sdbt_error:
371 rc = -ENOMEM;
372 goto allocate_sdbt_exit;
373 }
374
375 /*
376 * deallocate_sdbt() - deallocate all sampler memory
377 *
378 * For each online CPU all SDBT trees are deallocated.
379 * Returns the number of freed pages.
380 */
381 static int deallocate_sdbt(void)
382 {
383 int cpu;
384 int counter;
385
386 counter = 0;
387
388 for_each_online_cpu(cpu) {
389 unsigned long start;
390 unsigned long sdbt;
391 unsigned long *curr;
392 struct hws_cpu_buffer *cb;
393
394 cb = &per_cpu(sampler_cpu_buffer, cpu);
395
396 if (!cb->first_sdbt)
397 continue;
398
399 sdbt = cb->first_sdbt;
400 curr = (unsigned long *) sdbt;
401 start = sdbt;
402
403 /* we'll free the SDBT after all SDBs are processed... */
404 while (1) {
405 if (!*curr || !sdbt)
406 break;
407
408 /* watch for link entry reset if found */
409 if (is_link_entry(curr)) {
410 curr = get_next_sdbt(curr);
411 if (sdbt)
412 free_page(sdbt);
413
414 /* we are done if we reach the start */
415 if ((unsigned long) curr == start)
416 break;
417 else
418 sdbt = (unsigned long) curr;
419 } else {
420 /* process SDB pointer */
421 if (*curr) {
422 free_page(*curr);
423 curr++;
424 }
425 }
426 counter++;
427 }
428 cb->first_sdbt = 0;
429 }
430 return counter;
431 }
432
433 static int start_sampling(int cpu)
434 {
435 int rc;
436 struct hws_cpu_buffer *cb;
437
438 cb = &per_cpu(sampler_cpu_buffer, cpu);
439 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
440 if (rc) {
441 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
442 goto start_exit;
443 }
444
445 rc = -EINVAL;
446 if (!cb->qsi.es) {
447 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
448 goto start_exit;
449 }
450
451 if (!cb->qsi.cs) {
452 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
453 goto start_exit;
454 }
455
456 printk(KERN_INFO
457 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
458 cpu, interval);
459
460 rc = 0;
461
462 start_exit:
463 return rc;
464 }
465
466 static int stop_sampling(int cpu)
467 {
468 unsigned long v;
469 int rc;
470 struct hws_cpu_buffer *cb;
471
472 rc = smp_ctl_qsi(cpu);
473 WARN_ON(rc);
474
475 cb = &per_cpu(sampler_cpu_buffer, cpu);
476 if (!rc && !cb->qsi.es)
477 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
478
479 rc = smp_ctl_ssctl_stop(cpu);
480 if (rc) {
481 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
482 cpu, rc);
483 goto stop_exit;
484 }
485
486 printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
487
488 stop_exit:
489 v = cb->req_alert;
490 if (v)
491 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
492 " count=%lu.\n", cpu, v);
493
494 v = cb->loss_of_sample_data;
495 if (v)
496 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
497 " count=%lu.\n", cpu, v);
498
499 v = cb->invalid_entry_address;
500 if (v)
501 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
502 " count=%lu.\n", cpu, v);
503
504 v = cb->incorrect_sdbt_entry;
505 if (v)
506 printk(KERN_ERR
507 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
508 " count=%lu.\n", cpu, v);
509
510 v = cb->sample_auth_change_alert;
511 if (v)
512 printk(KERN_ERR
513 "hwsampler: CPU %d CPUMF Sample authorization change,"
514 " count=%lu.\n", cpu, v);
515
516 return rc;
517 }
518
519 static int check_hardware_prerequisites(void)
520 {
521 if (!test_facility(68))
522 return -EOPNOTSUPP;
523 return 0;
524 }
525 /*
526 * hws_oom_callback() - the OOM callback function
527 *
528 * In case the callback is invoked during memory allocation for the
529 * hw sampler, all obtained memory is deallocated and a flag is set
530 * so main sampler memory allocation can exit with a failure code.
531 * In case the callback is invoked during sampling the hw sampler
532 * is deactivated for all CPUs.
533 */
534 static int hws_oom_callback(struct notifier_block *nfb,
535 unsigned long dummy, void *parm)
536 {
537 unsigned long *freed;
538 int cpu;
539 struct hws_cpu_buffer *cb;
540
541 freed = parm;
542
543 mutex_lock(&hws_sem_oom);
544
545 if (hws_state == HWS_DEALLOCATED) {
546 /* during memory allocation */
547 if (oom_killer_was_active == 0) {
548 oom_killer_was_active = 1;
549 *freed += deallocate_sdbt();
550 }
551 } else {
552 int i;
553 cpu = get_cpu();
554 cb = &per_cpu(sampler_cpu_buffer, cpu);
555
556 if (!cb->oom) {
557 for_each_online_cpu(i) {
558 smp_ctl_ssctl_deactivate(i);
559 cb->oom = 1;
560 }
561 cb->finish = 1;
562
563 printk(KERN_INFO
564 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
565 cpu);
566 }
567 }
568
569 mutex_unlock(&hws_sem_oom);
570
571 return NOTIFY_OK;
572 }
573
574 static struct notifier_block hws_oom_notifier = {
575 .notifier_call = hws_oom_callback
576 };
577
578 static int hws_cpu_callback(struct notifier_block *nfb,
579 unsigned long action, void *hcpu)
580 {
581 /* We do not have sampler space available for all possible CPUs.
582 All CPUs should be online when hw sampling is activated. */
583 return NOTIFY_BAD;
584 }
585
586 static struct notifier_block hws_cpu_notifier = {
587 .notifier_call = hws_cpu_callback
588 };
589
590 /**
591 * hwsampler_deactivate() - set hardware sampling temporarily inactive
592 * @cpu: specifies the CPU to be set inactive.
593 *
594 * Returns 0 on success, !0 on failure.
595 */
596 int hwsampler_deactivate(unsigned int cpu)
597 {
598 /*
599 * Deactivate hw sampling temporarily and flush the buffer
600 * by pushing all the pending samples to oprofile buffer.
601 *
602 * This function can be called under one of the following conditions:
603 * Memory unmap, task is exiting.
604 */
605 int rc;
606 struct hws_cpu_buffer *cb;
607
608 rc = 0;
609 mutex_lock(&hws_sem);
610
611 cb = &per_cpu(sampler_cpu_buffer, cpu);
612 if (hws_state == HWS_STARTED) {
613 rc = smp_ctl_qsi(cpu);
614 WARN_ON(rc);
615 if (cb->qsi.cs) {
616 rc = smp_ctl_ssctl_deactivate(cpu);
617 if (rc) {
618 printk(KERN_INFO
619 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
620 cb->finish = 1;
621 hws_state = HWS_STOPPING;
622 } else {
623 hws_flush_all = 1;
624 /* Add work to queue to read pending samples.*/
625 queue_work_on(cpu, hws_wq, &cb->worker);
626 }
627 }
628 }
629 mutex_unlock(&hws_sem);
630
631 if (hws_wq)
632 flush_workqueue(hws_wq);
633
634 return rc;
635 }
636
637 /**
638 * hwsampler_activate() - activate/resume hardware sampling which was deactivated
639 * @cpu: specifies the CPU to be set active.
640 *
641 * Returns 0 on success, !0 on failure.
642 */
643 int hwsampler_activate(unsigned int cpu)
644 {
645 /*
646 * Re-activate hw sampling. This should be called in pair with
647 * hwsampler_deactivate().
648 */
649 int rc;
650 struct hws_cpu_buffer *cb;
651
652 rc = 0;
653 mutex_lock(&hws_sem);
654
655 cb = &per_cpu(sampler_cpu_buffer, cpu);
656 if (hws_state == HWS_STARTED) {
657 rc = smp_ctl_qsi(cpu);
658 WARN_ON(rc);
659 if (!cb->qsi.cs) {
660 hws_flush_all = 0;
661 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
662 if (rc) {
663 printk(KERN_ERR
664 "CPU %d, CPUMF activate sampling failed.\n",
665 cpu);
666 }
667 }
668 }
669
670 mutex_unlock(&hws_sem);
671
672 return rc;
673 }
674
675 static void hws_ext_handler(unsigned int ext_int_code,
676 unsigned int param32, unsigned long param64)
677 {
678 int cpu;
679 struct hws_cpu_buffer *cb;
680
681 kstat_cpu(smp_processor_id()).irqs[EXTINT_CPM]++;
682 cpu = smp_processor_id();
683 cb = &per_cpu(sampler_cpu_buffer, cpu);
684
685 atomic_xchg(
686 &cb->ext_params,
687 atomic_read(&cb->ext_params)
688 | S390_lowcore.ext_params);
689
690 if (hws_wq)
691 queue_work(hws_wq, &cb->worker);
692 }
693
694 static int check_qsi_on_setup(void)
695 {
696 int rc;
697 unsigned int cpu;
698 struct hws_cpu_buffer *cb;
699
700 for_each_online_cpu(cpu) {
701 cb = &per_cpu(sampler_cpu_buffer, cpu);
702 rc = smp_ctl_qsi(cpu);
703 WARN_ON(rc);
704 if (rc)
705 return -EOPNOTSUPP;
706
707 if (!cb->qsi.as) {
708 printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
709 return -EINVAL;
710 }
711
712 if (cb->qsi.es) {
713 printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
714 rc = smp_ctl_ssctl_stop(cpu);
715 if (rc)
716 return -EINVAL;
717
718 printk(KERN_INFO
719 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
720 }
721 }
722 return 0;
723 }
724
725 static int check_qsi_on_start(void)
726 {
727 unsigned int cpu;
728 int rc;
729 struct hws_cpu_buffer *cb;
730
731 for_each_online_cpu(cpu) {
732 cb = &per_cpu(sampler_cpu_buffer, cpu);
733 rc = smp_ctl_qsi(cpu);
734 WARN_ON(rc);
735
736 if (!cb->qsi.as)
737 return -EINVAL;
738
739 if (cb->qsi.es)
740 return -EINVAL;
741
742 if (cb->qsi.cs)
743 return -EINVAL;
744 }
745 return 0;
746 }
747
748 static void worker_on_start(unsigned int cpu)
749 {
750 struct hws_cpu_buffer *cb;
751
752 cb = &per_cpu(sampler_cpu_buffer, cpu);
753 cb->worker_entry = cb->first_sdbt;
754 }
755
756 static int worker_check_error(unsigned int cpu, int ext_params)
757 {
758 int rc;
759 unsigned long *sdbt;
760 struct hws_cpu_buffer *cb;
761
762 rc = 0;
763 cb = &per_cpu(sampler_cpu_buffer, cpu);
764 sdbt = (unsigned long *) cb->worker_entry;
765
766 if (!sdbt || !*sdbt)
767 return -EINVAL;
768
769 if (ext_params & EI_PRA)
770 cb->req_alert++;
771
772 if (ext_params & EI_LSDA)
773 cb->loss_of_sample_data++;
774
775 if (ext_params & EI_IEA) {
776 cb->invalid_entry_address++;
777 rc = -EINVAL;
778 }
779
780 if (ext_params & EI_ISE) {
781 cb->incorrect_sdbt_entry++;
782 rc = -EINVAL;
783 }
784
785 if (ext_params & EI_SACA) {
786 cb->sample_auth_change_alert++;
787 rc = -EINVAL;
788 }
789
790 return rc;
791 }
792
793 static void worker_on_finish(unsigned int cpu)
794 {
795 int rc, i;
796 struct hws_cpu_buffer *cb;
797
798 cb = &per_cpu(sampler_cpu_buffer, cpu);
799
800 if (cb->finish) {
801 rc = smp_ctl_qsi(cpu);
802 WARN_ON(rc);
803 if (cb->qsi.es) {
804 printk(KERN_INFO
805 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
806 cpu);
807 rc = smp_ctl_ssctl_stop(cpu);
808 if (rc)
809 printk(KERN_INFO
810 "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
811 cpu);
812
813 for_each_online_cpu(i) {
814 if (i == cpu)
815 continue;
816 if (!cb->finish) {
817 cb->finish = 1;
818 queue_work_on(i, hws_wq,
819 &cb->worker);
820 }
821 }
822 }
823 }
824 }
825
826 static void worker_on_interrupt(unsigned int cpu)
827 {
828 unsigned long *sdbt;
829 unsigned char done;
830 struct hws_cpu_buffer *cb;
831
832 cb = &per_cpu(sampler_cpu_buffer, cpu);
833
834 sdbt = (unsigned long *) cb->worker_entry;
835
836 done = 0;
837 /* do not proceed if stop was entered,
838 * forget the buffers not yet processed */
839 while (!done && !cb->stop_mode) {
840 unsigned long *trailer;
841 struct hws_trailer_entry *te;
842 unsigned long *dear = 0;
843
844 trailer = trailer_entry_ptr(*sdbt);
845 /* leave loop if no more work to do */
846 if (!(*trailer & BUFFER_FULL_MASK)) {
847 done = 1;
848 if (!hws_flush_all)
849 continue;
850 }
851
852 te = (struct hws_trailer_entry *)trailer;
853 cb->sample_overflow += te->overflow;
854
855 add_samples_to_oprofile(cpu, sdbt, dear);
856
857 /* reset trailer */
858 xchg((unsigned char *) te, 0x40);
859
860 /* advance to next sdb slot in current sdbt */
861 sdbt++;
862 /* in case link bit is set use address w/o link bit */
863 if (is_link_entry(sdbt))
864 sdbt = get_next_sdbt(sdbt);
865
866 cb->worker_entry = (unsigned long)sdbt;
867 }
868 }
869
870 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
871 unsigned long *dear)
872 {
873 struct hws_data_entry *sample_data_ptr;
874 unsigned long *trailer;
875
876 trailer = trailer_entry_ptr(*sdbt);
877 if (dear) {
878 if (dear > trailer)
879 return;
880 trailer = dear;
881 }
882
883 sample_data_ptr = (struct hws_data_entry *)(*sdbt);
884
885 while ((unsigned long *)sample_data_ptr < trailer) {
886 struct pt_regs *regs = NULL;
887 struct task_struct *tsk = NULL;
888
889 /*
890 * Check sampling mode, 1 indicates basic (=customer) sampling
891 * mode.
892 */
893 if (sample_data_ptr->def != 1) {
894 /* sample slot is not yet written */
895 break;
896 } else {
897 /* make sure we don't use it twice,
898 * the next time the sampler will set it again */
899 sample_data_ptr->def = 0;
900 }
901
902 /* Get pt_regs. */
903 if (sample_data_ptr->P == 1) {
904 /* userspace sample */
905 unsigned int pid = sample_data_ptr->prim_asn;
906 rcu_read_lock();
907 tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
908 if (tsk)
909 regs = task_pt_regs(tsk);
910 rcu_read_unlock();
911 } else {
912 /* kernelspace sample */
913 regs = task_pt_regs(current);
914 }
915
916 mutex_lock(&hws_sem);
917 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
918 !sample_data_ptr->P, tsk);
919 mutex_unlock(&hws_sem);
920
921 sample_data_ptr++;
922 }
923 }
924
925 static void worker(struct work_struct *work)
926 {
927 unsigned int cpu;
928 int ext_params;
929 struct hws_cpu_buffer *cb;
930
931 cb = container_of(work, struct hws_cpu_buffer, worker);
932 cpu = smp_processor_id();
933 ext_params = atomic_xchg(&cb->ext_params, 0);
934
935 if (!cb->worker_entry)
936 worker_on_start(cpu);
937
938 if (worker_check_error(cpu, ext_params))
939 return;
940
941 if (!cb->finish)
942 worker_on_interrupt(cpu);
943
944 if (cb->finish)
945 worker_on_finish(cpu);
946 }
947
948 /**
949 * hwsampler_allocate() - allocate memory for the hardware sampler
950 * @sdbt: number of SDBTs per online CPU (must be > 0)
951 * @sdb: number of SDBs per SDBT (minimum 1, maximum 511)
952 *
953 * Returns 0 on success, !0 on failure.
954 */
955 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
956 {
957 int cpu, rc;
958 mutex_lock(&hws_sem);
959
960 rc = -EINVAL;
961 if (hws_state != HWS_DEALLOCATED)
962 goto allocate_exit;
963
964 if (sdbt < 1)
965 goto allocate_exit;
966
967 if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
968 goto allocate_exit;
969
970 num_sdbt = sdbt;
971 num_sdb = sdb;
972
973 oom_killer_was_active = 0;
974 register_oom_notifier(&hws_oom_notifier);
975
976 for_each_online_cpu(cpu) {
977 if (allocate_sdbt(cpu)) {
978 unregister_oom_notifier(&hws_oom_notifier);
979 goto allocate_error;
980 }
981 }
982 unregister_oom_notifier(&hws_oom_notifier);
983 if (oom_killer_was_active)
984 goto allocate_error;
985
986 hws_state = HWS_STOPPED;
987 rc = 0;
988
989 allocate_exit:
990 mutex_unlock(&hws_sem);
991 return rc;
992
993 allocate_error:
994 rc = -ENOMEM;
995 printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
996 goto allocate_exit;
997 }
998
999 /**
1000 * hwsampler_deallocate() - deallocate hardware sampler memory
1001 *
1002 * Returns 0 on success, !0 on failure.
1003 */
1004 int hwsampler_deallocate()
1005 {
1006 int rc;
1007
1008 mutex_lock(&hws_sem);
1009
1010 rc = -EINVAL;
1011 if (hws_state != HWS_STOPPED)
1012 goto deallocate_exit;
1013
1014 ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1015 deallocate_sdbt();
1016
1017 hws_state = HWS_DEALLOCATED;
1018 rc = 0;
1019
1020 deallocate_exit:
1021 mutex_unlock(&hws_sem);
1022
1023 return rc;
1024 }
1025
1026 unsigned long hwsampler_query_min_interval(void)
1027 {
1028 return min_sampler_rate;
1029 }
1030
1031 unsigned long hwsampler_query_max_interval(void)
1032 {
1033 return max_sampler_rate;
1034 }
1035
1036 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1037 {
1038 struct hws_cpu_buffer *cb;
1039
1040 cb = &per_cpu(sampler_cpu_buffer, cpu);
1041
1042 return cb->sample_overflow;
1043 }
1044
1045 int hwsampler_setup()
1046 {
1047 int rc;
1048 int cpu;
1049 struct hws_cpu_buffer *cb;
1050
1051 mutex_lock(&hws_sem);
1052
1053 rc = -EINVAL;
1054 if (hws_state)
1055 goto setup_exit;
1056
1057 hws_state = HWS_INIT;
1058
1059 init_all_cpu_buffers();
1060
1061 rc = check_hardware_prerequisites();
1062 if (rc)
1063 goto setup_exit;
1064
1065 rc = check_qsi_on_setup();
1066 if (rc)
1067 goto setup_exit;
1068
1069 rc = -EINVAL;
1070 hws_wq = create_workqueue("hwsampler");
1071 if (!hws_wq)
1072 goto setup_exit;
1073
1074 register_cpu_notifier(&hws_cpu_notifier);
1075
1076 for_each_online_cpu(cpu) {
1077 cb = &per_cpu(sampler_cpu_buffer, cpu);
1078 INIT_WORK(&cb->worker, worker);
1079 rc = smp_ctl_qsi(cpu);
1080 WARN_ON(rc);
1081 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1082 if (min_sampler_rate) {
1083 printk(KERN_WARNING
1084 "hwsampler: different min sampler rate values.\n");
1085 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1086 min_sampler_rate =
1087 cb->qsi.min_sampl_rate;
1088 } else
1089 min_sampler_rate = cb->qsi.min_sampl_rate;
1090 }
1091 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1092 if (max_sampler_rate) {
1093 printk(KERN_WARNING
1094 "hwsampler: different max sampler rate values.\n");
1095 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1096 max_sampler_rate =
1097 cb->qsi.max_sampl_rate;
1098 } else
1099 max_sampler_rate = cb->qsi.max_sampl_rate;
1100 }
1101 }
1102 register_external_interrupt(0x1407, hws_ext_handler);
1103
1104 hws_state = HWS_DEALLOCATED;
1105 rc = 0;
1106
1107 setup_exit:
1108 mutex_unlock(&hws_sem);
1109 return rc;
1110 }
1111
1112 int hwsampler_shutdown()
1113 {
1114 int rc;
1115
1116 mutex_lock(&hws_sem);
1117
1118 rc = -EINVAL;
1119 if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1120 mutex_unlock(&hws_sem);
1121
1122 if (hws_wq)
1123 flush_workqueue(hws_wq);
1124
1125 mutex_lock(&hws_sem);
1126
1127 if (hws_state == HWS_STOPPED) {
1128 ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1129 deallocate_sdbt();
1130 }
1131 if (hws_wq) {
1132 destroy_workqueue(hws_wq);
1133 hws_wq = NULL;
1134 }
1135
1136 unregister_external_interrupt(0x1407, hws_ext_handler);
1137 hws_state = HWS_INIT;
1138 rc = 0;
1139 }
1140 mutex_unlock(&hws_sem);
1141
1142 unregister_cpu_notifier(&hws_cpu_notifier);
1143
1144 return rc;
1145 }
1146
1147 /**
1148 * hwsampler_start_all() - start hardware sampling on all online CPUs
1149 * @rate: specifies the used interval when samples are taken
1150 *
1151 * Returns 0 on success, !0 on failure.
1152 */
1153 int hwsampler_start_all(unsigned long rate)
1154 {
1155 int rc, cpu;
1156
1157 mutex_lock(&hws_sem);
1158
1159 hws_oom = 0;
1160
1161 rc = -EINVAL;
1162 if (hws_state != HWS_STOPPED)
1163 goto start_all_exit;
1164
1165 interval = rate;
1166
1167 /* fail if rate is not valid */
1168 if (interval < min_sampler_rate || interval > max_sampler_rate)
1169 goto start_all_exit;
1170
1171 rc = check_qsi_on_start();
1172 if (rc)
1173 goto start_all_exit;
1174
1175 rc = prepare_cpu_buffers();
1176 if (rc)
1177 goto start_all_exit;
1178
1179 for_each_online_cpu(cpu) {
1180 rc = start_sampling(cpu);
1181 if (rc)
1182 break;
1183 }
1184 if (rc) {
1185 for_each_online_cpu(cpu) {
1186 stop_sampling(cpu);
1187 }
1188 goto start_all_exit;
1189 }
1190 hws_state = HWS_STARTED;
1191 rc = 0;
1192
1193 start_all_exit:
1194 mutex_unlock(&hws_sem);
1195
1196 if (rc)
1197 return rc;
1198
1199 register_oom_notifier(&hws_oom_notifier);
1200 hws_oom = 1;
1201 hws_flush_all = 0;
1202 /* now let them in, 1407 CPUMF external interrupts */
1203 ctl_set_bit(0, 5); /* set CR0 bit 58 */
1204
1205 return 0;
1206 }
1207
1208 /**
1209 * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1210 *
1211 * Returns 0 on success, !0 on failure.
1212 */
1213 int hwsampler_stop_all()
1214 {
1215 int tmp_rc, rc, cpu;
1216 struct hws_cpu_buffer *cb;
1217
1218 mutex_lock(&hws_sem);
1219
1220 rc = 0;
1221 if (hws_state == HWS_INIT) {
1222 mutex_unlock(&hws_sem);
1223 return rc;
1224 }
1225 hws_state = HWS_STOPPING;
1226 mutex_unlock(&hws_sem);
1227
1228 for_each_online_cpu(cpu) {
1229 cb = &per_cpu(sampler_cpu_buffer, cpu);
1230 cb->stop_mode = 1;
1231 tmp_rc = stop_sampling(cpu);
1232 if (tmp_rc)
1233 rc = tmp_rc;
1234 }
1235
1236 if (hws_wq)
1237 flush_workqueue(hws_wq);
1238
1239 mutex_lock(&hws_sem);
1240 if (hws_oom) {
1241 unregister_oom_notifier(&hws_oom_notifier);
1242 hws_oom = 0;
1243 }
1244 hws_state = HWS_STOPPED;
1245 mutex_unlock(&hws_sem);
1246
1247 return rc;
1248 }
Linux kernel - Deliverables
This page took 0.080154 seconds and 5 git commands to generate.