projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'tpmdd-next-20160902' into next
[deliverable/linux.git] / drivers / bus / arm-cci.c
1 /*
2 * CCI cache coherent interconnect driver
3 *
4 * Copyright (C) 2013 ARM Ltd.
5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
12 * kind, whether express or implied; without even the implied warranty
13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 */
16
17 #include <linux/arm-cci.h>
18 #include <linux/io.h>
19 #include <linux/interrupt.h>
20 #include <linux/module.h>
21 #include <linux/of_address.h>
22 #include <linux/of_irq.h>
23 #include <linux/of_platform.h>
24 #include <linux/perf_event.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <linux/spinlock.h>
28
29 #include <asm/cacheflush.h>
30 #include <asm/smp_plat.h>
31
32 static void __iomem *cci_ctrl_base;
33 static unsigned long cci_ctrl_phys;
34
35 #ifdef CONFIG_ARM_CCI400_PORT_CTRL
36 struct cci_nb_ports {
37 unsigned int nb_ace;
38 unsigned int nb_ace_lite;
39 };
40
41 static const struct cci_nb_ports cci400_ports = {
42 .nb_ace = 2,
43 .nb_ace_lite = 3
44 };
45
46 #define CCI400_PORTS_DATA (&cci400_ports)
47 #else
48 #define CCI400_PORTS_DATA (NULL)
49 #endif
50
51 static const struct of_device_id arm_cci_matches[] = {
52 #ifdef CONFIG_ARM_CCI400_COMMON
53 {.compatible = "arm,cci-400", .data = CCI400_PORTS_DATA },
54 #endif
55 #ifdef CONFIG_ARM_CCI5xx_PMU
56 { .compatible = "arm,cci-500", },
57 { .compatible = "arm,cci-550", },
58 #endif
59 {},
60 };
61
62 #ifdef CONFIG_ARM_CCI_PMU
63
64 #define DRIVER_NAME "ARM-CCI"
65 #define DRIVER_NAME_PMU DRIVER_NAME " PMU"
66
67 #define CCI_PMCR 0x0100
68 #define CCI_PID2 0x0fe8
69
70 #define CCI_PMCR_CEN 0x00000001
71 #define CCI_PMCR_NCNT_MASK 0x0000f800
72 #define CCI_PMCR_NCNT_SHIFT 11
73
74 #define CCI_PID2_REV_MASK 0xf0
75 #define CCI_PID2_REV_SHIFT 4
76
77 #define CCI_PMU_EVT_SEL 0x000
78 #define CCI_PMU_CNTR 0x004
79 #define CCI_PMU_CNTR_CTRL 0x008
80 #define CCI_PMU_OVRFLW 0x00c
81
82 #define CCI_PMU_OVRFLW_FLAG 1
83
84 #define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size)
85 #define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model))
86 #define CCI_PMU_CNTR_MASK ((1ULL << 32) -1)
87 #define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1)
88
89 #define CCI_PMU_MAX_HW_CNTRS(model) \
90 ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs)
91
92 /* Types of interfaces that can generate events */
93 enum {
94 CCI_IF_SLAVE,
95 CCI_IF_MASTER,
96 #ifdef CONFIG_ARM_CCI5xx_PMU
97 CCI_IF_GLOBAL,
98 #endif
99 CCI_IF_MAX,
100 };
101
102 struct event_range {
103 u32 min;
104 u32 max;
105 };
106
107 struct cci_pmu_hw_events {
108 struct perf_event **events;
109 unsigned long *used_mask;
110 raw_spinlock_t pmu_lock;
111 };
112
113 struct cci_pmu;
114 /*
115 * struct cci_pmu_model:
116 * @fixed_hw_cntrs - Number of fixed event counters
117 * @num_hw_cntrs - Maximum number of programmable event counters
118 * @cntr_size - Size of an event counter mapping
119 */
120 struct cci_pmu_model {
121 char *name;
122 u32 fixed_hw_cntrs;
123 u32 num_hw_cntrs;
124 u32 cntr_size;
125 struct attribute **format_attrs;
126 struct attribute **event_attrs;
127 struct event_range event_ranges[CCI_IF_MAX];
128 int (*validate_hw_event)(struct cci_pmu *, unsigned long);
129 int (*get_event_idx)(struct cci_pmu *, struct cci_pmu_hw_events *, unsigned long);
130 void (*write_counters)(struct cci_pmu *, unsigned long *);
131 };
132
133 static struct cci_pmu_model cci_pmu_models[];
134
135 struct cci_pmu {
136 void __iomem *base;
137 struct pmu pmu;
138 int nr_irqs;
139 int *irqs;
140 unsigned long active_irqs;
141 const struct cci_pmu_model *model;
142 struct cci_pmu_hw_events hw_events;
143 struct platform_device *plat_device;
144 int num_cntrs;
145 atomic_t active_events;
146 struct mutex reserve_mutex;
147 struct list_head entry;
148 cpumask_t cpus;
149 };
150
151 #define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu))
152
153 static DEFINE_MUTEX(cci_pmu_mutex);
154 static LIST_HEAD(cci_pmu_list);
155
156 enum cci_models {
157 #ifdef CONFIG_ARM_CCI400_PMU
158 CCI400_R0,
159 CCI400_R1,
160 #endif
161 #ifdef CONFIG_ARM_CCI5xx_PMU
162 CCI500_R0,
163 CCI550_R0,
164 #endif
165 CCI_MODEL_MAX
166 };
167
168 static void pmu_write_counters(struct cci_pmu *cci_pmu,
169 unsigned long *mask);
170 static ssize_t cci_pmu_format_show(struct device *dev,
171 struct device_attribute *attr, char *buf);
172 static ssize_t cci_pmu_event_show(struct device *dev,
173 struct device_attribute *attr, char *buf);
174
175 #define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \
176 &((struct dev_ext_attribute[]) { \
177 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \
178 })[0].attr.attr
179
180 #define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \
181 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_format_show, (char *)_config)
182 #define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \
183 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config)
184
185 /* CCI400 PMU Specific definitions */
186
187 #ifdef CONFIG_ARM_CCI400_PMU
188
189 /* Port ids */
190 #define CCI400_PORT_S0 0
191 #define CCI400_PORT_S1 1
192 #define CCI400_PORT_S2 2
193 #define CCI400_PORT_S3 3
194 #define CCI400_PORT_S4 4
195 #define CCI400_PORT_M0 5
196 #define CCI400_PORT_M1 6
197 #define CCI400_PORT_M2 7
198
199 #define CCI400_R1_PX 5
200
201 /*
202 * Instead of an event id to monitor CCI cycles, a dedicated counter is
203 * provided. Use 0xff to represent CCI cycles and hope that no future revisions
204 * make use of this event in hardware.
205 */
206 enum cci400_perf_events {
207 CCI400_PMU_CYCLES = 0xff
208 };
209
210 #define CCI400_PMU_CYCLE_CNTR_IDX 0
211 #define CCI400_PMU_CNTR0_IDX 1
212
213 /*
214 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8
215 * ports and bits 4:0 are event codes. There are different event codes
216 * associated with each port type.
217 *
218 * Additionally, the range of events associated with the port types changed
219 * between Rev0 and Rev1.
220 *
221 * The constants below define the range of valid codes for each port type for
222 * the different revisions and are used to validate the event to be monitored.
223 */
224
225 #define CCI400_PMU_EVENT_MASK 0xffUL
226 #define CCI400_PMU_EVENT_SOURCE_SHIFT 5
227 #define CCI400_PMU_EVENT_SOURCE_MASK 0x7
228 #define CCI400_PMU_EVENT_CODE_SHIFT 0
229 #define CCI400_PMU_EVENT_CODE_MASK 0x1f
230 #define CCI400_PMU_EVENT_SOURCE(event) \
231 ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \
232 CCI400_PMU_EVENT_SOURCE_MASK)
233 #define CCI400_PMU_EVENT_CODE(event) \
234 ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK)
235
236 #define CCI400_R0_SLAVE_PORT_MIN_EV 0x00
237 #define CCI400_R0_SLAVE_PORT_MAX_EV 0x13
238 #define CCI400_R0_MASTER_PORT_MIN_EV 0x14
239 #define CCI400_R0_MASTER_PORT_MAX_EV 0x1a
240
241 #define CCI400_R1_SLAVE_PORT_MIN_EV 0x00
242 #define CCI400_R1_SLAVE_PORT_MAX_EV 0x14
243 #define CCI400_R1_MASTER_PORT_MIN_EV 0x00
244 #define CCI400_R1_MASTER_PORT_MAX_EV 0x11
245
246 #define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \
247 CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \
248 (unsigned long)_config)
249
250 static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
251 struct device_attribute *attr, char *buf);
252
253 static struct attribute *cci400_pmu_format_attrs[] = {
254 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),
255 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-7"),
256 NULL
257 };
258
259 static struct attribute *cci400_r0_pmu_event_attrs[] = {
260 /* Slave events */
261 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),
262 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),
263 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),
264 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),
265 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),
266 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),
267 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),
268 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
269 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),
270 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),
271 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),
272 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),
273 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),
274 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),
275 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),
276 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),
277 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),
278 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),
279 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),
280 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),
281 /* Master events */
282 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x14),
283 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard, 0x15),
284 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard, 0x16),
285 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full, 0x17),
286 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x18),
287 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x19),
288 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full, 0x1A),
289 /* Special event for cycles counter */
290 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),
291 NULL
292 };
293
294 static struct attribute *cci400_r1_pmu_event_attrs[] = {
295 /* Slave events */
296 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),
297 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),
298 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),
299 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),
300 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),
301 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),
302 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),
303 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
304 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),
305 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),
306 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),
307 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),
308 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),
309 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),
310 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),
311 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),
312 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),
313 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),
314 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),
315 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),
316 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard, 0x14),
317 /* Master events */
318 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x0),
319 CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard, 0x1),
320 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard, 0x2),
321 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full, 0x3),
322 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x4),
323 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x5),
324 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full, 0x6),
325 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full, 0x7),
326 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full, 0x8),
327 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0, 0x9),
328 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1, 0xA),
329 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2, 0xB),
330 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3, 0xC),
331 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0, 0xD),
332 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1, 0xE),
333 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2, 0xF),
334 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3, 0x10),
335 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard, 0x11),
336 /* Special event for cycles counter */
337 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),
338 NULL
339 };
340
341 static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
342 struct device_attribute *attr, char *buf)
343 {
344 struct dev_ext_attribute *eattr = container_of(attr,
345 struct dev_ext_attribute, attr);
346 return snprintf(buf, PAGE_SIZE, "config=0x%lx\n", (unsigned long)eattr->var);
347 }
348
349 static int cci400_get_event_idx(struct cci_pmu *cci_pmu,
350 struct cci_pmu_hw_events *hw,
351 unsigned long cci_event)
352 {
353 int idx;
354
355 /* cycles event idx is fixed */
356 if (cci_event == CCI400_PMU_CYCLES) {
357 if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX, hw->used_mask))
358 return -EAGAIN;
359
360 return CCI400_PMU_CYCLE_CNTR_IDX;
361 }
362
363 for (idx = CCI400_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx)
364 if (!test_and_set_bit(idx, hw->used_mask))
365 return idx;
366
367 /* No counters available */
368 return -EAGAIN;
369 }
370
371 static int cci400_validate_hw_event(struct cci_pmu *cci_pmu, unsigned long hw_event)
372 {
373 u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event);
374 u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event);
375 int if_type;
376
377 if (hw_event & ~CCI400_PMU_EVENT_MASK)
378 return -ENOENT;
379
380 if (hw_event == CCI400_PMU_CYCLES)
381 return hw_event;
382
383 switch (ev_source) {
384 case CCI400_PORT_S0:
385 case CCI400_PORT_S1:
386 case CCI400_PORT_S2:
387 case CCI400_PORT_S3:
388 case CCI400_PORT_S4:
389 /* Slave Interface */
390 if_type = CCI_IF_SLAVE;
391 break;
392 case CCI400_PORT_M0:
393 case CCI400_PORT_M1:
394 case CCI400_PORT_M2:
395 /* Master Interface */
396 if_type = CCI_IF_MASTER;
397 break;
398 default:
399 return -ENOENT;
400 }
401
402 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
403 ev_code <= cci_pmu->model->event_ranges[if_type].max)
404 return hw_event;
405
406 return -ENOENT;
407 }
408
409 static int probe_cci400_revision(void)
410 {
411 int rev;
412 rev = readl_relaxed(cci_ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK;
413 rev >>= CCI_PID2_REV_SHIFT;
414
415 if (rev < CCI400_R1_PX)
416 return CCI400_R0;
417 else
418 return CCI400_R1;
419 }
420
421 static const struct cci_pmu_model *probe_cci_model(struct platform_device *pdev)
422 {
423 if (platform_has_secure_cci_access())
424 return &cci_pmu_models[probe_cci400_revision()];
425 return NULL;
426 }
427 #else /* !CONFIG_ARM_CCI400_PMU */
428 static inline struct cci_pmu_model *probe_cci_model(struct platform_device *pdev)
429 {
430 return NULL;
431 }
432 #endif /* CONFIG_ARM_CCI400_PMU */
433
434 #ifdef CONFIG_ARM_CCI5xx_PMU
435
436 /*
437 * CCI5xx PMU event id is an 9-bit value made of two parts.
438 * bits [8:5] - Source for the event
439 * bits [4:0] - Event code (specific to type of interface)
440 *
441 *
442 */
443
444 /* Port ids */
445 #define CCI5xx_PORT_S0 0x0
446 #define CCI5xx_PORT_S1 0x1
447 #define CCI5xx_PORT_S2 0x2
448 #define CCI5xx_PORT_S3 0x3
449 #define CCI5xx_PORT_S4 0x4
450 #define CCI5xx_PORT_S5 0x5
451 #define CCI5xx_PORT_S6 0x6
452
453 #define CCI5xx_PORT_M0 0x8
454 #define CCI5xx_PORT_M1 0x9
455 #define CCI5xx_PORT_M2 0xa
456 #define CCI5xx_PORT_M3 0xb
457 #define CCI5xx_PORT_M4 0xc
458 #define CCI5xx_PORT_M5 0xd
459 #define CCI5xx_PORT_M6 0xe
460
461 #define CCI5xx_PORT_GLOBAL 0xf
462
463 #define CCI5xx_PMU_EVENT_MASK 0x1ffUL
464 #define CCI5xx_PMU_EVENT_SOURCE_SHIFT 0x5
465 #define CCI5xx_PMU_EVENT_SOURCE_MASK 0xf
466 #define CCI5xx_PMU_EVENT_CODE_SHIFT 0x0
467 #define CCI5xx_PMU_EVENT_CODE_MASK 0x1f
468
469 #define CCI5xx_PMU_EVENT_SOURCE(event) \
470 ((event >> CCI5xx_PMU_EVENT_SOURCE_SHIFT) & CCI5xx_PMU_EVENT_SOURCE_MASK)
471 #define CCI5xx_PMU_EVENT_CODE(event) \
472 ((event >> CCI5xx_PMU_EVENT_CODE_SHIFT) & CCI5xx_PMU_EVENT_CODE_MASK)
473
474 #define CCI5xx_SLAVE_PORT_MIN_EV 0x00
475 #define CCI5xx_SLAVE_PORT_MAX_EV 0x1f
476 #define CCI5xx_MASTER_PORT_MIN_EV 0x00
477 #define CCI5xx_MASTER_PORT_MAX_EV 0x06
478 #define CCI5xx_GLOBAL_PORT_MIN_EV 0x00
479 #define CCI5xx_GLOBAL_PORT_MAX_EV 0x0f
480
481
482 #define CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \
483 CCI_EXT_ATTR_ENTRY(_name, cci5xx_pmu_global_event_show, \
484 (unsigned long) _config)
485
486 static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
487 struct device_attribute *attr, char *buf);
488
489 static struct attribute *cci5xx_pmu_format_attrs[] = {
490 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),
491 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-8"),
492 NULL,
493 };
494
495 static struct attribute *cci5xx_pmu_event_attrs[] = {
496 /* Slave events */
497 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid, 0x0),
498 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev, 0x1),
499 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable, 0x2),
500 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc, 0x3),
501 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc, 0x4),
502 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate, 0x5),
503 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint, 0x6),
504 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
505 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval, 0x8),
506 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop, 0x9),
507 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid, 0xA),
508 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev, 0xB),
509 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable, 0xC),
510 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb, 0xD),
511 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu, 0xE),
512 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique, 0xF),
513 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict, 0x10),
514 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict, 0x11),
515 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat, 0x12),
516 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid, 0x13),
517 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read, 0x14),
518 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean, 0x15),
519 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low, 0x16),
520 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid, 0x17),
521 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall, 0x18),
522 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall, 0x19),
523 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall, 0x1A),
524 CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall, 0x1B),
525 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall, 0x1C),
526 CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall, 0x1D),
527 CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit, 0x1E),
528 CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit, 0x1F),
529
530 /* Master events */
531 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any, 0x0),
532 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any, 0x1),
533 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall, 0x2),
534 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall, 0x3),
535 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall, 0x4),
536 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall, 0x5),
537 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall, 0x6),
538
539 /* Global events */
540 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1, 0x0),
541 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3, 0x1),
542 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5, 0x2),
543 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7, 0x3),
544 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1, 0x4),
545 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3, 0x5),
546 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5, 0x6),
547 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7, 0x7),
548 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation, 0x8),
549 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy, 0x9),
550 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full, 0xA),
551 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB),
552 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC),
553 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD),
554 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snopp_rq_stall_tt_full, 0xE),
555 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF),
556 NULL
557 };
558
559 static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
560 struct device_attribute *attr, char *buf)
561 {
562 struct dev_ext_attribute *eattr = container_of(attr,
563 struct dev_ext_attribute, attr);
564 /* Global events have single fixed source code */
565 return snprintf(buf, PAGE_SIZE, "event=0x%lx,source=0x%x\n",
566 (unsigned long)eattr->var, CCI5xx_PORT_GLOBAL);
567 }
568
569 /*
570 * CCI500 provides 8 independent event counters that can count
571 * any of the events available.
572 * CCI500 PMU event source ids
573 * 0x0-0x6 - Slave interfaces
574 * 0x8-0xD - Master interfaces
575 * 0xf - Global Events
576 * 0x7,0xe - Reserved
577 */
578 static int cci500_validate_hw_event(struct cci_pmu *cci_pmu,
579 unsigned long hw_event)
580 {
581 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);
582 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);
583 int if_type;
584
585 if (hw_event & ~CCI5xx_PMU_EVENT_MASK)
586 return -ENOENT;
587
588 switch (ev_source) {
589 case CCI5xx_PORT_S0:
590 case CCI5xx_PORT_S1:
591 case CCI5xx_PORT_S2:
592 case CCI5xx_PORT_S3:
593 case CCI5xx_PORT_S4:
594 case CCI5xx_PORT_S5:
595 case CCI5xx_PORT_S6:
596 if_type = CCI_IF_SLAVE;
597 break;
598 case CCI5xx_PORT_M0:
599 case CCI5xx_PORT_M1:
600 case CCI5xx_PORT_M2:
601 case CCI5xx_PORT_M3:
602 case CCI5xx_PORT_M4:
603 case CCI5xx_PORT_M5:
604 if_type = CCI_IF_MASTER;
605 break;
606 case CCI5xx_PORT_GLOBAL:
607 if_type = CCI_IF_GLOBAL;
608 break;
609 default:
610 return -ENOENT;
611 }
612
613 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
614 ev_code <= cci_pmu->model->event_ranges[if_type].max)
615 return hw_event;
616
617 return -ENOENT;
618 }
619
620 /*
621 * CCI550 provides 8 independent event counters that can count
622 * any of the events available.
623 * CCI550 PMU event source ids
624 * 0x0-0x6 - Slave interfaces
625 * 0x8-0xe - Master interfaces
626 * 0xf - Global Events
627 * 0x7 - Reserved
628 */
629 static int cci550_validate_hw_event(struct cci_pmu *cci_pmu,
630 unsigned long hw_event)
631 {
632 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);
633 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);
634 int if_type;
635
636 if (hw_event & ~CCI5xx_PMU_EVENT_MASK)
637 return -ENOENT;
638
639 switch (ev_source) {
640 case CCI5xx_PORT_S0:
641 case CCI5xx_PORT_S1:
642 case CCI5xx_PORT_S2:
643 case CCI5xx_PORT_S3:
644 case CCI5xx_PORT_S4:
645 case CCI5xx_PORT_S5:
646 case CCI5xx_PORT_S6:
647 if_type = CCI_IF_SLAVE;
648 break;
649 case CCI5xx_PORT_M0:
650 case CCI5xx_PORT_M1:
651 case CCI5xx_PORT_M2:
652 case CCI5xx_PORT_M3:
653 case CCI5xx_PORT_M4:
654 case CCI5xx_PORT_M5:
655 case CCI5xx_PORT_M6:
656 if_type = CCI_IF_MASTER;
657 break;
658 case CCI5xx_PORT_GLOBAL:
659 if_type = CCI_IF_GLOBAL;
660 break;
661 default:
662 return -ENOENT;
663 }
664
665 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
666 ev_code <= cci_pmu->model->event_ranges[if_type].max)
667 return hw_event;
668
669 return -ENOENT;
670 }
671
672 #endif /* CONFIG_ARM_CCI5xx_PMU */
673
674 /*
675 * Program the CCI PMU counters which have PERF_HES_ARCH set
676 * with the event period and mark them ready before we enable
677 * PMU.
678 */
679 static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu)
680 {
681 int i;
682 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
683
684 DECLARE_BITMAP(mask, cci_pmu->num_cntrs);
685
686 bitmap_zero(mask, cci_pmu->num_cntrs);
687 for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) {
688 struct perf_event *event = cci_hw->events[i];
689
690 if (WARN_ON(!event))
691 continue;
692
693 /* Leave the events which are not counting */
694 if (event->hw.state & PERF_HES_STOPPED)
695 continue;
696 if (event->hw.state & PERF_HES_ARCH) {
697 set_bit(i, mask);
698 event->hw.state &= ~PERF_HES_ARCH;
699 }
700 }
701
702 pmu_write_counters(cci_pmu, mask);
703 }
704
705 /* Should be called with cci_pmu->hw_events->pmu_lock held */
706 static void __cci_pmu_enable_nosync(struct cci_pmu *cci_pmu)
707 {
708 u32 val;
709
710 /* Enable all the PMU counters. */
711 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) | CCI_PMCR_CEN;
712 writel(val, cci_ctrl_base + CCI_PMCR);
713 }
714
715 /* Should be called with cci_pmu->hw_events->pmu_lock held */
716 static void __cci_pmu_enable_sync(struct cci_pmu *cci_pmu)
717 {
718 cci_pmu_sync_counters(cci_pmu);
719 __cci_pmu_enable_nosync(cci_pmu);
720 }
721
722 /* Should be called with cci_pmu->hw_events->pmu_lock held */
723 static void __cci_pmu_disable(void)
724 {
725 u32 val;
726
727 /* Disable all the PMU counters. */
728 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN;
729 writel(val, cci_ctrl_base + CCI_PMCR);
730 }
731
732 static ssize_t cci_pmu_format_show(struct device *dev,
733 struct device_attribute *attr, char *buf)
734 {
735 struct dev_ext_attribute *eattr = container_of(attr,
736 struct dev_ext_attribute, attr);
737 return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var);
738 }
739
740 static ssize_t cci_pmu_event_show(struct device *dev,
741 struct device_attribute *attr, char *buf)
742 {
743 struct dev_ext_attribute *eattr = container_of(attr,
744 struct dev_ext_attribute, attr);
745 /* source parameter is mandatory for normal PMU events */
746 return snprintf(buf, PAGE_SIZE, "source=?,event=0x%lx\n",
747 (unsigned long)eattr->var);
748 }
749
750 static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx)
751 {
752 return 0 <= idx && idx <= CCI_PMU_CNTR_LAST(cci_pmu);
753 }
754
755 static u32 pmu_read_register(struct cci_pmu *cci_pmu, int idx, unsigned int offset)
756 {
757 return readl_relaxed(cci_pmu->base +
758 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);
759 }
760
761 static void pmu_write_register(struct cci_pmu *cci_pmu, u32 value,
762 int idx, unsigned int offset)
763 {
764 writel_relaxed(value, cci_pmu->base +
765 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);
766 }
767
768 static void pmu_disable_counter(struct cci_pmu *cci_pmu, int idx)
769 {
770 pmu_write_register(cci_pmu, 0, idx, CCI_PMU_CNTR_CTRL);
771 }
772
773 static void pmu_enable_counter(struct cci_pmu *cci_pmu, int idx)
774 {
775 pmu_write_register(cci_pmu, 1, idx, CCI_PMU_CNTR_CTRL);
776 }
777
778 static bool __maybe_unused
779 pmu_counter_is_enabled(struct cci_pmu *cci_pmu, int idx)
780 {
781 return (pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR_CTRL) & 0x1) != 0;
782 }
783
784 static void pmu_set_event(struct cci_pmu *cci_pmu, int idx, unsigned long event)
785 {
786 pmu_write_register(cci_pmu, event, idx, CCI_PMU_EVT_SEL);
787 }
788
789 /*
790 * For all counters on the CCI-PMU, disable any 'enabled' counters,
791 * saving the changed counters in the mask, so that we can restore
792 * it later using pmu_restore_counters. The mask is private to the
793 * caller. We cannot rely on the used_mask maintained by the CCI_PMU
794 * as it only tells us if the counter is assigned to perf_event or not.
795 * The state of the perf_event cannot be locked by the PMU layer, hence
796 * we check the individual counter status (which can be locked by
797 * cci_pm->hw_events->pmu_lock).
798 *
799 * @mask should be initialised to empty by the caller.
800 */
801 static void __maybe_unused
802 pmu_save_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
803 {
804 int i;
805
806 for (i = 0; i < cci_pmu->num_cntrs; i++) {
807 if (pmu_counter_is_enabled(cci_pmu, i)) {
808 set_bit(i, mask);
809 pmu_disable_counter(cci_pmu, i);
810 }
811 }
812 }
813
814 /*
815 * Restore the status of the counters. Reversal of the pmu_save_counters().
816 * For each counter set in the mask, enable the counter back.
817 */
818 static void __maybe_unused
819 pmu_restore_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
820 {
821 int i;
822
823 for_each_set_bit(i, mask, cci_pmu->num_cntrs)
824 pmu_enable_counter(cci_pmu, i);
825 }
826
827 /*
828 * Returns the number of programmable counters actually implemented
829 * by the cci
830 */
831 static u32 pmu_get_max_counters(void)
832 {
833 return (readl_relaxed(cci_ctrl_base + CCI_PMCR) &
834 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT;
835 }
836
837 static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event)
838 {
839 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
840 unsigned long cci_event = event->hw.config_base;
841 int idx;
842
843 if (cci_pmu->model->get_event_idx)
844 return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event);
845
846 /* Generic code to find an unused idx from the mask */
847 for(idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++)
848 if (!test_and_set_bit(idx, hw->used_mask))
849 return idx;
850
851 /* No counters available */
852 return -EAGAIN;
853 }
854
855 static int pmu_map_event(struct perf_event *event)
856 {
857 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
858
859 if (event->attr.type < PERF_TYPE_MAX ||
860 !cci_pmu->model->validate_hw_event)
861 return -ENOENT;
862
863 return cci_pmu->model->validate_hw_event(cci_pmu, event->attr.config);
864 }
865
866 static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler)
867 {
868 int i;
869 struct platform_device *pmu_device = cci_pmu->plat_device;
870
871 if (unlikely(!pmu_device))
872 return -ENODEV;
873
874 if (cci_pmu->nr_irqs < 1) {
875 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n");
876 return -ENODEV;
877 }
878
879 /*
880 * Register all available CCI PMU interrupts. In the interrupt handler
881 * we iterate over the counters checking for interrupt source (the
882 * overflowing counter) and clear it.
883 *
884 * This should allow handling of non-unique interrupt for the counters.
885 */
886 for (i = 0; i < cci_pmu->nr_irqs; i++) {
887 int err = request_irq(cci_pmu->irqs[i], handler, IRQF_SHARED,
888 "arm-cci-pmu", cci_pmu);
889 if (err) {
890 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n",
891 cci_pmu->irqs[i]);
892 return err;
893 }
894
895 set_bit(i, &cci_pmu->active_irqs);
896 }
897
898 return 0;
899 }
900
901 static void pmu_free_irq(struct cci_pmu *cci_pmu)
902 {
903 int i;
904
905 for (i = 0; i < cci_pmu->nr_irqs; i++) {
906 if (!test_and_clear_bit(i, &cci_pmu->active_irqs))
907 continue;
908
909 free_irq(cci_pmu->irqs[i], cci_pmu);
910 }
911 }
912
913 static u32 pmu_read_counter(struct perf_event *event)
914 {
915 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
916 struct hw_perf_event *hw_counter = &event->hw;
917 int idx = hw_counter->idx;
918 u32 value;
919
920 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
921 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
922 return 0;
923 }
924 value = pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR);
925
926 return value;
927 }
928
929 static void pmu_write_counter(struct cci_pmu *cci_pmu, u32 value, int idx)
930 {
931 pmu_write_register(cci_pmu, value, idx, CCI_PMU_CNTR);
932 }
933
934 static void __pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
935 {
936 int i;
937 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
938
939 for_each_set_bit(i, mask, cci_pmu->num_cntrs) {
940 struct perf_event *event = cci_hw->events[i];
941
942 if (WARN_ON(!event))
943 continue;
944 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);
945 }
946 }
947
948 static void pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
949 {
950 if (cci_pmu->model->write_counters)
951 cci_pmu->model->write_counters(cci_pmu, mask);
952 else
953 __pmu_write_counters(cci_pmu, mask);
954 }
955
956 #ifdef CONFIG_ARM_CCI5xx_PMU
957
958 /*
959 * CCI-500/CCI-550 has advanced power saving policies, which could gate the
960 * clocks to the PMU counters, which makes the writes to them ineffective.
961 * The only way to write to those counters is when the global counters
962 * are enabled and the particular counter is enabled.
963 *
964 * So we do the following :
965 *
966 * 1) Disable all the PMU counters, saving their current state
967 * 2) Enable the global PMU profiling, now that all counters are
968 * disabled.
969 *
970 * For each counter to be programmed, repeat steps 3-7:
971 *
972 * 3) Write an invalid event code to the event control register for the
973 counter, so that the counters are not modified.
974 * 4) Enable the counter control for the counter.
975 * 5) Set the counter value
976 * 6) Disable the counter
977 * 7) Restore the event in the target counter
978 *
979 * 8) Disable the global PMU.
980 * 9) Restore the status of the rest of the counters.
981 *
982 * We choose an event which for CCI-5xx is guaranteed not to count.
983 * We use the highest possible event code (0x1f) for the master interface 0.
984 */
985 #define CCI5xx_INVALID_EVENT ((CCI5xx_PORT_M0 << CCI5xx_PMU_EVENT_SOURCE_SHIFT) | \
986 (CCI5xx_PMU_EVENT_CODE_MASK << CCI5xx_PMU_EVENT_CODE_SHIFT))
987 static void cci5xx_pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
988 {
989 int i;
990 DECLARE_BITMAP(saved_mask, cci_pmu->num_cntrs);
991
992 bitmap_zero(saved_mask, cci_pmu->num_cntrs);
993 pmu_save_counters(cci_pmu, saved_mask);
994
995 /*
996 * Now that all the counters are disabled, we can safely turn the PMU on,
997 * without syncing the status of the counters
998 */
999 __cci_pmu_enable_nosync(cci_pmu);
1000
1001 for_each_set_bit(i, mask, cci_pmu->num_cntrs) {
1002 struct perf_event *event = cci_pmu->hw_events.events[i];
1003
1004 if (WARN_ON(!event))
1005 continue;
1006
1007 pmu_set_event(cci_pmu, i, CCI5xx_INVALID_EVENT);
1008 pmu_enable_counter(cci_pmu, i);
1009 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);
1010 pmu_disable_counter(cci_pmu, i);
1011 pmu_set_event(cci_pmu, i, event->hw.config_base);
1012 }
1013
1014 __cci_pmu_disable();
1015
1016 pmu_restore_counters(cci_pmu, saved_mask);
1017 }
1018
1019 #endif /* CONFIG_ARM_CCI5xx_PMU */
1020
1021 static u64 pmu_event_update(struct perf_event *event)
1022 {
1023 struct hw_perf_event *hwc = &event->hw;
1024 u64 delta, prev_raw_count, new_raw_count;
1025
1026 do {
1027 prev_raw_count = local64_read(&hwc->prev_count);
1028 new_raw_count = pmu_read_counter(event);
1029 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
1030 new_raw_count) != prev_raw_count);
1031
1032 delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK;
1033
1034 local64_add(delta, &event->count);
1035
1036 return new_raw_count;
1037 }
1038
1039 static void pmu_read(struct perf_event *event)
1040 {
1041 pmu_event_update(event);
1042 }
1043
1044 static void pmu_event_set_period(struct perf_event *event)
1045 {
1046 struct hw_perf_event *hwc = &event->hw;
1047 /*
1048 * The CCI PMU counters have a period of 2^32. To account for the
1049 * possiblity of extreme interrupt latency we program for a period of
1050 * half that. Hopefully we can handle the interrupt before another 2^31
1051 * events occur and the counter overtakes its previous value.
1052 */
1053 u64 val = 1ULL << 31;
1054 local64_set(&hwc->prev_count, val);
1055
1056 /*
1057 * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose
1058 * values needs to be sync-ed with the s/w state before the PMU is
1059 * enabled.
1060 * Mark this counter for sync.
1061 */
1062 hwc->state |= PERF_HES_ARCH;
1063 }
1064
1065 static irqreturn_t pmu_handle_irq(int irq_num, void *dev)
1066 {
1067 unsigned long flags;
1068 struct cci_pmu *cci_pmu = dev;
1069 struct cci_pmu_hw_events *events = &cci_pmu->hw_events;
1070 int idx, handled = IRQ_NONE;
1071
1072 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1073
1074 /* Disable the PMU while we walk through the counters */
1075 __cci_pmu_disable();
1076 /*
1077 * Iterate over counters and update the corresponding perf events.
1078 * This should work regardless of whether we have per-counter overflow
1079 * interrupt or a combined overflow interrupt.
1080 */
1081 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) {
1082 struct perf_event *event = events->events[idx];
1083
1084 if (!event)
1085 continue;
1086
1087 /* Did this counter overflow? */
1088 if (!(pmu_read_register(cci_pmu, idx, CCI_PMU_OVRFLW) &
1089 CCI_PMU_OVRFLW_FLAG))
1090 continue;
1091
1092 pmu_write_register(cci_pmu, CCI_PMU_OVRFLW_FLAG, idx,
1093 CCI_PMU_OVRFLW);
1094
1095 pmu_event_update(event);
1096 pmu_event_set_period(event);
1097 handled = IRQ_HANDLED;
1098 }
1099
1100 /* Enable the PMU and sync possibly overflowed counters */
1101 __cci_pmu_enable_sync(cci_pmu);
1102 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1103
1104 return IRQ_RETVAL(handled);
1105 }
1106
1107 static int cci_pmu_get_hw(struct cci_pmu *cci_pmu)
1108 {
1109 int ret = pmu_request_irq(cci_pmu, pmu_handle_irq);
1110 if (ret) {
1111 pmu_free_irq(cci_pmu);
1112 return ret;
1113 }
1114 return 0;
1115 }
1116
1117 static void cci_pmu_put_hw(struct cci_pmu *cci_pmu)
1118 {
1119 pmu_free_irq(cci_pmu);
1120 }
1121
1122 static void hw_perf_event_destroy(struct perf_event *event)
1123 {
1124 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1125 atomic_t *active_events = &cci_pmu->active_events;
1126 struct mutex *reserve_mutex = &cci_pmu->reserve_mutex;
1127
1128 if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) {
1129 cci_pmu_put_hw(cci_pmu);
1130 mutex_unlock(reserve_mutex);
1131 }
1132 }
1133
1134 static void cci_pmu_enable(struct pmu *pmu)
1135 {
1136 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1137 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1138 int enabled = bitmap_weight(hw_events->used_mask, cci_pmu->num_cntrs);
1139 unsigned long flags;
1140
1141 if (!enabled)
1142 return;
1143
1144 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1145 __cci_pmu_enable_sync(cci_pmu);
1146 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1147
1148 }
1149
1150 static void cci_pmu_disable(struct pmu *pmu)
1151 {
1152 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1153 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1154 unsigned long flags;
1155
1156 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1157 __cci_pmu_disable();
1158 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1159 }
1160
1161 /*
1162 * Check if the idx represents a non-programmable counter.
1163 * All the fixed event counters are mapped before the programmable
1164 * counters.
1165 */
1166 static bool pmu_fixed_hw_idx(struct cci_pmu *cci_pmu, int idx)
1167 {
1168 return (idx >= 0) && (idx < cci_pmu->model->fixed_hw_cntrs);
1169 }
1170
1171 static void cci_pmu_start(struct perf_event *event, int pmu_flags)
1172 {
1173 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1174 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1175 struct hw_perf_event *hwc = &event->hw;
1176 int idx = hwc->idx;
1177 unsigned long flags;
1178
1179 /*
1180 * To handle interrupt latency, we always reprogram the period
1181 * regardlesss of PERF_EF_RELOAD.
1182 */
1183 if (pmu_flags & PERF_EF_RELOAD)
1184 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
1185
1186 hwc->state = 0;
1187
1188 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
1189 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
1190 return;
1191 }
1192
1193 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1194
1195 /* Configure the counter unless you are counting a fixed event */
1196 if (!pmu_fixed_hw_idx(cci_pmu, idx))
1197 pmu_set_event(cci_pmu, idx, hwc->config_base);
1198
1199 pmu_event_set_period(event);
1200 pmu_enable_counter(cci_pmu, idx);
1201
1202 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1203 }
1204
1205 static void cci_pmu_stop(struct perf_event *event, int pmu_flags)
1206 {
1207 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1208 struct hw_perf_event *hwc = &event->hw;
1209 int idx = hwc->idx;
1210
1211 if (hwc->state & PERF_HES_STOPPED)
1212 return;
1213
1214 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
1215 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
1216 return;
1217 }
1218
1219 /*
1220 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See
1221 * cci_pmu_start()
1222 */
1223 pmu_disable_counter(cci_pmu, idx);
1224 pmu_event_update(event);
1225 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
1226 }
1227
1228 static int cci_pmu_add(struct perf_event *event, int flags)
1229 {
1230 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1231 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1232 struct hw_perf_event *hwc = &event->hw;
1233 int idx;
1234 int err = 0;
1235
1236 perf_pmu_disable(event->pmu);
1237
1238 /* If we don't have a space for the counter then finish early. */
1239 idx = pmu_get_event_idx(hw_events, event);
1240 if (idx < 0) {
1241 err = idx;
1242 goto out;
1243 }
1244
1245 event->hw.idx = idx;
1246 hw_events->events[idx] = event;
1247
1248 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1249 if (flags & PERF_EF_START)
1250 cci_pmu_start(event, PERF_EF_RELOAD);
1251
1252 /* Propagate our changes to the userspace mapping. */
1253 perf_event_update_userpage(event);
1254
1255 out:
1256 perf_pmu_enable(event->pmu);
1257 return err;
1258 }
1259
1260 static void cci_pmu_del(struct perf_event *event, int flags)
1261 {
1262 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1263 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1264 struct hw_perf_event *hwc = &event->hw;
1265 int idx = hwc->idx;
1266
1267 cci_pmu_stop(event, PERF_EF_UPDATE);
1268 hw_events->events[idx] = NULL;
1269 clear_bit(idx, hw_events->used_mask);
1270
1271 perf_event_update_userpage(event);
1272 }
1273
1274 static int
1275 validate_event(struct pmu *cci_pmu,
1276 struct cci_pmu_hw_events *hw_events,
1277 struct perf_event *event)
1278 {
1279 if (is_software_event(event))
1280 return 1;
1281
1282 /*
1283 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
1284 * core perf code won't check that the pmu->ctx == leader->ctx
1285 * until after pmu->event_init(event).
1286 */
1287 if (event->pmu != cci_pmu)
1288 return 0;
1289
1290 if (event->state < PERF_EVENT_STATE_OFF)
1291 return 1;
1292
1293 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
1294 return 1;
1295
1296 return pmu_get_event_idx(hw_events, event) >= 0;
1297 }
1298
1299 static int
1300 validate_group(struct perf_event *event)
1301 {
1302 struct perf_event *sibling, *leader = event->group_leader;
1303 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1304 unsigned long mask[BITS_TO_LONGS(cci_pmu->num_cntrs)];
1305 struct cci_pmu_hw_events fake_pmu = {
1306 /*
1307 * Initialise the fake PMU. We only need to populate the
1308 * used_mask for the purposes of validation.
1309 */
1310 .used_mask = mask,
1311 };
1312 memset(mask, 0, BITS_TO_LONGS(cci_pmu->num_cntrs) * sizeof(unsigned long));
1313
1314 if (!validate_event(event->pmu, &fake_pmu, leader))
1315 return -EINVAL;
1316
1317 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
1318 if (!validate_event(event->pmu, &fake_pmu, sibling))
1319 return -EINVAL;
1320 }
1321
1322 if (!validate_event(event->pmu, &fake_pmu, event))
1323 return -EINVAL;
1324
1325 return 0;
1326 }
1327
1328 static int
1329 __hw_perf_event_init(struct perf_event *event)
1330 {
1331 struct hw_perf_event *hwc = &event->hw;
1332 int mapping;
1333
1334 mapping = pmu_map_event(event);
1335
1336 if (mapping < 0) {
1337 pr_debug("event %x:%llx not supported\n", event->attr.type,
1338 event->attr.config);
1339 return mapping;
1340 }
1341
1342 /*
1343 * We don't assign an index until we actually place the event onto
1344 * hardware. Use -1 to signify that we haven't decided where to put it
1345 * yet.
1346 */
1347 hwc->idx = -1;
1348 hwc->config_base = 0;
1349 hwc->config = 0;
1350 hwc->event_base = 0;
1351
1352 /*
1353 * Store the event encoding into the config_base field.
1354 */
1355 hwc->config_base |= (unsigned long)mapping;
1356
1357 /*
1358 * Limit the sample_period to half of the counter width. That way, the
1359 * new counter value is far less likely to overtake the previous one
1360 * unless you have some serious IRQ latency issues.
1361 */
1362 hwc->sample_period = CCI_PMU_CNTR_MASK >> 1;
1363 hwc->last_period = hwc->sample_period;
1364 local64_set(&hwc->period_left, hwc->sample_period);
1365
1366 if (event->group_leader != event) {
1367 if (validate_group(event) != 0)
1368 return -EINVAL;
1369 }
1370
1371 return 0;
1372 }
1373
1374 static int cci_pmu_event_init(struct perf_event *event)
1375 {
1376 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1377 atomic_t *active_events = &cci_pmu->active_events;
1378 int err = 0;
1379 int cpu;
1380
1381 if (event->attr.type != event->pmu->type)
1382 return -ENOENT;
1383
1384 /* Shared by all CPUs, no meaningful state to sample */
1385 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
1386 return -EOPNOTSUPP;
1387
1388 /* We have no filtering of any kind */
1389 if (event->attr.exclude_user ||
1390 event->attr.exclude_kernel ||
1391 event->attr.exclude_hv ||
1392 event->attr.exclude_idle ||
1393 event->attr.exclude_host ||
1394 event->attr.exclude_guest)
1395 return -EINVAL;
1396
1397 /*
1398 * Following the example set by other "uncore" PMUs, we accept any CPU
1399 * and rewrite its affinity dynamically rather than having perf core
1400 * handle cpu == -1 and pid == -1 for this case.
1401 *
1402 * The perf core will pin online CPUs for the duration of this call and
1403 * the event being installed into its context, so the PMU's CPU can't
1404 * change under our feet.
1405 */
1406 cpu = cpumask_first(&cci_pmu->cpus);
1407 if (event->cpu < 0 || cpu < 0)
1408 return -EINVAL;
1409 event->cpu = cpu;
1410
1411 event->destroy = hw_perf_event_destroy;
1412 if (!atomic_inc_not_zero(active_events)) {
1413 mutex_lock(&cci_pmu->reserve_mutex);
1414 if (atomic_read(active_events) == 0)
1415 err = cci_pmu_get_hw(cci_pmu);
1416 if (!err)
1417 atomic_inc(active_events);
1418 mutex_unlock(&cci_pmu->reserve_mutex);
1419 }
1420 if (err)
1421 return err;
1422
1423 err = __hw_perf_event_init(event);
1424 if (err)
1425 hw_perf_event_destroy(event);
1426
1427 return err;
1428 }
1429
1430 static ssize_t pmu_cpumask_attr_show(struct device *dev,
1431 struct device_attribute *attr, char *buf)
1432 {
1433 struct pmu *pmu = dev_get_drvdata(dev);
1434 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1435
1436 int n = scnprintf(buf, PAGE_SIZE - 1, "%*pbl",
1437 cpumask_pr_args(&cci_pmu->cpus));
1438 buf[n++] = '\n';
1439 buf[n] = '\0';
1440 return n;
1441 }
1442
1443 static struct device_attribute pmu_cpumask_attr =
1444 __ATTR(cpumask, S_IRUGO, pmu_cpumask_attr_show, NULL);
1445
1446 static struct attribute *pmu_attrs[] = {
1447 &pmu_cpumask_attr.attr,
1448 NULL,
1449 };
1450
1451 static struct attribute_group pmu_attr_group = {
1452 .attrs = pmu_attrs,
1453 };
1454
1455 static struct attribute_group pmu_format_attr_group = {
1456 .name = "format",
1457 .attrs = NULL, /* Filled in cci_pmu_init_attrs */
1458 };
1459
1460 static struct attribute_group pmu_event_attr_group = {
1461 .name = "events",
1462 .attrs = NULL, /* Filled in cci_pmu_init_attrs */
1463 };
1464
1465 static const struct attribute_group *pmu_attr_groups[] = {
1466 &pmu_attr_group,
1467 &pmu_format_attr_group,
1468 &pmu_event_attr_group,
1469 NULL
1470 };
1471
1472 static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev)
1473 {
1474 const struct cci_pmu_model *model = cci_pmu->model;
1475 char *name = model->name;
1476 u32 num_cntrs;
1477
1478 pmu_event_attr_group.attrs = model->event_attrs;
1479 pmu_format_attr_group.attrs = model->format_attrs;
1480
1481 cci_pmu->pmu = (struct pmu) {
1482 .name = cci_pmu->model->name,
1483 .task_ctx_nr = perf_invalid_context,
1484 .pmu_enable = cci_pmu_enable,
1485 .pmu_disable = cci_pmu_disable,
1486 .event_init = cci_pmu_event_init,
1487 .add = cci_pmu_add,
1488 .del = cci_pmu_del,
1489 .start = cci_pmu_start,
1490 .stop = cci_pmu_stop,
1491 .read = pmu_read,
1492 .attr_groups = pmu_attr_groups,
1493 };
1494
1495 cci_pmu->plat_device = pdev;
1496 num_cntrs = pmu_get_max_counters();
1497 if (num_cntrs > cci_pmu->model->num_hw_cntrs) {
1498 dev_warn(&pdev->dev,
1499 "PMU implements more counters(%d) than supported by"
1500 " the model(%d), truncated.",
1501 num_cntrs, cci_pmu->model->num_hw_cntrs);
1502 num_cntrs = cci_pmu->model->num_hw_cntrs;
1503 }
1504 cci_pmu->num_cntrs = num_cntrs + cci_pmu->model->fixed_hw_cntrs;
1505
1506 return perf_pmu_register(&cci_pmu->pmu, name, -1);
1507 }
1508
1509 static int cci_pmu_offline_cpu(unsigned int cpu)
1510 {
1511 struct cci_pmu *cci_pmu;
1512 unsigned int target;
1513
1514 mutex_lock(&cci_pmu_mutex);
1515 list_for_each_entry(cci_pmu, &cci_pmu_list, entry) {
1516 if (!cpumask_test_and_clear_cpu(cpu, &cci_pmu->cpus))
1517 continue;
1518 target = cpumask_any_but(cpu_online_mask, cpu);
1519 if (target >= nr_cpu_ids)
1520 continue;
1521 /*
1522 * TODO: migrate context once core races on event->ctx have
1523 * been fixed.
1524 */
1525 cpumask_set_cpu(target, &cci_pmu->cpus);
1526 }
1527 mutex_unlock(&cci_pmu_mutex);
1528 return 0;
1529 }
1530
1531 static struct cci_pmu_model cci_pmu_models[] = {
1532 #ifdef CONFIG_ARM_CCI400_PMU
1533 [CCI400_R0] = {
1534 .name = "CCI_400",
1535 .fixed_hw_cntrs = 1, /* Cycle counter */
1536 .num_hw_cntrs = 4,
1537 .cntr_size = SZ_4K,
1538 .format_attrs = cci400_pmu_format_attrs,
1539 .event_attrs = cci400_r0_pmu_event_attrs,
1540 .event_ranges = {
1541 [CCI_IF_SLAVE] = {
1542 CCI400_R0_SLAVE_PORT_MIN_EV,
1543 CCI400_R0_SLAVE_PORT_MAX_EV,
1544 },
1545 [CCI_IF_MASTER] = {
1546 CCI400_R0_MASTER_PORT_MIN_EV,
1547 CCI400_R0_MASTER_PORT_MAX_EV,
1548 },
1549 },
1550 .validate_hw_event = cci400_validate_hw_event,
1551 .get_event_idx = cci400_get_event_idx,
1552 },
1553 [CCI400_R1] = {
1554 .name = "CCI_400_r1",
1555 .fixed_hw_cntrs = 1, /* Cycle counter */
1556 .num_hw_cntrs = 4,
1557 .cntr_size = SZ_4K,
1558 .format_attrs = cci400_pmu_format_attrs,
1559 .event_attrs = cci400_r1_pmu_event_attrs,
1560 .event_ranges = {
1561 [CCI_IF_SLAVE] = {
1562 CCI400_R1_SLAVE_PORT_MIN_EV,
1563 CCI400_R1_SLAVE_PORT_MAX_EV,
1564 },
1565 [CCI_IF_MASTER] = {
1566 CCI400_R1_MASTER_PORT_MIN_EV,
1567 CCI400_R1_MASTER_PORT_MAX_EV,
1568 },
1569 },
1570 .validate_hw_event = cci400_validate_hw_event,
1571 .get_event_idx = cci400_get_event_idx,
1572 },
1573 #endif
1574 #ifdef CONFIG_ARM_CCI5xx_PMU
1575 [CCI500_R0] = {
1576 .name = "CCI_500",
1577 .fixed_hw_cntrs = 0,
1578 .num_hw_cntrs = 8,
1579 .cntr_size = SZ_64K,
1580 .format_attrs = cci5xx_pmu_format_attrs,
1581 .event_attrs = cci5xx_pmu_event_attrs,
1582 .event_ranges = {
1583 [CCI_IF_SLAVE] = {
1584 CCI5xx_SLAVE_PORT_MIN_EV,
1585 CCI5xx_SLAVE_PORT_MAX_EV,
1586 },
1587 [CCI_IF_MASTER] = {
1588 CCI5xx_MASTER_PORT_MIN_EV,
1589 CCI5xx_MASTER_PORT_MAX_EV,
1590 },
1591 [CCI_IF_GLOBAL] = {
1592 CCI5xx_GLOBAL_PORT_MIN_EV,
1593 CCI5xx_GLOBAL_PORT_MAX_EV,
1594 },
1595 },
1596 .validate_hw_event = cci500_validate_hw_event,
1597 .write_counters = cci5xx_pmu_write_counters,
1598 },
1599 [CCI550_R0] = {
1600 .name = "CCI_550",
1601 .fixed_hw_cntrs = 0,
1602 .num_hw_cntrs = 8,
1603 .cntr_size = SZ_64K,
1604 .format_attrs = cci5xx_pmu_format_attrs,
1605 .event_attrs = cci5xx_pmu_event_attrs,
1606 .event_ranges = {
1607 [CCI_IF_SLAVE] = {
1608 CCI5xx_SLAVE_PORT_MIN_EV,
1609 CCI5xx_SLAVE_PORT_MAX_EV,
1610 },
1611 [CCI_IF_MASTER] = {
1612 CCI5xx_MASTER_PORT_MIN_EV,
1613 CCI5xx_MASTER_PORT_MAX_EV,
1614 },
1615 [CCI_IF_GLOBAL] = {
1616 CCI5xx_GLOBAL_PORT_MIN_EV,
1617 CCI5xx_GLOBAL_PORT_MAX_EV,
1618 },
1619 },
1620 .validate_hw_event = cci550_validate_hw_event,
1621 .write_counters = cci5xx_pmu_write_counters,
1622 },
1623 #endif
1624 };
1625
1626 static const struct of_device_id arm_cci_pmu_matches[] = {
1627 #ifdef CONFIG_ARM_CCI400_PMU
1628 {
1629 .compatible = "arm,cci-400-pmu",
1630 .data = NULL,
1631 },
1632 {
1633 .compatible = "arm,cci-400-pmu,r0",
1634 .data = &cci_pmu_models[CCI400_R0],
1635 },
1636 {
1637 .compatible = "arm,cci-400-pmu,r1",
1638 .data = &cci_pmu_models[CCI400_R1],
1639 },
1640 #endif
1641 #ifdef CONFIG_ARM_CCI5xx_PMU
1642 {
1643 .compatible = "arm,cci-500-pmu,r0",
1644 .data = &cci_pmu_models[CCI500_R0],
1645 },
1646 {
1647 .compatible = "arm,cci-550-pmu,r0",
1648 .data = &cci_pmu_models[CCI550_R0],
1649 },
1650 #endif
1651 {},
1652 };
1653
1654 static inline const struct cci_pmu_model *get_cci_model(struct platform_device *pdev)
1655 {
1656 const struct of_device_id *match = of_match_node(arm_cci_pmu_matches,
1657 pdev->dev.of_node);
1658 if (!match)
1659 return NULL;
1660 if (match->data)
1661 return match->data;
1662
1663 dev_warn(&pdev->dev, "DEPRECATED compatible property,"
1664 "requires secure access to CCI registers");
1665 return probe_cci_model(pdev);
1666 }
1667
1668 static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs)
1669 {
1670 int i;
1671
1672 for (i = 0; i < nr_irqs; i++)
1673 if (irq == irqs[i])
1674 return true;
1675
1676 return false;
1677 }
1678
1679 static struct cci_pmu *cci_pmu_alloc(struct platform_device *pdev)
1680 {
1681 struct cci_pmu *cci_pmu;
1682 const struct cci_pmu_model *model;
1683
1684 /*
1685 * All allocations are devm_* hence we don't have to free
1686 * them explicitly on an error, as it would end up in driver
1687 * detach.
1688 */
1689 model = get_cci_model(pdev);
1690 if (!model) {
1691 dev_warn(&pdev->dev, "CCI PMU version not supported\n");
1692 return ERR_PTR(-ENODEV);
1693 }
1694
1695 cci_pmu = devm_kzalloc(&pdev->dev, sizeof(*cci_pmu), GFP_KERNEL);
1696 if (!cci_pmu)
1697 return ERR_PTR(-ENOMEM);
1698
1699 cci_pmu->model = model;
1700 cci_pmu->irqs = devm_kcalloc(&pdev->dev, CCI_PMU_MAX_HW_CNTRS(model),
1701 sizeof(*cci_pmu->irqs), GFP_KERNEL);
1702 if (!cci_pmu->irqs)
1703 return ERR_PTR(-ENOMEM);
1704 cci_pmu->hw_events.events = devm_kcalloc(&pdev->dev,
1705 CCI_PMU_MAX_HW_CNTRS(model),
1706 sizeof(*cci_pmu->hw_events.events),
1707 GFP_KERNEL);
1708 if (!cci_pmu->hw_events.events)
1709 return ERR_PTR(-ENOMEM);
1710 cci_pmu->hw_events.used_mask = devm_kcalloc(&pdev->dev,
1711 BITS_TO_LONGS(CCI_PMU_MAX_HW_CNTRS(model)),
1712 sizeof(*cci_pmu->hw_events.used_mask),
1713 GFP_KERNEL);
1714 if (!cci_pmu->hw_events.used_mask)
1715 return ERR_PTR(-ENOMEM);
1716
1717 return cci_pmu;
1718 }
1719
1720
1721 static int cci_pmu_probe(struct platform_device *pdev)
1722 {
1723 struct resource *res;
1724 struct cci_pmu *cci_pmu;
1725 int i, ret, irq;
1726
1727 cci_pmu = cci_pmu_alloc(pdev);
1728 if (IS_ERR(cci_pmu))
1729 return PTR_ERR(cci_pmu);
1730
1731 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1732 cci_pmu->base = devm_ioremap_resource(&pdev->dev, res);
1733 if (IS_ERR(cci_pmu->base))
1734 return -ENOMEM;
1735
1736 /*
1737 * CCI PMU has one overflow interrupt per counter; but some may be tied
1738 * together to a common interrupt.
1739 */
1740 cci_pmu->nr_irqs = 0;
1741 for (i = 0; i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model); i++) {
1742 irq = platform_get_irq(pdev, i);
1743 if (irq < 0)
1744 break;
1745
1746 if (is_duplicate_irq(irq, cci_pmu->irqs, cci_pmu->nr_irqs))
1747 continue;
1748
1749 cci_pmu->irqs[cci_pmu->nr_irqs++] = irq;
1750 }
1751
1752 /*
1753 * Ensure that the device tree has as many interrupts as the number
1754 * of counters.
1755 */
1756 if (i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)) {
1757 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n",
1758 i, CCI_PMU_MAX_HW_CNTRS(cci_pmu->model));
1759 return -EINVAL;
1760 }
1761
1762 raw_spin_lock_init(&cci_pmu->hw_events.pmu_lock);
1763 mutex_init(&cci_pmu->reserve_mutex);
1764 atomic_set(&cci_pmu->active_events, 0);
1765 cpumask_set_cpu(smp_processor_id(), &cci_pmu->cpus);
1766
1767 ret = cci_pmu_init(cci_pmu, pdev);
1768 if (ret)
1769 return ret;
1770
1771 mutex_lock(&cci_pmu_mutex);
1772 list_add(&cci_pmu->entry, &cci_pmu_list);
1773 mutex_unlock(&cci_pmu_mutex);
1774
1775 pr_info("ARM %s PMU driver probed", cci_pmu->model->name);
1776 return 0;
1777 }
1778
1779 static int cci_platform_probe(struct platform_device *pdev)
1780 {
1781 if (!cci_probed())
1782 return -ENODEV;
1783
1784 return of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
1785 }
1786
1787 static struct platform_driver cci_pmu_driver = {
1788 .driver = {
1789 .name = DRIVER_NAME_PMU,
1790 .of_match_table = arm_cci_pmu_matches,
1791 },
1792 .probe = cci_pmu_probe,
1793 };
1794
1795 static struct platform_driver cci_platform_driver = {
1796 .driver = {
1797 .name = DRIVER_NAME,
1798 .of_match_table = arm_cci_matches,
1799 },
1800 .probe = cci_platform_probe,
1801 };
1802
1803 static int __init cci_platform_init(void)
1804 {
1805 int ret;
1806
1807 ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,
1808 "AP_PERF_ARM_CCI_ONLINE", NULL,
1809 cci_pmu_offline_cpu);
1810 if (ret)
1811 return ret;
1812
1813 ret = platform_driver_register(&cci_pmu_driver);
1814 if (ret)
1815 return ret;
1816
1817 return platform_driver_register(&cci_platform_driver);
1818 }
1819
1820 #else /* !CONFIG_ARM_CCI_PMU */
1821
1822 static int __init cci_platform_init(void)
1823 {
1824 return 0;
1825 }
1826
1827 #endif /* CONFIG_ARM_CCI_PMU */
1828
1829 #ifdef CONFIG_ARM_CCI400_PORT_CTRL
1830
1831 #define CCI_PORT_CTRL 0x0
1832 #define CCI_CTRL_STATUS 0xc
1833
1834 #define CCI_ENABLE_SNOOP_REQ 0x1
1835 #define CCI_ENABLE_DVM_REQ 0x2
1836 #define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ)
1837
1838 enum cci_ace_port_type {
1839 ACE_INVALID_PORT = 0x0,
1840 ACE_PORT,
1841 ACE_LITE_PORT,
1842 };
1843
1844 struct cci_ace_port {
1845 void __iomem *base;
1846 unsigned long phys;
1847 enum cci_ace_port_type type;
1848 struct device_node *dn;
1849 };
1850
1851 static struct cci_ace_port *ports;
1852 static unsigned int nb_cci_ports;
1853
1854 struct cpu_port {
1855 u64 mpidr;
1856 u32 port;
1857 };
1858
1859 /*
1860 * Use the port MSB as valid flag, shift can be made dynamic
1861 * by computing number of bits required for port indexes.
1862 * Code disabling CCI cpu ports runs with D-cache invalidated
1863 * and SCTLR bit clear so data accesses must be kept to a minimum
1864 * to improve performance; for now shift is left static to
1865 * avoid one more data access while disabling the CCI port.
1866 */
1867 #define PORT_VALID_SHIFT 31
1868 #define PORT_VALID (0x1 << PORT_VALID_SHIFT)
1869
1870 static inline void init_cpu_port(struct cpu_port *port, u32 index, u64 mpidr)
1871 {
1872 port->port = PORT_VALID | index;
1873 port->mpidr = mpidr;
1874 }
1875
1876 static inline bool cpu_port_is_valid(struct cpu_port *port)
1877 {
1878 return !!(port->port & PORT_VALID);
1879 }
1880
1881 static inline bool cpu_port_match(struct cpu_port *port, u64 mpidr)
1882 {
1883 return port->mpidr == (mpidr & MPIDR_HWID_BITMASK);
1884 }
1885
1886 static struct cpu_port cpu_port[NR_CPUS];
1887
1888 /**
1889 * __cci_ace_get_port - Function to retrieve the port index connected to
1890 * a cpu or device.
1891 *
1892 * @dn: device node of the device to look-up
1893 * @type: port type
1894 *
1895 * Return value:
1896 * - CCI port index if success
1897 * - -ENODEV if failure
1898 */
1899 static int __cci_ace_get_port(struct device_node *dn, int type)
1900 {
1901 int i;
1902 bool ace_match;
1903 struct device_node *cci_portn;
1904
1905 cci_portn = of_parse_phandle(dn, "cci-control-port", 0);
1906 for (i = 0; i < nb_cci_ports; i++) {
1907 ace_match = ports[i].type == type;
1908 if (ace_match && cci_portn == ports[i].dn)
1909 return i;
1910 }
1911 return -ENODEV;
1912 }
1913
1914 int cci_ace_get_port(struct device_node *dn)
1915 {
1916 return __cci_ace_get_port(dn, ACE_LITE_PORT);
1917 }
1918 EXPORT_SYMBOL_GPL(cci_ace_get_port);
1919
1920 static void cci_ace_init_ports(void)
1921 {
1922 int port, cpu;
1923 struct device_node *cpun;
1924
1925 /*
1926 * Port index look-up speeds up the function disabling ports by CPU,
1927 * since the logical to port index mapping is done once and does
1928 * not change after system boot.
1929 * The stashed index array is initialized for all possible CPUs
1930 * at probe time.
1931 */
1932 for_each_possible_cpu(cpu) {
1933 /* too early to use cpu->of_node */
1934 cpun = of_get_cpu_node(cpu, NULL);
1935
1936 if (WARN(!cpun, "Missing cpu device node\n"))
1937 continue;
1938
1939 port = __cci_ace_get_port(cpun, ACE_PORT);
1940 if (port < 0)
1941 continue;
1942
1943 init_cpu_port(&cpu_port[cpu], port, cpu_logical_map(cpu));
1944 }
1945
1946 for_each_possible_cpu(cpu) {
1947 WARN(!cpu_port_is_valid(&cpu_port[cpu]),
1948 "CPU %u does not have an associated CCI port\n",
1949 cpu);
1950 }
1951 }
1952 /*
1953 * Functions to enable/disable a CCI interconnect slave port
1954 *
1955 * They are called by low-level power management code to disable slave
1956 * interfaces snoops and DVM broadcast.
1957 * Since they may execute with cache data allocation disabled and
1958 * after the caches have been cleaned and invalidated the functions provide
1959 * no explicit locking since they may run with D-cache disabled, so normal
1960 * cacheable kernel locks based on ldrex/strex may not work.
1961 * Locking has to be provided by BSP implementations to ensure proper
1962 * operations.
1963 */
1964
1965 /**
1966 * cci_port_control() - function to control a CCI port
1967 *
1968 * @port: index of the port to setup
1969 * @enable: if true enables the port, if false disables it
1970 */
1971 static void notrace cci_port_control(unsigned int port, bool enable)
1972 {
1973 void __iomem *base = ports[port].base;
1974
1975 writel_relaxed(enable ? CCI_ENABLE_REQ : 0, base + CCI_PORT_CTRL);
1976 /*
1977 * This function is called from power down procedures
1978 * and must not execute any instruction that might
1979 * cause the processor to be put in a quiescent state
1980 * (eg wfi). Hence, cpu_relax() can not be added to this
1981 * read loop to optimize power, since it might hide possibly
1982 * disruptive operations.
1983 */
1984 while (readl_relaxed(cci_ctrl_base + CCI_CTRL_STATUS) & 0x1)
1985 ;
1986 }
1987
1988 /**
1989 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU
1990 * reference
1991 *
1992 * @mpidr: mpidr of the CPU whose CCI port should be disabled
1993 *
1994 * Disabling a CCI port for a CPU implies disabling the CCI port
1995 * controlling that CPU cluster. Code disabling CPU CCI ports
1996 * must make sure that the CPU running the code is the last active CPU
1997 * in the cluster ie all other CPUs are quiescent in a low power state.
1998 *
1999 * Return:
2000 * 0 on success
2001 * -ENODEV on port look-up failure
2002 */
2003 int notrace cci_disable_port_by_cpu(u64 mpidr)
2004 {
2005 int cpu;
2006 bool is_valid;
2007 for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
2008 is_valid = cpu_port_is_valid(&cpu_port[cpu]);
2009 if (is_valid && cpu_port_match(&cpu_port[cpu], mpidr)) {
2010 cci_port_control(cpu_port[cpu].port, false);
2011 return 0;
2012 }
2013 }
2014 return -ENODEV;
2015 }
2016 EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu);
2017
2018 /**
2019 * cci_enable_port_for_self() - enable a CCI port for calling CPU
2020 *
2021 * Enabling a CCI port for the calling CPU implies enabling the CCI
2022 * port controlling that CPU's cluster. Caller must make sure that the
2023 * CPU running the code is the first active CPU in the cluster and all
2024 * other CPUs are quiescent in a low power state or waiting for this CPU
2025 * to complete the CCI initialization.
2026 *
2027 * Because this is called when the MMU is still off and with no stack,
2028 * the code must be position independent and ideally rely on callee
2029 * clobbered registers only. To achieve this we must code this function
2030 * entirely in assembler.
2031 *
2032 * On success this returns with the proper CCI port enabled. In case of
2033 * any failure this never returns as the inability to enable the CCI is
2034 * fatal and there is no possible recovery at this stage.
2035 */
2036 asmlinkage void __naked cci_enable_port_for_self(void)
2037 {
2038 asm volatile ("\n"
2039 " .arch armv7-a\n"
2040 " mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n"
2041 " and r0, r0, #"__stringify(MPIDR_HWID_BITMASK)" \n"
2042 " adr r1, 5f \n"
2043 " ldr r2, [r1] \n"
2044 " add r1, r1, r2 @ &cpu_port \n"
2045 " add ip, r1, %[sizeof_cpu_port] \n"
2046
2047 /* Loop over the cpu_port array looking for a matching MPIDR */
2048 "1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n"
2049 " cmp r2, r0 @ compare MPIDR \n"
2050 " bne 2f \n"
2051
2052 /* Found a match, now test port validity */
2053 " ldr r3, [r1, %[offsetof_cpu_port_port]] \n"
2054 " tst r3, #"__stringify(PORT_VALID)" \n"
2055 " bne 3f \n"
2056
2057 /* no match, loop with the next cpu_port entry */
2058 "2: add r1, r1, %[sizeof_struct_cpu_port] \n"
2059 " cmp r1, ip @ done? \n"
2060 " blo 1b \n"
2061
2062 /* CCI port not found -- cheaply try to stall this CPU */
2063 "cci_port_not_found: \n"
2064 " wfi \n"
2065 " wfe \n"
2066 " b cci_port_not_found \n"
2067
2068 /* Use matched port index to look up the corresponding ports entry */
2069 "3: bic r3, r3, #"__stringify(PORT_VALID)" \n"
2070 " adr r0, 6f \n"
2071 " ldmia r0, {r1, r2} \n"
2072 " sub r1, r1, r0 @ virt - phys \n"
2073 " ldr r0, [r0, r2] @ *(&ports) \n"
2074 " mov r2, %[sizeof_struct_ace_port] \n"
2075 " mla r0, r2, r3, r0 @ &ports[index] \n"
2076 " sub r0, r0, r1 @ virt_to_phys() \n"
2077
2078 /* Enable the CCI port */
2079 " ldr r0, [r0, %[offsetof_port_phys]] \n"
2080 " mov r3, %[cci_enable_req]\n"
2081 " str r3, [r0, #"__stringify(CCI_PORT_CTRL)"] \n"
2082
2083 /* poll the status reg for completion */
2084 " adr r1, 7f \n"
2085 " ldr r0, [r1] \n"
2086 " ldr r0, [r0, r1] @ cci_ctrl_base \n"
2087 "4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS)"] \n"
2088 " tst r1, %[cci_control_status_bits] \n"
2089 " bne 4b \n"
2090
2091 " mov r0, #0 \n"
2092 " bx lr \n"
2093
2094 " .align 2 \n"
2095 "5: .word cpu_port - . \n"
2096 "6: .word . \n"
2097 " .word ports - 6b \n"
2098 "7: .word cci_ctrl_phys - . \n"
2099 : :
2100 [sizeof_cpu_port] "i" (sizeof(cpu_port)),
2101 [cci_enable_req] "i" cpu_to_le32(CCI_ENABLE_REQ),
2102 [cci_control_status_bits] "i" cpu_to_le32(1),
2103 #ifndef __ARMEB__
2104 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)),
2105 #else
2106 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)+4),
2107 #endif
2108 [offsetof_cpu_port_port] "i" (offsetof(struct cpu_port, port)),
2109 [sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)),
2110 [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)),
2111 [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) );
2112
2113 unreachable();
2114 }
2115
2116 /**
2117 * __cci_control_port_by_device() - function to control a CCI port by device
2118 * reference
2119 *
2120 * @dn: device node pointer of the device whose CCI port should be
2121 * controlled
2122 * @enable: if true enables the port, if false disables it
2123 *
2124 * Return:
2125 * 0 on success
2126 * -ENODEV on port look-up failure
2127 */
2128 int notrace __cci_control_port_by_device(struct device_node *dn, bool enable)
2129 {
2130 int port;
2131
2132 if (!dn)
2133 return -ENODEV;
2134
2135 port = __cci_ace_get_port(dn, ACE_LITE_PORT);
2136 if (WARN_ONCE(port < 0, "node %s ACE lite port look-up failure\n",
2137 dn->full_name))
2138 return -ENODEV;
2139 cci_port_control(port, enable);
2140 return 0;
2141 }
2142 EXPORT_SYMBOL_GPL(__cci_control_port_by_device);
2143
2144 /**
2145 * __cci_control_port_by_index() - function to control a CCI port by port index
2146 *
2147 * @port: port index previously retrieved with cci_ace_get_port()
2148 * @enable: if true enables the port, if false disables it
2149 *
2150 * Return:
2151 * 0 on success
2152 * -ENODEV on port index out of range
2153 * -EPERM if operation carried out on an ACE PORT
2154 */
2155 int notrace __cci_control_port_by_index(u32 port, bool enable)
2156 {
2157 if (port >= nb_cci_ports || ports[port].type == ACE_INVALID_PORT)
2158 return -ENODEV;
2159 /*
2160 * CCI control for ports connected to CPUS is extremely fragile
2161 * and must be made to go through a specific and controlled
2162 * interface (ie cci_disable_port_by_cpu(); control by general purpose
2163 * indexing is therefore disabled for ACE ports.
2164 */
2165 if (ports[port].type == ACE_PORT)
2166 return -EPERM;
2167
2168 cci_port_control(port, enable);
2169 return 0;
2170 }
2171 EXPORT_SYMBOL_GPL(__cci_control_port_by_index);
2172
2173 static const struct of_device_id arm_cci_ctrl_if_matches[] = {
2174 {.compatible = "arm,cci-400-ctrl-if", },
2175 {},
2176 };
2177
2178 static int cci_probe_ports(struct device_node *np)
2179 {
2180 struct cci_nb_ports const *cci_config;
2181 int ret, i, nb_ace = 0, nb_ace_lite = 0;
2182 struct device_node *cp;
2183 struct resource res;
2184 const char *match_str;
2185 bool is_ace;
2186
2187
2188 cci_config = of_match_node(arm_cci_matches, np)->data;
2189 if (!cci_config)
2190 return -ENODEV;
2191
2192 nb_cci_ports = cci_config->nb_ace + cci_config->nb_ace_lite;
2193
2194 ports = kcalloc(nb_cci_ports, sizeof(*ports), GFP_KERNEL);
2195 if (!ports)
2196 return -ENOMEM;
2197
2198 for_each_child_of_node(np, cp) {
2199 if (!of_match_node(arm_cci_ctrl_if_matches, cp))
2200 continue;
2201
2202 i = nb_ace + nb_ace_lite;
2203
2204 if (i >= nb_cci_ports)
2205 break;
2206
2207 if (of_property_read_string(cp, "interface-type",
2208 &match_str)) {
2209 WARN(1, "node %s missing interface-type property\n",
2210 cp->full_name);
2211 continue;
2212 }
2213 is_ace = strcmp(match_str, "ace") == 0;
2214 if (!is_ace && strcmp(match_str, "ace-lite")) {
2215 WARN(1, "node %s containing invalid interface-type property, skipping it\n",
2216 cp->full_name);
2217 continue;
2218 }
2219
2220 ret = of_address_to_resource(cp, 0, &res);
2221 if (!ret) {
2222 ports[i].base = ioremap(res.start, resource_size(&res));
2223 ports[i].phys = res.start;
2224 }
2225 if (ret || !ports[i].base) {
2226 WARN(1, "unable to ioremap CCI port %d\n", i);
2227 continue;
2228 }
2229
2230 if (is_ace) {
2231 if (WARN_ON(nb_ace >= cci_config->nb_ace))
2232 continue;
2233 ports[i].type = ACE_PORT;
2234 ++nb_ace;
2235 } else {
2236 if (WARN_ON(nb_ace_lite >= cci_config->nb_ace_lite))
2237 continue;
2238 ports[i].type = ACE_LITE_PORT;
2239 ++nb_ace_lite;
2240 }
2241 ports[i].dn = cp;
2242 }
2243
2244 /* initialize a stashed array of ACE ports to speed-up look-up */
2245 cci_ace_init_ports();
2246
2247 /*
2248 * Multi-cluster systems may need this data when non-coherent, during
2249 * cluster power-up/power-down. Make sure it reaches main memory.
2250 */
2251 sync_cache_w(&cci_ctrl_base);
2252 sync_cache_w(&cci_ctrl_phys);
2253 sync_cache_w(&ports);
2254 sync_cache_w(&cpu_port);
2255 __sync_cache_range_w(ports, sizeof(*ports) * nb_cci_ports);
2256 pr_info("ARM CCI driver probed\n");
2257
2258 return 0;
2259 }
2260 #else /* !CONFIG_ARM_CCI400_PORT_CTRL */
2261 static inline int cci_probe_ports(struct device_node *np)
2262 {
2263 return 0;
2264 }
2265 #endif /* CONFIG_ARM_CCI400_PORT_CTRL */
2266
2267 static int cci_probe(void)
2268 {
2269 int ret;
2270 struct device_node *np;
2271 struct resource res;
2272
2273 np = of_find_matching_node(NULL, arm_cci_matches);
2274 if(!np || !of_device_is_available(np))
2275 return -ENODEV;
2276
2277 ret = of_address_to_resource(np, 0, &res);
2278 if (!ret) {
2279 cci_ctrl_base = ioremap(res.start, resource_size(&res));
2280 cci_ctrl_phys = res.start;
2281 }
2282 if (ret || !cci_ctrl_base) {
2283 WARN(1, "unable to ioremap CCI ctrl\n");
2284 return -ENXIO;
2285 }
2286
2287 return cci_probe_ports(np);
2288 }
2289
2290 static int cci_init_status = -EAGAIN;
2291 static DEFINE_MUTEX(cci_probing);
2292
2293 static int cci_init(void)
2294 {
2295 if (cci_init_status != -EAGAIN)
2296 return cci_init_status;
2297
2298 mutex_lock(&cci_probing);
2299 if (cci_init_status == -EAGAIN)
2300 cci_init_status = cci_probe();
2301 mutex_unlock(&cci_probing);
2302 return cci_init_status;
2303 }
2304
2305 /*
2306 * To sort out early init calls ordering a helper function is provided to
2307 * check if the CCI driver has beed initialized. Function check if the driver
2308 * has been initialized, if not it calls the init function that probes
2309 * the driver and updates the return value.
2310 */
2311 bool cci_probed(void)
2312 {
2313 return cci_init() == 0;
2314 }
2315 EXPORT_SYMBOL_GPL(cci_probed);
2316
2317 early_initcall(cci_init);
2318 core_initcall(cci_platform_init);
2319 MODULE_LICENSE("GPL");
2320 MODULE_DESCRIPTION("ARM CCI support");
Linux kernel - Deliverables
This page took 0.133927 seconds and 5 git commands to generate.