2 * CCI cache coherent interconnect driver
4 * Copyright (C) 2013 ARM Ltd.
5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
12 * kind, whether express or implied; without even the implied warranty
13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
17 #include <linux/arm-cci.h>
19 #include <linux/interrupt.h>
20 #include <linux/module.h>
21 #include <linux/of_address.h>
22 #include <linux/of_irq.h>
23 #include <linux/of_platform.h>
24 #include <linux/perf_event.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <linux/spinlock.h>
29 #include <asm/cacheflush.h>
30 #include <asm/smp_plat.h>
32 static void __iomem
*cci_ctrl_base
;
33 static unsigned long cci_ctrl_phys
;
35 #ifdef CONFIG_ARM_CCI400_PORT_CTRL
38 unsigned int nb_ace_lite
;
41 static const struct cci_nb_ports cci400_ports
= {
46 #define CCI400_PORTS_DATA (&cci400_ports)
48 #define CCI400_PORTS_DATA (NULL)
51 static const struct of_device_id arm_cci_matches
[] = {
52 #ifdef CONFIG_ARM_CCI400_COMMON
53 {.compatible
= "arm,cci-400", .data
= CCI400_PORTS_DATA
},
55 #ifdef CONFIG_ARM_CCI500_PMU
56 { .compatible
= "arm,cci-500", },
61 #ifdef CONFIG_ARM_CCI_PMU
63 #define DRIVER_NAME "ARM-CCI"
64 #define DRIVER_NAME_PMU DRIVER_NAME " PMU"
66 #define CCI_PMCR 0x0100
67 #define CCI_PID2 0x0fe8
69 #define CCI_PMCR_CEN 0x00000001
70 #define CCI_PMCR_NCNT_MASK 0x0000f800
71 #define CCI_PMCR_NCNT_SHIFT 11
73 #define CCI_PID2_REV_MASK 0xf0
74 #define CCI_PID2_REV_SHIFT 4
76 #define CCI_PMU_EVT_SEL 0x000
77 #define CCI_PMU_CNTR 0x004
78 #define CCI_PMU_CNTR_CTRL 0x008
79 #define CCI_PMU_OVRFLW 0x00c
81 #define CCI_PMU_OVRFLW_FLAG 1
83 #define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size)
84 #define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model))
85 #define CCI_PMU_CNTR_MASK ((1ULL << 32) -1)
86 #define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1)
88 #define CCI_PMU_MAX_HW_CNTRS(model) \
89 ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs)
91 /* Types of interfaces that can generate events */
95 #ifdef CONFIG_ARM_CCI500_PMU
106 struct cci_pmu_hw_events
{
107 struct perf_event
**events
;
108 unsigned long *used_mask
;
109 raw_spinlock_t pmu_lock
;
114 * struct cci_pmu_model:
115 * @fixed_hw_cntrs - Number of fixed event counters
116 * @num_hw_cntrs - Maximum number of programmable event counters
117 * @cntr_size - Size of an event counter mapping
119 struct cci_pmu_model
{
124 struct attribute
**format_attrs
;
125 struct attribute
**event_attrs
;
126 struct event_range event_ranges
[CCI_IF_MAX
];
127 int (*validate_hw_event
)(struct cci_pmu
*, unsigned long);
128 int (*get_event_idx
)(struct cci_pmu
*, struct cci_pmu_hw_events
*, unsigned long);
129 void (*write_counters
)(struct cci_pmu
*, unsigned long *);
132 static struct cci_pmu_model cci_pmu_models
[];
139 unsigned long active_irqs
;
140 const struct cci_pmu_model
*model
;
141 struct cci_pmu_hw_events hw_events
;
142 struct platform_device
*plat_device
;
144 atomic_t active_events
;
145 struct mutex reserve_mutex
;
146 struct notifier_block cpu_nb
;
150 #define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu))
153 #ifdef CONFIG_ARM_CCI400_PMU
157 #ifdef CONFIG_ARM_CCI500_PMU
163 static void pmu_write_counters(struct cci_pmu
*cci_pmu
,
164 unsigned long *mask
);
165 static ssize_t
cci_pmu_format_show(struct device
*dev
,
166 struct device_attribute
*attr
, char *buf
);
167 static ssize_t
cci_pmu_event_show(struct device
*dev
,
168 struct device_attribute
*attr
, char *buf
);
170 #define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \
171 &((struct dev_ext_attribute[]) { \
172 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \
175 #define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \
176 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_format_show, (char *)_config)
177 #define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \
178 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config)
180 /* CCI400 PMU Specific definitions */
182 #ifdef CONFIG_ARM_CCI400_PMU
185 #define CCI400_PORT_S0 0
186 #define CCI400_PORT_S1 1
187 #define CCI400_PORT_S2 2
188 #define CCI400_PORT_S3 3
189 #define CCI400_PORT_S4 4
190 #define CCI400_PORT_M0 5
191 #define CCI400_PORT_M1 6
192 #define CCI400_PORT_M2 7
194 #define CCI400_R1_PX 5
197 * Instead of an event id to monitor CCI cycles, a dedicated counter is
198 * provided. Use 0xff to represent CCI cycles and hope that no future revisions
199 * make use of this event in hardware.
201 enum cci400_perf_events
{
202 CCI400_PMU_CYCLES
= 0xff
205 #define CCI400_PMU_CYCLE_CNTR_IDX 0
206 #define CCI400_PMU_CNTR0_IDX 1
209 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8
210 * ports and bits 4:0 are event codes. There are different event codes
211 * associated with each port type.
213 * Additionally, the range of events associated with the port types changed
214 * between Rev0 and Rev1.
216 * The constants below define the range of valid codes for each port type for
217 * the different revisions and are used to validate the event to be monitored.
220 #define CCI400_PMU_EVENT_MASK 0xffUL
221 #define CCI400_PMU_EVENT_SOURCE_SHIFT 5
222 #define CCI400_PMU_EVENT_SOURCE_MASK 0x7
223 #define CCI400_PMU_EVENT_CODE_SHIFT 0
224 #define CCI400_PMU_EVENT_CODE_MASK 0x1f
225 #define CCI400_PMU_EVENT_SOURCE(event) \
226 ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \
227 CCI400_PMU_EVENT_SOURCE_MASK)
228 #define CCI400_PMU_EVENT_CODE(event) \
229 ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK)
231 #define CCI400_R0_SLAVE_PORT_MIN_EV 0x00
232 #define CCI400_R0_SLAVE_PORT_MAX_EV 0x13
233 #define CCI400_R0_MASTER_PORT_MIN_EV 0x14
234 #define CCI400_R0_MASTER_PORT_MAX_EV 0x1a
236 #define CCI400_R1_SLAVE_PORT_MIN_EV 0x00
237 #define CCI400_R1_SLAVE_PORT_MAX_EV 0x14
238 #define CCI400_R1_MASTER_PORT_MIN_EV 0x00
239 #define CCI400_R1_MASTER_PORT_MAX_EV 0x11
241 #define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \
242 CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \
243 (unsigned long)_config)
245 static ssize_t
cci400_pmu_cycle_event_show(struct device
*dev
,
246 struct device_attribute
*attr
, char *buf
);
248 static struct attribute
*cci400_pmu_format_attrs
[] = {
249 CCI_FORMAT_EXT_ATTR_ENTRY(event
, "config:0-4"),
250 CCI_FORMAT_EXT_ATTR_ENTRY(source
, "config:5-7"),
254 static struct attribute
*cci400_r0_pmu_event_attrs
[] = {
256 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any
, 0x0),
257 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device
, 0x01),
258 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable
, 0x2),
259 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable
, 0x3),
260 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance
, 0x4),
261 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier
, 0x5),
262 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier
, 0x6),
263 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg
, 0x7),
264 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync
, 0x8),
265 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full
, 0x9),
266 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop
, 0xA),
267 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l
, 0xB),
268 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any
, 0xC),
269 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device
, 0xD),
270 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable
, 0xE),
271 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean
, 0xF),
272 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique
, 0x10),
273 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique
, 0x11),
274 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict
, 0x12),
275 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full
, 0x13),
277 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch
, 0x14),
278 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard
, 0x15),
279 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard
, 0x16),
280 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full
, 0x17),
281 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard
, 0x18),
282 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard
, 0x19),
283 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full
, 0x1A),
284 /* Special event for cycles counter */
285 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles
, 0xff),
289 static struct attribute
*cci400_r1_pmu_event_attrs
[] = {
291 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any
, 0x0),
292 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device
, 0x01),
293 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable
, 0x2),
294 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable
, 0x3),
295 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance
, 0x4),
296 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier
, 0x5),
297 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier
, 0x6),
298 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg
, 0x7),
299 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync
, 0x8),
300 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full
, 0x9),
301 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop
, 0xA),
302 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l
, 0xB),
303 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any
, 0xC),
304 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device
, 0xD),
305 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable
, 0xE),
306 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean
, 0xF),
307 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique
, 0x10),
308 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique
, 0x11),
309 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict
, 0x12),
310 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full
, 0x13),
311 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard
, 0x14),
313 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch
, 0x0),
314 CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard
, 0x1),
315 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard
, 0x2),
316 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full
, 0x3),
317 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard
, 0x4),
318 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard
, 0x5),
319 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full
, 0x6),
320 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full
, 0x7),
321 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full
, 0x8),
322 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0
, 0x9),
323 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1
, 0xA),
324 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2
, 0xB),
325 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3
, 0xC),
326 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0
, 0xD),
327 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1
, 0xE),
328 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2
, 0xF),
329 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3
, 0x10),
330 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard
, 0x11),
331 /* Special event for cycles counter */
332 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles
, 0xff),
336 static ssize_t
cci400_pmu_cycle_event_show(struct device
*dev
,
337 struct device_attribute
*attr
, char *buf
)
339 struct dev_ext_attribute
*eattr
= container_of(attr
,
340 struct dev_ext_attribute
, attr
);
341 return snprintf(buf
, PAGE_SIZE
, "config=0x%lx\n", (unsigned long)eattr
->var
);
344 static int cci400_get_event_idx(struct cci_pmu
*cci_pmu
,
345 struct cci_pmu_hw_events
*hw
,
346 unsigned long cci_event
)
350 /* cycles event idx is fixed */
351 if (cci_event
== CCI400_PMU_CYCLES
) {
352 if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX
, hw
->used_mask
))
355 return CCI400_PMU_CYCLE_CNTR_IDX
;
358 for (idx
= CCI400_PMU_CNTR0_IDX
; idx
<= CCI_PMU_CNTR_LAST(cci_pmu
); ++idx
)
359 if (!test_and_set_bit(idx
, hw
->used_mask
))
362 /* No counters available */
366 static int cci400_validate_hw_event(struct cci_pmu
*cci_pmu
, unsigned long hw_event
)
368 u8 ev_source
= CCI400_PMU_EVENT_SOURCE(hw_event
);
369 u8 ev_code
= CCI400_PMU_EVENT_CODE(hw_event
);
372 if (hw_event
& ~CCI400_PMU_EVENT_MASK
)
375 if (hw_event
== CCI400_PMU_CYCLES
)
384 /* Slave Interface */
385 if_type
= CCI_IF_SLAVE
;
390 /* Master Interface */
391 if_type
= CCI_IF_MASTER
;
397 if (ev_code
>= cci_pmu
->model
->event_ranges
[if_type
].min
&&
398 ev_code
<= cci_pmu
->model
->event_ranges
[if_type
].max
)
404 static int probe_cci400_revision(void)
407 rev
= readl_relaxed(cci_ctrl_base
+ CCI_PID2
) & CCI_PID2_REV_MASK
;
408 rev
>>= CCI_PID2_REV_SHIFT
;
410 if (rev
< CCI400_R1_PX
)
416 static const struct cci_pmu_model
*probe_cci_model(struct platform_device
*pdev
)
418 if (platform_has_secure_cci_access())
419 return &cci_pmu_models
[probe_cci400_revision()];
422 #else /* !CONFIG_ARM_CCI400_PMU */
423 static inline struct cci_pmu_model
*probe_cci_model(struct platform_device
*pdev
)
427 #endif /* CONFIG_ARM_CCI400_PMU */
429 #ifdef CONFIG_ARM_CCI500_PMU
432 * CCI500 provides 8 independent event counters that can count
433 * any of the events available.
435 * CCI500 PMU event id is an 9-bit value made of two parts.
436 * bits [8:5] - Source for the event
437 * 0x0-0x6 - Slave interfaces
438 * 0x8-0xD - Master interfaces
439 * 0xf - Global Events
442 * bits [4:0] - Event code (specific to type of interface)
446 #define CCI500_PORT_S0 0x0
447 #define CCI500_PORT_S1 0x1
448 #define CCI500_PORT_S2 0x2
449 #define CCI500_PORT_S3 0x3
450 #define CCI500_PORT_S4 0x4
451 #define CCI500_PORT_S5 0x5
452 #define CCI500_PORT_S6 0x6
454 #define CCI500_PORT_M0 0x8
455 #define CCI500_PORT_M1 0x9
456 #define CCI500_PORT_M2 0xa
457 #define CCI500_PORT_M3 0xb
458 #define CCI500_PORT_M4 0xc
459 #define CCI500_PORT_M5 0xd
461 #define CCI500_PORT_GLOBAL 0xf
463 #define CCI500_PMU_EVENT_MASK 0x1ffUL
464 #define CCI500_PMU_EVENT_SOURCE_SHIFT 0x5
465 #define CCI500_PMU_EVENT_SOURCE_MASK 0xf
466 #define CCI500_PMU_EVENT_CODE_SHIFT 0x0
467 #define CCI500_PMU_EVENT_CODE_MASK 0x1f
469 #define CCI500_PMU_EVENT_SOURCE(event) \
470 ((event >> CCI500_PMU_EVENT_SOURCE_SHIFT) & CCI500_PMU_EVENT_SOURCE_MASK)
471 #define CCI500_PMU_EVENT_CODE(event) \
472 ((event >> CCI500_PMU_EVENT_CODE_SHIFT) & CCI500_PMU_EVENT_CODE_MASK)
474 #define CCI500_SLAVE_PORT_MIN_EV 0x00
475 #define CCI500_SLAVE_PORT_MAX_EV 0x1f
476 #define CCI500_MASTER_PORT_MIN_EV 0x00
477 #define CCI500_MASTER_PORT_MAX_EV 0x06
478 #define CCI500_GLOBAL_PORT_MIN_EV 0x00
479 #define CCI500_GLOBAL_PORT_MAX_EV 0x0f
482 #define CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \
483 CCI_EXT_ATTR_ENTRY(_name, cci500_pmu_global_event_show, \
484 (unsigned long) _config)
486 static ssize_t
cci500_pmu_global_event_show(struct device
*dev
,
487 struct device_attribute
*attr
, char *buf
);
489 static struct attribute
*cci500_pmu_format_attrs
[] = {
490 CCI_FORMAT_EXT_ATTR_ENTRY(event
, "config:0-4"),
491 CCI_FORMAT_EXT_ATTR_ENTRY(source
, "config:5-8"),
495 static struct attribute
*cci500_pmu_event_attrs
[] = {
497 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid
, 0x0),
498 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev
, 0x1),
499 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable
, 0x2),
500 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc
, 0x3),
501 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc
, 0x4),
502 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate
, 0x5),
503 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint
, 0x6),
504 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg
, 0x7),
505 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval
, 0x8),
506 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop
, 0x9),
507 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid
, 0xA),
508 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev
, 0xB),
509 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable
, 0xC),
510 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb
, 0xD),
511 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu
, 0xE),
512 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique
, 0xF),
513 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict
, 0x10),
514 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict
, 0x11),
515 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat
, 0x12),
516 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid
, 0x13),
517 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read
, 0x14),
518 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean
, 0x15),
519 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low
, 0x16),
520 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid
, 0x17),
521 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall
, 0x18),
522 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall
, 0x19),
523 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall
, 0x1A),
524 CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall
, 0x1B),
525 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall
, 0x1C),
526 CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall
, 0x1D),
527 CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit
, 0x1E),
528 CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit
, 0x1F),
531 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any
, 0x0),
532 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any
, 0x1),
533 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall
, 0x2),
534 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall
, 0x3),
535 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall
, 0x4),
536 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall
, 0x5),
537 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall
, 0x6),
540 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1
, 0x0),
541 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3
, 0x1),
542 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5
, 0x2),
543 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7
, 0x3),
544 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1
, 0x4),
545 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3
, 0x5),
546 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5
, 0x6),
547 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7
, 0x7),
548 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation
, 0x8),
549 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy
, 0x9),
550 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full
, 0xA),
551 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq
, 0xB),
552 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs
, 0xC),
553 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard
, 0xD),
554 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snopp_rq_stall_tt_full
, 0xE),
555 CCI500_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot
, 0xF),
559 static ssize_t
cci500_pmu_global_event_show(struct device
*dev
,
560 struct device_attribute
*attr
, char *buf
)
562 struct dev_ext_attribute
*eattr
= container_of(attr
,
563 struct dev_ext_attribute
, attr
);
564 /* Global events have single fixed source code */
565 return snprintf(buf
, PAGE_SIZE
, "event=0x%lx,source=0x%x\n",
566 (unsigned long)eattr
->var
, CCI500_PORT_GLOBAL
);
569 static int cci500_validate_hw_event(struct cci_pmu
*cci_pmu
,
570 unsigned long hw_event
)
572 u32 ev_source
= CCI500_PMU_EVENT_SOURCE(hw_event
);
573 u32 ev_code
= CCI500_PMU_EVENT_CODE(hw_event
);
576 if (hw_event
& ~CCI500_PMU_EVENT_MASK
)
587 if_type
= CCI_IF_SLAVE
;
595 if_type
= CCI_IF_MASTER
;
597 case CCI500_PORT_GLOBAL
:
598 if_type
= CCI_IF_GLOBAL
;
604 if (ev_code
>= cci_pmu
->model
->event_ranges
[if_type
].min
&&
605 ev_code
<= cci_pmu
->model
->event_ranges
[if_type
].max
)
610 #endif /* CONFIG_ARM_CCI500_PMU */
613 * Program the CCI PMU counters which have PERF_HES_ARCH set
614 * with the event period and mark them ready before we enable
617 void cci_pmu_sync_counters(struct cci_pmu
*cci_pmu
)
620 struct cci_pmu_hw_events
*cci_hw
= &cci_pmu
->hw_events
;
622 DECLARE_BITMAP(mask
, cci_pmu
->num_cntrs
);
624 bitmap_zero(mask
, cci_pmu
->num_cntrs
);
625 for_each_set_bit(i
, cci_pmu
->hw_events
.used_mask
, cci_pmu
->num_cntrs
) {
626 struct perf_event
*event
= cci_hw
->events
[i
];
631 /* Leave the events which are not counting */
632 if (event
->hw
.state
& PERF_HES_STOPPED
)
634 if (event
->hw
.state
& PERF_HES_ARCH
) {
636 event
->hw
.state
&= ~PERF_HES_ARCH
;
640 pmu_write_counters(cci_pmu
, mask
);
643 /* Should be called with cci_pmu->hw_events->pmu_lock held */
644 static void __cci_pmu_enable_nosync(struct cci_pmu
*cci_pmu
)
648 /* Enable all the PMU counters. */
649 val
= readl_relaxed(cci_ctrl_base
+ CCI_PMCR
) | CCI_PMCR_CEN
;
650 writel(val
, cci_ctrl_base
+ CCI_PMCR
);
653 /* Should be called with cci_pmu->hw_events->pmu_lock held */
654 static void __cci_pmu_enable_sync(struct cci_pmu
*cci_pmu
)
656 cci_pmu_sync_counters(cci_pmu
);
657 __cci_pmu_enable_nosync(cci_pmu
);
660 /* Should be called with cci_pmu->hw_events->pmu_lock held */
661 static void __cci_pmu_disable(void)
665 /* Disable all the PMU counters. */
666 val
= readl_relaxed(cci_ctrl_base
+ CCI_PMCR
) & ~CCI_PMCR_CEN
;
667 writel(val
, cci_ctrl_base
+ CCI_PMCR
);
670 static ssize_t
cci_pmu_format_show(struct device
*dev
,
671 struct device_attribute
*attr
, char *buf
)
673 struct dev_ext_attribute
*eattr
= container_of(attr
,
674 struct dev_ext_attribute
, attr
);
675 return snprintf(buf
, PAGE_SIZE
, "%s\n", (char *)eattr
->var
);
678 static ssize_t
cci_pmu_event_show(struct device
*dev
,
679 struct device_attribute
*attr
, char *buf
)
681 struct dev_ext_attribute
*eattr
= container_of(attr
,
682 struct dev_ext_attribute
, attr
);
683 /* source parameter is mandatory for normal PMU events */
684 return snprintf(buf
, PAGE_SIZE
, "source=?,event=0x%lx\n",
685 (unsigned long)eattr
->var
);
688 static int pmu_is_valid_counter(struct cci_pmu
*cci_pmu
, int idx
)
690 return 0 <= idx
&& idx
<= CCI_PMU_CNTR_LAST(cci_pmu
);
693 static u32
pmu_read_register(struct cci_pmu
*cci_pmu
, int idx
, unsigned int offset
)
695 return readl_relaxed(cci_pmu
->base
+
696 CCI_PMU_CNTR_BASE(cci_pmu
->model
, idx
) + offset
);
699 static void pmu_write_register(struct cci_pmu
*cci_pmu
, u32 value
,
700 int idx
, unsigned int offset
)
702 return writel_relaxed(value
, cci_pmu
->base
+
703 CCI_PMU_CNTR_BASE(cci_pmu
->model
, idx
) + offset
);
706 static void pmu_disable_counter(struct cci_pmu
*cci_pmu
, int idx
)
708 pmu_write_register(cci_pmu
, 0, idx
, CCI_PMU_CNTR_CTRL
);
711 static void pmu_enable_counter(struct cci_pmu
*cci_pmu
, int idx
)
713 pmu_write_register(cci_pmu
, 1, idx
, CCI_PMU_CNTR_CTRL
);
716 static bool __maybe_unused
717 pmu_counter_is_enabled(struct cci_pmu
*cci_pmu
, int idx
)
719 return (pmu_read_register(cci_pmu
, idx
, CCI_PMU_CNTR_CTRL
) & 0x1) != 0;
722 static void pmu_set_event(struct cci_pmu
*cci_pmu
, int idx
, unsigned long event
)
724 pmu_write_register(cci_pmu
, event
, idx
, CCI_PMU_EVT_SEL
);
728 * For all counters on the CCI-PMU, disable any 'enabled' counters,
729 * saving the changed counters in the mask, so that we can restore
730 * it later using pmu_restore_counters. The mask is private to the
731 * caller. We cannot rely on the used_mask maintained by the CCI_PMU
732 * as it only tells us if the counter is assigned to perf_event or not.
733 * The state of the perf_event cannot be locked by the PMU layer, hence
734 * we check the individual counter status (which can be locked by
735 * cci_pm->hw_events->pmu_lock).
737 * @mask should be initialised to empty by the caller.
739 static void __maybe_unused
740 pmu_save_counters(struct cci_pmu
*cci_pmu
, unsigned long *mask
)
744 for (i
= 0; i
< cci_pmu
->num_cntrs
; i
++) {
745 if (pmu_counter_is_enabled(cci_pmu
, i
)) {
747 pmu_disable_counter(cci_pmu
, i
);
753 * Restore the status of the counters. Reversal of the pmu_save_counters().
754 * For each counter set in the mask, enable the counter back.
756 static void __maybe_unused
757 pmu_restore_counters(struct cci_pmu
*cci_pmu
, unsigned long *mask
)
761 for_each_set_bit(i
, mask
, cci_pmu
->num_cntrs
)
762 pmu_enable_counter(cci_pmu
, i
);
766 * Returns the number of programmable counters actually implemented
769 static u32
pmu_get_max_counters(void)
771 return (readl_relaxed(cci_ctrl_base
+ CCI_PMCR
) &
772 CCI_PMCR_NCNT_MASK
) >> CCI_PMCR_NCNT_SHIFT
;
775 static int pmu_get_event_idx(struct cci_pmu_hw_events
*hw
, struct perf_event
*event
)
777 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
778 unsigned long cci_event
= event
->hw
.config_base
;
781 if (cci_pmu
->model
->get_event_idx
)
782 return cci_pmu
->model
->get_event_idx(cci_pmu
, hw
, cci_event
);
784 /* Generic code to find an unused idx from the mask */
785 for(idx
= 0; idx
<= CCI_PMU_CNTR_LAST(cci_pmu
); idx
++)
786 if (!test_and_set_bit(idx
, hw
->used_mask
))
789 /* No counters available */
793 static int pmu_map_event(struct perf_event
*event
)
795 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
797 if (event
->attr
.type
< PERF_TYPE_MAX
||
798 !cci_pmu
->model
->validate_hw_event
)
801 return cci_pmu
->model
->validate_hw_event(cci_pmu
, event
->attr
.config
);
804 static int pmu_request_irq(struct cci_pmu
*cci_pmu
, irq_handler_t handler
)
807 struct platform_device
*pmu_device
= cci_pmu
->plat_device
;
809 if (unlikely(!pmu_device
))
812 if (cci_pmu
->nr_irqs
< 1) {
813 dev_err(&pmu_device
->dev
, "no irqs for CCI PMUs defined\n");
818 * Register all available CCI PMU interrupts. In the interrupt handler
819 * we iterate over the counters checking for interrupt source (the
820 * overflowing counter) and clear it.
822 * This should allow handling of non-unique interrupt for the counters.
824 for (i
= 0; i
< cci_pmu
->nr_irqs
; i
++) {
825 int err
= request_irq(cci_pmu
->irqs
[i
], handler
, IRQF_SHARED
,
826 "arm-cci-pmu", cci_pmu
);
828 dev_err(&pmu_device
->dev
, "unable to request IRQ%d for ARM CCI PMU counters\n",
833 set_bit(i
, &cci_pmu
->active_irqs
);
839 static void pmu_free_irq(struct cci_pmu
*cci_pmu
)
843 for (i
= 0; i
< cci_pmu
->nr_irqs
; i
++) {
844 if (!test_and_clear_bit(i
, &cci_pmu
->active_irqs
))
847 free_irq(cci_pmu
->irqs
[i
], cci_pmu
);
851 static u32
pmu_read_counter(struct perf_event
*event
)
853 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
854 struct hw_perf_event
*hw_counter
= &event
->hw
;
855 int idx
= hw_counter
->idx
;
858 if (unlikely(!pmu_is_valid_counter(cci_pmu
, idx
))) {
859 dev_err(&cci_pmu
->plat_device
->dev
, "Invalid CCI PMU counter %d\n", idx
);
862 value
= pmu_read_register(cci_pmu
, idx
, CCI_PMU_CNTR
);
867 static void pmu_write_counter(struct cci_pmu
*cci_pmu
, u32 value
, int idx
)
869 pmu_write_register(cci_pmu
, value
, idx
, CCI_PMU_CNTR
);
872 static void __pmu_write_counters(struct cci_pmu
*cci_pmu
, unsigned long *mask
)
875 struct cci_pmu_hw_events
*cci_hw
= &cci_pmu
->hw_events
;
877 for_each_set_bit(i
, mask
, cci_pmu
->num_cntrs
) {
878 struct perf_event
*event
= cci_hw
->events
[i
];
882 pmu_write_counter(cci_pmu
, local64_read(&event
->hw
.prev_count
), i
);
886 static void pmu_write_counters(struct cci_pmu
*cci_pmu
, unsigned long *mask
)
888 if (cci_pmu
->model
->write_counters
)
889 cci_pmu
->model
->write_counters(cci_pmu
, mask
);
891 __pmu_write_counters(cci_pmu
, mask
);
894 #ifdef CONFIG_ARM_CCI500_PMU
897 * CCI-500 has advanced power saving policies, which could gate the
898 * clocks to the PMU counters, which makes the writes to them ineffective.
899 * The only way to write to those counters is when the global counters
900 * are enabled and the particular counter is enabled.
902 * So we do the following :
904 * 1) Disable all the PMU counters, saving their current state
905 * 2) Enable the global PMU profiling, now that all counters are
908 * For each counter to be programmed, repeat steps 3-7:
910 * 3) Write an invalid event code to the event control register for the
911 counter, so that the counters are not modified.
912 * 4) Enable the counter control for the counter.
913 * 5) Set the counter value
914 * 6) Disable the counter
915 * 7) Restore the event in the target counter
917 * 8) Disable the global PMU.
918 * 9) Restore the status of the rest of the counters.
920 * We choose an event which for CCI-500 is guaranteed not to count.
921 * We use the highest possible event code (0x1f) for the master interface 0.
923 #define CCI500_INVALID_EVENT ((CCI500_PORT_M0 << CCI500_PMU_EVENT_SOURCE_SHIFT) | \
924 (CCI500_PMU_EVENT_CODE_MASK << CCI500_PMU_EVENT_CODE_SHIFT))
925 static void cci500_pmu_write_counters(struct cci_pmu
*cci_pmu
, unsigned long *mask
)
928 DECLARE_BITMAP(saved_mask
, cci_pmu
->num_cntrs
);
930 bitmap_zero(saved_mask
, cci_pmu
->num_cntrs
);
931 pmu_save_counters(cci_pmu
, saved_mask
);
934 * Now that all the counters are disabled, we can safely turn the PMU on,
935 * without syncing the status of the counters
937 __cci_pmu_enable_nosync(cci_pmu
);
939 for_each_set_bit(i
, mask
, cci_pmu
->num_cntrs
) {
940 struct perf_event
*event
= cci_pmu
->hw_events
.events
[i
];
945 pmu_set_event(cci_pmu
, i
, CCI500_INVALID_EVENT
);
946 pmu_enable_counter(cci_pmu
, i
);
947 pmu_write_counter(cci_pmu
, local64_read(&event
->hw
.prev_count
), i
);
948 pmu_disable_counter(cci_pmu
, i
);
949 pmu_set_event(cci_pmu
, i
, event
->hw
.config_base
);
954 pmu_restore_counters(cci_pmu
, saved_mask
);
957 #endif /* CONFIG_ARM_CCI500_PMU */
959 static u64
pmu_event_update(struct perf_event
*event
)
961 struct hw_perf_event
*hwc
= &event
->hw
;
962 u64 delta
, prev_raw_count
, new_raw_count
;
965 prev_raw_count
= local64_read(&hwc
->prev_count
);
966 new_raw_count
= pmu_read_counter(event
);
967 } while (local64_cmpxchg(&hwc
->prev_count
, prev_raw_count
,
968 new_raw_count
) != prev_raw_count
);
970 delta
= (new_raw_count
- prev_raw_count
) & CCI_PMU_CNTR_MASK
;
972 local64_add(delta
, &event
->count
);
974 return new_raw_count
;
977 static void pmu_read(struct perf_event
*event
)
979 pmu_event_update(event
);
982 void pmu_event_set_period(struct perf_event
*event
)
984 struct hw_perf_event
*hwc
= &event
->hw
;
986 * The CCI PMU counters have a period of 2^32. To account for the
987 * possiblity of extreme interrupt latency we program for a period of
988 * half that. Hopefully we can handle the interrupt before another 2^31
989 * events occur and the counter overtakes its previous value.
991 u64 val
= 1ULL << 31;
992 local64_set(&hwc
->prev_count
, val
);
995 * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose
996 * values needs to be sync-ed with the s/w state before the PMU is
998 * Mark this counter for sync.
1000 hwc
->state
|= PERF_HES_ARCH
;
1003 static irqreturn_t
pmu_handle_irq(int irq_num
, void *dev
)
1005 unsigned long flags
;
1006 struct cci_pmu
*cci_pmu
= dev
;
1007 struct cci_pmu_hw_events
*events
= &cci_pmu
->hw_events
;
1008 int idx
, handled
= IRQ_NONE
;
1010 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
1012 /* Disable the PMU while we walk through the counters */
1013 __cci_pmu_disable();
1015 * Iterate over counters and update the corresponding perf events.
1016 * This should work regardless of whether we have per-counter overflow
1017 * interrupt or a combined overflow interrupt.
1019 for (idx
= 0; idx
<= CCI_PMU_CNTR_LAST(cci_pmu
); idx
++) {
1020 struct perf_event
*event
= events
->events
[idx
];
1021 struct hw_perf_event
*hw_counter
;
1026 hw_counter
= &event
->hw
;
1028 /* Did this counter overflow? */
1029 if (!(pmu_read_register(cci_pmu
, idx
, CCI_PMU_OVRFLW
) &
1030 CCI_PMU_OVRFLW_FLAG
))
1033 pmu_write_register(cci_pmu
, CCI_PMU_OVRFLW_FLAG
, idx
,
1036 pmu_event_update(event
);
1037 pmu_event_set_period(event
);
1038 handled
= IRQ_HANDLED
;
1041 /* Enable the PMU and sync possibly overflowed counters */
1042 __cci_pmu_enable_sync(cci_pmu
);
1043 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
1045 return IRQ_RETVAL(handled
);
1048 static int cci_pmu_get_hw(struct cci_pmu
*cci_pmu
)
1050 int ret
= pmu_request_irq(cci_pmu
, pmu_handle_irq
);
1052 pmu_free_irq(cci_pmu
);
1058 static void cci_pmu_put_hw(struct cci_pmu
*cci_pmu
)
1060 pmu_free_irq(cci_pmu
);
1063 static void hw_perf_event_destroy(struct perf_event
*event
)
1065 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
1066 atomic_t
*active_events
= &cci_pmu
->active_events
;
1067 struct mutex
*reserve_mutex
= &cci_pmu
->reserve_mutex
;
1069 if (atomic_dec_and_mutex_lock(active_events
, reserve_mutex
)) {
1070 cci_pmu_put_hw(cci_pmu
);
1071 mutex_unlock(reserve_mutex
);
1075 static void cci_pmu_enable(struct pmu
*pmu
)
1077 struct cci_pmu
*cci_pmu
= to_cci_pmu(pmu
);
1078 struct cci_pmu_hw_events
*hw_events
= &cci_pmu
->hw_events
;
1079 int enabled
= bitmap_weight(hw_events
->used_mask
, cci_pmu
->num_cntrs
);
1080 unsigned long flags
;
1085 raw_spin_lock_irqsave(&hw_events
->pmu_lock
, flags
);
1086 __cci_pmu_enable_sync(cci_pmu
);
1087 raw_spin_unlock_irqrestore(&hw_events
->pmu_lock
, flags
);
1091 static void cci_pmu_disable(struct pmu
*pmu
)
1093 struct cci_pmu
*cci_pmu
= to_cci_pmu(pmu
);
1094 struct cci_pmu_hw_events
*hw_events
= &cci_pmu
->hw_events
;
1095 unsigned long flags
;
1097 raw_spin_lock_irqsave(&hw_events
->pmu_lock
, flags
);
1098 __cci_pmu_disable();
1099 raw_spin_unlock_irqrestore(&hw_events
->pmu_lock
, flags
);
1103 * Check if the idx represents a non-programmable counter.
1104 * All the fixed event counters are mapped before the programmable
1107 static bool pmu_fixed_hw_idx(struct cci_pmu
*cci_pmu
, int idx
)
1109 return (idx
>= 0) && (idx
< cci_pmu
->model
->fixed_hw_cntrs
);
1112 static void cci_pmu_start(struct perf_event
*event
, int pmu_flags
)
1114 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
1115 struct cci_pmu_hw_events
*hw_events
= &cci_pmu
->hw_events
;
1116 struct hw_perf_event
*hwc
= &event
->hw
;
1118 unsigned long flags
;
1121 * To handle interrupt latency, we always reprogram the period
1122 * regardlesss of PERF_EF_RELOAD.
1124 if (pmu_flags
& PERF_EF_RELOAD
)
1125 WARN_ON_ONCE(!(hwc
->state
& PERF_HES_UPTODATE
));
1129 if (unlikely(!pmu_is_valid_counter(cci_pmu
, idx
))) {
1130 dev_err(&cci_pmu
->plat_device
->dev
, "Invalid CCI PMU counter %d\n", idx
);
1134 raw_spin_lock_irqsave(&hw_events
->pmu_lock
, flags
);
1136 /* Configure the counter unless you are counting a fixed event */
1137 if (!pmu_fixed_hw_idx(cci_pmu
, idx
))
1138 pmu_set_event(cci_pmu
, idx
, hwc
->config_base
);
1140 pmu_event_set_period(event
);
1141 pmu_enable_counter(cci_pmu
, idx
);
1143 raw_spin_unlock_irqrestore(&hw_events
->pmu_lock
, flags
);
1146 static void cci_pmu_stop(struct perf_event
*event
, int pmu_flags
)
1148 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
1149 struct hw_perf_event
*hwc
= &event
->hw
;
1152 if (hwc
->state
& PERF_HES_STOPPED
)
1155 if (unlikely(!pmu_is_valid_counter(cci_pmu
, idx
))) {
1156 dev_err(&cci_pmu
->plat_device
->dev
, "Invalid CCI PMU counter %d\n", idx
);
1161 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See
1164 pmu_disable_counter(cci_pmu
, idx
);
1165 pmu_event_update(event
);
1166 hwc
->state
|= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
1169 static int cci_pmu_add(struct perf_event
*event
, int flags
)
1171 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
1172 struct cci_pmu_hw_events
*hw_events
= &cci_pmu
->hw_events
;
1173 struct hw_perf_event
*hwc
= &event
->hw
;
1177 perf_pmu_disable(event
->pmu
);
1179 /* If we don't have a space for the counter then finish early. */
1180 idx
= pmu_get_event_idx(hw_events
, event
);
1186 event
->hw
.idx
= idx
;
1187 hw_events
->events
[idx
] = event
;
1189 hwc
->state
= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
1190 if (flags
& PERF_EF_START
)
1191 cci_pmu_start(event
, PERF_EF_RELOAD
);
1193 /* Propagate our changes to the userspace mapping. */
1194 perf_event_update_userpage(event
);
1197 perf_pmu_enable(event
->pmu
);
1201 static void cci_pmu_del(struct perf_event
*event
, int flags
)
1203 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
1204 struct cci_pmu_hw_events
*hw_events
= &cci_pmu
->hw_events
;
1205 struct hw_perf_event
*hwc
= &event
->hw
;
1208 cci_pmu_stop(event
, PERF_EF_UPDATE
);
1209 hw_events
->events
[idx
] = NULL
;
1210 clear_bit(idx
, hw_events
->used_mask
);
1212 perf_event_update_userpage(event
);
1216 validate_event(struct pmu
*cci_pmu
,
1217 struct cci_pmu_hw_events
*hw_events
,
1218 struct perf_event
*event
)
1220 if (is_software_event(event
))
1224 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
1225 * core perf code won't check that the pmu->ctx == leader->ctx
1226 * until after pmu->event_init(event).
1228 if (event
->pmu
!= cci_pmu
)
1231 if (event
->state
< PERF_EVENT_STATE_OFF
)
1234 if (event
->state
== PERF_EVENT_STATE_OFF
&& !event
->attr
.enable_on_exec
)
1237 return pmu_get_event_idx(hw_events
, event
) >= 0;
1241 validate_group(struct perf_event
*event
)
1243 struct perf_event
*sibling
, *leader
= event
->group_leader
;
1244 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
1245 unsigned long mask
[BITS_TO_LONGS(cci_pmu
->num_cntrs
)];
1246 struct cci_pmu_hw_events fake_pmu
= {
1248 * Initialise the fake PMU. We only need to populate the
1249 * used_mask for the purposes of validation.
1253 memset(mask
, 0, BITS_TO_LONGS(cci_pmu
->num_cntrs
) * sizeof(unsigned long));
1255 if (!validate_event(event
->pmu
, &fake_pmu
, leader
))
1258 list_for_each_entry(sibling
, &leader
->sibling_list
, group_entry
) {
1259 if (!validate_event(event
->pmu
, &fake_pmu
, sibling
))
1263 if (!validate_event(event
->pmu
, &fake_pmu
, event
))
1270 __hw_perf_event_init(struct perf_event
*event
)
1272 struct hw_perf_event
*hwc
= &event
->hw
;
1275 mapping
= pmu_map_event(event
);
1278 pr_debug("event %x:%llx not supported\n", event
->attr
.type
,
1279 event
->attr
.config
);
1284 * We don't assign an index until we actually place the event onto
1285 * hardware. Use -1 to signify that we haven't decided where to put it
1289 hwc
->config_base
= 0;
1291 hwc
->event_base
= 0;
1294 * Store the event encoding into the config_base field.
1296 hwc
->config_base
|= (unsigned long)mapping
;
1299 * Limit the sample_period to half of the counter width. That way, the
1300 * new counter value is far less likely to overtake the previous one
1301 * unless you have some serious IRQ latency issues.
1303 hwc
->sample_period
= CCI_PMU_CNTR_MASK
>> 1;
1304 hwc
->last_period
= hwc
->sample_period
;
1305 local64_set(&hwc
->period_left
, hwc
->sample_period
);
1307 if (event
->group_leader
!= event
) {
1308 if (validate_group(event
) != 0)
1315 static int cci_pmu_event_init(struct perf_event
*event
)
1317 struct cci_pmu
*cci_pmu
= to_cci_pmu(event
->pmu
);
1318 atomic_t
*active_events
= &cci_pmu
->active_events
;
1322 if (event
->attr
.type
!= event
->pmu
->type
)
1325 /* Shared by all CPUs, no meaningful state to sample */
1326 if (is_sampling_event(event
) || event
->attach_state
& PERF_ATTACH_TASK
)
1329 /* We have no filtering of any kind */
1330 if (event
->attr
.exclude_user
||
1331 event
->attr
.exclude_kernel
||
1332 event
->attr
.exclude_hv
||
1333 event
->attr
.exclude_idle
||
1334 event
->attr
.exclude_host
||
1335 event
->attr
.exclude_guest
)
1339 * Following the example set by other "uncore" PMUs, we accept any CPU
1340 * and rewrite its affinity dynamically rather than having perf core
1341 * handle cpu == -1 and pid == -1 for this case.
1343 * The perf core will pin online CPUs for the duration of this call and
1344 * the event being installed into its context, so the PMU's CPU can't
1345 * change under our feet.
1347 cpu
= cpumask_first(&cci_pmu
->cpus
);
1348 if (event
->cpu
< 0 || cpu
< 0)
1352 event
->destroy
= hw_perf_event_destroy
;
1353 if (!atomic_inc_not_zero(active_events
)) {
1354 mutex_lock(&cci_pmu
->reserve_mutex
);
1355 if (atomic_read(active_events
) == 0)
1356 err
= cci_pmu_get_hw(cci_pmu
);
1358 atomic_inc(active_events
);
1359 mutex_unlock(&cci_pmu
->reserve_mutex
);
1364 err
= __hw_perf_event_init(event
);
1366 hw_perf_event_destroy(event
);
1371 static ssize_t
pmu_cpumask_attr_show(struct device
*dev
,
1372 struct device_attribute
*attr
, char *buf
)
1374 struct pmu
*pmu
= dev_get_drvdata(dev
);
1375 struct cci_pmu
*cci_pmu
= to_cci_pmu(pmu
);
1377 int n
= scnprintf(buf
, PAGE_SIZE
- 1, "%*pbl",
1378 cpumask_pr_args(&cci_pmu
->cpus
));
1384 static struct device_attribute pmu_cpumask_attr
=
1385 __ATTR(cpumask
, S_IRUGO
, pmu_cpumask_attr_show
, NULL
);
1387 static struct attribute
*pmu_attrs
[] = {
1388 &pmu_cpumask_attr
.attr
,
1392 static struct attribute_group pmu_attr_group
= {
1396 static struct attribute_group pmu_format_attr_group
= {
1398 .attrs
= NULL
, /* Filled in cci_pmu_init_attrs */
1401 static struct attribute_group pmu_event_attr_group
= {
1403 .attrs
= NULL
, /* Filled in cci_pmu_init_attrs */
1406 static const struct attribute_group
*pmu_attr_groups
[] = {
1408 &pmu_format_attr_group
,
1409 &pmu_event_attr_group
,
1413 static int cci_pmu_init(struct cci_pmu
*cci_pmu
, struct platform_device
*pdev
)
1415 const struct cci_pmu_model
*model
= cci_pmu
->model
;
1416 char *name
= model
->name
;
1419 pmu_event_attr_group
.attrs
= model
->event_attrs
;
1420 pmu_format_attr_group
.attrs
= model
->format_attrs
;
1422 cci_pmu
->pmu
= (struct pmu
) {
1423 .name
= cci_pmu
->model
->name
,
1424 .task_ctx_nr
= perf_invalid_context
,
1425 .pmu_enable
= cci_pmu_enable
,
1426 .pmu_disable
= cci_pmu_disable
,
1427 .event_init
= cci_pmu_event_init
,
1430 .start
= cci_pmu_start
,
1431 .stop
= cci_pmu_stop
,
1433 .attr_groups
= pmu_attr_groups
,
1436 cci_pmu
->plat_device
= pdev
;
1437 num_cntrs
= pmu_get_max_counters();
1438 if (num_cntrs
> cci_pmu
->model
->num_hw_cntrs
) {
1439 dev_warn(&pdev
->dev
,
1440 "PMU implements more counters(%d) than supported by"
1441 " the model(%d), truncated.",
1442 num_cntrs
, cci_pmu
->model
->num_hw_cntrs
);
1443 num_cntrs
= cci_pmu
->model
->num_hw_cntrs
;
1445 cci_pmu
->num_cntrs
= num_cntrs
+ cci_pmu
->model
->fixed_hw_cntrs
;
1447 return perf_pmu_register(&cci_pmu
->pmu
, name
, -1);
1450 static int cci_pmu_cpu_notifier(struct notifier_block
*self
,
1451 unsigned long action
, void *hcpu
)
1453 struct cci_pmu
*cci_pmu
= container_of(self
,
1454 struct cci_pmu
, cpu_nb
);
1455 unsigned int cpu
= (long)hcpu
;
1456 unsigned int target
;
1458 switch (action
& ~CPU_TASKS_FROZEN
) {
1459 case CPU_DOWN_PREPARE
:
1460 if (!cpumask_test_and_clear_cpu(cpu
, &cci_pmu
->cpus
))
1462 target
= cpumask_any_but(cpu_online_mask
, cpu
);
1463 if (target
>= nr_cpu_ids
) // UP, last CPU
1466 * TODO: migrate context once core races on event->ctx have
1469 cpumask_set_cpu(target
, &cci_pmu
->cpus
);
1477 static struct cci_pmu_model cci_pmu_models
[] = {
1478 #ifdef CONFIG_ARM_CCI400_PMU
1481 .fixed_hw_cntrs
= 1, /* Cycle counter */
1484 .format_attrs
= cci400_pmu_format_attrs
,
1485 .event_attrs
= cci400_r0_pmu_event_attrs
,
1488 CCI400_R0_SLAVE_PORT_MIN_EV
,
1489 CCI400_R0_SLAVE_PORT_MAX_EV
,
1492 CCI400_R0_MASTER_PORT_MIN_EV
,
1493 CCI400_R0_MASTER_PORT_MAX_EV
,
1496 .validate_hw_event
= cci400_validate_hw_event
,
1497 .get_event_idx
= cci400_get_event_idx
,
1500 .name
= "CCI_400_r1",
1501 .fixed_hw_cntrs
= 1, /* Cycle counter */
1504 .format_attrs
= cci400_pmu_format_attrs
,
1505 .event_attrs
= cci400_r1_pmu_event_attrs
,
1508 CCI400_R1_SLAVE_PORT_MIN_EV
,
1509 CCI400_R1_SLAVE_PORT_MAX_EV
,
1512 CCI400_R1_MASTER_PORT_MIN_EV
,
1513 CCI400_R1_MASTER_PORT_MAX_EV
,
1516 .validate_hw_event
= cci400_validate_hw_event
,
1517 .get_event_idx
= cci400_get_event_idx
,
1520 #ifdef CONFIG_ARM_CCI500_PMU
1523 .fixed_hw_cntrs
= 0,
1525 .cntr_size
= SZ_64K
,
1526 .format_attrs
= cci500_pmu_format_attrs
,
1527 .event_attrs
= cci500_pmu_event_attrs
,
1530 CCI500_SLAVE_PORT_MIN_EV
,
1531 CCI500_SLAVE_PORT_MAX_EV
,
1534 CCI500_MASTER_PORT_MIN_EV
,
1535 CCI500_MASTER_PORT_MAX_EV
,
1538 CCI500_GLOBAL_PORT_MIN_EV
,
1539 CCI500_GLOBAL_PORT_MAX_EV
,
1542 .validate_hw_event
= cci500_validate_hw_event
,
1543 .write_counters
= cci500_pmu_write_counters
,
1548 static const struct of_device_id arm_cci_pmu_matches
[] = {
1549 #ifdef CONFIG_ARM_CCI400_PMU
1551 .compatible
= "arm,cci-400-pmu",
1555 .compatible
= "arm,cci-400-pmu,r0",
1556 .data
= &cci_pmu_models
[CCI400_R0
],
1559 .compatible
= "arm,cci-400-pmu,r1",
1560 .data
= &cci_pmu_models
[CCI400_R1
],
1563 #ifdef CONFIG_ARM_CCI500_PMU
1565 .compatible
= "arm,cci-500-pmu,r0",
1566 .data
= &cci_pmu_models
[CCI500_R0
],
1572 static inline const struct cci_pmu_model
*get_cci_model(struct platform_device
*pdev
)
1574 const struct of_device_id
*match
= of_match_node(arm_cci_pmu_matches
,
1581 dev_warn(&pdev
->dev
, "DEPRECATED compatible property,"
1582 "requires secure access to CCI registers");
1583 return probe_cci_model(pdev
);
1586 static bool is_duplicate_irq(int irq
, int *irqs
, int nr_irqs
)
1590 for (i
= 0; i
< nr_irqs
; i
++)
1597 static struct cci_pmu
*cci_pmu_alloc(struct platform_device
*pdev
)
1599 struct cci_pmu
*cci_pmu
;
1600 const struct cci_pmu_model
*model
;
1603 * All allocations are devm_* hence we don't have to free
1604 * them explicitly on an error, as it would end up in driver
1607 model
= get_cci_model(pdev
);
1609 dev_warn(&pdev
->dev
, "CCI PMU version not supported\n");
1610 return ERR_PTR(-ENODEV
);
1613 cci_pmu
= devm_kzalloc(&pdev
->dev
, sizeof(*cci_pmu
), GFP_KERNEL
);
1615 return ERR_PTR(-ENOMEM
);
1617 cci_pmu
->model
= model
;
1618 cci_pmu
->irqs
= devm_kcalloc(&pdev
->dev
, CCI_PMU_MAX_HW_CNTRS(model
),
1619 sizeof(*cci_pmu
->irqs
), GFP_KERNEL
);
1621 return ERR_PTR(-ENOMEM
);
1622 cci_pmu
->hw_events
.events
= devm_kcalloc(&pdev
->dev
,
1623 CCI_PMU_MAX_HW_CNTRS(model
),
1624 sizeof(*cci_pmu
->hw_events
.events
),
1626 if (!cci_pmu
->hw_events
.events
)
1627 return ERR_PTR(-ENOMEM
);
1628 cci_pmu
->hw_events
.used_mask
= devm_kcalloc(&pdev
->dev
,
1629 BITS_TO_LONGS(CCI_PMU_MAX_HW_CNTRS(model
)),
1630 sizeof(*cci_pmu
->hw_events
.used_mask
),
1632 if (!cci_pmu
->hw_events
.used_mask
)
1633 return ERR_PTR(-ENOMEM
);
1639 static int cci_pmu_probe(struct platform_device
*pdev
)
1641 struct resource
*res
;
1642 struct cci_pmu
*cci_pmu
;
1645 cci_pmu
= cci_pmu_alloc(pdev
);
1646 if (IS_ERR(cci_pmu
))
1647 return PTR_ERR(cci_pmu
);
1649 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1650 cci_pmu
->base
= devm_ioremap_resource(&pdev
->dev
, res
);
1651 if (IS_ERR(cci_pmu
->base
))
1655 * CCI PMU has one overflow interrupt per counter; but some may be tied
1656 * together to a common interrupt.
1658 cci_pmu
->nr_irqs
= 0;
1659 for (i
= 0; i
< CCI_PMU_MAX_HW_CNTRS(cci_pmu
->model
); i
++) {
1660 irq
= platform_get_irq(pdev
, i
);
1664 if (is_duplicate_irq(irq
, cci_pmu
->irqs
, cci_pmu
->nr_irqs
))
1667 cci_pmu
->irqs
[cci_pmu
->nr_irqs
++] = irq
;
1671 * Ensure that the device tree has as many interrupts as the number
1674 if (i
< CCI_PMU_MAX_HW_CNTRS(cci_pmu
->model
)) {
1675 dev_warn(&pdev
->dev
, "In-correct number of interrupts: %d, should be %d\n",
1676 i
, CCI_PMU_MAX_HW_CNTRS(cci_pmu
->model
));
1680 raw_spin_lock_init(&cci_pmu
->hw_events
.pmu_lock
);
1681 mutex_init(&cci_pmu
->reserve_mutex
);
1682 atomic_set(&cci_pmu
->active_events
, 0);
1683 cpumask_set_cpu(smp_processor_id(), &cci_pmu
->cpus
);
1685 cci_pmu
->cpu_nb
= (struct notifier_block
) {
1686 .notifier_call
= cci_pmu_cpu_notifier
,
1688 * to migrate uncore events, our notifier should be executed
1689 * before perf core's notifier.
1691 .priority
= CPU_PRI_PERF
+ 1,
1694 ret
= register_cpu_notifier(&cci_pmu
->cpu_nb
);
1698 ret
= cci_pmu_init(cci_pmu
, pdev
);
1700 unregister_cpu_notifier(&cci_pmu
->cpu_nb
);
1704 pr_info("ARM %s PMU driver probed", cci_pmu
->model
->name
);
1708 static int cci_platform_probe(struct platform_device
*pdev
)
1713 return of_platform_populate(pdev
->dev
.of_node
, NULL
, NULL
, &pdev
->dev
);
1716 static struct platform_driver cci_pmu_driver
= {
1718 .name
= DRIVER_NAME_PMU
,
1719 .of_match_table
= arm_cci_pmu_matches
,
1721 .probe
= cci_pmu_probe
,
1724 static struct platform_driver cci_platform_driver
= {
1726 .name
= DRIVER_NAME
,
1727 .of_match_table
= arm_cci_matches
,
1729 .probe
= cci_platform_probe
,
1732 static int __init
cci_platform_init(void)
1736 ret
= platform_driver_register(&cci_pmu_driver
);
1740 return platform_driver_register(&cci_platform_driver
);
1743 #else /* !CONFIG_ARM_CCI_PMU */
1745 static int __init
cci_platform_init(void)
1750 #endif /* CONFIG_ARM_CCI_PMU */
1752 #ifdef CONFIG_ARM_CCI400_PORT_CTRL
1754 #define CCI_PORT_CTRL 0x0
1755 #define CCI_CTRL_STATUS 0xc
1757 #define CCI_ENABLE_SNOOP_REQ 0x1
1758 #define CCI_ENABLE_DVM_REQ 0x2
1759 #define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ)
1761 enum cci_ace_port_type
{
1762 ACE_INVALID_PORT
= 0x0,
1767 struct cci_ace_port
{
1770 enum cci_ace_port_type type
;
1771 struct device_node
*dn
;
1774 static struct cci_ace_port
*ports
;
1775 static unsigned int nb_cci_ports
;
1783 * Use the port MSB as valid flag, shift can be made dynamic
1784 * by computing number of bits required for port indexes.
1785 * Code disabling CCI cpu ports runs with D-cache invalidated
1786 * and SCTLR bit clear so data accesses must be kept to a minimum
1787 * to improve performance; for now shift is left static to
1788 * avoid one more data access while disabling the CCI port.
1790 #define PORT_VALID_SHIFT 31
1791 #define PORT_VALID (0x1 << PORT_VALID_SHIFT)
1793 static inline void init_cpu_port(struct cpu_port
*port
, u32 index
, u64 mpidr
)
1795 port
->port
= PORT_VALID
| index
;
1796 port
->mpidr
= mpidr
;
1799 static inline bool cpu_port_is_valid(struct cpu_port
*port
)
1801 return !!(port
->port
& PORT_VALID
);
1804 static inline bool cpu_port_match(struct cpu_port
*port
, u64 mpidr
)
1806 return port
->mpidr
== (mpidr
& MPIDR_HWID_BITMASK
);
1809 static struct cpu_port cpu_port
[NR_CPUS
];
1812 * __cci_ace_get_port - Function to retrieve the port index connected to
1815 * @dn: device node of the device to look-up
1819 * - CCI port index if success
1820 * - -ENODEV if failure
1822 static int __cci_ace_get_port(struct device_node
*dn
, int type
)
1826 struct device_node
*cci_portn
;
1828 cci_portn
= of_parse_phandle(dn
, "cci-control-port", 0);
1829 for (i
= 0; i
< nb_cci_ports
; i
++) {
1830 ace_match
= ports
[i
].type
== type
;
1831 if (ace_match
&& cci_portn
== ports
[i
].dn
)
1837 int cci_ace_get_port(struct device_node
*dn
)
1839 return __cci_ace_get_port(dn
, ACE_LITE_PORT
);
1841 EXPORT_SYMBOL_GPL(cci_ace_get_port
);
1843 static void cci_ace_init_ports(void)
1846 struct device_node
*cpun
;
1849 * Port index look-up speeds up the function disabling ports by CPU,
1850 * since the logical to port index mapping is done once and does
1851 * not change after system boot.
1852 * The stashed index array is initialized for all possible CPUs
1855 for_each_possible_cpu(cpu
) {
1856 /* too early to use cpu->of_node */
1857 cpun
= of_get_cpu_node(cpu
, NULL
);
1859 if (WARN(!cpun
, "Missing cpu device node\n"))
1862 port
= __cci_ace_get_port(cpun
, ACE_PORT
);
1866 init_cpu_port(&cpu_port
[cpu
], port
, cpu_logical_map(cpu
));
1869 for_each_possible_cpu(cpu
) {
1870 WARN(!cpu_port_is_valid(&cpu_port
[cpu
]),
1871 "CPU %u does not have an associated CCI port\n",
1876 * Functions to enable/disable a CCI interconnect slave port
1878 * They are called by low-level power management code to disable slave
1879 * interfaces snoops and DVM broadcast.
1880 * Since they may execute with cache data allocation disabled and
1881 * after the caches have been cleaned and invalidated the functions provide
1882 * no explicit locking since they may run with D-cache disabled, so normal
1883 * cacheable kernel locks based on ldrex/strex may not work.
1884 * Locking has to be provided by BSP implementations to ensure proper
1889 * cci_port_control() - function to control a CCI port
1891 * @port: index of the port to setup
1892 * @enable: if true enables the port, if false disables it
1894 static void notrace
cci_port_control(unsigned int port
, bool enable
)
1896 void __iomem
*base
= ports
[port
].base
;
1898 writel_relaxed(enable
? CCI_ENABLE_REQ
: 0, base
+ CCI_PORT_CTRL
);
1900 * This function is called from power down procedures
1901 * and must not execute any instruction that might
1902 * cause the processor to be put in a quiescent state
1903 * (eg wfi). Hence, cpu_relax() can not be added to this
1904 * read loop to optimize power, since it might hide possibly
1905 * disruptive operations.
1907 while (readl_relaxed(cci_ctrl_base
+ CCI_CTRL_STATUS
) & 0x1)
1912 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU
1915 * @mpidr: mpidr of the CPU whose CCI port should be disabled
1917 * Disabling a CCI port for a CPU implies disabling the CCI port
1918 * controlling that CPU cluster. Code disabling CPU CCI ports
1919 * must make sure that the CPU running the code is the last active CPU
1920 * in the cluster ie all other CPUs are quiescent in a low power state.
1924 * -ENODEV on port look-up failure
1926 int notrace
cci_disable_port_by_cpu(u64 mpidr
)
1930 for (cpu
= 0; cpu
< nr_cpu_ids
; cpu
++) {
1931 is_valid
= cpu_port_is_valid(&cpu_port
[cpu
]);
1932 if (is_valid
&& cpu_port_match(&cpu_port
[cpu
], mpidr
)) {
1933 cci_port_control(cpu_port
[cpu
].port
, false);
1939 EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu
);
1942 * cci_enable_port_for_self() - enable a CCI port for calling CPU
1944 * Enabling a CCI port for the calling CPU implies enabling the CCI
1945 * port controlling that CPU's cluster. Caller must make sure that the
1946 * CPU running the code is the first active CPU in the cluster and all
1947 * other CPUs are quiescent in a low power state or waiting for this CPU
1948 * to complete the CCI initialization.
1950 * Because this is called when the MMU is still off and with no stack,
1951 * the code must be position independent and ideally rely on callee
1952 * clobbered registers only. To achieve this we must code this function
1953 * entirely in assembler.
1955 * On success this returns with the proper CCI port enabled. In case of
1956 * any failure this never returns as the inability to enable the CCI is
1957 * fatal and there is no possible recovery at this stage.
1959 asmlinkage
void __naked
cci_enable_port_for_self(void)
1963 " mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n"
1964 " and r0, r0, #"__stringify(MPIDR_HWID_BITMASK
)" \n"
1967 " add r1, r1, r2 @ &cpu_port \n"
1968 " add ip, r1, %[sizeof_cpu_port] \n"
1970 /* Loop over the cpu_port array looking for a matching MPIDR */
1971 "1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n"
1972 " cmp r2, r0 @ compare MPIDR \n"
1975 /* Found a match, now test port validity */
1976 " ldr r3, [r1, %[offsetof_cpu_port_port]] \n"
1977 " tst r3, #"__stringify(PORT_VALID
)" \n"
1980 /* no match, loop with the next cpu_port entry */
1981 "2: add r1, r1, %[sizeof_struct_cpu_port] \n"
1982 " cmp r1, ip @ done? \n"
1985 /* CCI port not found -- cheaply try to stall this CPU */
1986 "cci_port_not_found: \n"
1989 " b cci_port_not_found \n"
1991 /* Use matched port index to look up the corresponding ports entry */
1992 "3: bic r3, r3, #"__stringify(PORT_VALID
)" \n"
1994 " ldmia r0, {r1, r2} \n"
1995 " sub r1, r1, r0 @ virt - phys \n"
1996 " ldr r0, [r0, r2] @ *(&ports) \n"
1997 " mov r2, %[sizeof_struct_ace_port] \n"
1998 " mla r0, r2, r3, r0 @ &ports[index] \n"
1999 " sub r0, r0, r1 @ virt_to_phys() \n"
2001 /* Enable the CCI port */
2002 " ldr r0, [r0, %[offsetof_port_phys]] \n"
2003 " mov r3, %[cci_enable_req]\n"
2004 " str r3, [r0, #"__stringify(CCI_PORT_CTRL
)"] \n"
2006 /* poll the status reg for completion */
2009 " ldr r0, [r0, r1] @ cci_ctrl_base \n"
2010 "4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS
)"] \n"
2011 " tst r1, %[cci_control_status_bits] \n"
2018 "5: .word cpu_port - . \n"
2020 " .word ports - 6b \n"
2021 "7: .word cci_ctrl_phys - . \n"
2023 [sizeof_cpu_port
] "i" (sizeof(cpu_port
)),
2024 [cci_enable_req
] "i" cpu_to_le32(CCI_ENABLE_REQ
),
2025 [cci_control_status_bits
] "i" cpu_to_le32(1),
2027 [offsetof_cpu_port_mpidr_lsb
] "i" (offsetof(struct cpu_port
, mpidr
)),
2029 [offsetof_cpu_port_mpidr_lsb
] "i" (offsetof(struct cpu_port
, mpidr
)+4),
2031 [offsetof_cpu_port_port
] "i" (offsetof(struct cpu_port
, port
)),
2032 [sizeof_struct_cpu_port
] "i" (sizeof(struct cpu_port
)),
2033 [sizeof_struct_ace_port
] "i" (sizeof(struct cci_ace_port
)),
2034 [offsetof_port_phys
] "i" (offsetof(struct cci_ace_port
, phys
)) );
2040 * __cci_control_port_by_device() - function to control a CCI port by device
2043 * @dn: device node pointer of the device whose CCI port should be
2045 * @enable: if true enables the port, if false disables it
2049 * -ENODEV on port look-up failure
2051 int notrace
__cci_control_port_by_device(struct device_node
*dn
, bool enable
)
2058 port
= __cci_ace_get_port(dn
, ACE_LITE_PORT
);
2059 if (WARN_ONCE(port
< 0, "node %s ACE lite port look-up failure\n",
2062 cci_port_control(port
, enable
);
2065 EXPORT_SYMBOL_GPL(__cci_control_port_by_device
);
2068 * __cci_control_port_by_index() - function to control a CCI port by port index
2070 * @port: port index previously retrieved with cci_ace_get_port()
2071 * @enable: if true enables the port, if false disables it
2075 * -ENODEV on port index out of range
2076 * -EPERM if operation carried out on an ACE PORT
2078 int notrace
__cci_control_port_by_index(u32 port
, bool enable
)
2080 if (port
>= nb_cci_ports
|| ports
[port
].type
== ACE_INVALID_PORT
)
2083 * CCI control for ports connected to CPUS is extremely fragile
2084 * and must be made to go through a specific and controlled
2085 * interface (ie cci_disable_port_by_cpu(); control by general purpose
2086 * indexing is therefore disabled for ACE ports.
2088 if (ports
[port
].type
== ACE_PORT
)
2091 cci_port_control(port
, enable
);
2094 EXPORT_SYMBOL_GPL(__cci_control_port_by_index
);
2096 static const struct of_device_id arm_cci_ctrl_if_matches
[] = {
2097 {.compatible
= "arm,cci-400-ctrl-if", },
2101 static int cci_probe_ports(struct device_node
*np
)
2103 struct cci_nb_ports
const *cci_config
;
2104 int ret
, i
, nb_ace
= 0, nb_ace_lite
= 0;
2105 struct device_node
*cp
;
2106 struct resource res
;
2107 const char *match_str
;
2111 cci_config
= of_match_node(arm_cci_matches
, np
)->data
;
2115 nb_cci_ports
= cci_config
->nb_ace
+ cci_config
->nb_ace_lite
;
2117 ports
= kcalloc(nb_cci_ports
, sizeof(*ports
), GFP_KERNEL
);
2121 for_each_child_of_node(np
, cp
) {
2122 if (!of_match_node(arm_cci_ctrl_if_matches
, cp
))
2125 i
= nb_ace
+ nb_ace_lite
;
2127 if (i
>= nb_cci_ports
)
2130 if (of_property_read_string(cp
, "interface-type",
2132 WARN(1, "node %s missing interface-type property\n",
2136 is_ace
= strcmp(match_str
, "ace") == 0;
2137 if (!is_ace
&& strcmp(match_str
, "ace-lite")) {
2138 WARN(1, "node %s containing invalid interface-type property, skipping it\n",
2143 ret
= of_address_to_resource(cp
, 0, &res
);
2145 ports
[i
].base
= ioremap(res
.start
, resource_size(&res
));
2146 ports
[i
].phys
= res
.start
;
2148 if (ret
|| !ports
[i
].base
) {
2149 WARN(1, "unable to ioremap CCI port %d\n", i
);
2154 if (WARN_ON(nb_ace
>= cci_config
->nb_ace
))
2156 ports
[i
].type
= ACE_PORT
;
2159 if (WARN_ON(nb_ace_lite
>= cci_config
->nb_ace_lite
))
2161 ports
[i
].type
= ACE_LITE_PORT
;
2167 /* initialize a stashed array of ACE ports to speed-up look-up */
2168 cci_ace_init_ports();
2171 * Multi-cluster systems may need this data when non-coherent, during
2172 * cluster power-up/power-down. Make sure it reaches main memory.
2174 sync_cache_w(&cci_ctrl_base
);
2175 sync_cache_w(&cci_ctrl_phys
);
2176 sync_cache_w(&ports
);
2177 sync_cache_w(&cpu_port
);
2178 __sync_cache_range_w(ports
, sizeof(*ports
) * nb_cci_ports
);
2179 pr_info("ARM CCI driver probed\n");
2183 #else /* !CONFIG_ARM_CCI400_PORT_CTRL */
2184 static inline int cci_probe_ports(struct device_node
*np
)
2188 #endif /* CONFIG_ARM_CCI400_PORT_CTRL */
2190 static int cci_probe(void)
2193 struct device_node
*np
;
2194 struct resource res
;
2196 np
= of_find_matching_node(NULL
, arm_cci_matches
);
2197 if(!np
|| !of_device_is_available(np
))
2200 ret
= of_address_to_resource(np
, 0, &res
);
2202 cci_ctrl_base
= ioremap(res
.start
, resource_size(&res
));
2203 cci_ctrl_phys
= res
.start
;
2205 if (ret
|| !cci_ctrl_base
) {
2206 WARN(1, "unable to ioremap CCI ctrl\n");
2210 return cci_probe_ports(np
);
2213 static int cci_init_status
= -EAGAIN
;
2214 static DEFINE_MUTEX(cci_probing
);
2216 static int cci_init(void)
2218 if (cci_init_status
!= -EAGAIN
)
2219 return cci_init_status
;
2221 mutex_lock(&cci_probing
);
2222 if (cci_init_status
== -EAGAIN
)
2223 cci_init_status
= cci_probe();
2224 mutex_unlock(&cci_probing
);
2225 return cci_init_status
;
2229 * To sort out early init calls ordering a helper function is provided to
2230 * check if the CCI driver has beed initialized. Function check if the driver
2231 * has been initialized, if not it calls the init function that probes
2232 * the driver and updates the return value.
2234 bool cci_probed(void)
2236 return cci_init() == 0;
2238 EXPORT_SYMBOL_GPL(cci_probed
);
2240 early_initcall(cci_init
);
2241 core_initcall(cci_platform_init
);
2242 MODULE_LICENSE("GPL");
2243 MODULE_DESCRIPTION("ARM CCI support");