Commit | Line | Data |
---|---|---|
ed9b8cef AP |
1 | /* |
2 | * VGICv3 MMIO handling functions | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | */ | |
13 | ||
14 | #include <linux/irqchip/arm-gic-v3.h> | |
15 | #include <linux/kvm.h> | |
16 | #include <linux/kvm_host.h> | |
17 | #include <kvm/iodev.h> | |
18 | #include <kvm/arm_vgic.h> | |
19 | ||
20 | #include <asm/kvm_emulate.h> | |
21 | ||
22 | #include "vgic.h" | |
23 | #include "vgic-mmio.h" | |
24 | ||
741972d8 | 25 | /* extract @num bytes at @offset bytes offset in data */ |
424c3383 AP |
26 | unsigned long extract_bytes(unsigned long data, unsigned int offset, |
27 | unsigned int num) | |
741972d8 AP |
28 | { |
29 | return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0); | |
30 | } | |
31 | ||
0aa1de57 | 32 | /* allows updates of any half of a 64-bit register (or the whole thing) */ |
424c3383 AP |
33 | u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len, |
34 | unsigned long val) | |
0aa1de57 AP |
35 | { |
36 | int lower = (offset & 4) * 8; | |
37 | int upper = lower + 8 * len - 1; | |
38 | ||
39 | reg &= ~GENMASK_ULL(upper, lower); | |
40 | val &= GENMASK_ULL(len * 8 - 1, 0); | |
41 | ||
42 | return reg | ((u64)val << lower); | |
43 | } | |
44 | ||
59c5ab40 AP |
45 | bool vgic_has_its(struct kvm *kvm) |
46 | { | |
47 | struct vgic_dist *dist = &kvm->arch.vgic; | |
48 | ||
49 | if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) | |
50 | return false; | |
51 | ||
1085fdc6 | 52 | return dist->has_its; |
59c5ab40 AP |
53 | } |
54 | ||
fd59ed3b AP |
55 | static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, |
56 | gpa_t addr, unsigned int len) | |
57 | { | |
58 | u32 value = 0; | |
59 | ||
60 | switch (addr & 0x0c) { | |
61 | case GICD_CTLR: | |
62 | if (vcpu->kvm->arch.vgic.enabled) | |
63 | value |= GICD_CTLR_ENABLE_SS_G1; | |
64 | value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS; | |
65 | break; | |
66 | case GICD_TYPER: | |
67 | value = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; | |
68 | value = (value >> 5) - 1; | |
0e4e82f1 AP |
69 | if (vgic_has_its(vcpu->kvm)) { |
70 | value |= (INTERRUPT_ID_BITS_ITS - 1) << 19; | |
71 | value |= GICD_TYPER_LPIS; | |
72 | } else { | |
73 | value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19; | |
74 | } | |
fd59ed3b AP |
75 | break; |
76 | case GICD_IIDR: | |
77 | value = (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0); | |
78 | break; | |
79 | default: | |
80 | return 0; | |
81 | } | |
82 | ||
83 | return value; | |
84 | } | |
85 | ||
86 | static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu, | |
87 | gpa_t addr, unsigned int len, | |
88 | unsigned long val) | |
89 | { | |
90 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
91 | bool was_enabled = dist->enabled; | |
92 | ||
93 | switch (addr & 0x0c) { | |
94 | case GICD_CTLR: | |
95 | dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; | |
96 | ||
97 | if (!was_enabled && dist->enabled) | |
98 | vgic_kick_vcpus(vcpu->kvm); | |
99 | break; | |
100 | case GICD_TYPER: | |
101 | case GICD_IIDR: | |
102 | return; | |
103 | } | |
104 | } | |
105 | ||
78a714ab AP |
106 | static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu, |
107 | gpa_t addr, unsigned int len) | |
108 | { | |
109 | int intid = VGIC_ADDR_TO_INTID(addr, 64); | |
110 | struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid); | |
5dd4b924 | 111 | unsigned long ret = 0; |
78a714ab AP |
112 | |
113 | if (!irq) | |
114 | return 0; | |
115 | ||
116 | /* The upper word is RAZ for us. */ | |
5dd4b924 AP |
117 | if (!(addr & 4)) |
118 | ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len); | |
78a714ab | 119 | |
5dd4b924 AP |
120 | vgic_put_irq(vcpu->kvm, irq); |
121 | return ret; | |
78a714ab AP |
122 | } |
123 | ||
124 | static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu, | |
125 | gpa_t addr, unsigned int len, | |
126 | unsigned long val) | |
127 | { | |
128 | int intid = VGIC_ADDR_TO_INTID(addr, 64); | |
5dd4b924 | 129 | struct vgic_irq *irq; |
78a714ab AP |
130 | |
131 | /* The upper word is WI for us since we don't implement Aff3. */ | |
132 | if (addr & 4) | |
133 | return; | |
134 | ||
5dd4b924 AP |
135 | irq = vgic_get_irq(vcpu->kvm, NULL, intid); |
136 | ||
137 | if (!irq) | |
138 | return; | |
139 | ||
78a714ab AP |
140 | spin_lock(&irq->irq_lock); |
141 | ||
142 | /* We only care about and preserve Aff0, Aff1 and Aff2. */ | |
143 | irq->mpidr = val & GENMASK(23, 0); | |
144 | irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr); | |
145 | ||
146 | spin_unlock(&irq->irq_lock); | |
5dd4b924 | 147 | vgic_put_irq(vcpu->kvm, irq); |
78a714ab AP |
148 | } |
149 | ||
59c5ab40 AP |
150 | static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu, |
151 | gpa_t addr, unsigned int len) | |
152 | { | |
153 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
154 | ||
155 | return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0; | |
156 | } | |
157 | ||
158 | ||
159 | static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, | |
160 | gpa_t addr, unsigned int len, | |
161 | unsigned long val) | |
162 | { | |
163 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
164 | bool was_enabled = vgic_cpu->lpis_enabled; | |
165 | ||
166 | if (!vgic_has_its(vcpu->kvm)) | |
167 | return; | |
168 | ||
169 | vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS; | |
170 | ||
0e4e82f1 AP |
171 | if (!was_enabled && vgic_cpu->lpis_enabled) |
172 | vgic_enable_lpis(vcpu); | |
59c5ab40 AP |
173 | } |
174 | ||
741972d8 AP |
175 | static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu, |
176 | gpa_t addr, unsigned int len) | |
177 | { | |
178 | unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu); | |
179 | int target_vcpu_id = vcpu->vcpu_id; | |
180 | u64 value; | |
181 | ||
182 | value = (mpidr & GENMASK(23, 0)) << 32; | |
183 | value |= ((target_vcpu_id & 0xffff) << 8); | |
184 | if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1) | |
185 | value |= GICR_TYPER_LAST; | |
0e4e82f1 AP |
186 | if (vgic_has_its(vcpu->kvm)) |
187 | value |= GICR_TYPER_PLPIS; | |
741972d8 AP |
188 | |
189 | return extract_bytes(value, addr & 7, len); | |
190 | } | |
191 | ||
192 | static unsigned long vgic_mmio_read_v3r_iidr(struct kvm_vcpu *vcpu, | |
193 | gpa_t addr, unsigned int len) | |
194 | { | |
195 | return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0); | |
196 | } | |
197 | ||
54f59d2b AP |
198 | static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu, |
199 | gpa_t addr, unsigned int len) | |
200 | { | |
201 | switch (addr & 0xffff) { | |
202 | case GICD_PIDR2: | |
203 | /* report a GICv3 compliant implementation */ | |
204 | return 0x3b; | |
205 | } | |
206 | ||
207 | return 0; | |
208 | } | |
209 | ||
0aa1de57 AP |
210 | /* We want to avoid outer shareable. */ |
211 | u64 vgic_sanitise_shareability(u64 field) | |
212 | { | |
213 | switch (field) { | |
214 | case GIC_BASER_OuterShareable: | |
215 | return GIC_BASER_InnerShareable; | |
216 | default: | |
217 | return field; | |
218 | } | |
219 | } | |
220 | ||
221 | /* Avoid any inner non-cacheable mapping. */ | |
222 | u64 vgic_sanitise_inner_cacheability(u64 field) | |
223 | { | |
224 | switch (field) { | |
225 | case GIC_BASER_CACHE_nCnB: | |
226 | case GIC_BASER_CACHE_nC: | |
227 | return GIC_BASER_CACHE_RaWb; | |
228 | default: | |
229 | return field; | |
230 | } | |
231 | } | |
232 | ||
233 | /* Non-cacheable or same-as-inner are OK. */ | |
234 | u64 vgic_sanitise_outer_cacheability(u64 field) | |
235 | { | |
236 | switch (field) { | |
237 | case GIC_BASER_CACHE_SameAsInner: | |
238 | case GIC_BASER_CACHE_nC: | |
239 | return field; | |
240 | default: | |
241 | return GIC_BASER_CACHE_nC; | |
242 | } | |
243 | } | |
244 | ||
245 | u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift, | |
246 | u64 (*sanitise_fn)(u64)) | |
247 | { | |
248 | u64 field = (reg & field_mask) >> field_shift; | |
249 | ||
250 | field = sanitise_fn(field) << field_shift; | |
251 | return (reg & ~field_mask) | field; | |
252 | } | |
253 | ||
254 | #define PROPBASER_RES0_MASK \ | |
255 | (GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5)) | |
256 | #define PENDBASER_RES0_MASK \ | |
257 | (BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) | \ | |
258 | GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0)) | |
259 | ||
260 | static u64 vgic_sanitise_pendbaser(u64 reg) | |
261 | { | |
262 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK, | |
263 | GICR_PENDBASER_SHAREABILITY_SHIFT, | |
264 | vgic_sanitise_shareability); | |
265 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK, | |
266 | GICR_PENDBASER_INNER_CACHEABILITY_SHIFT, | |
267 | vgic_sanitise_inner_cacheability); | |
268 | reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK, | |
269 | GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT, | |
270 | vgic_sanitise_outer_cacheability); | |
271 | ||
272 | reg &= ~PENDBASER_RES0_MASK; | |
273 | reg &= ~GENMASK_ULL(51, 48); | |
274 | ||
275 | return reg; | |
276 | } | |
277 | ||
278 | static u64 vgic_sanitise_propbaser(u64 reg) | |
279 | { | |
280 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK, | |
281 | GICR_PROPBASER_SHAREABILITY_SHIFT, | |
282 | vgic_sanitise_shareability); | |
283 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK, | |
284 | GICR_PROPBASER_INNER_CACHEABILITY_SHIFT, | |
285 | vgic_sanitise_inner_cacheability); | |
286 | reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK, | |
287 | GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT, | |
288 | vgic_sanitise_outer_cacheability); | |
289 | ||
290 | reg &= ~PROPBASER_RES0_MASK; | |
291 | reg &= ~GENMASK_ULL(51, 48); | |
292 | return reg; | |
293 | } | |
294 | ||
295 | static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu, | |
296 | gpa_t addr, unsigned int len) | |
297 | { | |
298 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
299 | ||
300 | return extract_bytes(dist->propbaser, addr & 7, len); | |
301 | } | |
302 | ||
303 | static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu, | |
304 | gpa_t addr, unsigned int len, | |
305 | unsigned long val) | |
306 | { | |
307 | struct vgic_dist *dist = &vcpu->kvm->arch.vgic; | |
308 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
d9ae449b | 309 | u64 old_propbaser, propbaser; |
0aa1de57 AP |
310 | |
311 | /* Storing a value with LPIs already enabled is undefined */ | |
312 | if (vgic_cpu->lpis_enabled) | |
313 | return; | |
314 | ||
d9ae449b CD |
315 | do { |
316 | old_propbaser = dist->propbaser; | |
317 | propbaser = old_propbaser; | |
318 | propbaser = update_64bit_reg(propbaser, addr & 4, len, val); | |
319 | propbaser = vgic_sanitise_propbaser(propbaser); | |
320 | } while (cmpxchg64(&dist->propbaser, old_propbaser, | |
321 | propbaser) != old_propbaser); | |
0aa1de57 AP |
322 | } |
323 | ||
324 | static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu, | |
325 | gpa_t addr, unsigned int len) | |
326 | { | |
327 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
328 | ||
329 | return extract_bytes(vgic_cpu->pendbaser, addr & 7, len); | |
330 | } | |
331 | ||
332 | static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, | |
333 | gpa_t addr, unsigned int len, | |
334 | unsigned long val) | |
335 | { | |
336 | struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; | |
d9ae449b | 337 | u64 old_pendbaser, pendbaser; |
0aa1de57 AP |
338 | |
339 | /* Storing a value with LPIs already enabled is undefined */ | |
340 | if (vgic_cpu->lpis_enabled) | |
341 | return; | |
342 | ||
d9ae449b CD |
343 | do { |
344 | old_pendbaser = vgic_cpu->pendbaser; | |
345 | pendbaser = old_pendbaser; | |
346 | pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val); | |
347 | pendbaser = vgic_sanitise_pendbaser(pendbaser); | |
348 | } while (cmpxchg64(&vgic_cpu->pendbaser, old_pendbaser, | |
349 | pendbaser) != old_pendbaser); | |
0aa1de57 AP |
350 | } |
351 | ||
ed9b8cef AP |
352 | /* |
353 | * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the | |
354 | * redistributors, while SPIs are covered by registers in the distributor | |
355 | * block. Trying to set private IRQs in this block gets ignored. | |
356 | * We take some special care here to fix the calculation of the register | |
357 | * offset. | |
358 | */ | |
359 | #define REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(off, rd, wr, bpi, acc) \ | |
360 | { \ | |
361 | .reg_offset = off, \ | |
362 | .bits_per_irq = bpi, \ | |
363 | .len = (bpi * VGIC_NR_PRIVATE_IRQS) / 8, \ | |
364 | .access_flags = acc, \ | |
365 | .read = vgic_mmio_read_raz, \ | |
366 | .write = vgic_mmio_write_wi, \ | |
367 | }, { \ | |
368 | .reg_offset = off + (bpi * VGIC_NR_PRIVATE_IRQS) / 8, \ | |
369 | .bits_per_irq = bpi, \ | |
370 | .len = (bpi * (1024 - VGIC_NR_PRIVATE_IRQS)) / 8, \ | |
371 | .access_flags = acc, \ | |
372 | .read = rd, \ | |
373 | .write = wr, \ | |
374 | } | |
375 | ||
376 | static const struct vgic_register_region vgic_v3_dist_registers[] = { | |
377 | REGISTER_DESC_WITH_LENGTH(GICD_CTLR, | |
fd59ed3b | 378 | vgic_mmio_read_v3_misc, vgic_mmio_write_v3_misc, 16, |
ed9b8cef AP |
379 | VGIC_ACCESS_32bit), |
380 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGROUPR, | |
381 | vgic_mmio_read_rao, vgic_mmio_write_wi, 1, | |
382 | VGIC_ACCESS_32bit), | |
383 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISENABLER, | |
384 | vgic_mmio_read_enable, vgic_mmio_write_senable, 1, | |
385 | VGIC_ACCESS_32bit), | |
386 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICENABLER, | |
387 | vgic_mmio_read_enable, vgic_mmio_write_cenable, 1, | |
388 | VGIC_ACCESS_32bit), | |
389 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISPENDR, | |
390 | vgic_mmio_read_pending, vgic_mmio_write_spending, 1, | |
391 | VGIC_ACCESS_32bit), | |
392 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICPENDR, | |
393 | vgic_mmio_read_pending, vgic_mmio_write_cpending, 1, | |
394 | VGIC_ACCESS_32bit), | |
395 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISACTIVER, | |
396 | vgic_mmio_read_active, vgic_mmio_write_sactive, 1, | |
397 | VGIC_ACCESS_32bit), | |
398 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICACTIVER, | |
399 | vgic_mmio_read_active, vgic_mmio_write_cactive, 1, | |
400 | VGIC_ACCESS_32bit), | |
401 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IPRIORITYR, | |
402 | vgic_mmio_read_priority, vgic_mmio_write_priority, 8, | |
403 | VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), | |
404 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ITARGETSR, | |
405 | vgic_mmio_read_raz, vgic_mmio_write_wi, 8, | |
406 | VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), | |
407 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICFGR, | |
408 | vgic_mmio_read_config, vgic_mmio_write_config, 2, | |
409 | VGIC_ACCESS_32bit), | |
410 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR, | |
411 | vgic_mmio_read_raz, vgic_mmio_write_wi, 1, | |
412 | VGIC_ACCESS_32bit), | |
413 | REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER, | |
78a714ab | 414 | vgic_mmio_read_irouter, vgic_mmio_write_irouter, 64, |
ed9b8cef AP |
415 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
416 | REGISTER_DESC_WITH_LENGTH(GICD_IDREGS, | |
54f59d2b | 417 | vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, |
ed9b8cef AP |
418 | VGIC_ACCESS_32bit), |
419 | }; | |
420 | ||
421 | static const struct vgic_register_region vgic_v3_rdbase_registers[] = { | |
422 | REGISTER_DESC_WITH_LENGTH(GICR_CTLR, | |
59c5ab40 | 423 | vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4, |
ed9b8cef AP |
424 | VGIC_ACCESS_32bit), |
425 | REGISTER_DESC_WITH_LENGTH(GICR_IIDR, | |
741972d8 | 426 | vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4, |
ed9b8cef AP |
427 | VGIC_ACCESS_32bit), |
428 | REGISTER_DESC_WITH_LENGTH(GICR_TYPER, | |
741972d8 | 429 | vgic_mmio_read_v3r_typer, vgic_mmio_write_wi, 8, |
ed9b8cef AP |
430 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
431 | REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER, | |
0aa1de57 | 432 | vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8, |
ed9b8cef AP |
433 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
434 | REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER, | |
0aa1de57 | 435 | vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8, |
ed9b8cef AP |
436 | VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), |
437 | REGISTER_DESC_WITH_LENGTH(GICR_IDREGS, | |
54f59d2b | 438 | vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, |
ed9b8cef AP |
439 | VGIC_ACCESS_32bit), |
440 | }; | |
441 | ||
442 | static const struct vgic_register_region vgic_v3_sgibase_registers[] = { | |
443 | REGISTER_DESC_WITH_LENGTH(GICR_IGROUPR0, | |
444 | vgic_mmio_read_rao, vgic_mmio_write_wi, 4, | |
445 | VGIC_ACCESS_32bit), | |
446 | REGISTER_DESC_WITH_LENGTH(GICR_ISENABLER0, | |
447 | vgic_mmio_read_enable, vgic_mmio_write_senable, 4, | |
448 | VGIC_ACCESS_32bit), | |
449 | REGISTER_DESC_WITH_LENGTH(GICR_ICENABLER0, | |
450 | vgic_mmio_read_enable, vgic_mmio_write_cenable, 4, | |
451 | VGIC_ACCESS_32bit), | |
452 | REGISTER_DESC_WITH_LENGTH(GICR_ISPENDR0, | |
453 | vgic_mmio_read_pending, vgic_mmio_write_spending, 4, | |
454 | VGIC_ACCESS_32bit), | |
455 | REGISTER_DESC_WITH_LENGTH(GICR_ICPENDR0, | |
456 | vgic_mmio_read_pending, vgic_mmio_write_cpending, 4, | |
457 | VGIC_ACCESS_32bit), | |
458 | REGISTER_DESC_WITH_LENGTH(GICR_ISACTIVER0, | |
459 | vgic_mmio_read_active, vgic_mmio_write_sactive, 4, | |
460 | VGIC_ACCESS_32bit), | |
461 | REGISTER_DESC_WITH_LENGTH(GICR_ICACTIVER0, | |
462 | vgic_mmio_read_active, vgic_mmio_write_cactive, 4, | |
463 | VGIC_ACCESS_32bit), | |
464 | REGISTER_DESC_WITH_LENGTH(GICR_IPRIORITYR0, | |
465 | vgic_mmio_read_priority, vgic_mmio_write_priority, 32, | |
466 | VGIC_ACCESS_32bit | VGIC_ACCESS_8bit), | |
467 | REGISTER_DESC_WITH_LENGTH(GICR_ICFGR0, | |
468 | vgic_mmio_read_config, vgic_mmio_write_config, 8, | |
469 | VGIC_ACCESS_32bit), | |
470 | REGISTER_DESC_WITH_LENGTH(GICR_IGRPMODR0, | |
471 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
472 | VGIC_ACCESS_32bit), | |
473 | REGISTER_DESC_WITH_LENGTH(GICR_NSACR, | |
474 | vgic_mmio_read_raz, vgic_mmio_write_wi, 4, | |
475 | VGIC_ACCESS_32bit), | |
476 | }; | |
477 | ||
478 | unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev) | |
479 | { | |
480 | dev->regions = vgic_v3_dist_registers; | |
481 | dev->nr_regions = ARRAY_SIZE(vgic_v3_dist_registers); | |
482 | ||
483 | kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops); | |
484 | ||
485 | return SZ_64K; | |
486 | } | |
487 | ||
488 | int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t redist_base_address) | |
489 | { | |
ed9b8cef | 490 | struct kvm_vcpu *vcpu; |
ed9b8cef AP |
491 | int c, ret = 0; |
492 | ||
ed9b8cef AP |
493 | kvm_for_each_vcpu(c, vcpu, kvm) { |
494 | gpa_t rd_base = redist_base_address + c * SZ_64K * 2; | |
495 | gpa_t sgi_base = rd_base + SZ_64K; | |
8f6cdc1c AP |
496 | struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev; |
497 | struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev; | |
ed9b8cef AP |
498 | |
499 | kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops); | |
500 | rd_dev->base_addr = rd_base; | |
59c5ab40 | 501 | rd_dev->iodev_type = IODEV_REDIST; |
ed9b8cef AP |
502 | rd_dev->regions = vgic_v3_rdbase_registers; |
503 | rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers); | |
504 | rd_dev->redist_vcpu = vcpu; | |
505 | ||
506 | mutex_lock(&kvm->slots_lock); | |
507 | ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base, | |
508 | SZ_64K, &rd_dev->dev); | |
509 | mutex_unlock(&kvm->slots_lock); | |
510 | ||
511 | if (ret) | |
512 | break; | |
513 | ||
514 | kvm_iodevice_init(&sgi_dev->dev, &kvm_io_gic_ops); | |
515 | sgi_dev->base_addr = sgi_base; | |
59c5ab40 | 516 | sgi_dev->iodev_type = IODEV_REDIST; |
ed9b8cef AP |
517 | sgi_dev->regions = vgic_v3_sgibase_registers; |
518 | sgi_dev->nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers); | |
519 | sgi_dev->redist_vcpu = vcpu; | |
520 | ||
521 | mutex_lock(&kvm->slots_lock); | |
522 | ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, sgi_base, | |
523 | SZ_64K, &sgi_dev->dev); | |
524 | mutex_unlock(&kvm->slots_lock); | |
525 | if (ret) { | |
526 | kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, | |
527 | &rd_dev->dev); | |
528 | break; | |
529 | } | |
530 | } | |
531 | ||
532 | if (ret) { | |
533 | /* The current c failed, so we start with the previous one. */ | |
534 | for (c--; c >= 0; c--) { | |
8f6cdc1c AP |
535 | struct vgic_cpu *vgic_cpu; |
536 | ||
537 | vcpu = kvm_get_vcpu(kvm, c); | |
538 | vgic_cpu = &vcpu->arch.vgic_cpu; | |
ed9b8cef | 539 | kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, |
8f6cdc1c | 540 | &vgic_cpu->rd_iodev.dev); |
ed9b8cef | 541 | kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, |
8f6cdc1c | 542 | &vgic_cpu->sgi_iodev.dev); |
ed9b8cef | 543 | } |
ed9b8cef AP |
544 | } |
545 | ||
546 | return ret; | |
547 | } | |
621ecd8d AP |
548 | |
549 | /* | |
550 | * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI | |
551 | * generation register ICC_SGI1R_EL1) with a given VCPU. | |
552 | * If the VCPU's MPIDR matches, return the level0 affinity, otherwise | |
553 | * return -1. | |
554 | */ | |
555 | static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu) | |
556 | { | |
557 | unsigned long affinity; | |
558 | int level0; | |
559 | ||
560 | /* | |
561 | * Split the current VCPU's MPIDR into affinity level 0 and the | |
562 | * rest as this is what we have to compare against. | |
563 | */ | |
564 | affinity = kvm_vcpu_get_mpidr_aff(vcpu); | |
565 | level0 = MPIDR_AFFINITY_LEVEL(affinity, 0); | |
566 | affinity &= ~MPIDR_LEVEL_MASK; | |
567 | ||
568 | /* bail out if the upper three levels don't match */ | |
569 | if (sgi_aff != affinity) | |
570 | return -1; | |
571 | ||
572 | /* Is this VCPU's bit set in the mask ? */ | |
573 | if (!(sgi_cpu_mask & BIT(level0))) | |
574 | return -1; | |
575 | ||
576 | return level0; | |
577 | } | |
578 | ||
579 | /* | |
580 | * The ICC_SGI* registers encode the affinity differently from the MPIDR, | |
581 | * so provide a wrapper to use the existing defines to isolate a certain | |
582 | * affinity level. | |
583 | */ | |
584 | #define SGI_AFFINITY_LEVEL(reg, level) \ | |
585 | ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \ | |
586 | >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level)) | |
587 | ||
588 | /** | |
589 | * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs | |
590 | * @vcpu: The VCPU requesting a SGI | |
591 | * @reg: The value written into the ICC_SGI1R_EL1 register by that VCPU | |
592 | * | |
593 | * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register. | |
594 | * This will trap in sys_regs.c and call this function. | |
595 | * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the | |
596 | * target processors as well as a bitmask of 16 Aff0 CPUs. | |
597 | * If the interrupt routing mode bit is not set, we iterate over all VCPUs to | |
598 | * check for matching ones. If this bit is set, we signal all, but not the | |
599 | * calling VCPU. | |
600 | */ | |
601 | void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg) | |
602 | { | |
603 | struct kvm *kvm = vcpu->kvm; | |
604 | struct kvm_vcpu *c_vcpu; | |
605 | u16 target_cpus; | |
606 | u64 mpidr; | |
607 | int sgi, c; | |
608 | int vcpu_id = vcpu->vcpu_id; | |
609 | bool broadcast; | |
610 | ||
611 | sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT; | |
612 | broadcast = reg & BIT(ICC_SGI1R_IRQ_ROUTING_MODE_BIT); | |
613 | target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT; | |
614 | mpidr = SGI_AFFINITY_LEVEL(reg, 3); | |
615 | mpidr |= SGI_AFFINITY_LEVEL(reg, 2); | |
616 | mpidr |= SGI_AFFINITY_LEVEL(reg, 1); | |
617 | ||
618 | /* | |
619 | * We iterate over all VCPUs to find the MPIDRs matching the request. | |
620 | * If we have handled one CPU, we clear its bit to detect early | |
621 | * if we are already finished. This avoids iterating through all | |
622 | * VCPUs when most of the times we just signal a single VCPU. | |
623 | */ | |
624 | kvm_for_each_vcpu(c, c_vcpu, kvm) { | |
625 | struct vgic_irq *irq; | |
626 | ||
627 | /* Exit early if we have dealt with all requested CPUs */ | |
628 | if (!broadcast && target_cpus == 0) | |
629 | break; | |
630 | ||
631 | /* Don't signal the calling VCPU */ | |
632 | if (broadcast && c == vcpu_id) | |
633 | continue; | |
634 | ||
635 | if (!broadcast) { | |
636 | int level0; | |
637 | ||
638 | level0 = match_mpidr(mpidr, target_cpus, c_vcpu); | |
639 | if (level0 == -1) | |
640 | continue; | |
641 | ||
642 | /* remove this matching VCPU from the mask */ | |
643 | target_cpus &= ~BIT(level0); | |
644 | } | |
645 | ||
646 | irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi); | |
647 | ||
648 | spin_lock(&irq->irq_lock); | |
649 | irq->pending = true; | |
650 | ||
651 | vgic_queue_irq_unlock(vcpu->kvm, irq); | |
5dd4b924 | 652 | vgic_put_irq(vcpu->kvm, irq); |
621ecd8d AP |
653 | } |
654 | } |