projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'regmap/for-next'
[deliverable/linux.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
a374e892 26#include <linux/random.h>
b0c632db 27#include <linux/slab.h>
ba5c1e9b 28#include <linux/timer.h>
41408c28 29#include <linux/vmalloc.h>
15c9705f 30#include <linux/bitmap.h>
cbb870c8 31#include <asm/asm-offsets.h>
b0c632db 32#include <asm/lowcore.h>
fd5ada04 33#include <asm/stp.h>
b0c632db 34#include <asm/pgtable.h>
1e133ab2 35#include <asm/gmap.h>
f5daba1d 36#include <asm/nmi.h>
a0616cde 37#include <asm/switch_to.h>
6d3da241 38#include <asm/isc.h>
1526bf9c 39#include <asm/sclp.h>
0a763c78 40#include <asm/cpacf.h>
221bb8a4 41#include <asm/timex.h>
8f2abe6a 42#include "kvm-s390.h"
b0c632db
HC		 43#include "gaccess.h"
44
ea2cdd27
DH		 45#define KMSG_COMPONENT "kvm-s390"
46#undef pr_fmt
47#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
48
5786fffa
CH		 49#define CREATE_TRACE_POINTS
50#include "trace.h"
ade38c31 51#include "trace-s390.h"
5786fffa 52
41408c28 53#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 54#define LOCAL_IRQS 32
55#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
56 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 57
b0c632db
HC		 58#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
59
60struct kvm_stats_debugfs_item debugfs_entries[] = {
61 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 62 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 63 { "exit_validity", VCPU_STAT(exit_validity) },
64 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
65 { "exit_external_request", VCPU_STAT(exit_external_request) },
66 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 67 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 68 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 69 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
70 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 71 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 72 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 73 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 74 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 75 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 76 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 77 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 78 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
79 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 80 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 81 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 82 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
83 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
84 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
85 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
86 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
87 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
88 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 89 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 90 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
91 { "instruction_spx", VCPU_STAT(instruction_spx) },
92 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
93 { "instruction_stap", VCPU_STAT(instruction_stap) },
94 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 95 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 96 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
97 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 98 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 99 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
100 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 101 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 102 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 103 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 104 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 105 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 106 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 107 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 108 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
109 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 110 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 111 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
112 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 113 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 114 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
115 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
116 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 117 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
118 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
119 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 120 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 121 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 122 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 123 { "diagnose_258", VCPU_STAT(diagnose_258) },
124 { "diagnose_308", VCPU_STAT(diagnose_308) },
125 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 126 { NULL }
127};
128
a411edf1
DH		 129/* allow nested virtualization in KVM (if enabled by user space) */
130static int nested;
131module_param(nested, int, S_IRUGO);
132MODULE_PARM_DESC(nested, "Nested virtualization support");
133
9d8d5786 134/* upper facilities limit for kvm */
f6c1d359 135unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
b0c632db 136
9d8d5786 137unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 138{
9d8d5786
MM		 139 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
140 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 141}
142
15c9705f
DH		 143/* available cpu features supported by kvm */
144static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 145/* available subfunctions indicated via query / "test bit" */
146static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 147
9d8d5786 148static struct gmap_notifier gmap_notifier;
a3508fbe 149static struct gmap_notifier vsie_gmap_notifier;
78f26131 150debug_info_t *kvm_s390_dbf;
9d8d5786 151
b0c632db 152/* Section: not file related */
13a34e06 153int kvm_arch_hardware_enable(void)
b0c632db
HC		 154{
155 /* every s390 is virtualization enabled ;-) */
10474ae8 156 return 0;
b0c632db
HC		 157}
158
414d3b07
MS		 159static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
160 unsigned long end);
2c70fe44 161
fdf03650
FZ		 162/*
163 * This callback is executed during stop_machine(). All CPUs are therefore
164 * temporarily stopped. In order not to change guest behavior, we have to
165 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
166 * so a CPU won't be stopped while calculating with the epoch.
167 */
168static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
169 void *v)
170{
171 struct kvm *kvm;
172 struct kvm_vcpu *vcpu;
173 int i;
174 unsigned long long *delta = v;
175
176 list_for_each_entry(kvm, &vm_list, vm_list) {
177 kvm->arch.epoch -= *delta;
178 kvm_for_each_vcpu(i, vcpu, kvm) {
179 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 180 if (vcpu->arch.cputm_enabled)
181 vcpu->arch.cputm_start += *delta;
91473b48
DH		 182 if (vcpu->arch.vsie_block)
183 vcpu->arch.vsie_block->epoch -= *delta;
fdf03650
FZ		 184 }
185 }
186 return NOTIFY_OK;
187}
188
189static struct notifier_block kvm_clock_notifier = {
190 .notifier_call = kvm_clock_sync,
191};
192
b0c632db
HC		 193int kvm_arch_hardware_setup(void)
194{
2c70fe44 195 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 196 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 197 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
198 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 199 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
200 &kvm_clock_notifier);
b0c632db
HC		 201 return 0;
202}
203
204void kvm_arch_hardware_unsetup(void)
205{
b2d73b2a 206 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 207 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 208 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
209 &kvm_clock_notifier);
b0c632db
HC		 210}
211
22be5a13
DH		 212static void allow_cpu_feat(unsigned long nr)
213{
214 set_bit_inv(nr, kvm_s390_available_cpu_feat);
215}
216
0a763c78
DH		 217static inline int plo_test_bit(unsigned char nr)
218{
219 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
220 int cc = 3; /* subfunction not available */
221
222 asm volatile(
223 /* Parameter registers are ignored for "test bit" */
224 " plo 0,0,0,0(0)\n"
225 " ipm %0\n"
226 " srl %0,28\n"
227 : "=d" (cc)
228 : "d" (r0)
229 : "cc");
230 return cc == 0;
231}
232
22be5a13
DH		 233static void kvm_s390_cpu_feat_init(void)
234{
0a763c78
DH		 235 int i;
236
237 for (i = 0; i < 256; ++i) {
238 if (plo_test_bit(i))
239 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
240 }
241
242 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 243 ptff(kvm_s390_available_subfunc.ptff,
244 sizeof(kvm_s390_available_subfunc.ptff),
245 PTFF_QAF);
0a763c78
DH		 246
247 if (test_facility(17)) { /* MSA */
69c0e360
MS		 248 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
249 kvm_s390_available_subfunc.kmac);
250 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
251 kvm_s390_available_subfunc.kmc);
252 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
253 kvm_s390_available_subfunc.km);
254 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
255 kvm_s390_available_subfunc.kimd);
256 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
257 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 258 }
259 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 260 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
261 kvm_s390_available_subfunc.pckmo);
0a763c78 262 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 263 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
264 kvm_s390_available_subfunc.kmctr);
265 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
266 kvm_s390_available_subfunc.kmf);
267 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
268 kvm_s390_available_subfunc.kmo);
269 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
270 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 271 }
272 if (test_facility(57)) /* MSA5 */
69c0e360
MS		 273 __cpacf_query(CPACF_PPNO, (cpacf_mask_t *)
274 kvm_s390_available_subfunc.ppno);
0a763c78 275
22be5a13
DH		 276 if (MACHINE_HAS_ESOP)
277 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 278 /*
279 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
280 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
281 */
282 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 283 !test_facility(3) || !nested)
a3508fbe
DH		 284 return;
285 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 286 if (sclp.has_64bscao)
287 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 288 if (sclp.has_siif)
289 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 290 if (sclp.has_gpere)
291 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 292 if (sclp.has_gsls)
293 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 294 if (sclp.has_ib)
295 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 296 if (sclp.has_cei)
297 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 298 if (sclp.has_ibs)
299 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
5d3876a8
DH		 300 /*
301 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
302 * all skey handling functions read/set the skey from the PGSTE
303 * instead of the real storage key.
304 *
305 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
306 * pages being detected as preserved although they are resident.
307 *
308 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
309 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
310 *
311 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
312 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
313 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
314 *
315 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
316 * cannot easily shadow the SCA because of the ipte lock.
317 */
22be5a13
DH		 318}
319
b0c632db
HC		 320int kvm_arch_init(void *opaque)
321{
78f26131
CB		 322 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
323 if (!kvm_s390_dbf)
324 return -ENOMEM;
325
326 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
327 debug_unregister(kvm_s390_dbf);
328 return -ENOMEM;
329 }
330
22be5a13
DH		 331 kvm_s390_cpu_feat_init();
332
84877d93
CH		 333 /* Register floating interrupt controller interface. */
334 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 335}
336
78f26131
CB		 337void kvm_arch_exit(void)
338{
339 debug_unregister(kvm_s390_dbf);
340}
341
b0c632db
HC		 342/* Section: device related */
343long kvm_arch_dev_ioctl(struct file *filp,
344 unsigned int ioctl, unsigned long arg)
345{
346 if (ioctl == KVM_S390_ENABLE_SIE)
347 return s390_enable_sie();
348 return -EINVAL;
349}
350
784aa3d7 351int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 352{
d7b0b5eb
CO		 353 int r;
354
2bd0ac4e 355 switch (ext) {
d7b0b5eb 356 case KVM_CAP_S390_PSW:
b6cf8788 357 case KVM_CAP_S390_GMAP:
52e16b18 358 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 359#ifdef CONFIG_KVM_S390_UCONTROL
360 case KVM_CAP_S390_UCONTROL:
361#endif
3c038e6b 362 case KVM_CAP_ASYNC_PF:
60b413c9 363 case KVM_CAP_SYNC_REGS:
14eebd91 364 case KVM_CAP_ONE_REG:
d6712df9 365 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 366 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 367 case KVM_CAP_IOEVENTFD:
c05c4186 368 case KVM_CAP_DEVICE_CTRL:
d938dc55 369 case KVM_CAP_ENABLE_CAP_VM:
78599d90 370 case KVM_CAP_S390_IRQCHIP:
f2061656 371 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 372 case KVM_CAP_MP_STATE:
47b43c52 373 case KVM_CAP_S390_INJECT_IRQ:
2444b352 374 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 375 case KVM_CAP_S390_USER_STSI:
30ee2a98 376 case KVM_CAP_S390_SKEYS:
816c7667 377 case KVM_CAP_S390_IRQ_STATE:
6502a34c 378 case KVM_CAP_S390_USER_INSTR0:
d7b0b5eb
CO		 379 r = 1;
380 break;
41408c28
TH		 381 case KVM_CAP_S390_MEM_OP:
382 r = MEM_OP_MAX_SIZE;
383 break;
e726b1bd
CB		 384 case KVM_CAP_NR_VCPUS:
385 case KVM_CAP_MAX_VCPUS:
76a6dd72
DH		 386 r = KVM_S390_BSCA_CPU_SLOTS;
387 if (sclp.has_esca && sclp.has_64bscao)
388 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 389 break;
e1e2e605
NW		 390 case KVM_CAP_NR_MEMSLOTS:
391 r = KVM_USER_MEM_SLOTS;
392 break;
1526bf9c 393 case KVM_CAP_S390_COW:
abf09bed 394 r = MACHINE_HAS_ESOP;
1526bf9c 395 break;
68c55750
EF		 396 case KVM_CAP_S390_VECTOR_REGISTERS:
397 r = MACHINE_HAS_VX;
398 break;
c6e5f166
FZ		 399 case KVM_CAP_S390_RI:
400 r = test_facility(64);
401 break;
2bd0ac4e 402 default:
d7b0b5eb 403 r = 0;
2bd0ac4e 404 }
d7b0b5eb 405 return r;
b0c632db
HC		 406}
407
15f36ebd
JH		 408static void kvm_s390_sync_dirty_log(struct kvm *kvm,
409 struct kvm_memory_slot *memslot)
410{
411 gfn_t cur_gfn, last_gfn;
412 unsigned long address;
413 struct gmap *gmap = kvm->arch.gmap;
414
15f36ebd
JH		 415 /* Loop over all guest pages */
416 last_gfn = memslot->base_gfn + memslot->npages;
417 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
418 address = gfn_to_hva_memslot(memslot, cur_gfn);
419
1e133ab2 420 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 421 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 422 if (fatal_signal_pending(current))
423 return;
70c88a00 424 cond_resched();
15f36ebd 425 }
15f36ebd
JH		 426}
427
b0c632db 428/* Section: vm related */
a6e2f683
ED		 429static void sca_del_vcpu(struct kvm_vcpu *vcpu);
430
b0c632db
HC		 431/*
432 * Get (and clear) the dirty memory log for a memory slot.
433 */
434int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
435 struct kvm_dirty_log *log)
436{
15f36ebd
JH		 437 int r;
438 unsigned long n;
9f6b8029 439 struct kvm_memslots *slots;
15f36ebd
JH		 440 struct kvm_memory_slot *memslot;
441 int is_dirty = 0;
442
443 mutex_lock(&kvm->slots_lock);
444
445 r = -EINVAL;
446 if (log->slot >= KVM_USER_MEM_SLOTS)
447 goto out;
448
9f6b8029
PB		 449 slots = kvm_memslots(kvm);
450 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 451 r = -ENOENT;
452 if (!memslot->dirty_bitmap)
453 goto out;
454
455 kvm_s390_sync_dirty_log(kvm, memslot);
456 r = kvm_get_dirty_log(kvm, log, &is_dirty);
457 if (r)
458 goto out;
459
460 /* Clear the dirty log */
461 if (is_dirty) {
462 n = kvm_dirty_bitmap_bytes(memslot);
463 memset(memslot->dirty_bitmap, 0, n);
464 }
465 r = 0;
466out:
467 mutex_unlock(&kvm->slots_lock);
468 return r;
b0c632db
HC		 469}
470
6502a34c
DH		 471static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
472{
473 unsigned int i;
474 struct kvm_vcpu *vcpu;
475
476 kvm_for_each_vcpu(i, vcpu, kvm) {
477 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
478 }
479}
480
d938dc55
CH		 481static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
482{
483 int r;
484
485 if (cap->flags)
486 return -EINVAL;
487
488 switch (cap->cap) {
84223598 489 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 490 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 491 kvm->arch.use_irqchip = 1;
492 r = 0;
493 break;
2444b352 494 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 495 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 496 kvm->arch.user_sigp = 1;
497 r = 0;
498 break;
68c55750 499 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 500 mutex_lock(&kvm->lock);
a03825bb 501 if (kvm->created_vcpus) {
5967c17b
DH		 502 r = -EBUSY;
503 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 504 set_kvm_facility(kvm->arch.model.fac_mask, 129);
505 set_kvm_facility(kvm->arch.model.fac_list, 129);
18280d8b
MM		 506 r = 0;
507 } else
508 r = -EINVAL;
5967c17b 509 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 510 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
511 r ? "(not available)" : "(success)");
68c55750 512 break;
c6e5f166
FZ		 513 case KVM_CAP_S390_RI:
514 r = -EINVAL;
515 mutex_lock(&kvm->lock);
a03825bb 516 if (kvm->created_vcpus) {
c6e5f166
FZ		 517 r = -EBUSY;
518 } else if (test_facility(64)) {
c54f0d6a
DH		 519 set_kvm_facility(kvm->arch.model.fac_mask, 64);
520 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 521 r = 0;
522 }
523 mutex_unlock(&kvm->lock);
524 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
525 r ? "(not available)" : "(success)");
526 break;
e44fc8c9 527 case KVM_CAP_S390_USER_STSI:
c92ea7b9 528 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 529 kvm->arch.user_stsi = 1;
530 r = 0;
531 break;
6502a34c
DH		 532 case KVM_CAP_S390_USER_INSTR0:
533 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
534 kvm->arch.user_instr0 = 1;
535 icpt_operexc_on_all_vcpus(kvm);
536 r = 0;
537 break;
d938dc55
CH		 538 default:
539 r = -EINVAL;
540 break;
541 }
542 return r;
543}
544
8c0a7ce6
DD		 545static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
546{
547 int ret;
548
549 switch (attr->attr) {
550 case KVM_S390_VM_MEM_LIMIT_SIZE:
551 ret = 0;
c92ea7b9 552 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 553 kvm->arch.mem_limit);
554 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 555 ret = -EFAULT;
556 break;
557 default:
558 ret = -ENXIO;
559 break;
560 }
561 return ret;
562}
563
564static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 565{
566 int ret;
567 unsigned int idx;
568 switch (attr->attr) {
569 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 570 ret = -ENXIO;
c24cc9c8 571 if (!sclp.has_cmma)
e6db1d61
DD		 572 break;
573
4f718eab 574 ret = -EBUSY;
c92ea7b9 575 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 576 mutex_lock(&kvm->lock);
a03825bb 577 if (!kvm->created_vcpus) {
4f718eab
DD		 578 kvm->arch.use_cmma = 1;
579 ret = 0;
580 }
581 mutex_unlock(&kvm->lock);
582 break;
583 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 584 ret = -ENXIO;
585 if (!sclp.has_cmma)
586 break;
c3489155
DD		 587 ret = -EINVAL;
588 if (!kvm->arch.use_cmma)
589 break;
590
c92ea7b9 591 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 592 mutex_lock(&kvm->lock);
593 idx = srcu_read_lock(&kvm->srcu);
a13cff31 594 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 595 srcu_read_unlock(&kvm->srcu, idx);
596 mutex_unlock(&kvm->lock);
597 ret = 0;
598 break;
8c0a7ce6
DD		 599 case KVM_S390_VM_MEM_LIMIT_SIZE: {
600 unsigned long new_limit;
601
602 if (kvm_is_ucontrol(kvm))
603 return -EINVAL;
604
605 if (get_user(new_limit, (u64 __user *)attr->addr))
606 return -EFAULT;
607
a3a92c31
DD		 608 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
609 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 610 return -E2BIG;
611
a3a92c31
DD		 612 if (!new_limit)
613 return -EINVAL;
614
6ea427bb 615 /* gmap_create takes last usable address */
a3a92c31
DD		 616 if (new_limit != KVM_S390_NO_MEM_LIMIT)
617 new_limit -= 1;
618
8c0a7ce6
DD		 619 ret = -EBUSY;
620 mutex_lock(&kvm->lock);
a03825bb 621 if (!kvm->created_vcpus) {
6ea427bb
MS		 622 /* gmap_create will round the limit up */
623 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 624
625 if (!new) {
626 ret = -ENOMEM;
627 } else {
6ea427bb 628 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 629 new->private = kvm;
630 kvm->arch.gmap = new;
631 ret = 0;
632 }
633 }
634 mutex_unlock(&kvm->lock);
a3a92c31
DD		 635 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
636 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
637 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 638 break;
639 }
4f718eab
DD		 640 default:
641 ret = -ENXIO;
642 break;
643 }
644 return ret;
645}
646
a374e892
TK		 647static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
648
649static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
650{
651 struct kvm_vcpu *vcpu;
652 int i;
653
9d8d5786 654 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 655 return -EINVAL;
656
657 mutex_lock(&kvm->lock);
658 switch (attr->attr) {
659 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
660 get_random_bytes(
661 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
662 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
663 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 664 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 665 break;
666 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
667 get_random_bytes(
668 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
669 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
670 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 671 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 672 break;
673 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
674 kvm->arch.crypto.aes_kw = 0;
675 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
676 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 677 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 678 break;
679 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
680 kvm->arch.crypto.dea_kw = 0;
681 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
682 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 683 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 684 break;
685 default:
686 mutex_unlock(&kvm->lock);
687 return -ENXIO;
688 }
689
690 kvm_for_each_vcpu(i, vcpu, kvm) {
691 kvm_s390_vcpu_crypto_setup(vcpu);
692 exit_sie(vcpu);
693 }
694 mutex_unlock(&kvm->lock);
695 return 0;
696}
697
72f25020
JH		 698static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
699{
700 u8 gtod_high;
701
702 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
703 sizeof(gtod_high)))
704 return -EFAULT;
705
706 if (gtod_high != 0)
707 return -EINVAL;
58c383c6 708 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 709
710 return 0;
711}
712
713static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
714{
5a3d883a 715 u64 gtod;
72f25020
JH		 716
717 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
718 return -EFAULT;
719
25ed1675 720 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 721 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 722 return 0;
723}
724
725static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
726{
727 int ret;
728
729 if (attr->flags)
730 return -EINVAL;
731
732 switch (attr->attr) {
733 case KVM_S390_VM_TOD_HIGH:
734 ret = kvm_s390_set_tod_high(kvm, attr);
735 break;
736 case KVM_S390_VM_TOD_LOW:
737 ret = kvm_s390_set_tod_low(kvm, attr);
738 break;
739 default:
740 ret = -ENXIO;
741 break;
742 }
743 return ret;
744}
745
746static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
747{
748 u8 gtod_high = 0;
749
750 if (copy_to_user((void __user *)attr->addr, &gtod_high,
751 sizeof(gtod_high)))
752 return -EFAULT;
58c383c6 753 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 754
755 return 0;
756}
757
758static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
759{
5a3d883a 760 u64 gtod;
72f25020 761
60417fcc 762 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 763 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
764 return -EFAULT;
58c383c6 765 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 766
767 return 0;
768}
769
770static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
771{
772 int ret;
773
774 if (attr->flags)
775 return -EINVAL;
776
777 switch (attr->attr) {
778 case KVM_S390_VM_TOD_HIGH:
779 ret = kvm_s390_get_tod_high(kvm, attr);
780 break;
781 case KVM_S390_VM_TOD_LOW:
782 ret = kvm_s390_get_tod_low(kvm, attr);
783 break;
784 default:
785 ret = -ENXIO;
786 break;
787 }
788 return ret;
789}
790
658b6eda
MM		 791static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
792{
793 struct kvm_s390_vm_cpu_processor *proc;
053dd230 794 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 795 int ret = 0;
796
797 mutex_lock(&kvm->lock);
a03825bb 798 if (kvm->created_vcpus) {
658b6eda
MM		 799 ret = -EBUSY;
800 goto out;
801 }
802 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
803 if (!proc) {
804 ret = -ENOMEM;
805 goto out;
806 }
807 if (!copy_from_user(proc, (void __user *)attr->addr,
808 sizeof(*proc))) {
9bb0ec09 809 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 810 lowest_ibc = sclp.ibc >> 16 & 0xfff;
811 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 812 if (lowest_ibc && proc->ibc) {
053dd230
DH		 813 if (proc->ibc > unblocked_ibc)
814 kvm->arch.model.ibc = unblocked_ibc;
815 else if (proc->ibc < lowest_ibc)
816 kvm->arch.model.ibc = lowest_ibc;
817 else
818 kvm->arch.model.ibc = proc->ibc;
819 }
c54f0d6a 820 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 821 S390_ARCH_FAC_LIST_SIZE_BYTE);
822 } else
823 ret = -EFAULT;
824 kfree(proc);
825out:
826 mutex_unlock(&kvm->lock);
827 return ret;
828}
829
15c9705f
DH		 830static int kvm_s390_set_processor_feat(struct kvm *kvm,
831 struct kvm_device_attr *attr)
832{
833 struct kvm_s390_vm_cpu_feat data;
834 int ret = -EBUSY;
835
836 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
837 return -EFAULT;
838 if (!bitmap_subset((unsigned long *) data.feat,
839 kvm_s390_available_cpu_feat,
840 KVM_S390_VM_CPU_FEAT_NR_BITS))
841 return -EINVAL;
842
843 mutex_lock(&kvm->lock);
844 if (!atomic_read(&kvm->online_vcpus)) {
845 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
846 KVM_S390_VM_CPU_FEAT_NR_BITS);
847 ret = 0;
848 }
849 mutex_unlock(&kvm->lock);
850 return ret;
851}
852
0a763c78
DH		 853static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
854 struct kvm_device_attr *attr)
855{
856 /*
857 * Once supported by kernel + hw, we have to store the subfunctions
858 * in kvm->arch and remember that user space configured them.
859 */
860 return -ENXIO;
861}
862
658b6eda
MM		 863static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
864{
865 int ret = -ENXIO;
866
867 switch (attr->attr) {
868 case KVM_S390_VM_CPU_PROCESSOR:
869 ret = kvm_s390_set_processor(kvm, attr);
870 break;
15c9705f
DH		 871 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
872 ret = kvm_s390_set_processor_feat(kvm, attr);
873 break;
0a763c78
DH		 874 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
875 ret = kvm_s390_set_processor_subfunc(kvm, attr);
876 break;
658b6eda
MM		 877 }
878 return ret;
879}
880
881static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
882{
883 struct kvm_s390_vm_cpu_processor *proc;
884 int ret = 0;
885
886 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
887 if (!proc) {
888 ret = -ENOMEM;
889 goto out;
890 }
9bb0ec09 891 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 892 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 893 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
894 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 895 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
896 ret = -EFAULT;
897 kfree(proc);
898out:
899 return ret;
900}
901
902static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
903{
904 struct kvm_s390_vm_cpu_machine *mach;
905 int ret = 0;
906
907 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
908 if (!mach) {
909 ret = -ENOMEM;
910 goto out;
911 }
912 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 913 mach->ibc = sclp.ibc;
c54f0d6a 914 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 915 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 916 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 917 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 918 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
919 ret = -EFAULT;
920 kfree(mach);
921out:
922 return ret;
923}
924
15c9705f
DH		 925static int kvm_s390_get_processor_feat(struct kvm *kvm,
926 struct kvm_device_attr *attr)
927{
928 struct kvm_s390_vm_cpu_feat data;
929
930 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
931 KVM_S390_VM_CPU_FEAT_NR_BITS);
932 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
933 return -EFAULT;
934 return 0;
935}
936
937static int kvm_s390_get_machine_feat(struct kvm *kvm,
938 struct kvm_device_attr *attr)
939{
940 struct kvm_s390_vm_cpu_feat data;
941
942 bitmap_copy((unsigned long *) data.feat,
943 kvm_s390_available_cpu_feat,
944 KVM_S390_VM_CPU_FEAT_NR_BITS);
945 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
946 return -EFAULT;
947 return 0;
948}
949
0a763c78
DH		 950static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
951 struct kvm_device_attr *attr)
952{
953 /*
954 * Once we can actually configure subfunctions (kernel + hw support),
955 * we have to check if they were already set by user space, if so copy
956 * them from kvm->arch.
957 */
958 return -ENXIO;
959}
960
961static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
962 struct kvm_device_attr *attr)
963{
964 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
965 sizeof(struct kvm_s390_vm_cpu_subfunc)))
966 return -EFAULT;
967 return 0;
968}
658b6eda
MM		 969static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
970{
971 int ret = -ENXIO;
972
973 switch (attr->attr) {
974 case KVM_S390_VM_CPU_PROCESSOR:
975 ret = kvm_s390_get_processor(kvm, attr);
976 break;
977 case KVM_S390_VM_CPU_MACHINE:
978 ret = kvm_s390_get_machine(kvm, attr);
979 break;
15c9705f
DH		 980 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
981 ret = kvm_s390_get_processor_feat(kvm, attr);
982 break;
983 case KVM_S390_VM_CPU_MACHINE_FEAT:
984 ret = kvm_s390_get_machine_feat(kvm, attr);
985 break;
0a763c78
DH		 986 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
987 ret = kvm_s390_get_processor_subfunc(kvm, attr);
988 break;
989 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
990 ret = kvm_s390_get_machine_subfunc(kvm, attr);
991 break;
658b6eda
MM		 992 }
993 return ret;
994}
995
f2061656
DD		 996static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
997{
998 int ret;
999
1000 switch (attr->group) {
4f718eab 1001 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1002 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1003 break;
72f25020
JH		 1004 case KVM_S390_VM_TOD:
1005 ret = kvm_s390_set_tod(kvm, attr);
1006 break;
658b6eda
MM		 1007 case KVM_S390_VM_CPU_MODEL:
1008 ret = kvm_s390_set_cpu_model(kvm, attr);
1009 break;
a374e892
TK		 1010 case KVM_S390_VM_CRYPTO:
1011 ret = kvm_s390_vm_set_crypto(kvm, attr);
1012 break;
f2061656
DD		 1013 default:
1014 ret = -ENXIO;
1015 break;
1016 }
1017
1018 return ret;
1019}
1020
1021static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1022{
8c0a7ce6
DD		 1023 int ret;
1024
1025 switch (attr->group) {
1026 case KVM_S390_VM_MEM_CTRL:
1027 ret = kvm_s390_get_mem_control(kvm, attr);
1028 break;
72f25020
JH		 1029 case KVM_S390_VM_TOD:
1030 ret = kvm_s390_get_tod(kvm, attr);
1031 break;
658b6eda
MM		 1032 case KVM_S390_VM_CPU_MODEL:
1033 ret = kvm_s390_get_cpu_model(kvm, attr);
1034 break;
8c0a7ce6
DD		 1035 default:
1036 ret = -ENXIO;
1037 break;
1038 }
1039
1040 return ret;
f2061656
DD		 1041}
1042
1043static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1044{
1045 int ret;
1046
1047 switch (attr->group) {
4f718eab
DD		 1048 case KVM_S390_VM_MEM_CTRL:
1049 switch (attr->attr) {
1050 case KVM_S390_VM_MEM_ENABLE_CMMA:
1051 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1052 ret = sclp.has_cmma ? 0 : -ENXIO;
1053 break;
8c0a7ce6 1054 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1055 ret = 0;
1056 break;
1057 default:
1058 ret = -ENXIO;
1059 break;
1060 }
1061 break;
72f25020
JH		 1062 case KVM_S390_VM_TOD:
1063 switch (attr->attr) {
1064 case KVM_S390_VM_TOD_LOW:
1065 case KVM_S390_VM_TOD_HIGH:
1066 ret = 0;
1067 break;
1068 default:
1069 ret = -ENXIO;
1070 break;
1071 }
1072 break;
658b6eda
MM		 1073 case KVM_S390_VM_CPU_MODEL:
1074 switch (attr->attr) {
1075 case KVM_S390_VM_CPU_PROCESSOR:
1076 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1077 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1078 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1079 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1080 ret = 0;
1081 break;
0a763c78
DH		 1082 /* configuring subfunctions is not supported yet */
1083 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1084 default:
1085 ret = -ENXIO;
1086 break;
1087 }
1088 break;
a374e892
TK		 1089 case KVM_S390_VM_CRYPTO:
1090 switch (attr->attr) {
1091 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1092 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1093 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1094 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1095 ret = 0;
1096 break;
1097 default:
1098 ret = -ENXIO;
1099 break;
1100 }
1101 break;
f2061656
DD		 1102 default:
1103 ret = -ENXIO;
1104 break;
1105 }
1106
1107 return ret;
1108}
1109
30ee2a98
JH		 1110static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1111{
1112 uint8_t *keys;
1113 uint64_t hva;
30ee2a98
JH		 1114 int i, r = 0;
1115
1116 if (args->flags != 0)
1117 return -EINVAL;
1118
1119 /* Is this guest using storage keys? */
1120 if (!mm_use_skey(current->mm))
1121 return KVM_S390_GET_SKEYS_NONE;
1122
1123 /* Enforce sane limit on memory allocation */
1124 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1125 return -EINVAL;
1126
1127 keys = kmalloc_array(args->count, sizeof(uint8_t),
1128 GFP_KERNEL | __GFP_NOWARN);
1129 if (!keys)
1130 keys = vmalloc(sizeof(uint8_t) * args->count);
1131 if (!keys)
1132 return -ENOMEM;
1133
d3ed1cee 1134 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1135 for (i = 0; i < args->count; i++) {
1136 hva = gfn_to_hva(kvm, args->start_gfn + i);
1137 if (kvm_is_error_hva(hva)) {
1138 r = -EFAULT;
d3ed1cee 1139 break;
30ee2a98
JH		 1140 }
1141
154c8c19
DH		 1142 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1143 if (r)
d3ed1cee 1144 break;
30ee2a98 1145 }
d3ed1cee
MS		 1146 up_read(&current->mm->mmap_sem);
1147
1148 if (!r) {
1149 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1150 sizeof(uint8_t) * args->count);
1151 if (r)
1152 r = -EFAULT;
30ee2a98
JH		 1153 }
1154
30ee2a98
JH		 1155 kvfree(keys);
1156 return r;
1157}
1158
1159static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1160{
1161 uint8_t *keys;
1162 uint64_t hva;
1163 int i, r = 0;
1164
1165 if (args->flags != 0)
1166 return -EINVAL;
1167
1168 /* Enforce sane limit on memory allocation */
1169 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1170 return -EINVAL;
1171
1172 keys = kmalloc_array(args->count, sizeof(uint8_t),
1173 GFP_KERNEL | __GFP_NOWARN);
1174 if (!keys)
1175 keys = vmalloc(sizeof(uint8_t) * args->count);
1176 if (!keys)
1177 return -ENOMEM;
1178
1179 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1180 sizeof(uint8_t) * args->count);
1181 if (r) {
1182 r = -EFAULT;
1183 goto out;
1184 }
1185
1186 /* Enable storage key handling for the guest */
14d4a425
DD		 1187 r = s390_enable_skey();
1188 if (r)
1189 goto out;
30ee2a98 1190
d3ed1cee 1191 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1192 for (i = 0; i < args->count; i++) {
1193 hva = gfn_to_hva(kvm, args->start_gfn + i);
1194 if (kvm_is_error_hva(hva)) {
1195 r = -EFAULT;
d3ed1cee 1196 break;
30ee2a98
JH		 1197 }
1198
1199 /* Lowest order bit is reserved */
1200 if (keys[i] & 0x01) {
1201 r = -EINVAL;
d3ed1cee 1202 break;
30ee2a98
JH		 1203 }
1204
fe69eabf 1205 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1206 if (r)
d3ed1cee 1207 break;
30ee2a98 1208 }
d3ed1cee 1209 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1210out:
1211 kvfree(keys);
1212 return r;
1213}
1214
b0c632db
HC		 1215long kvm_arch_vm_ioctl(struct file *filp,
1216 unsigned int ioctl, unsigned long arg)
1217{
1218 struct kvm *kvm = filp->private_data;
1219 void __user *argp = (void __user *)arg;
f2061656 1220 struct kvm_device_attr attr;
b0c632db
HC		 1221 int r;
1222
1223 switch (ioctl) {
ba5c1e9b
CO		 1224 case KVM_S390_INTERRUPT: {
1225 struct kvm_s390_interrupt s390int;
1226
1227 r = -EFAULT;
1228 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1229 break;
1230 r = kvm_s390_inject_vm(kvm, &s390int);
1231 break;
1232 }
d938dc55
CH		 1233 case KVM_ENABLE_CAP: {
1234 struct kvm_enable_cap cap;
1235 r = -EFAULT;
1236 if (copy_from_user(&cap, argp, sizeof(cap)))
1237 break;
1238 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1239 break;
1240 }
84223598
CH		 1241 case KVM_CREATE_IRQCHIP: {
1242 struct kvm_irq_routing_entry routing;
1243
1244 r = -EINVAL;
1245 if (kvm->arch.use_irqchip) {
1246 /* Set up dummy routing. */
1247 memset(&routing, 0, sizeof(routing));
152b2839 1248 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1249 }
1250 break;
1251 }
f2061656
DD		 1252 case KVM_SET_DEVICE_ATTR: {
1253 r = -EFAULT;
1254 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1255 break;
1256 r = kvm_s390_vm_set_attr(kvm, &attr);
1257 break;
1258 }
1259 case KVM_GET_DEVICE_ATTR: {
1260 r = -EFAULT;
1261 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1262 break;
1263 r = kvm_s390_vm_get_attr(kvm, &attr);
1264 break;
1265 }
1266 case KVM_HAS_DEVICE_ATTR: {
1267 r = -EFAULT;
1268 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1269 break;
1270 r = kvm_s390_vm_has_attr(kvm, &attr);
1271 break;
1272 }
30ee2a98
JH		 1273 case KVM_S390_GET_SKEYS: {
1274 struct kvm_s390_skeys args;
1275
1276 r = -EFAULT;
1277 if (copy_from_user(&args, argp,
1278 sizeof(struct kvm_s390_skeys)))
1279 break;
1280 r = kvm_s390_get_skeys(kvm, &args);
1281 break;
1282 }
1283 case KVM_S390_SET_SKEYS: {
1284 struct kvm_s390_skeys args;
1285
1286 r = -EFAULT;
1287 if (copy_from_user(&args, argp,
1288 sizeof(struct kvm_s390_skeys)))
1289 break;
1290 r = kvm_s390_set_skeys(kvm, &args);
1291 break;
1292 }
b0c632db 1293 default:
367e1319 1294 r = -ENOTTY;
b0c632db
HC		 1295 }
1296
1297 return r;
1298}
1299
45c9b47c
TK		 1300static int kvm_s390_query_ap_config(u8 *config)
1301{
1302 u32 fcn_code = 0x04000000UL;
86044c8c 1303 u32 cc = 0;
45c9b47c 1304
86044c8c 1305 memset(config, 0, 128);
45c9b47c
TK		 1306 asm volatile(
1307 "lgr 0,%1\n"
1308 "lgr 2,%2\n"
1309 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1310 "0: ipm %0\n"
45c9b47c 1311 "srl %0,28\n"
86044c8c
CB		 1312 "1:\n"
1313 EX_TABLE(0b, 1b)
1314 : "+r" (cc)
45c9b47c
TK		 1315 : "r" (fcn_code), "r" (config)
1316 : "cc", "0", "2", "memory"
1317 );
1318
1319 return cc;
1320}
1321
1322static int kvm_s390_apxa_installed(void)
1323{
1324 u8 config[128];
1325 int cc;
1326
a6aacc3f 1327 if (test_facility(12)) {
45c9b47c
TK		 1328 cc = kvm_s390_query_ap_config(config);
1329
1330 if (cc)
1331 pr_err("PQAP(QCI) failed with cc=%d", cc);
1332 else
1333 return config[0] & 0x40;
1334 }
1335
1336 return 0;
1337}
1338
1339static void kvm_s390_set_crycb_format(struct kvm *kvm)
1340{
1341 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1342
1343 if (kvm_s390_apxa_installed())
1344 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1345 else
1346 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1347}
1348
9bb0ec09 1349static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1350{
9bb0ec09
DH		 1351 struct cpuid cpuid;
1352
1353 get_cpu_id(&cpuid);
1354 cpuid.version = 0xff;
1355 return *((u64 *) &cpuid);
9d8d5786
MM		 1356}
1357
c54f0d6a 1358static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1359{
9d8d5786 1360 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1361 return;
5102ee87 1362
c54f0d6a 1363 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1364 kvm_s390_set_crycb_format(kvm);
5102ee87 1365
ed6f76b4
TK		 1366 /* Enable AES/DEA protected key functions by default */
1367 kvm->arch.crypto.aes_kw = 1;
1368 kvm->arch.crypto.dea_kw = 1;
1369 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1370 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1371 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1372 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1373}
1374
7d43bafc
ED		 1375static void sca_dispose(struct kvm *kvm)
1376{
1377 if (kvm->arch.use_esca)
5e044315 1378 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1379 else
1380 free_page((unsigned long)(kvm->arch.sca));
1381 kvm->arch.sca = NULL;
1382}
1383
e08b9637 1384int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1385{
76a6dd72 1386 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1387 int i, rc;
b0c632db 1388 char debug_name[16];
f6c137ff 1389 static unsigned long sca_offset;
b0c632db 1390
e08b9637
CO		 1391 rc = -EINVAL;
1392#ifdef CONFIG_KVM_S390_UCONTROL
1393 if (type & ~KVM_VM_S390_UCONTROL)
1394 goto out_err;
1395 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1396 goto out_err;
1397#else
1398 if (type)
1399 goto out_err;
1400#endif
1401
b0c632db
HC		 1402 rc = s390_enable_sie();
1403 if (rc)
d89f5eff 1404 goto out_err;
b0c632db 1405
b290411a
CO		 1406 rc = -ENOMEM;
1407
7d0a5e62
JF		 1408 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1409
7d43bafc 1410 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1411 if (!sclp.has_64bscao)
1412 alloc_flags |= GFP_DMA;
5e044315 1413 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1414 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1415 if (!kvm->arch.sca)
d89f5eff 1416 goto out_err;
f6c137ff 1417 spin_lock(&kvm_lock);
c5c2c393 1418 sca_offset += 16;
bc784cce 1419 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1420 sca_offset = 0;
bc784cce
ED		 1421 kvm->arch.sca = (struct bsca_block *)
1422 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1423 spin_unlock(&kvm_lock);
b0c632db
HC		 1424
1425 sprintf(debug_name, "kvm-%u", current->pid);
1426
1cb9cf72 1427 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1428 if (!kvm->arch.dbf)
40f5b735 1429 goto out_err;
b0c632db 1430
c54f0d6a
DH		 1431 kvm->arch.sie_page2 =
1432 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1433 if (!kvm->arch.sie_page2)
40f5b735 1434 goto out_err;
9d8d5786 1435
fb5bf93f 1436 /* Populate the facility mask initially. */
c54f0d6a 1437 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
94422ee8 1438 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1439 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1440 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1441 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1442 else
c54f0d6a 1443 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1444 }
1445
981467c9 1446 /* Populate the facility list initially. */
c54f0d6a
DH		 1447 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1448 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1449 S390_ARCH_FAC_LIST_SIZE_BYTE);
1450
95ca2cb5
JF		 1451 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1452 set_kvm_facility(kvm->arch.model.fac_list, 74);
1453
9bb0ec09 1454 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1455 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1456
c54f0d6a 1457 kvm_s390_crypto_init(kvm);
5102ee87 1458
ba5c1e9b 1459 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1460 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1461 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1462 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1463 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1464
b0c632db 1465 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1466 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1467
e08b9637
CO		 1468 if (type & KVM_VM_S390_UCONTROL) {
1469 kvm->arch.gmap = NULL;
a3a92c31 1470 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1471 } else {
32e6b236
GH		 1472 if (sclp.hamax == U64_MAX)
1473 kvm->arch.mem_limit = TASK_MAX_SIZE;
1474 else
1475 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1476 sclp.hamax + 1);
6ea427bb 1477 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1478 if (!kvm->arch.gmap)
40f5b735 1479 goto out_err;
2c70fe44 1480 kvm->arch.gmap->private = kvm;
24eb3a82 1481 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1482 }
fa6b7fe9
CH		 1483
1484 kvm->arch.css_support = 0;
84223598 1485 kvm->arch.use_irqchip = 0;
72f25020 1486 kvm->arch.epoch = 0;
fa6b7fe9 1487
8ad35755 1488 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 1489 kvm_s390_vsie_init(kvm);
8335713a 1490 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1491
d89f5eff 1492 return 0;
40f5b735 1493out_err:
c54f0d6a 1494 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1495 debug_unregister(kvm->arch.dbf);
7d43bafc 1496 sca_dispose(kvm);
78f26131 1497 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1498 return rc;
b0c632db
HC		 1499}
1500
d329c035
CB		 1501void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1502{
1503 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1504 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1505 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1506 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1507 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1508 sca_del_vcpu(vcpu);
27e0393f
CO		 1509
1510 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 1511 gmap_remove(vcpu->arch.gmap);
27e0393f 1512
e6db1d61 1513 if (vcpu->kvm->arch.use_cmma)
b31605c1 1514 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1515 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1516
6692cef3 1517 kvm_vcpu_uninit(vcpu);
b110feaf 1518 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1519}
1520
1521static void kvm_free_vcpus(struct kvm *kvm)
1522{
1523 unsigned int i;
988a2cae 1524 struct kvm_vcpu *vcpu;
d329c035 1525
988a2cae
GN		 1526 kvm_for_each_vcpu(i, vcpu, kvm)
1527 kvm_arch_vcpu_destroy(vcpu);
1528
1529 mutex_lock(&kvm->lock);
1530 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1531 kvm->vcpus[i] = NULL;
1532
1533 atomic_set(&kvm->online_vcpus, 0);
1534 mutex_unlock(&kvm->lock);
d329c035
CB		 1535}
1536
b0c632db
HC		 1537void kvm_arch_destroy_vm(struct kvm *kvm)
1538{
d329c035 1539 kvm_free_vcpus(kvm);
7d43bafc 1540 sca_dispose(kvm);
d329c035 1541 debug_unregister(kvm->arch.dbf);
c54f0d6a 1542 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 1543 if (!kvm_is_ucontrol(kvm))
6ea427bb 1544 gmap_remove(kvm->arch.gmap);
841b91c5 1545 kvm_s390_destroy_adapters(kvm);
67335e63 1546 kvm_s390_clear_float_irqs(kvm);
a3508fbe 1547 kvm_s390_vsie_destroy(kvm);
8335713a 1548 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1549}
1550
1551/* Section: vcpu related */
dafd032a
DD		 1552static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1553{
6ea427bb 1554 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 1555 if (!vcpu->arch.gmap)
1556 return -ENOMEM;
1557 vcpu->arch.gmap->private = vcpu->kvm;
1558
1559 return 0;
1560}
1561
a6e2f683
ED		 1562static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1563{
5e044315 1564 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1565 if (vcpu->kvm->arch.use_esca) {
1566 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1567
7d43bafc 1568 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1569 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1570 } else {
1571 struct bsca_block *sca = vcpu->kvm->arch.sca;
1572
1573 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1574 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1575 }
5e044315 1576 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1577}
1578
eaa78f34 1579static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1580{
eaa78f34
DH		 1581 read_lock(&vcpu->kvm->arch.sca_lock);
1582 if (vcpu->kvm->arch.use_esca) {
1583 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1584
eaa78f34 1585 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1586 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1587 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1588 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1589 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1590 } else {
eaa78f34 1591 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1592
eaa78f34 1593 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1594 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1595 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1596 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1597 }
eaa78f34 1598 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1599}
1600
1601/* Basic SCA to Extended SCA data copy routines */
1602static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1603{
1604 d->sda = s->sda;
1605 d->sigp_ctrl.c = s->sigp_ctrl.c;
1606 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1607}
1608
1609static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1610{
1611 int i;
1612
1613 d->ipte_control = s->ipte_control;
1614 d->mcn[0] = s->mcn;
1615 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1616 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1617}
1618
1619static int sca_switch_to_extended(struct kvm *kvm)
1620{
1621 struct bsca_block *old_sca = kvm->arch.sca;
1622 struct esca_block *new_sca;
1623 struct kvm_vcpu *vcpu;
1624 unsigned int vcpu_idx;
1625 u32 scaol, scaoh;
1626
1627 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1628 if (!new_sca)
1629 return -ENOMEM;
1630
1631 scaoh = (u32)((u64)(new_sca) >> 32);
1632 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1633
1634 kvm_s390_vcpu_block_all(kvm);
1635 write_lock(&kvm->arch.sca_lock);
1636
1637 sca_copy_b_to_e(new_sca, old_sca);
1638
1639 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1640 vcpu->arch.sie_block->scaoh = scaoh;
1641 vcpu->arch.sie_block->scaol = scaol;
1642 vcpu->arch.sie_block->ecb2 |= 0x04U;
1643 }
1644 kvm->arch.sca = new_sca;
1645 kvm->arch.use_esca = 1;
1646
1647 write_unlock(&kvm->arch.sca_lock);
1648 kvm_s390_vcpu_unblock_all(kvm);
1649
1650 free_page((unsigned long)old_sca);
1651
8335713a
CB		 1652 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1653 old_sca, kvm->arch.sca);
5e044315 1654 return 0;
a6e2f683
ED		 1655}
1656
1657static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1658{
5e044315
ED		 1659 int rc;
1660
1661 if (id < KVM_S390_BSCA_CPU_SLOTS)
1662 return true;
76a6dd72 1663 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1664 return false;
1665
1666 mutex_lock(&kvm->lock);
1667 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1668 mutex_unlock(&kvm->lock);
1669
1670 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1671}
1672
b0c632db
HC		 1673int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1674{
3c038e6b
DD		 1675 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1676 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1677 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1678 KVM_SYNC_GPRS |
9eed0735 1679 KVM_SYNC_ACRS |
b028ee3e
DH		 1680 KVM_SYNC_CRS |
1681 KVM_SYNC_ARCH0 |
1682 KVM_SYNC_PFAULT;
75a4615c 1683 kvm_s390_set_prefix(vcpu, 0);
c6e5f166
FZ		 1684 if (test_kvm_facility(vcpu->kvm, 64))
1685 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1686 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1687 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1688 */
1689 if (MACHINE_HAS_VX)
68c55750 1690 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1691 else
1692 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1693
1694 if (kvm_is_ucontrol(vcpu->kvm))
1695 return __kvm_ucontrol_vcpu_init(vcpu);
1696
b0c632db
HC		 1697 return 0;
1698}
1699
db0758b2
DH		 1700/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1701static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1702{
1703 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1704 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1705 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1706 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1707}
1708
1709/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1710static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1711{
1712 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1713 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1714 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1715 vcpu->arch.cputm_start = 0;
9c23a131 1716 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1717}
1718
1719/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1720static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1721{
1722 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1723 vcpu->arch.cputm_enabled = true;
1724 __start_cpu_timer_accounting(vcpu);
1725}
1726
1727/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1728static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1729{
1730 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1731 __stop_cpu_timer_accounting(vcpu);
1732 vcpu->arch.cputm_enabled = false;
1733}
1734
1735static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1736{
1737 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1738 __enable_cpu_timer_accounting(vcpu);
1739 preempt_enable();
1740}
1741
1742static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1743{
1744 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1745 __disable_cpu_timer_accounting(vcpu);
1746 preempt_enable();
1747}
1748
4287f247
DH		 1749/* set the cpu timer - may only be called from the VCPU thread itself */
1750void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1751{
db0758b2 1752 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1753 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1754 if (vcpu->arch.cputm_enabled)
1755 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1756 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1757 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1758 preempt_enable();
4287f247
DH		 1759}
1760
db0758b2 1761/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1762__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1763{
9c23a131 1764 unsigned int seq;
db0758b2 1765 __u64 value;
db0758b2
DH		 1766
1767 if (unlikely(!vcpu->arch.cputm_enabled))
1768 return vcpu->arch.sie_block->cputm;
1769
9c23a131
DH		 1770 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1771 do {
1772 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1773 /*
1774 * If the writer would ever execute a read in the critical
1775 * section, e.g. in irq context, we have a deadlock.
1776 */
1777 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1778 value = vcpu->arch.sie_block->cputm;
1779 /* if cputm_start is 0, accounting is being started/stopped */
1780 if (likely(vcpu->arch.cputm_start))
1781 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1782 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1783 preempt_enable();
db0758b2 1784 return value;
4287f247
DH		 1785}
1786
b0c632db
HC		 1787void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1788{
9977e886 1789 /* Save host register state */
d0164ee2 1790 save_fpu_regs();
9abc2a08
DH		 1791 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1792 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1793
6fd8e67d
DH		 1794 if (MACHINE_HAS_VX)
1795 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1796 else
1797 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1798 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1799 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1800 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1801 current->thread.fpu.fpc = 0;
1802
1803 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1804 restore_access_regs(vcpu->run->s.regs.acrs);
37d9df98 1805 gmap_enable(vcpu->arch.enabled_gmap);
805de8f4 1806 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1807 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1808 __start_cpu_timer_accounting(vcpu);
01a745ac 1809 vcpu->cpu = cpu;
b0c632db
HC		 1810}
1811
1812void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1813{
01a745ac 1814 vcpu->cpu = -1;
5ebda316 1815 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1816 __stop_cpu_timer_accounting(vcpu);
805de8f4 1817 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
37d9df98
DH		 1818 vcpu->arch.enabled_gmap = gmap_get_enabled();
1819 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 1820
9abc2a08 1821 /* Save guest register state */
d0164ee2 1822 save_fpu_regs();
9abc2a08 1823 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1824
9abc2a08
DH		 1825 /* Restore host register state */
1826 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1827 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1828
1829 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1830 restore_access_regs(vcpu->arch.host_acrs);
1831}
1832
1833static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1834{
1835 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1836 vcpu->arch.sie_block->gpsw.mask = 0UL;
1837 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1838 kvm_s390_set_prefix(vcpu, 0);
4287f247 1839 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1840 vcpu->arch.sie_block->ckc = 0UL;
1841 vcpu->arch.sie_block->todpr = 0;
1842 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1843 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1844 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1845 /* make sure the new fpc will be lazily loaded */
1846 save_fpu_regs();
1847 current->thread.fpu.fpc = 0;
b0c632db 1848 vcpu->arch.sie_block->gbea = 1;
672550fb 1849 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1850 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1851 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1852 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1853 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1854 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1855}
1856
31928aa5 1857void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1858{
72f25020 1859 mutex_lock(&vcpu->kvm->lock);
fdf03650 1860 preempt_disable();
72f25020 1861 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1862 preempt_enable();
72f25020 1863 mutex_unlock(&vcpu->kvm->lock);
25508824 1864 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1865 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1866 sca_add_vcpu(vcpu);
25508824 1867 }
6502a34c
DH		 1868 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1869 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 1870 /* make vcpu_load load the right gmap on the first trigger */
1871 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 1872}
1873
5102ee87
TK		 1874static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1875{
9d8d5786 1876 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1877 return;
1878
a374e892
TK		 1879 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1880
1881 if (vcpu->kvm->arch.crypto.aes_kw)
1882 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1883 if (vcpu->kvm->arch.crypto.dea_kw)
1884 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1885
5102ee87
TK		 1886 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1887}
1888
b31605c1
DD		 1889void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1890{
1891 free_page(vcpu->arch.sie_block->cbrlo);
1892 vcpu->arch.sie_block->cbrlo = 0;
1893}
1894
1895int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1896{
1897 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1898 if (!vcpu->arch.sie_block->cbrlo)
1899 return -ENOMEM;
1900
1901 vcpu->arch.sie_block->ecb2 |= 0x80;
1902 vcpu->arch.sie_block->ecb2 &= ~0x08;
1903 return 0;
1904}
1905
91520f1a
MM		 1906static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1907{
1908 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1909
91520f1a 1910 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1911 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1912 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1913}
1914
b0c632db
HC		 1915int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1916{
b31605c1 1917 int rc = 0;
b31288fa 1918
9e6dabef
CH		 1919 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1920 CPUSTAT_SM |
a4a4f191
GH		 1921 CPUSTAT_STOPPED);
1922
53df84f8 1923 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1924 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1925 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1926 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1927
91520f1a
MM		 1928 kvm_s390_vcpu_setup_model(vcpu);
1929
bdab09f3
DH		 1930 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1931 if (MACHINE_HAS_ESOP)
1932 vcpu->arch.sie_block->ecb |= 0x02;
bd50e8ec
DH		 1933 if (test_kvm_facility(vcpu->kvm, 9))
1934 vcpu->arch.sie_block->ecb |= 0x04;
f597d24e 1935 if (test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1936 vcpu->arch.sie_block->ecb |= 0x10;
1937
873b425e 1938 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
d6af0b49 1939 vcpu->arch.sie_block->ecb2 |= 0x08;
48ee7d3a
DH		 1940 vcpu->arch.sie_block->eca = 0x1002000U;
1941 if (sclp.has_cei)
1942 vcpu->arch.sie_block->eca |= 0x80000000U;
11ad65b7
DH		 1943 if (sclp.has_ib)
1944 vcpu->arch.sie_block->eca |= 0x40000000U;
37c5f6c8 1945 if (sclp.has_siif)
217a4406 1946 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1947 if (sclp.has_sigpif)
ea5f4969 1948 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1949 if (test_kvm_facility(vcpu->kvm, 64))
1950 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1951 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1952 vcpu->arch.sie_block->eca |= 0x00020000;
1953 vcpu->arch.sie_block->ecd |= 0x20000000;
1954 }
c6e5f166 1955 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1956 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1957
e6db1d61 1958 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1959 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1960 if (rc)
1961 return rc;
b31288fa 1962 }
0ac96caf 1963 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1964 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1965
5102ee87
TK		 1966 kvm_s390_vcpu_crypto_setup(vcpu);
1967
b31605c1 1968 return rc;
b0c632db
HC		 1969}
1970
1971struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1972 unsigned int id)
1973{
4d47555a 1974 struct kvm_vcpu *vcpu;
7feb6bb8 1975 struct sie_page *sie_page;
4d47555a
CO		 1976 int rc = -EINVAL;
1977
4215825e 1978 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1979 goto out;
1980
1981 rc = -ENOMEM;
b0c632db 1982
b110feaf 1983 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1984 if (!vcpu)
4d47555a 1985 goto out;
b0c632db 1986
7feb6bb8
MM		 1987 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1988 if (!sie_page)
b0c632db
HC		 1989 goto out_free_cpu;
1990
7feb6bb8
MM		 1991 vcpu->arch.sie_block = &sie_page->sie_block;
1992 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1993
efed1104
DH		 1994 /* the real guest size will always be smaller than msl */
1995 vcpu->arch.sie_block->mso = 0;
1996 vcpu->arch.sie_block->msl = sclp.hamax;
1997
b0c632db 1998 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1999 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 2000 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 2001 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 2002 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 2003 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 2004
b0c632db
HC		 2005 rc = kvm_vcpu_init(vcpu, kvm, id);
2006 if (rc)
9abc2a08 2007 goto out_free_sie_block;
8335713a 2008 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 2009 vcpu->arch.sie_block);
ade38c31 2010 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 2011
b0c632db 2012 return vcpu;
7b06bf2f
WY		 2013out_free_sie_block:
2014 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 2015out_free_cpu:
b110feaf 2016 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 2017out:
b0c632db
HC		 2018 return ERR_PTR(rc);
2019}
2020
b0c632db
HC		 2021int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2022{
9a022067 2023 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 2024}
2025
27406cd5 2026void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 2027{
805de8f4 2028 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 2029 exit_sie(vcpu);
49b99e1e
CB		 2030}
2031
27406cd5 2032void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 2033{
805de8f4 2034 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 2035}
2036
8e236546
CB		 2037static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2038{
805de8f4 2039 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 2040 exit_sie(vcpu);
8e236546
CB		 2041}
2042
2043static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2044{
9bf9fde2 2045 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 2046}
2047
49b99e1e
CB		 2048/*
2049 * Kick a guest cpu out of SIE and wait until SIE is not running.
2050 * If the CPU is not running (e.g. waiting as idle) the function will
2051 * return immediately. */
2052void exit_sie(struct kvm_vcpu *vcpu)
2053{
805de8f4 2054 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 2055 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2056 cpu_relax();
2057}
2058
8e236546
CB		 2059/* Kick a guest cpu out of SIE to process a request synchronously */
2060void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 2061{
8e236546
CB		 2062 kvm_make_request(req, vcpu);
2063 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 2064}
2065
414d3b07
MS		 2066static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2067 unsigned long end)
2c70fe44 2068{
2c70fe44
CB		 2069 struct kvm *kvm = gmap->private;
2070 struct kvm_vcpu *vcpu;
414d3b07
MS		 2071 unsigned long prefix;
2072 int i;
2c70fe44 2073
65d0b0d4
DH		 2074 if (gmap_is_shadow(gmap))
2075 return;
414d3b07
MS		 2076 if (start >= 1UL << 31)
2077 /* We are only interested in prefix pages */
2078 return;
2c70fe44
CB		 2079 kvm_for_each_vcpu(i, vcpu, kvm) {
2080 /* match against both prefix pages */
414d3b07
MS		 2081 prefix = kvm_s390_get_prefix(vcpu);
2082 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2083 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2084 start, end);
8e236546 2085 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 2086 }
2087 }
2088}
2089
b6d33834
CD		 2090int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2091{
2092 /* kvm common code refers to this, but never calls it */
2093 BUG();
2094 return 0;
2095}
2096
14eebd91
CO		 2097static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2098 struct kvm_one_reg *reg)
2099{
2100 int r = -EINVAL;
2101
2102 switch (reg->id) {
29b7c71b
CO		 2103 case KVM_REG_S390_TODPR:
2104 r = put_user(vcpu->arch.sie_block->todpr,
2105 (u32 __user *)reg->addr);
2106 break;
2107 case KVM_REG_S390_EPOCHDIFF:
2108 r = put_user(vcpu->arch.sie_block->epoch,
2109 (u64 __user *)reg->addr);
2110 break;
46a6dd1c 2111 case KVM_REG_S390_CPU_TIMER:
4287f247 2112 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2113 (u64 __user *)reg->addr);
2114 break;
2115 case KVM_REG_S390_CLOCK_COMP:
2116 r = put_user(vcpu->arch.sie_block->ckc,
2117 (u64 __user *)reg->addr);
2118 break;
536336c2
DD		 2119 case KVM_REG_S390_PFTOKEN:
2120 r = put_user(vcpu->arch.pfault_token,
2121 (u64 __user *)reg->addr);
2122 break;
2123 case KVM_REG_S390_PFCOMPARE:
2124 r = put_user(vcpu->arch.pfault_compare,
2125 (u64 __user *)reg->addr);
2126 break;
2127 case KVM_REG_S390_PFSELECT:
2128 r = put_user(vcpu->arch.pfault_select,
2129 (u64 __user *)reg->addr);
2130 break;
672550fb
CB		 2131 case KVM_REG_S390_PP:
2132 r = put_user(vcpu->arch.sie_block->pp,
2133 (u64 __user *)reg->addr);
2134 break;
afa45ff5
CB		 2135 case KVM_REG_S390_GBEA:
2136 r = put_user(vcpu->arch.sie_block->gbea,
2137 (u64 __user *)reg->addr);
2138 break;
14eebd91
CO		 2139 default:
2140 break;
2141 }
2142
2143 return r;
2144}
2145
2146static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2147 struct kvm_one_reg *reg)
2148{
2149 int r = -EINVAL;
4287f247 2150 __u64 val;
14eebd91
CO		 2151
2152 switch (reg->id) {
29b7c71b
CO		 2153 case KVM_REG_S390_TODPR:
2154 r = get_user(vcpu->arch.sie_block->todpr,
2155 (u32 __user *)reg->addr);
2156 break;
2157 case KVM_REG_S390_EPOCHDIFF:
2158 r = get_user(vcpu->arch.sie_block->epoch,
2159 (u64 __user *)reg->addr);
2160 break;
46a6dd1c 2161 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2162 r = get_user(val, (u64 __user *)reg->addr);
2163 if (!r)
2164 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2165 break;
2166 case KVM_REG_S390_CLOCK_COMP:
2167 r = get_user(vcpu->arch.sie_block->ckc,
2168 (u64 __user *)reg->addr);
2169 break;
536336c2
DD		 2170 case KVM_REG_S390_PFTOKEN:
2171 r = get_user(vcpu->arch.pfault_token,
2172 (u64 __user *)reg->addr);
9fbd8082
DH		 2173 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2174 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2175 break;
2176 case KVM_REG_S390_PFCOMPARE:
2177 r = get_user(vcpu->arch.pfault_compare,
2178 (u64 __user *)reg->addr);
2179 break;
2180 case KVM_REG_S390_PFSELECT:
2181 r = get_user(vcpu->arch.pfault_select,
2182 (u64 __user *)reg->addr);
2183 break;
672550fb
CB		 2184 case KVM_REG_S390_PP:
2185 r = get_user(vcpu->arch.sie_block->pp,
2186 (u64 __user *)reg->addr);
2187 break;
afa45ff5
CB		 2188 case KVM_REG_S390_GBEA:
2189 r = get_user(vcpu->arch.sie_block->gbea,
2190 (u64 __user *)reg->addr);
2191 break;
14eebd91
CO		 2192 default:
2193 break;
2194 }
2195
2196 return r;
2197}
b6d33834 2198
b0c632db
HC		 2199static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2200{
b0c632db 2201 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2202 return 0;
2203}
2204
2205int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2206{
5a32c1af 2207 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2208 return 0;
2209}
2210
2211int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2212{
5a32c1af 2213 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2214 return 0;
2215}
2216
2217int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2218 struct kvm_sregs *sregs)
2219{
59674c1a 2220 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2221 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 2222 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 2223 return 0;
2224}
2225
2226int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2227 struct kvm_sregs *sregs)
2228{
59674c1a 2229 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2230 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2231 return 0;
2232}
2233
2234int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2235{
9abc2a08
DH		 2236 /* make sure the new values will be lazily loaded */
2237 save_fpu_regs();
4725c860
MS		 2238 if (test_fp_ctl(fpu->fpc))
2239 return -EINVAL;
9abc2a08
DH		 2240 current->thread.fpu.fpc = fpu->fpc;
2241 if (MACHINE_HAS_VX)
2242 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2243 else
2244 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2245 return 0;
2246}
2247
2248int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2249{
9abc2a08
DH		 2250 /* make sure we have the latest values */
2251 save_fpu_regs();
2252 if (MACHINE_HAS_VX)
2253 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2254 else
2255 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2256 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 2257 return 0;
2258}
2259
2260static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2261{
2262 int rc = 0;
2263
7a42fdc2 2264 if (!is_vcpu_stopped(vcpu))
b0c632db 2265 rc = -EBUSY;
d7b0b5eb
CO		 2266 else {
2267 vcpu->run->psw_mask = psw.mask;
2268 vcpu->run->psw_addr = psw.addr;
2269 }
b0c632db
HC		 2270 return rc;
2271}
2272
2273int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2274 struct kvm_translation *tr)
2275{
2276 return -EINVAL; /* not implemented yet */
2277}
2278
27291e21
DH		 2279#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2280 KVM_GUESTDBG_USE_HW_BP | \
2281 KVM_GUESTDBG_ENABLE)
2282
d0bfb940
JK		 2283int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2284 struct kvm_guest_debug *dbg)
b0c632db 2285{
27291e21
DH		 2286 int rc = 0;
2287
2288 vcpu->guest_debug = 0;
2289 kvm_s390_clear_bp_data(vcpu);
2290
2de3bfc2 2291 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2292 return -EINVAL;
89b5b4de
DH		 2293 if (!sclp.has_gpere)
2294 return -EINVAL;
27291e21
DH		 2295
2296 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2297 vcpu->guest_debug = dbg->control;
2298 /* enforce guest PER */
805de8f4 2299 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2300
2301 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2302 rc = kvm_s390_import_bp_data(vcpu, dbg);
2303 } else {
805de8f4 2304 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2305 vcpu->arch.guestdbg.last_bp = 0;
2306 }
2307
2308 if (rc) {
2309 vcpu->guest_debug = 0;
2310 kvm_s390_clear_bp_data(vcpu);
805de8f4 2311 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2312 }
2313
2314 return rc;
b0c632db
HC		 2315}
2316
62d9f0db
MT		 2317int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2318 struct kvm_mp_state *mp_state)
2319{
6352e4d2
DH		 2320 /* CHECK_STOP and LOAD are not supported yet */
2321 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2322 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2323}
2324
2325int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2326 struct kvm_mp_state *mp_state)
2327{
6352e4d2
DH		 2328 int rc = 0;
2329
2330 /* user space knows about this interface - let it control the state */
2331 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2332
2333 switch (mp_state->mp_state) {
2334 case KVM_MP_STATE_STOPPED:
2335 kvm_s390_vcpu_stop(vcpu);
2336 break;
2337 case KVM_MP_STATE_OPERATING:
2338 kvm_s390_vcpu_start(vcpu);
2339 break;
2340 case KVM_MP_STATE_LOAD:
2341 case KVM_MP_STATE_CHECK_STOP:
2342 /* fall through - CHECK_STOP and LOAD are not supported yet */
2343 default:
2344 rc = -ENXIO;
2345 }
2346
2347 return rc;
62d9f0db
MT		 2348}
2349
8ad35755
DH		 2350static bool ibs_enabled(struct kvm_vcpu *vcpu)
2351{
2352 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2353}
2354
2c70fe44
CB		 2355static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2356{
8ad35755 2357retry:
8e236546 2358 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2359 if (!vcpu->requests)
2360 return 0;
2c70fe44
CB		 2361 /*
2362 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2363 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2364 * This ensures that the ipte instruction for this request has
2365 * already finished. We might race against a second unmapper that
2366 * wants to set the blocking bit. Lets just retry the request loop.
2367 */
8ad35755 2368 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2369 int rc;
b2d73b2a
MS		 2370 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2371 kvm_s390_get_prefix(vcpu),
2372 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 2373 if (rc) {
2374 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 2375 return rc;
aca411a4 2376 }
8ad35755 2377 goto retry;
2c70fe44 2378 }
8ad35755 2379
d3d692c8
DH		 2380 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2381 vcpu->arch.sie_block->ihcpu = 0xffff;
2382 goto retry;
2383 }
2384
8ad35755
DH		 2385 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2386 if (!ibs_enabled(vcpu)) {
2387 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2388 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2389 &vcpu->arch.sie_block->cpuflags);
2390 }
2391 goto retry;
2c70fe44 2392 }
8ad35755
DH		 2393
2394 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2395 if (ibs_enabled(vcpu)) {
2396 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2397 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2398 &vcpu->arch.sie_block->cpuflags);
2399 }
2400 goto retry;
2401 }
2402
6502a34c
DH		 2403 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2404 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2405 goto retry;
2406 }
2407
0759d068
DH		 2408 /* nothing to do, just clear the request */
2409 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2410
2c70fe44
CB		 2411 return 0;
2412}
2413
25ed1675
DH		 2414void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2415{
2416 struct kvm_vcpu *vcpu;
2417 int i;
2418
2419 mutex_lock(&kvm->lock);
2420 preempt_disable();
2421 kvm->arch.epoch = tod - get_tod_clock();
2422 kvm_s390_vcpu_block_all(kvm);
2423 kvm_for_each_vcpu(i, vcpu, kvm)
2424 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2425 kvm_s390_vcpu_unblock_all(kvm);
2426 preempt_enable();
2427 mutex_unlock(&kvm->lock);
2428}
2429
fa576c58
TH		 2430/**
2431 * kvm_arch_fault_in_page - fault-in guest page if necessary
2432 * @vcpu: The corresponding virtual cpu
2433 * @gpa: Guest physical address
2434 * @writable: Whether the page should be writable or not
2435 *
2436 * Make sure that a guest page has been faulted-in on the host.
2437 *
2438 * Return: Zero on success, negative error code otherwise.
2439 */
2440long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2441{
527e30b4
MS		 2442 return gmap_fault(vcpu->arch.gmap, gpa,
2443 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2444}
2445
3c038e6b
DD		 2446static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2447 unsigned long token)
2448{
2449 struct kvm_s390_interrupt inti;
383d0b05 2450 struct kvm_s390_irq irq;
3c038e6b
DD		 2451
2452 if (start_token) {
383d0b05
JF		 2453 irq.u.ext.ext_params2 = token;
2454 irq.type = KVM_S390_INT_PFAULT_INIT;
2455 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2456 } else {
2457 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2458 inti.parm64 = token;
3c038e6b
DD		 2459 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2460 }
2461}
2462
2463void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2464 struct kvm_async_pf *work)
2465{
2466 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2467 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2468}
2469
2470void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2471 struct kvm_async_pf *work)
2472{
2473 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2474 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2475}
2476
2477void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2478 struct kvm_async_pf *work)
2479{
2480 /* s390 will always inject the page directly */
2481}
2482
2483bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2484{
2485 /*
2486 * s390 will always inject the page directly,
2487 * but we still want check_async_completion to cleanup
2488 */
2489 return true;
2490}
2491
2492static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2493{
2494 hva_t hva;
2495 struct kvm_arch_async_pf arch;
2496 int rc;
2497
2498 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2499 return 0;
2500 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2501 vcpu->arch.pfault_compare)
2502 return 0;
2503 if (psw_extint_disabled(vcpu))
2504 return 0;
9a022067 2505 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2506 return 0;
2507 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2508 return 0;
2509 if (!vcpu->arch.gmap->pfault_enabled)
2510 return 0;
2511
81480cc1
HC		 2512 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2513 hva += current->thread.gmap_addr & ~PAGE_MASK;
2514 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2515 return 0;
2516
2517 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2518 return rc;
2519}
2520
3fb4c40f 2521static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2522{
3fb4c40f 2523 int rc, cpuflags;
e168bf8d 2524
3c038e6b
DD		 2525 /*
2526 * On s390 notifications for arriving pages will be delivered directly
2527 * to the guest but the house keeping for completed pfaults is
2528 * handled outside the worker.
2529 */
2530 kvm_check_async_pf_completion(vcpu);
2531
7ec7c8c7
CB		 2532 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2533 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2534
2535 if (need_resched())
2536 schedule();
2537
d3a73acb 2538 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2539 s390_handle_mcck();
2540
79395031
JF		 2541 if (!kvm_is_ucontrol(vcpu->kvm)) {
2542 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2543 if (rc)
2544 return rc;
2545 }
0ff31867 2546
2c70fe44
CB		 2547 rc = kvm_s390_handle_requests(vcpu);
2548 if (rc)
2549 return rc;
2550
27291e21
DH		 2551 if (guestdbg_enabled(vcpu)) {
2552 kvm_s390_backup_guest_per_regs(vcpu);
2553 kvm_s390_patch_guest_per_regs(vcpu);
2554 }
2555
b0c632db 2556 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2557 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2558 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2559 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2560
3fb4c40f
TH		 2561 return 0;
2562}
2563
492d8642
TH		 2564static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2565{
56317920
DH		 2566 struct kvm_s390_pgm_info pgm_info = {
2567 .code = PGM_ADDRESSING,
2568 };
2569 u8 opcode, ilen;
492d8642
TH		 2570 int rc;
2571
2572 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2573 trace_kvm_s390_sie_fault(vcpu);
2574
2575 /*
2576 * We want to inject an addressing exception, which is defined as a
2577 * suppressing or terminating exception. However, since we came here
2578 * by a DAT access exception, the PSW still points to the faulting
2579 * instruction since DAT exceptions are nullifying. So we've got
2580 * to look up the current opcode to get the length of the instruction
2581 * to be able to forward the PSW.
2582 */
65977322 2583 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2584 ilen = insn_length(opcode);
9b0d721a
DH		 2585 if (rc < 0) {
2586 return rc;
2587 } else if (rc) {
2588 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2589 * Forward by arbitrary ilc, injection will take care of
2590 * nullification if necessary.
2591 */
2592 pgm_info = vcpu->arch.pgm;
2593 ilen = 4;
2594 }
56317920
DH		 2595 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2596 kvm_s390_forward_psw(vcpu, ilen);
2597 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2598}
2599
3fb4c40f
TH		 2600static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2601{
2b29a9fd
DD		 2602 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2603 vcpu->arch.sie_block->icptcode);
2604 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2605
27291e21
DH		 2606 if (guestdbg_enabled(vcpu))
2607 kvm_s390_restore_guest_per_regs(vcpu);
2608
7ec7c8c7
CB		 2609 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2610 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2611
2612 if (vcpu->arch.sie_block->icptcode > 0) {
2613 int rc = kvm_handle_sie_intercept(vcpu);
2614
2615 if (rc != -EOPNOTSUPP)
2616 return rc;
2617 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2618 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2619 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2620 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2621 return -EREMOTE;
2622 } else if (exit_reason != -EFAULT) {
2623 vcpu->stat.exit_null++;
2624 return 0;
210b1607
TH		 2625 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2626 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2627 vcpu->run->s390_ucontrol.trans_exc_code =
2628 current->thread.gmap_addr;
2629 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2630 return -EREMOTE;
24eb3a82 2631 } else if (current->thread.gmap_pfault) {
3c038e6b 2632 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2633 current->thread.gmap_pfault = 0;
71f116bf
DH		 2634 if (kvm_arch_setup_async_pf(vcpu))
2635 return 0;
2636 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2637 }
71f116bf 2638 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2639}
2640
2641static int __vcpu_run(struct kvm_vcpu *vcpu)
2642{
2643 int rc, exit_reason;
2644
800c1065
TH		 2645 /*
2646 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2647 * ning the guest), so that memslots (and other stuff) are protected
2648 */
2649 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2650
a76ccff6
TH		 2651 do {
2652 rc = vcpu_pre_run(vcpu);
2653 if (rc)
2654 break;
3fb4c40f 2655
800c1065 2656 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2657 /*
2658 * As PF_VCPU will be used in fault handler, between
2659 * guest_enter and guest_exit should be no uaccess.
2660 */
0097d12e 2661 local_irq_disable();
6edaa530 2662 guest_enter_irqoff();
db0758b2 2663 __disable_cpu_timer_accounting(vcpu);
0097d12e 2664 local_irq_enable();
a76ccff6
TH		 2665 exit_reason = sie64a(vcpu->arch.sie_block,
2666 vcpu->run->s.regs.gprs);
0097d12e 2667 local_irq_disable();
db0758b2 2668 __enable_cpu_timer_accounting(vcpu);
6edaa530 2669 guest_exit_irqoff();
0097d12e 2670 local_irq_enable();
800c1065 2671 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2672
2673 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2674 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2675
800c1065 2676 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2677 return rc;
b0c632db
HC		 2678}
2679
b028ee3e
DH		 2680static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2681{
2682 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2683 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2684 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2685 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2686 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2687 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2688 /* some control register changes require a tlb flush */
2689 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2690 }
2691 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2692 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2693 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2694 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2695 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2696 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2697 }
2698 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2699 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2700 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2701 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2702 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2703 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2704 }
2705 kvm_run->kvm_dirty_regs = 0;
2706}
2707
2708static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2709{
2710 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2711 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2712 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2713 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2714 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2715 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2716 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2717 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2718 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2719 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2720 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2721 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2722}
2723
b0c632db
HC		 2724int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2725{
8f2abe6a 2726 int rc;
b0c632db
HC		 2727 sigset_t sigsaved;
2728
27291e21
DH		 2729 if (guestdbg_exit_pending(vcpu)) {
2730 kvm_s390_prepare_debug_exit(vcpu);
2731 return 0;
2732 }
2733
b0c632db
HC		 2734 if (vcpu->sigset_active)
2735 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2736
6352e4d2
DH		 2737 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2738 kvm_s390_vcpu_start(vcpu);
2739 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2740 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2741 vcpu->vcpu_id);
2742 return -EINVAL;
2743 }
b0c632db 2744
b028ee3e 2745 sync_regs(vcpu, kvm_run);
db0758b2 2746 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2747
dab4079d 2748 might_fault();
a76ccff6 2749 rc = __vcpu_run(vcpu);
9ace903d 2750
b1d16c49
CE		 2751 if (signal_pending(current) && !rc) {
2752 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2753 rc = -EINTR;
b1d16c49 2754 }
8f2abe6a 2755
27291e21
DH		 2756 if (guestdbg_exit_pending(vcpu) && !rc) {
2757 kvm_s390_prepare_debug_exit(vcpu);
2758 rc = 0;
2759 }
2760
8f2abe6a 2761 if (rc == -EREMOTE) {
71f116bf 2762 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2763 rc = 0;
2764 }
b0c632db 2765
db0758b2 2766 disable_cpu_timer_accounting(vcpu);
b028ee3e 2767 store_regs(vcpu, kvm_run);
d7b0b5eb 2768
b0c632db
HC		 2769 if (vcpu->sigset_active)
2770 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2771
b0c632db 2772 vcpu->stat.exit_userspace++;
7e8e6ab4 2773 return rc;
b0c632db
HC		 2774}
2775
b0c632db
HC		 2776/*
2777 * store status at address
2778 * we use have two special cases:
2779 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2780 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2781 */
d0bce605 2782int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2783{
092670cd 2784 unsigned char archmode = 1;
9abc2a08 2785 freg_t fprs[NUM_FPRS];
fda902cb 2786 unsigned int px;
4287f247 2787 u64 clkcomp, cputm;
d0bce605 2788 int rc;
b0c632db 2789
d9a3a09a 2790 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2791 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2792 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2793 return -EFAULT;
d9a3a09a 2794 gpa = 0;
d0bce605
HC		 2795 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2796 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2797 return -EFAULT;
d9a3a09a
MS		 2798 gpa = px;
2799 } else
2800 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2801
2802 /* manually convert vector registers if necessary */
2803 if (MACHINE_HAS_VX) {
9522b37f 2804 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2805 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2806 fprs, 128);
2807 } else {
2808 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2809 vcpu->run->s.regs.fprs, 128);
9abc2a08 2810 }
d9a3a09a 2811 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2812 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2813 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2814 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2815 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2816 &px, 4);
d9a3a09a 2817 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2818 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2819 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2820 &vcpu->arch.sie_block->todpr, 4);
4287f247 2821 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2822 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2823 &cputm, 8);
178bd789 2824 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2825 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2826 &clkcomp, 8);
d9a3a09a 2827 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2828 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2829 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2830 &vcpu->arch.sie_block->gcr, 128);
2831 return rc ? -EFAULT : 0;
b0c632db
HC		 2832}
2833
e879892c
TH		 2834int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2835{
2836 /*
2837 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2838 * copying in vcpu load/put. Lets update our copies before we save
2839 * it into the save area
2840 */
d0164ee2 2841 save_fpu_regs();
9abc2a08 2842 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2843 save_access_regs(vcpu->run->s.regs.acrs);
2844
2845 return kvm_s390_store_status_unloaded(vcpu, addr);
2846}
2847
bc17de7c
EF		 2848/*
2849 * store additional status at address
2850 */
2851int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2852 unsigned long gpa)
2853{
2854 /* Only bits 0-53 are used for address formation */
2855 if (!(gpa & ~0x3ff))
2856 return 0;
2857
2858 return write_guest_abs(vcpu, gpa & ~0x3ff,
2859 (void *)&vcpu->run->s.regs.vrs, 512);
2860}
2861
2862int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2863{
2864 if (!test_kvm_facility(vcpu->kvm, 129))
2865 return 0;
2866
2867 /*
2868 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2869 * copying in vcpu load/put. We can simply call save_fpu_regs()
2870 * to save the current register state because we are in the
2871 * middle of a load/put cycle.
2872 *
2873 * Let's update our copies before we save it into the save area.
bc17de7c 2874 */
d0164ee2 2875 save_fpu_regs();
bc17de7c
EF		 2876
2877 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2878}
2879
8ad35755
DH		 2880static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2881{
2882 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2883 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2884}
2885
2886static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2887{
2888 unsigned int i;
2889 struct kvm_vcpu *vcpu;
2890
2891 kvm_for_each_vcpu(i, vcpu, kvm) {
2892 __disable_ibs_on_vcpu(vcpu);
2893 }
2894}
2895
2896static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2897{
09a400e7
DH		 2898 if (!sclp.has_ibs)
2899 return;
8ad35755 2900 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2901 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2902}
2903
6852d7b6
DH		 2904void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2905{
8ad35755
DH		 2906 int i, online_vcpus, started_vcpus = 0;
2907
2908 if (!is_vcpu_stopped(vcpu))
2909 return;
2910
6852d7b6 2911 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2912 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2913 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2914 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2915
2916 for (i = 0; i < online_vcpus; i++) {
2917 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2918 started_vcpus++;
2919 }
2920
2921 if (started_vcpus == 0) {
2922 /* we're the only active VCPU -> speed it up */
2923 __enable_ibs_on_vcpu(vcpu);
2924 } else if (started_vcpus == 1) {
2925 /*
2926 * As we are starting a second VCPU, we have to disable
2927 * the IBS facility on all VCPUs to remove potentially
2928 * oustanding ENABLE requests.
2929 */
2930 __disable_ibs_on_all_vcpus(vcpu->kvm);
2931 }
2932
805de8f4 2933 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2934 /*
2935 * Another VCPU might have used IBS while we were offline.
2936 * Let's play safe and flush the VCPU at startup.
2937 */
d3d692c8 2938 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2939 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2940 return;
6852d7b6
DH		 2941}
2942
2943void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2944{
8ad35755
DH		 2945 int i, online_vcpus, started_vcpus = 0;
2946 struct kvm_vcpu *started_vcpu = NULL;
2947
2948 if (is_vcpu_stopped(vcpu))
2949 return;
2950
6852d7b6 2951 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2952 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2953 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2954 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2955
32f5ff63 2956 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2957 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2958
805de8f4 2959 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2960 __disable_ibs_on_vcpu(vcpu);
2961
2962 for (i = 0; i < online_vcpus; i++) {
2963 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2964 started_vcpus++;
2965 started_vcpu = vcpu->kvm->vcpus[i];
2966 }
2967 }
2968
2969 if (started_vcpus == 1) {
2970 /*
2971 * As we only have one VCPU left, we want to enable the
2972 * IBS facility for that VCPU to speed it up.
2973 */
2974 __enable_ibs_on_vcpu(started_vcpu);
2975 }
2976
433b9ee4 2977 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2978 return;
6852d7b6
DH		 2979}
2980
d6712df9
CH		 2981static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2982 struct kvm_enable_cap *cap)
2983{
2984 int r;
2985
2986 if (cap->flags)
2987 return -EINVAL;
2988
2989 switch (cap->cap) {
fa6b7fe9
CH		 2990 case KVM_CAP_S390_CSS_SUPPORT:
2991 if (!vcpu->kvm->arch.css_support) {
2992 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2993 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2994 trace_kvm_s390_enable_css(vcpu->kvm);
2995 }
2996 r = 0;
2997 break;
d6712df9
CH		 2998 default:
2999 r = -EINVAL;
3000 break;
3001 }
3002 return r;
3003}
3004
41408c28
TH		 3005static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3006 struct kvm_s390_mem_op *mop)
3007{
3008 void __user *uaddr = (void __user *)mop->buf;
3009 void *tmpbuf = NULL;
3010 int r, srcu_idx;
3011 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3012 | KVM_S390_MEMOP_F_CHECK_ONLY;
3013
3014 if (mop->flags & ~supported_flags)
3015 return -EINVAL;
3016
3017 if (mop->size > MEM_OP_MAX_SIZE)
3018 return -E2BIG;
3019
3020 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3021 tmpbuf = vmalloc(mop->size);
3022 if (!tmpbuf)
3023 return -ENOMEM;
3024 }
3025
3026 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3027
3028 switch (mop->op) {
3029 case KVM_S390_MEMOP_LOGICAL_READ:
3030 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3031 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3032 mop->size, GACC_FETCH);
41408c28
TH		 3033 break;
3034 }
3035 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3036 if (r == 0) {
3037 if (copy_to_user(uaddr, tmpbuf, mop->size))
3038 r = -EFAULT;
3039 }
3040 break;
3041 case KVM_S390_MEMOP_LOGICAL_WRITE:
3042 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3043 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3044 mop->size, GACC_STORE);
41408c28
TH		 3045 break;
3046 }
3047 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3048 r = -EFAULT;
3049 break;
3050 }
3051 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3052 break;
3053 default:
3054 r = -EINVAL;
3055 }
3056
3057 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3058
3059 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3060 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3061
3062 vfree(tmpbuf);
3063 return r;
3064}
3065
b0c632db
HC		 3066long kvm_arch_vcpu_ioctl(struct file *filp,
3067 unsigned int ioctl, unsigned long arg)
3068{
3069 struct kvm_vcpu *vcpu = filp->private_data;
3070 void __user *argp = (void __user *)arg;
800c1065 3071 int idx;
bc923cc9 3072 long r;
b0c632db 3073
93736624 3074 switch (ioctl) {
47b43c52
JF		 3075 case KVM_S390_IRQ: {
3076 struct kvm_s390_irq s390irq;
3077
3078 r = -EFAULT;
3079 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3080 break;
3081 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3082 break;
3083 }
93736624 3084 case KVM_S390_INTERRUPT: {
ba5c1e9b 3085 struct kvm_s390_interrupt s390int;
383d0b05 3086 struct kvm_s390_irq s390irq;
ba5c1e9b 3087
93736624 3088 r = -EFAULT;
ba5c1e9b 3089 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 3090 break;
383d0b05
JF		 3091 if (s390int_to_s390irq(&s390int, &s390irq))
3092 return -EINVAL;
3093 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 3094 break;
ba5c1e9b 3095 }
b0c632db 3096 case KVM_S390_STORE_STATUS:
800c1065 3097 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3098 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3099 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3100 break;
b0c632db
HC		 3101 case KVM_S390_SET_INITIAL_PSW: {
3102 psw_t psw;
3103
bc923cc9 3104 r = -EFAULT;
b0c632db 3105 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3106 break;
3107 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3108 break;
b0c632db
HC		 3109 }
3110 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3111 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3112 break;
14eebd91
CO		 3113 case KVM_SET_ONE_REG:
3114 case KVM_GET_ONE_REG: {
3115 struct kvm_one_reg reg;
3116 r = -EFAULT;
3117 if (copy_from_user(&reg, argp, sizeof(reg)))
3118 break;
3119 if (ioctl == KVM_SET_ONE_REG)
3120 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3121 else
3122 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3123 break;
3124 }
27e0393f
CO		 3125#ifdef CONFIG_KVM_S390_UCONTROL
3126 case KVM_S390_UCAS_MAP: {
3127 struct kvm_s390_ucas_mapping ucasmap;
3128
3129 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3130 r = -EFAULT;
3131 break;
3132 }
3133
3134 if (!kvm_is_ucontrol(vcpu->kvm)) {
3135 r = -EINVAL;
3136 break;
3137 }
3138
3139 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3140 ucasmap.vcpu_addr, ucasmap.length);
3141 break;
3142 }
3143 case KVM_S390_UCAS_UNMAP: {
3144 struct kvm_s390_ucas_mapping ucasmap;
3145
3146 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3147 r = -EFAULT;
3148 break;
3149 }
3150
3151 if (!kvm_is_ucontrol(vcpu->kvm)) {
3152 r = -EINVAL;
3153 break;
3154 }
3155
3156 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3157 ucasmap.length);
3158 break;
3159 }
3160#endif
ccc7910f 3161 case KVM_S390_VCPU_FAULT: {
527e30b4 3162 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3163 break;
3164 }
d6712df9
CH		 3165 case KVM_ENABLE_CAP:
3166 {
3167 struct kvm_enable_cap cap;
3168 r = -EFAULT;
3169 if (copy_from_user(&cap, argp, sizeof(cap)))
3170 break;
3171 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3172 break;
3173 }
41408c28
TH		 3174 case KVM_S390_MEM_OP: {
3175 struct kvm_s390_mem_op mem_op;
3176
3177 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3178 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3179 else
3180 r = -EFAULT;
3181 break;
3182 }
816c7667
JF		 3183 case KVM_S390_SET_IRQ_STATE: {
3184 struct kvm_s390_irq_state irq_state;
3185
3186 r = -EFAULT;
3187 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3188 break;
3189 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3190 irq_state.len == 0 ||
3191 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3192 r = -EINVAL;
3193 break;
3194 }
3195 r = kvm_s390_set_irq_state(vcpu,
3196 (void __user *) irq_state.buf,
3197 irq_state.len);
3198 break;
3199 }
3200 case KVM_S390_GET_IRQ_STATE: {
3201 struct kvm_s390_irq_state irq_state;
3202
3203 r = -EFAULT;
3204 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3205 break;
3206 if (irq_state.len == 0) {
3207 r = -EINVAL;
3208 break;
3209 }
3210 r = kvm_s390_get_irq_state(vcpu,
3211 (__u8 __user *) irq_state.buf,
3212 irq_state.len);
3213 break;
3214 }
b0c632db 3215 default:
3e6afcf1 3216 r = -ENOTTY;
b0c632db 3217 }
bc923cc9 3218 return r;
b0c632db
HC		 3219}
3220
5b1c1493
CO		 3221int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3222{
3223#ifdef CONFIG_KVM_S390_UCONTROL
3224 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3225 && (kvm_is_ucontrol(vcpu->kvm))) {
3226 vmf->page = virt_to_page(vcpu->arch.sie_block);
3227 get_page(vmf->page);
3228 return 0;
3229 }
3230#endif
3231 return VM_FAULT_SIGBUS;
3232}
3233
5587027c
AK		 3234int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3235 unsigned long npages)
db3fe4eb
TY		 3236{
3237 return 0;
3238}
3239
b0c632db 3240/* Section: memory related */
f7784b8e
MT		 3241int kvm_arch_prepare_memory_region(struct kvm *kvm,
3242 struct kvm_memory_slot *memslot,
09170a49 3243 const struct kvm_userspace_memory_region *mem,
7b6195a9 3244 enum kvm_mr_change change)
b0c632db 3245{
dd2887e7
NW		 3246 /* A few sanity checks. We can have memory slots which have to be
3247 located/ended at a segment boundary (1MB). The memory in userland is
3248 ok to be fragmented into various different vmas. It is okay to mmap()
3249 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3250
598841ca 3251 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3252 return -EINVAL;
3253
598841ca 3254 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3255 return -EINVAL;
3256
a3a92c31
DD		 3257 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3258 return -EINVAL;
3259
f7784b8e
MT		 3260 return 0;
3261}
3262
3263void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3264 const struct kvm_userspace_memory_region *mem,
8482644a 3265 const struct kvm_memory_slot *old,
f36f3f28 3266 const struct kvm_memory_slot *new,
8482644a 3267 enum kvm_mr_change change)
f7784b8e 3268{
f7850c92 3269 int rc;
f7784b8e 3270
2cef4deb
CB		 3271 /* If the basics of the memslot do not change, we do not want
3272 * to update the gmap. Every update causes several unnecessary
3273 * segment translation exceptions. This is usually handled just
3274 * fine by the normal fault handler + gmap, but it will also
3275 * cause faults on the prefix page of running guest CPUs.
3276 */
3277 if (old->userspace_addr == mem->userspace_addr &&
3278 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3279 old->npages * PAGE_SIZE == mem->memory_size)
3280 return;
598841ca
CO		 3281
3282 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3283 mem->guest_phys_addr, mem->memory_size);
3284 if (rc)
ea2cdd27 3285 pr_warn("failed to commit memory region\n");
598841ca 3286 return;
b0c632db
HC		 3287}
3288
60a37709
AY		 3289static inline unsigned long nonhyp_mask(int i)
3290{
3291 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3292
3293 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3294}
3295
3491caf2
CB		 3296void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3297{
3298 vcpu->valid_wakeup = false;
3299}
3300
b0c632db
HC		 3301static int __init kvm_s390_init(void)
3302{
60a37709
AY		 3303 int i;
3304
07197fd0
DH		 3305 if (!sclp.has_sief2) {
3306 pr_info("SIE not available\n");
3307 return -ENODEV;
3308 }
3309
60a37709
AY		 3310 for (i = 0; i < 16; i++)
3311 kvm_s390_fac_list_mask[i] |=
3312 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3313
9d8d5786 3314 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3315}
3316
3317static void __exit kvm_s390_exit(void)
3318{
3319 kvm_exit();
3320}
3321
3322module_init(kvm_s390_init);
3323module_exit(kvm_s390_exit);
566af940
CH		 3324
3325/*
3326 * Enable autoloading of the kvm module.
3327 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3328 * since x86 takes a different approach.
3329 */
3330#include <linux/miscdevice.h>
3331MODULE_ALIAS_MISCDEV(KVM_MINOR);
3332MODULE_ALIAS("devname:kvm");
Linux kernel - Deliverables
This page took 0.695319 seconds and 5 git commands to generate.