arch/mips/kvm/entry.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Generation of main entry point for the guest, exception handling.
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.
   9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  10  *
  11  * Copyright (C) 2016 Imagination Technologies Ltd.
  12  */
  13
  14 #include <linux/kvm_host.h>
  15 #include <asm/msa.h>
  16 #include <asm/setup.h>
  17 #include <asm/uasm.h>
  18
  19 /* Register names */
  20 #define ZERO            0
  21 #define AT              1
  22 #define V0              2
  23 #define V1              3
  24 #define A0              4
  25 #define A1              5
  26
  27 #if _MIPS_SIM == _MIPS_SIM_ABI32
  28 #define T0              8
  29 #define T1              9
  30 #define T2              10
  31 #define T3              11
  32 #endif /* _MIPS_SIM == _MIPS_SIM_ABI32 */
  33
  34 #if _MIPS_SIM == _MIPS_SIM_ABI64 || _MIPS_SIM == _MIPS_SIM_NABI32
  35 #define T0              12
  36 #define T1              13
  37 #define T2              14
  38 #define T3              15
  39 #endif /* _MIPS_SIM == _MIPS_SIM_ABI64 || _MIPS_SIM == _MIPS_SIM_NABI32 */
  40
  41 #define S0              16
  42 #define S1              17
  43 #define T9              25
  44 #define K0              26
  45 #define K1              27
  46 #define GP              28
  47 #define SP              29
  48 #define RA              31
  49
  50 /* Some CP0 registers */
  51 #define C0_HWRENA       7, 0
  52 #define C0_BADVADDR     8, 0
  53 #define C0_ENTRYHI      10, 0
  54 #define C0_STATUS       12, 0
  55 #define C0_CAUSE        13, 0
  56 #define C0_EPC          14, 0
  57 #define C0_EBASE        15, 1
  58 #define C0_CONFIG5      16, 5
  59 #define C0_DDATA_LO     28, 3
  60 #define C0_ERROREPC     30, 0
  61
  62 #define CALLFRAME_SIZ   32
  63
  64 static unsigned int scratch_vcpu[2] = { C0_DDATA_LO };
  65 static unsigned int scratch_tmp[2] = { C0_ERROREPC };
  66
  67 enum label_id {
  68         label_fpu_1 = 1,
  69         label_msa_1,
  70         label_return_to_host,
  71         label_kernel_asid,
  72         label_exit_common,
  73 };
  74
  75 UASM_L_LA(_fpu_1)
  76 UASM_L_LA(_msa_1)
  77 UASM_L_LA(_return_to_host)
  78 UASM_L_LA(_kernel_asid)
  79 UASM_L_LA(_exit_common)
  80
  81 static void *kvm_mips_build_enter_guest(void *addr);
  82 static void *kvm_mips_build_ret_from_exit(void *addr);
  83 static void *kvm_mips_build_ret_to_guest(void *addr);
  84 static void *kvm_mips_build_ret_to_host(void *addr);
  85
  86 /**
  87  * kvm_mips_entry_setup() - Perform global setup for entry code.
  88  *
  89  * Perform global setup for entry code, such as choosing a scratch register.
  90  *
  91  * Returns:     0 on success.
  92  *              -errno on failure.
  93  */
  94 int kvm_mips_entry_setup(void)
  95 {
  96         /*
  97          * We prefer to use KScratchN registers if they are available over the
  98          * defaults above, which may not work on all cores.
  99          */
 100         unsigned int kscratch_mask = cpu_data[0].kscratch_mask & 0xfc;
 101
 102         /* Pick a scratch register for storing VCPU */
 103         if (kscratch_mask) {
 104                 scratch_vcpu[0] = 31;
 105                 scratch_vcpu[1] = ffs(kscratch_mask) - 1;
 106                 kscratch_mask &= ~BIT(scratch_vcpu[1]);
 107         }
 108
 109         /* Pick a scratch register to use as a temp for saving state */
 110         if (kscratch_mask) {
 111                 scratch_tmp[0] = 31;
 112                 scratch_tmp[1] = ffs(kscratch_mask) - 1;
 113                 kscratch_mask &= ~BIT(scratch_tmp[1]);
 114         }
 115
 116         return 0;
 117 }
 118
 119 static void kvm_mips_build_save_scratch(u32 **p, unsigned int tmp,
 120                                         unsigned int frame)
 121 {
 122         /* Save the VCPU scratch register value in cp0_epc of the stack frame */
 123         UASM_i_MFC0(p, tmp, scratch_vcpu[0], scratch_vcpu[1]);
 124         UASM_i_SW(p, tmp, offsetof(struct pt_regs, cp0_epc), frame);
 125
 126         /* Save the temp scratch register value in cp0_cause of stack frame */
 127         if (scratch_tmp[0] == 31) {
 128                 UASM_i_MFC0(p, tmp, scratch_tmp[0], scratch_tmp[1]);
 129                 UASM_i_SW(p, tmp, offsetof(struct pt_regs, cp0_cause), frame);
 130         }
 131 }
 132
 133 static void kvm_mips_build_restore_scratch(u32 **p, unsigned int tmp,
 134                                            unsigned int frame)
 135 {
 136         /*
 137          * Restore host scratch register values saved by
 138          * kvm_mips_build_save_scratch().
 139          */
 140         UASM_i_LW(p, tmp, offsetof(struct pt_regs, cp0_epc), frame);
 141         UASM_i_MTC0(p, tmp, scratch_vcpu[0], scratch_vcpu[1]);
 142
 143         if (scratch_tmp[0] == 31) {
 144                 UASM_i_LW(p, tmp, offsetof(struct pt_regs, cp0_cause), frame);
 145                 UASM_i_MTC0(p, tmp, scratch_tmp[0], scratch_tmp[1]);
 146         }
 147 }
 148
 149 /**
 150  * kvm_mips_build_vcpu_run() - Assemble function to start running a guest VCPU.
 151  * @addr:       Address to start writing code.
 152  *
 153  * Assemble the start of the vcpu_run function to run a guest VCPU. The function
 154  * conforms to the following prototype:
 155  *
 156  * int vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu);
 157  *
 158  * The exit from the guest and return to the caller is handled by the code
 159  * generated by kvm_mips_build_ret_to_host().
 160  *
 161  * Returns:     Next address after end of written function.
 162  */
 163 void *kvm_mips_build_vcpu_run(void *addr)
 164 {
 165         u32 *p = addr;
 166         unsigned int i;
 167
 168         /*
 169          * A0: run
 170          * A1: vcpu
 171          */
 172
 173         /* k0/k1 not being used in host kernel context */
 174         UASM_i_ADDIU(&p, K1, SP, -(int)sizeof(struct pt_regs));
 175         for (i = 16; i < 32; ++i) {
 176                 if (i == 24)
 177                         i = 28;
 178                 UASM_i_SW(&p, i, offsetof(struct pt_regs, regs[i]), K1);
 179         }
 180
 181         /* Save host status */
 182         uasm_i_mfc0(&p, V0, C0_STATUS);
 183         UASM_i_SW(&p, V0, offsetof(struct pt_regs, cp0_status), K1);
 184
 185         /* Save scratch registers, will be used to store pointer to vcpu etc */
 186         kvm_mips_build_save_scratch(&p, V1, K1);
 187
 188         /* VCPU scratch register has pointer to vcpu */
 189         UASM_i_MTC0(&p, A1, scratch_vcpu[0], scratch_vcpu[1]);
 190
 191         /* Offset into vcpu->arch */
 192         UASM_i_ADDIU(&p, K1, A1, offsetof(struct kvm_vcpu, arch));
 193
 194         /*
 195          * Save the host stack to VCPU, used for exception processing
 196          * when we exit from the Guest
 197          */
 198         UASM_i_SW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1);
 199
 200         /* Save the kernel gp as well */
 201         UASM_i_SW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1);
 202
 203         /*
 204          * Setup status register for running the guest in UM, interrupts
 205          * are disabled
 206          */
 207         UASM_i_LA(&p, K0, ST0_EXL | KSU_USER | ST0_BEV);
 208         uasm_i_mtc0(&p, K0, C0_STATUS);
 209         uasm_i_ehb(&p);
 210
 211         /* load up the new EBASE */
 212         UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, guest_ebase), K1);
 213         uasm_i_mtc0(&p, K0, C0_EBASE);
 214
 215         /*
 216          * Now that the new EBASE has been loaded, unset BEV, set
 217          * interrupt mask as it was but make sure that timer interrupts
 218          * are enabled
 219          */
 220         uasm_i_addiu(&p, K0, ZERO, ST0_EXL | KSU_USER | ST0_IE);
 221         uasm_i_andi(&p, V0, V0, ST0_IM);
 222         uasm_i_or(&p, K0, K0, V0);
 223         uasm_i_mtc0(&p, K0, C0_STATUS);
 224         uasm_i_ehb(&p);
 225
 226         p = kvm_mips_build_enter_guest(p);
 227
 228         return p;
 229 }
 230
 231 /**
 232  * kvm_mips_build_enter_guest() - Assemble code to resume guest execution.
 233  * @addr:       Address to start writing code.
 234  *
 235  * Assemble the code to resume guest execution. This code is common between the
 236  * initial entry into the guest from the host, and returning from the exit
 237  * handler back to the guest.
 238  *
 239  * Returns:     Next address after end of written function.
 240  */
 241 static void *kvm_mips_build_enter_guest(void *addr)
 242 {
 243         u32 *p = addr;
 244         unsigned int i;
 245         struct uasm_label labels[2];
 246         struct uasm_reloc relocs[2];
 247         struct uasm_label *l = labels;
 248         struct uasm_reloc *r = relocs;
 249
 250         memset(labels, 0, sizeof(labels));
 251         memset(relocs, 0, sizeof(relocs));
 252
 253         /* Set Guest EPC */
 254         UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, pc), K1);
 255         UASM_i_MTC0(&p, T0, C0_EPC);
 256
 257         /* Set the ASID for the Guest Kernel */
 258         UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, cop0), K1);
 259         UASM_i_LW(&p, T0, offsetof(struct mips_coproc, reg[MIPS_CP0_STATUS][0]),
 260                   T0);
 261         uasm_i_andi(&p, T0, T0, KSU_USER | ST0_ERL | ST0_EXL);
 262         uasm_i_xori(&p, T0, T0, KSU_USER);
 263         uasm_il_bnez(&p, &r, T0, label_kernel_asid);
 264          UASM_i_ADDIU(&p, T1, K1,
 265                       offsetof(struct kvm_vcpu_arch, guest_kernel_asid));
 266         /* else user */
 267         UASM_i_ADDIU(&p, T1, K1,
 268                      offsetof(struct kvm_vcpu_arch, guest_user_asid));
 269         uasm_l_kernel_asid(&l, p);
 270
 271         /* t1: contains the base of the ASID array, need to get the cpu id  */
 272         /* smp_processor_id */
 273         uasm_i_lw(&p, T2, offsetof(struct thread_info, cpu), GP);
 274         /* x4 */
 275         uasm_i_sll(&p, T2, T2, 2);
 276         UASM_i_ADDU(&p, T3, T1, T2);
 277         uasm_i_lw(&p, K0, 0, T3);
 278 #ifdef CONFIG_MIPS_ASID_BITS_VARIABLE
 279         /* x sizeof(struct cpuinfo_mips)/4 */
 280         uasm_i_addiu(&p, T3, ZERO, sizeof(struct cpuinfo_mips)/4);
 281         uasm_i_mul(&p, T2, T2, T3);
 282
 283         UASM_i_LA_mostly(&p, AT, (long)&cpu_data[0].asid_mask);
 284         UASM_i_ADDU(&p, AT, AT, T2);
 285         UASM_i_LW(&p, T2, uasm_rel_lo((long)&cpu_data[0].asid_mask), AT);
 286         uasm_i_and(&p, K0, K0, T2);
 287 #else
 288         uasm_i_andi(&p, K0, K0, MIPS_ENTRYHI_ASID);
 289 #endif
 290         uasm_i_mtc0(&p, K0, C0_ENTRYHI);
 291         uasm_i_ehb(&p);
 292
 293         /* Disable RDHWR access */
 294         uasm_i_mtc0(&p, ZERO, C0_HWRENA);
 295
 296         /* load the guest context from VCPU and return */
 297         for (i = 1; i < 32; ++i) {
 298                 /* Guest k0/k1 loaded later */
 299                 if (i == K0 || i == K1)
 300                         continue;
 301                 UASM_i_LW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1);
 302         }
 303
 304 #ifndef CONFIG_CPU_MIPSR6
 305         /* Restore hi/lo */
 306         UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, hi), K1);
 307         uasm_i_mthi(&p, K0);
 308
 309         UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, lo), K1);
 310         uasm_i_mtlo(&p, K0);
 311 #endif
 312
 313         /* Restore the guest's k0/k1 registers */
 314         UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, gprs[K0]), K1);
 315         UASM_i_LW(&p, K1, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1);
 316
 317         /* Jump to guest */
 318         uasm_i_eret(&p);
 319
 320         uasm_resolve_relocs(relocs, labels);
 321
 322         return p;
 323 }
 324
 325 /**
 326  * kvm_mips_build_exception() - Assemble first level guest exception handler.
 327  * @addr:       Address to start writing code.
 328  * @handler:    Address of common handler (within range of @addr).
 329  *
 330  * Assemble exception vector code for guest execution. The generated vector will
 331  * branch to the common exception handler generated by kvm_mips_build_exit().
 332  *
 333  * Returns:     Next address after end of written function.
 334  */
 335 void *kvm_mips_build_exception(void *addr, void *handler)
 336 {
 337         u32 *p = addr;
 338         struct uasm_label labels[2];
 339         struct uasm_reloc relocs[2];
 340         struct uasm_label *l = labels;
 341         struct uasm_reloc *r = relocs;
 342
 343         memset(labels, 0, sizeof(labels));
 344         memset(relocs, 0, sizeof(relocs));
 345
 346         /* Save guest k1 into scratch register */
 347         UASM_i_MTC0(&p, K1, scratch_tmp[0], scratch_tmp[1]);
 348
 349         /* Get the VCPU pointer from the VCPU scratch register */
 350         UASM_i_MFC0(&p, K1, scratch_vcpu[0], scratch_vcpu[1]);
 351         UASM_i_ADDIU(&p, K1, K1, offsetof(struct kvm_vcpu, arch));
 352
 353         /* Save guest k0 into VCPU structure */
 354         UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, gprs[K0]), K1);
 355
 356         /* Branch to the common handler */
 357         uasm_il_b(&p, &r, label_exit_common);
 358          uasm_i_nop(&p);
 359
 360         uasm_l_exit_common(&l, handler);
 361         uasm_resolve_relocs(relocs, labels);
 362
 363         return p;
 364 }
 365
 366 /**
 367  * kvm_mips_build_exit() - Assemble common guest exit handler.
 368  * @addr:       Address to start writing code.
 369  *
 370  * Assemble the generic guest exit handling code. This is called by the
 371  * exception vectors (generated by kvm_mips_build_exception()), and calls
 372  * kvm_mips_handle_exit(), then either resumes the guest or returns to the host
 373  * depending on the return value.
 374  *
 375  * Returns:     Next address after end of written function.
 376  */
 377 void *kvm_mips_build_exit(void *addr)
 378 {
 379         u32 *p = addr;
 380         unsigned int i;
 381         struct uasm_label labels[3];
 382         struct uasm_reloc relocs[3];
 383         struct uasm_label *l = labels;
 384         struct uasm_reloc *r = relocs;
 385
 386         memset(labels, 0, sizeof(labels));
 387         memset(relocs, 0, sizeof(relocs));
 388
 389         /*
 390          * Generic Guest exception handler. We end up here when the guest
 391          * does something that causes a trap to kernel mode.
 392          *
 393          * Both k0/k1 registers will have already been saved (k0 into the vcpu
 394          * structure, and k1 into the scratch_tmp register).
 395          *
 396          * The k1 register will already contain the kvm_vcpu_arch pointer.
 397          */
 398
 399         /* Start saving Guest context to VCPU */
 400         for (i = 0; i < 32; ++i) {
 401                 /* Guest k0/k1 saved later */
 402                 if (i == K0 || i == K1)
 403                         continue;
 404                 UASM_i_SW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1);
 405         }
 406
 407 #ifndef CONFIG_CPU_MIPSR6
 408         /* We need to save hi/lo and restore them on the way out */
 409         uasm_i_mfhi(&p, T0);
 410         UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, hi), K1);
 411
 412         uasm_i_mflo(&p, T0);
 413         UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, lo), K1);
 414 #endif
 415
 416         /* Finally save guest k1 to VCPU */
 417         uasm_i_ehb(&p);
 418         UASM_i_MFC0(&p, T0, scratch_tmp[0], scratch_tmp[1]);
 419         UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1);
 420
 421         /* Now that context has been saved, we can use other registers */
 422
 423         /* Restore vcpu */
 424         UASM_i_MFC0(&p, A1, scratch_vcpu[0], scratch_vcpu[1]);
 425         uasm_i_move(&p, S1, A1);
 426
 427         /* Restore run (vcpu->run) */
 428         UASM_i_LW(&p, A0, offsetof(struct kvm_vcpu, run), A1);
 429         /* Save pointer to run in s0, will be saved by the compiler */
 430         uasm_i_move(&p, S0, A0);
 431
 432         /*
 433          * Save Host level EPC, BadVaddr and Cause to VCPU, useful to process
 434          * the exception
 435          */
 436         UASM_i_MFC0(&p, K0, C0_EPC);
 437         UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, pc), K1);
 438
 439         UASM_i_MFC0(&p, K0, C0_BADVADDR);
 440         UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_badvaddr),
 441                   K1);
 442
 443         uasm_i_mfc0(&p, K0, C0_CAUSE);
 444         uasm_i_sw(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_cause), K1);
 445
 446         /* Now restore the host state just enough to run the handlers */
 447
 448         /* Switch EBASE to the one used by Linux */
 449         /* load up the host EBASE */
 450         uasm_i_mfc0(&p, V0, C0_STATUS);
 451
 452         uasm_i_lui(&p, AT, ST0_BEV >> 16);
 453         uasm_i_or(&p, K0, V0, AT);
 454
 455         uasm_i_mtc0(&p, K0, C0_STATUS);
 456         uasm_i_ehb(&p);
 457
 458         UASM_i_LA_mostly(&p, K0, (long)&ebase);
 459         UASM_i_LW(&p, K0, uasm_rel_lo((long)&ebase), K0);
 460         uasm_i_mtc0(&p, K0, C0_EBASE);
 461
 462         if (raw_cpu_has_fpu) {
 463                 /*
 464                  * If FPU is enabled, save FCR31 and clear it so that later
 465                  * ctc1's don't trigger FPE for pending exceptions.
 466                  */
 467                 uasm_i_lui(&p, AT, ST0_CU1 >> 16);
 468                 uasm_i_and(&p, V1, V0, AT);
 469                 uasm_il_beqz(&p, &r, V1, label_fpu_1);
 470                  uasm_i_nop(&p);
 471                 uasm_i_cfc1(&p, T0, 31);
 472                 uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.fcr31),
 473                           K1);
 474                 uasm_i_ctc1(&p, ZERO, 31);
 475                 uasm_l_fpu_1(&l, p);
 476         }
 477
 478         if (cpu_has_msa) {
 479                 /*
 480                  * If MSA is enabled, save MSACSR and clear it so that later
 481                  * instructions don't trigger MSAFPE for pending exceptions.
 482                  */
 483                 uasm_i_mfc0(&p, T0, C0_CONFIG5);
 484                 uasm_i_ext(&p, T0, T0, 27, 1); /* MIPS_CONF5_MSAEN */
 485                 uasm_il_beqz(&p, &r, T0, label_msa_1);
 486                  uasm_i_nop(&p);
 487                 uasm_i_cfcmsa(&p, T0, MSA_CSR);
 488                 uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.msacsr),
 489                           K1);
 490                 uasm_i_ctcmsa(&p, MSA_CSR, ZERO);
 491                 uasm_l_msa_1(&l, p);
 492         }
 493
 494         /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */
 495         uasm_i_addiu(&p, AT, ZERO, ~(ST0_EXL | KSU_USER | ST0_IE));
 496         uasm_i_and(&p, V0, V0, AT);
 497         uasm_i_lui(&p, AT, ST0_CU0 >> 16);
 498         uasm_i_or(&p, V0, V0, AT);
 499         uasm_i_mtc0(&p, V0, C0_STATUS);
 500         uasm_i_ehb(&p);
 501
 502         /* Load up host GP */
 503         UASM_i_LW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1);
 504
 505         /* Need a stack before we can jump to "C" */
 506         UASM_i_LW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1);
 507
 508         /* Saved host state */
 509         UASM_i_ADDIU(&p, SP, SP, -(int)sizeof(struct pt_regs));
 510
 511         /*
 512          * XXXKYMA do we need to load the host ASID, maybe not because the
 513          * kernel entries are marked GLOBAL, need to verify
 514          */
 515
 516         /* Restore host scratch registers, as we'll have clobbered them */
 517         kvm_mips_build_restore_scratch(&p, K0, SP);
 518
 519         /* Restore RDHWR access */
 520         UASM_i_LA_mostly(&p, K0, (long)&hwrena);
 521         uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0);
 522         uasm_i_mtc0(&p, K0, C0_HWRENA);
 523
 524         /* Jump to handler */
 525         /*
 526          * XXXKYMA: not sure if this is safe, how large is the stack??
 527          * Now jump to the kvm_mips_handle_exit() to see if we can deal
 528          * with this in the kernel
 529          */
 530         UASM_i_LA(&p, T9, (unsigned long)kvm_mips_handle_exit);
 531         uasm_i_jalr(&p, RA, T9);
 532          UASM_i_ADDIU(&p, SP, SP, -CALLFRAME_SIZ);
 533
 534         uasm_resolve_relocs(relocs, labels);
 535
 536         p = kvm_mips_build_ret_from_exit(p);
 537
 538         return p;
 539 }
 540
 541 /**
 542  * kvm_mips_build_ret_from_exit() - Assemble guest exit return handler.
 543  * @addr:       Address to start writing code.
 544  *
 545  * Assemble the code to handle the return from kvm_mips_handle_exit(), either
 546  * resuming the guest or returning to the host depending on the return value.
 547  *
 548  * Returns:     Next address after end of written function.
 549  */
 550 static void *kvm_mips_build_ret_from_exit(void *addr)
 551 {
 552         u32 *p = addr;
 553         struct uasm_label labels[2];
 554         struct uasm_reloc relocs[2];
 555         struct uasm_label *l = labels;
 556         struct uasm_reloc *r = relocs;
 557
 558         memset(labels, 0, sizeof(labels));
 559         memset(relocs, 0, sizeof(relocs));
 560
 561         /* Return from handler Make sure interrupts are disabled */
 562         uasm_i_di(&p, ZERO);
 563         uasm_i_ehb(&p);
 564
 565         /*
 566          * XXXKYMA: k0/k1 could have been blown away if we processed
 567          * an exception while we were handling the exception from the
 568          * guest, reload k1
 569          */
 570
 571         uasm_i_move(&p, K1, S1);
 572         UASM_i_ADDIU(&p, K1, K1, offsetof(struct kvm_vcpu, arch));
 573
 574         /*
 575          * Check return value, should tell us if we are returning to the
 576          * host (handle I/O etc)or resuming the guest
 577          */
 578         uasm_i_andi(&p, T0, V0, RESUME_HOST);
 579         uasm_il_bnez(&p, &r, T0, label_return_to_host);
 580          uasm_i_nop(&p);
 581
 582         p = kvm_mips_build_ret_to_guest(p);
 583
 584         uasm_l_return_to_host(&l, p);
 585         p = kvm_mips_build_ret_to_host(p);
 586
 587         uasm_resolve_relocs(relocs, labels);
 588
 589         return p;
 590 }
 591
 592 /**
 593  * kvm_mips_build_ret_to_guest() - Assemble code to return to the guest.
 594  * @addr:       Address to start writing code.
 595  *
 596  * Assemble the code to handle return from the guest exit handler
 597  * (kvm_mips_handle_exit()) back to the guest.
 598  *
 599  * Returns:     Next address after end of written function.
 600  */
 601 static void *kvm_mips_build_ret_to_guest(void *addr)
 602 {
 603         u32 *p = addr;
 604
 605         /* Put the saved pointer to vcpu (s1) back into the scratch register */
 606         UASM_i_MTC0(&p, S1, scratch_vcpu[0], scratch_vcpu[1]);
 607
 608         /* Load up the Guest EBASE to minimize the window where BEV is set */
 609         UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, guest_ebase), K1);
 610
 611         /* Switch EBASE back to the one used by KVM */
 612         uasm_i_mfc0(&p, V1, C0_STATUS);
 613         uasm_i_lui(&p, AT, ST0_BEV >> 16);
 614         uasm_i_or(&p, K0, V1, AT);
 615         uasm_i_mtc0(&p, K0, C0_STATUS);
 616         uasm_i_ehb(&p);
 617         uasm_i_mtc0(&p, T0, C0_EBASE);
 618
 619         /* Setup status register for running guest in UM */
 620         uasm_i_ori(&p, V1, V1, ST0_EXL | KSU_USER | ST0_IE);
 621         UASM_i_LA(&p, AT, ~(ST0_CU0 | ST0_MX));
 622         uasm_i_and(&p, V1, V1, AT);
 623         uasm_i_mtc0(&p, V1, C0_STATUS);
 624         uasm_i_ehb(&p);
 625
 626         p = kvm_mips_build_enter_guest(p);
 627
 628         return p;
 629 }
 630
 631 /**
 632  * kvm_mips_build_ret_to_host() - Assemble code to return to the host.
 633  * @addr:       Address to start writing code.
 634  *
 635  * Assemble the code to handle return from the guest exit handler
 636  * (kvm_mips_handle_exit()) back to the host, i.e. to the caller of the vcpu_run
 637  * function generated by kvm_mips_build_vcpu_run().
 638  *
 639  * Returns:     Next address after end of written function.
 640  */
 641 static void *kvm_mips_build_ret_to_host(void *addr)
 642 {
 643         u32 *p = addr;
 644         unsigned int i;
 645
 646         /* EBASE is already pointing to Linux */
 647         UASM_i_LW(&p, K1, offsetof(struct kvm_vcpu_arch, host_stack), K1);
 648         UASM_i_ADDIU(&p, K1, K1, -(int)sizeof(struct pt_regs));
 649
 650         /*
 651          * r2/v0 is the return code, shift it down by 2 (arithmetic)
 652          * to recover the err code
 653          */
 654         uasm_i_sra(&p, K0, V0, 2);
 655         uasm_i_move(&p, V0, K0);
 656
 657         /* Load context saved on the host stack */
 658         for (i = 16; i < 31; ++i) {
 659                 if (i == 24)
 660                         i = 28;
 661                 UASM_i_LW(&p, i, offsetof(struct pt_regs, regs[i]), K1);
 662         }
 663
 664         /* Restore RDHWR access */
 665         UASM_i_LA_mostly(&p, K0, (long)&hwrena);
 666         uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0);
 667         uasm_i_mtc0(&p, K0, C0_HWRENA);
 668
 669         /* Restore RA, which is the address we will return to */
 670         UASM_i_LW(&p, RA, offsetof(struct pt_regs, regs[RA]), K1);
 671         uasm_i_jr(&p, RA);
 672          uasm_i_nop(&p);
 673
 674         return p;
 675 }
 676