arch/s390/kernel/fpu.c

   1 /*
   2  * In-kernel vector facility support functions
   3  *
   4  * Copyright IBM Corp. 2015
   5  * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/cpu.h>
   9 #include <linux/sched.h>
  10 #include <asm/fpu/types.h>
  11 #include <asm/fpu/api.h>
  12
  13 /*
  14  * Per-CPU variable to maintain FPU register ranges that are in use
  15  * by the kernel.
  16  */
  17 static DEFINE_PER_CPU(u32, kernel_fpu_state);
  18
  19 #define KERNEL_FPU_STATE_MASK   (KERNEL_FPU_MASK|KERNEL_FPC)
  20
  21
  22 void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
  23 {
  24         if (!__this_cpu_read(kernel_fpu_state)) {
  25                 /*
  26                  * Save user space FPU state and register contents.  Multiple
  27                  * calls because of interruptions do not matter and return
  28                  * immediately.  This also sets CIF_FPU to lazy restore FP/VX
  29                  * register contents when returning to user space.
  30                  */
  31                 save_fpu_regs();
  32         }
  33
  34         /* Update flags to use the vector facility for KERNEL_FPR */
  35         if (MACHINE_HAS_VX && (state->mask & KERNEL_FPR)) {
  36                 flags |= KERNEL_VXR_LOW | KERNEL_FPC;
  37                 flags &= ~KERNEL_FPR;
  38         }
  39
  40         /* Save and update current kernel VX state */
  41         state->mask = __this_cpu_read(kernel_fpu_state);
  42         __this_cpu_or(kernel_fpu_state, flags & KERNEL_FPU_STATE_MASK);
  43
  44         /*
  45          * If this is the first call to __kernel_fpu_begin(), no additional
  46          * work is required.
  47          */
  48         if (!(state->mask & KERNEL_FPU_STATE_MASK))
  49                 return;
  50
  51         /*
  52          * If KERNEL_FPR is still set, the vector facility is not available
  53          * and, thus, save floating-point control and registers only.
  54          */
  55         if (state->mask & KERNEL_FPR) {
  56                 asm volatile("stfpc %0" : "=Q" (state->fpc));
  57                 asm volatile("std 0,%0" : "=Q" (state->fprs[0]));
  58                 asm volatile("std 1,%0" : "=Q" (state->fprs[1]));
  59                 asm volatile("std 2,%0" : "=Q" (state->fprs[2]));
  60                 asm volatile("std 3,%0" : "=Q" (state->fprs[3]));
  61                 asm volatile("std 4,%0" : "=Q" (state->fprs[4]));
  62                 asm volatile("std 5,%0" : "=Q" (state->fprs[5]));
  63                 asm volatile("std 6,%0" : "=Q" (state->fprs[6]));
  64                 asm volatile("std 7,%0" : "=Q" (state->fprs[7]));
  65                 asm volatile("std 8,%0" : "=Q" (state->fprs[8]));
  66                 asm volatile("std 9,%0" : "=Q" (state->fprs[9]));
  67                 asm volatile("std 10,%0" : "=Q" (state->fprs[10]));
  68                 asm volatile("std 11,%0" : "=Q" (state->fprs[11]));
  69                 asm volatile("std 12,%0" : "=Q" (state->fprs[12]));
  70                 asm volatile("std 13,%0" : "=Q" (state->fprs[13]));
  71                 asm volatile("std 14,%0" : "=Q" (state->fprs[14]));
  72                 asm volatile("std 15,%0" : "=Q" (state->fprs[15]));
  73                 return;
  74         }
  75
  76         /*
  77          * If this is a nested call to __kernel_fpu_begin(), check the saved
  78          * state mask to save and later restore the vector registers that
  79          * are already in use.  Let's start with checking floating-point
  80          * controls.
  81          */
  82         if (state->mask & KERNEL_FPC)
  83                 asm volatile("stfpc %0" : "=m" (state->fpc));
  84
  85         /* Test and save vector registers */
  86         asm volatile (
  87                 /*
  88                  * Test if any vector register must be saved and, if so,
  89                  * test if all register can be saved.
  90                  */
  91                 "       tmll    %[m],15\n"      /* KERNEL_VXR_MASK */
  92                 "       jz      20f\n"          /* no work -> done */
  93                 "       la      1,%[vxrs]\n"    /* load save area */
  94                 "       jo      18f\n"          /* -> save V0..V31 */
  95
  96                 /*
  97                  * Test if V8..V23 can be saved at once... this speeds up
  98                  * for KERNEL_fpu_MID only. Otherwise continue to split the
  99                  * range of vector registers into two halves and test them
 100                  * separately.
 101                  */
 102                 "       tmll    %[m],6\n"       /* KERNEL_VXR_MID */
 103                 "       jo      17f\n"          /* -> save V8..V23 */
 104
 105                 /* Test and save the first half of 16 vector registers */
 106                 "1:     tmll    %[m],3\n"       /* KERNEL_VXR_LOW */
 107                 "       jz      10f\n"          /* -> KERNEL_VXR_HIGH */
 108                 "       jo      2f\n"           /* 11 -> save V0..V15 */
 109                 "       brc     4,3f\n"         /* 01 -> save V0..V7  */
 110                 "       brc     2,4f\n"         /* 10 -> save V8..V15 */
 111
 112                 /* Test and save the second half of 16 vector registers */
 113                 "10:    tmll    %[m],12\n"      /* KERNEL_VXR_HIGH */
 114                 "       jo      19f\n"          /* 11 -> save V16..V31 */
 115                 "       brc     4,11f\n"        /* 01 -> save V16..V23  */
 116                 "       brc     2,12f\n"        /* 10 -> save V24..V31 */
 117                 "       j       20f\n"          /* 00 -> done */
 118
 119                 /*
 120                  * Below are the vstm combinations to save multiple vector
 121                  * registers at once.
 122                  */
 123                 "2:     .word   0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */
 124                 "       j       10b\n"                  /* -> VXR_HIGH */
 125                 "3:     .word   0xe707,0x1000,0x003e\n" /* vstm 0,7,0(1) */
 126                 "       j       10b\n"                  /* -> VXR_HIGH */
 127                 "4:     .word   0xe78f,0x1080,0x003e\n" /* vstm 8,15,128(1) */
 128                 "       j       10b\n"                  /* -> VXR_HIGH */
 129                 "\n"
 130                 "11:    .word   0xe707,0x1100,0x0c3e\n" /* vstm 16,23,256(1) */
 131                 "       j       20f\n"                  /* -> done */
 132                 "12:    .word   0xe78f,0x1180,0x0c3e\n" /* vstm 24,31,384(1) */
 133                 "       j       20f\n"                  /* -> done */
 134                 "\n"
 135                 "17:    .word   0xe787,0x1080,0x043e\n" /* vstm 8,23,128(1) */
 136                 "       nill    %[m],249\n"             /* m &= ~VXR_MID    */
 137                 "       j       1b\n"                   /* -> VXR_LOW */
 138                 "\n"
 139                 "18:    .word   0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */
 140                 "19:    .word   0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */
 141                 "20:"
 142                 : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
 143                 : [m] "d" (state->mask)
 144                 : "1", "cc");
 145 }
 146 EXPORT_SYMBOL(__kernel_fpu_begin);
 147
 148 void __kernel_fpu_end(struct kernel_fpu *state)
 149 {
 150         /* Just update the per-CPU state if there is nothing to restore */
 151         if (!(state->mask & KERNEL_FPU_STATE_MASK))
 152                 goto update_fpu_state;
 153
 154         /*
 155          * If KERNEL_FPR is specified, the vector facility is not available
 156          * and, thus, restore floating-point control and registers only.
 157          */
 158         if (state->mask & KERNEL_FPR) {
 159                 asm volatile("lfpc %0" : : "Q" (state->fpc));
 160                 asm volatile("ld 0,%0" : : "Q" (state->fprs[0]));
 161                 asm volatile("ld 1,%0" : : "Q" (state->fprs[1]));
 162                 asm volatile("ld 2,%0" : : "Q" (state->fprs[2]));
 163                 asm volatile("ld 3,%0" : : "Q" (state->fprs[3]));
 164                 asm volatile("ld 4,%0" : : "Q" (state->fprs[4]));
 165                 asm volatile("ld 5,%0" : : "Q" (state->fprs[5]));
 166                 asm volatile("ld 6,%0" : : "Q" (state->fprs[6]));
 167                 asm volatile("ld 7,%0" : : "Q" (state->fprs[7]));
 168                 asm volatile("ld 8,%0" : : "Q" (state->fprs[8]));
 169                 asm volatile("ld 9,%0" : : "Q" (state->fprs[9]));
 170                 asm volatile("ld 10,%0" : : "Q" (state->fprs[10]));
 171                 asm volatile("ld 11,%0" : : "Q" (state->fprs[11]));
 172                 asm volatile("ld 12,%0" : : "Q" (state->fprs[12]));
 173                 asm volatile("ld 13,%0" : : "Q" (state->fprs[13]));
 174                 asm volatile("ld 14,%0" : : "Q" (state->fprs[14]));
 175                 asm volatile("ld 15,%0" : : "Q" (state->fprs[15]));
 176                 goto update_fpu_state;
 177         }
 178
 179         /* Test and restore floating-point controls */
 180         if (state->mask & KERNEL_FPC)
 181                 asm volatile("lfpc %0" : : "Q" (state->fpc));
 182
 183         /* Test and restore (load) vector registers */
 184         asm volatile (
 185                 /*
 186                  * Test if any vector registers must be loaded and, if so,
 187                  * test if all registers can be loaded at once.
 188                  */
 189                 "       tmll    %[m],15\n"      /* KERNEL_VXR_MASK */
 190                 "       jz      20f\n"          /* no work -> done */
 191                 "       la      1,%[vxrs]\n"    /* load load area */
 192                 "       jo      18f\n"          /* -> load V0..V31 */
 193
 194                 /*
 195                  * Test if V8..V23 can be restored at once... this speeds up
 196                  * for KERNEL_VXR_MID only. Otherwise continue to split the
 197                  * range of vector registers into two halves and test them
 198                  * separately.
 199                  */
 200                 "       tmll    %[m],6\n"       /* KERNEL_VXR_MID */
 201                 "       jo      17f\n"          /* -> load V8..V23 */
 202
 203                 /* Test and load the first half of 16 vector registers */
 204                 "1:     tmll    %[m],3\n"       /* KERNEL_VXR_LOW */
 205                 "       jz      10f\n"          /* -> KERNEL_VXR_HIGH */
 206                 "       jo      2f\n"           /* 11 -> load V0..V15 */
 207                 "       brc     4,3f\n"         /* 01 -> load V0..V7  */
 208                 "       brc     2,4f\n"         /* 10 -> load V8..V15 */
 209
 210                 /* Test and load the second half of 16 vector registers */
 211                 "10:    tmll    %[m],12\n"      /* KERNEL_VXR_HIGH */
 212                 "       jo      19f\n"          /* 11 -> load V16..V31 */
 213                 "       brc     4,11f\n"        /* 01 -> load V16..V23  */
 214                 "       brc     2,12f\n"        /* 10 -> load V24..V31 */
 215                 "       j       20f\n"          /* 00 -> done */
 216
 217                 /*
 218                  * Below are the vstm combinations to load multiple vector
 219                  * registers at once.
 220                  */
 221                 "2:     .word   0xe70f,0x1000,0x0036\n" /* vlm 0,15,0(1) */
 222                 "       j       10b\n"                  /* -> VXR_HIGH */
 223                 "3:     .word   0xe707,0x1000,0x0036\n" /* vlm 0,7,0(1) */
 224                 "       j       10b\n"                  /* -> VXR_HIGH */
 225                 "4:     .word   0xe78f,0x1080,0x0036\n" /* vlm 8,15,128(1) */
 226                 "       j       10b\n"                  /* -> VXR_HIGH */
 227                 "\n"
 228                 "11:    .word   0xe707,0x1100,0x0c36\n" /* vlm 16,23,256(1) */
 229                 "       j       20f\n"                  /* -> done */
 230                 "12:    .word   0xe78f,0x1180,0x0c36\n" /* vlm 24,31,384(1) */
 231                 "       j       20f\n"                  /* -> done */
 232                 "\n"
 233                 "17:    .word   0xe787,0x1080,0x0436\n" /* vlm 8,23,128(1) */
 234                 "       nill    %[m],249\n"             /* m &= ~VXR_MID    */
 235                 "       j       1b\n"                   /* -> VXR_LOW */
 236                 "\n"
 237                 "18:    .word   0xe70f,0x1000,0x0036\n" /* vlm 0,15,0(1) */
 238                 "19:    .word   0xe70f,0x1100,0x0c36\n" /* vlm 16,31,256(1) */
 239                 "20:"
 240                 :
 241                 : [vxrs] "Q" (*(struct vx_array *) &state->vxrs),
 242                   [m] "d" (state->mask)
 243                 : "1", "cc");
 244
 245 update_fpu_state:
 246         /* Update current kernel VX state */
 247         __this_cpu_write(kernel_fpu_state, state->mask);
 248 }
 249 EXPORT_SYMBOL(__kernel_fpu_end);