arch/sparc/mm/hypersparc.S

   1 /* $Id: hypersparc.S,v 1.18 2001/12/21 04:56:15 davem Exp $
   2  * hypersparc.S: High speed Hypersparc mmu/cache operations.
   3  *
   4  * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
   5  */
   6
   7 #include <asm/ptrace.h>
   8 #include <asm/psr.h>
   9 #include <asm/asm_offsets.h>
  10 #include <asm/asi.h>
  11 #include <asm/page.h>
  12 #include <asm/pgtsrmmu.h>
  13 #include <linux/config.h>
  14 #include <linux/init.h>
  15
  16         .text
  17         .align  4
  18
  19         .globl  hypersparc_flush_cache_all, hypersparc_flush_cache_mm
  20         .globl  hypersparc_flush_cache_range, hypersparc_flush_cache_page
  21         .globl  hypersparc_flush_page_to_ram
  22         .globl  hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns
  23         .globl  hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm
  24         .globl  hypersparc_flush_tlb_range, hypersparc_flush_tlb_page
  25
  26 hypersparc_flush_cache_all:
  27         WINDOW_FLUSH(%g4, %g5)
  28         sethi   %hi(vac_cache_size), %g4
  29         ld      [%g4 + %lo(vac_cache_size)], %g5
  30         sethi   %hi(vac_line_size), %g1
  31         ld      [%g1 + %lo(vac_line_size)], %g2
  32 1:
  33         subcc   %g5, %g2, %g5                   ! hyper_flush_unconditional_combined
  34         bne     1b
  35          sta    %g0, [%g5] ASI_M_FLUSH_CTX
  36         retl
  37          sta    %g0, [%g0] ASI_M_FLUSH_IWHOLE   ! hyper_flush_whole_icache
  38
  39         /* We expand the window flush to get maximum performance. */
  40 hypersparc_flush_cache_mm:
  41 #ifndef CONFIG_SMP
  42         ld      [%o0 + AOFF_mm_context], %g1
  43         cmp     %g1, -1
  44         be      hypersparc_flush_cache_mm_out
  45 #endif
  46         WINDOW_FLUSH(%g4, %g5)
  47
  48         sethi   %hi(vac_line_size), %g1
  49         ld      [%g1 + %lo(vac_line_size)], %o1
  50         sethi   %hi(vac_cache_size), %g2
  51         ld      [%g2 + %lo(vac_cache_size)], %o0
  52         add     %o1, %o1, %g1
  53         add     %o1, %g1, %g2
  54         add     %o1, %g2, %g3
  55         add     %o1, %g3, %g4
  56         add     %o1, %g4, %g5
  57         add     %o1, %g5, %o4
  58         add     %o1, %o4, %o5
  59
  60         /* BLAMMO! */
  61 1:
  62         subcc   %o0, %o5, %o0                           ! hyper_flush_cache_user
  63         sta     %g0, [%o0 + %g0] ASI_M_FLUSH_USER
  64         sta     %g0, [%o0 + %o1] ASI_M_FLUSH_USER
  65         sta     %g0, [%o0 + %g1] ASI_M_FLUSH_USER
  66         sta     %g0, [%o0 + %g2] ASI_M_FLUSH_USER
  67         sta     %g0, [%o0 + %g3] ASI_M_FLUSH_USER
  68         sta     %g0, [%o0 + %g4] ASI_M_FLUSH_USER
  69         sta     %g0, [%o0 + %g5] ASI_M_FLUSH_USER
  70         bne     1b
  71          sta    %g0, [%o0 + %o4] ASI_M_FLUSH_USER
  72 hypersparc_flush_cache_mm_out:
  73         retl
  74          nop
  75
  76         /* The things we do for performance... */
  77 hypersparc_flush_cache_range:
  78         ld      [%o0 + 0x0], %o0                /* XXX vma->vm_mm, GROSS XXX */
  79 #ifndef CONFIG_SMP
  80         ld      [%o0 + AOFF_mm_context], %g1
  81         cmp     %g1, -1
  82         be      hypersparc_flush_cache_range_out
  83 #endif
  84         WINDOW_FLUSH(%g4, %g5)
  85
  86         sethi   %hi(vac_line_size), %g1
  87         ld      [%g1 + %lo(vac_line_size)], %o4
  88         sethi   %hi(vac_cache_size), %g2
  89         ld      [%g2 + %lo(vac_cache_size)], %o3
  90
  91         /* Here comes the fun part... */
  92         add     %o2, (PAGE_SIZE - 1), %o2
  93         andn    %o1, (PAGE_SIZE - 1), %o1
  94         add     %o4, %o4, %o5
  95         andn    %o2, (PAGE_SIZE - 1), %o2
  96         add     %o4, %o5, %g1
  97         sub     %o2, %o1, %g4
  98         add     %o4, %g1, %g2
  99         sll     %o3, 2, %g5
 100         add     %o4, %g2, %g3
 101         cmp     %g4, %g5
 102         add     %o4, %g3, %g4
 103         blu     0f
 104          add    %o4, %g4, %g5
 105         add     %o4, %g5, %g7
 106
 107         /* Flush entire user space, believe it or not this is quicker
 108          * than page at a time flushings for range > (cache_size<<2).
 109          */
 110 1:
 111         subcc   %o3, %g7, %o3
 112         sta     %g0, [%o3 + %g0] ASI_M_FLUSH_USER
 113         sta     %g0, [%o3 + %o4] ASI_M_FLUSH_USER
 114         sta     %g0, [%o3 + %o5] ASI_M_FLUSH_USER
 115         sta     %g0, [%o3 + %g1] ASI_M_FLUSH_USER
 116         sta     %g0, [%o3 + %g2] ASI_M_FLUSH_USER
 117         sta     %g0, [%o3 + %g3] ASI_M_FLUSH_USER
 118         sta     %g0, [%o3 + %g4] ASI_M_FLUSH_USER
 119         bne     1b
 120          sta    %g0, [%o3 + %g5] ASI_M_FLUSH_USER
 121         retl
 122          nop
 123
 124         /* Below our threshold, flush one page at a time. */
 125 0:
 126         ld      [%o0 + AOFF_mm_context], %o0
 127         mov     SRMMU_CTX_REG, %g7
 128         lda     [%g7] ASI_M_MMUREGS, %o3
 129         sta     %o0, [%g7] ASI_M_MMUREGS
 130         add     %o2, -PAGE_SIZE, %o0
 131 1:
 132         or      %o0, 0x400, %g7
 133         lda     [%g7] ASI_M_FLUSH_PROBE, %g7
 134         orcc    %g7, 0, %g0
 135         be,a    3f
 136          mov    %o0, %o2
 137         add     %o4, %g5, %g7
 138 2:
 139         sub     %o2, %g7, %o2
 140         sta     %g0, [%o2 + %g0] ASI_M_FLUSH_PAGE
 141         sta     %g0, [%o2 + %o4] ASI_M_FLUSH_PAGE
 142         sta     %g0, [%o2 + %o5] ASI_M_FLUSH_PAGE
 143         sta     %g0, [%o2 + %g1] ASI_M_FLUSH_PAGE
 144         sta     %g0, [%o2 + %g2] ASI_M_FLUSH_PAGE
 145         sta     %g0, [%o2 + %g3] ASI_M_FLUSH_PAGE
 146         andcc   %o2, 0xffc, %g0
 147         sta     %g0, [%o2 + %g4] ASI_M_FLUSH_PAGE
 148         bne     2b
 149          sta    %g0, [%o2 + %g5] ASI_M_FLUSH_PAGE
 150 3:
 151         cmp     %o2, %o1
 152         bne     1b
 153          add    %o2, -PAGE_SIZE, %o0
 154         mov     SRMMU_FAULT_STATUS, %g5
 155         lda     [%g5] ASI_M_MMUREGS, %g0
 156         mov     SRMMU_CTX_REG, %g7
 157         sta     %o3, [%g7] ASI_M_MMUREGS
 158 hypersparc_flush_cache_range_out:
 159         retl
 160          nop
 161
 162         /* HyperSparc requires a valid mapping where we are about to flush
 163          * in order to check for a physical tag match during the flush.
 164          */
 165         /* Verified, my ass... */
 166 hypersparc_flush_cache_page:
 167         ld      [%o0 + 0x0], %o0                /* XXX vma->vm_mm, GROSS XXX */
 168         ld      [%o0 + AOFF_mm_context], %g2
 169 #ifndef CONFIG_SMP
 170         cmp     %g2, -1
 171         be      hypersparc_flush_cache_page_out
 172 #endif
 173         WINDOW_FLUSH(%g4, %g5)
 174
 175         sethi   %hi(vac_line_size), %g1
 176         ld      [%g1 + %lo(vac_line_size)], %o4
 177         mov     SRMMU_CTX_REG, %o3
 178         andn    %o1, (PAGE_SIZE - 1), %o1
 179         lda     [%o3] ASI_M_MMUREGS, %o2
 180         sta     %g2, [%o3] ASI_M_MMUREGS
 181         or      %o1, 0x400, %o5
 182         lda     [%o5] ASI_M_FLUSH_PROBE, %g1
 183         orcc    %g0, %g1, %g0
 184         be      2f
 185          add    %o4, %o4, %o5
 186         sub     %o1, -PAGE_SIZE, %o1
 187         add     %o4, %o5, %g1
 188         add     %o4, %g1, %g2
 189         add     %o4, %g2, %g3
 190         add     %o4, %g3, %g4
 191         add     %o4, %g4, %g5
 192         add     %o4, %g5, %g7
 193
 194         /* BLAMMO! */
 195 1:
 196         sub     %o1, %g7, %o1
 197         sta     %g0, [%o1 + %g0] ASI_M_FLUSH_PAGE
 198         sta     %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
 199         sta     %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
 200         sta     %g0, [%o1 + %g1] ASI_M_FLUSH_PAGE
 201         sta     %g0, [%o1 + %g2] ASI_M_FLUSH_PAGE
 202         sta     %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
 203         andcc   %o1, 0xffc, %g0
 204         sta     %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
 205         bne     1b
 206          sta    %g0, [%o1 + %g5] ASI_M_FLUSH_PAGE
 207 2:
 208         mov     SRMMU_FAULT_STATUS, %g7
 209         mov     SRMMU_CTX_REG, %g4
 210         lda     [%g7] ASI_M_MMUREGS, %g0
 211         sta     %o2, [%g4] ASI_M_MMUREGS
 212 hypersparc_flush_cache_page_out:
 213         retl
 214          nop
 215
 216 hypersparc_flush_sig_insns:
 217         flush   %o1
 218         retl
 219          flush  %o1 + 4
 220
 221         /* HyperSparc is copy-back. */
 222 hypersparc_flush_page_to_ram:
 223         sethi   %hi(vac_line_size), %g1
 224         ld      [%g1 + %lo(vac_line_size)], %o4
 225         andn    %o0, (PAGE_SIZE - 1), %o0
 226         add     %o4, %o4, %o5
 227         or      %o0, 0x400, %g7
 228         lda     [%g7] ASI_M_FLUSH_PROBE, %g5
 229         add     %o4, %o5, %g1
 230         orcc    %g5, 0, %g0
 231         be      2f
 232          add    %o4, %g1, %g2
 233         add     %o4, %g2, %g3
 234         sub     %o0, -PAGE_SIZE, %o0
 235         add     %o4, %g3, %g4
 236         add     %o4, %g4, %g5
 237         add     %o4, %g5, %g7
 238
 239         /* BLAMMO! */
 240 1:
 241         sub     %o0, %g7, %o0
 242         sta     %g0, [%o0 + %g0] ASI_M_FLUSH_PAGE
 243         sta     %g0, [%o0 + %o4] ASI_M_FLUSH_PAGE
 244         sta     %g0, [%o0 + %o5] ASI_M_FLUSH_PAGE
 245         sta     %g0, [%o0 + %g1] ASI_M_FLUSH_PAGE
 246         sta     %g0, [%o0 + %g2] ASI_M_FLUSH_PAGE
 247         sta     %g0, [%o0 + %g3] ASI_M_FLUSH_PAGE
 248         andcc   %o0, 0xffc, %g0
 249         sta     %g0, [%o0 + %g4] ASI_M_FLUSH_PAGE
 250         bne     1b
 251          sta    %g0, [%o0 + %g5] ASI_M_FLUSH_PAGE
 252 2:
 253         mov     SRMMU_FAULT_STATUS, %g1
 254         retl
 255          lda    [%g1] ASI_M_MMUREGS, %g0
 256
 257         /* HyperSparc is IO cache coherent. */
 258 hypersparc_flush_page_for_dma:
 259         retl
 260          nop
 261
 262         /* It was noted that at boot time a TLB flush all in a delay slot
 263          * can deliver an illegal instruction to the processor if the timing
 264          * is just right...
 265          */
 266 hypersparc_flush_tlb_all:
 267         mov     0x400, %g1
 268         sta     %g0, [%g1] ASI_M_FLUSH_PROBE
 269         retl
 270          nop
 271
 272 hypersparc_flush_tlb_mm:
 273         mov     SRMMU_CTX_REG, %g1
 274         ld      [%o0 + AOFF_mm_context], %o1
 275         lda     [%g1] ASI_M_MMUREGS, %g5
 276 #ifndef CONFIG_SMP
 277         cmp     %o1, -1
 278         be      hypersparc_flush_tlb_mm_out
 279 #endif
 280          mov    0x300, %g2
 281         sta     %o1, [%g1] ASI_M_MMUREGS
 282         sta     %g0, [%g2] ASI_M_FLUSH_PROBE
 283 hypersparc_flush_tlb_mm_out:
 284         retl
 285          sta    %g5, [%g1] ASI_M_MMUREGS
 286
 287 hypersparc_flush_tlb_range:
 288         ld      [%o0 + 0x00], %o0       /* XXX vma->vm_mm GROSS XXX */
 289         mov     SRMMU_CTX_REG, %g1
 290         ld      [%o0 + AOFF_mm_context], %o3
 291         lda     [%g1] ASI_M_MMUREGS, %g5
 292 #ifndef CONFIG_SMP
 293         cmp     %o3, -1
 294         be      hypersparc_flush_tlb_range_out
 295 #endif
 296          sethi  %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4
 297         sta     %o3, [%g1] ASI_M_MMUREGS
 298         and     %o1, %o4, %o1
 299         add     %o1, 0x200, %o1
 300         sta     %g0, [%o1] ASI_M_FLUSH_PROBE
 301 1:
 302         sub     %o1, %o4, %o1
 303         cmp     %o1, %o2
 304         blu,a   1b
 305          sta    %g0, [%o1] ASI_M_FLUSH_PROBE
 306 hypersparc_flush_tlb_range_out:
 307         retl
 308          sta    %g5, [%g1] ASI_M_MMUREGS
 309
 310 hypersparc_flush_tlb_page:
 311         ld      [%o0 + 0x00], %o0       /* XXX vma->vm_mm GROSS XXX */
 312         mov     SRMMU_CTX_REG, %g1
 313         ld      [%o0 + AOFF_mm_context], %o3
 314         andn    %o1, (PAGE_SIZE - 1), %o1
 315 #ifndef CONFIG_SMP
 316         cmp     %o3, -1
 317         be      hypersparc_flush_tlb_page_out
 318 #endif
 319          lda    [%g1] ASI_M_MMUREGS, %g5
 320         sta     %o3, [%g1] ASI_M_MMUREGS
 321         sta     %g0, [%o1] ASI_M_FLUSH_PROBE
 322 hypersparc_flush_tlb_page_out:
 323         retl
 324          sta    %g5, [%g1] ASI_M_MMUREGS
 325
 326         __INIT
 327
 328         /* High speed page clear/copy. */
 329 hypersparc_bzero_1page:
 330 /* NOTE: This routine has to be shorter than 40insns --jj */
 331         clr     %g1
 332         mov     32, %g2
 333         mov     64, %g3
 334         mov     96, %g4
 335         mov     128, %g5
 336         mov     160, %g7
 337         mov     192, %o2
 338         mov     224, %o3
 339         mov     16, %o1
 340 1:
 341         stda    %g0, [%o0 + %g0] ASI_M_BFILL
 342         stda    %g0, [%o0 + %g2] ASI_M_BFILL
 343         stda    %g0, [%o0 + %g3] ASI_M_BFILL
 344         stda    %g0, [%o0 + %g4] ASI_M_BFILL
 345         stda    %g0, [%o0 + %g5] ASI_M_BFILL
 346         stda    %g0, [%o0 + %g7] ASI_M_BFILL
 347         stda    %g0, [%o0 + %o2] ASI_M_BFILL
 348         stda    %g0, [%o0 + %o3] ASI_M_BFILL
 349         subcc   %o1, 1, %o1
 350         bne     1b
 351          add    %o0, 256, %o0
 352
 353         retl
 354          nop
 355
 356 hypersparc_copy_1page:
 357 /* NOTE: This routine has to be shorter than 70insns --jj */
 358         sub     %o1, %o0, %o2           ! difference
 359         mov     16, %g1
 360 1:
 361         sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 362         add     %o0, 32, %o0
 363         sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 364         add     %o0, 32, %o0
 365         sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 366         add     %o0, 32, %o0
 367         sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 368         add     %o0, 32, %o0
 369         sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 370         add     %o0, 32, %o0
 371         sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 372         add     %o0, 32, %o0
 373         sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 374         add     %o0, 32, %o0
 375         sta     %o0, [%o0 + %o2] ASI_M_BCOPY
 376         subcc   %g1, 1, %g1
 377         bne     1b
 378          add    %o0, 32, %o0
 379
 380         retl
 381          nop
 382
 383         .globl  hypersparc_setup_blockops
 384 hypersparc_setup_blockops:
 385         sethi   %hi(bzero_1page), %o0
 386         or      %o0, %lo(bzero_1page), %o0
 387         sethi   %hi(hypersparc_bzero_1page), %o1
 388         or      %o1, %lo(hypersparc_bzero_1page), %o1
 389         sethi   %hi(hypersparc_copy_1page), %o2
 390         or      %o2, %lo(hypersparc_copy_1page), %o2
 391         ld      [%o1], %o4
 392 1:
 393         add     %o1, 4, %o1
 394         st      %o4, [%o0]
 395         add     %o0, 4, %o0
 396         cmp     %o1, %o2
 397         bne     1b
 398          ld     [%o1], %o4
 399         sethi   %hi(__copy_1page), %o0
 400         or      %o0, %lo(__copy_1page), %o0
 401         sethi   %hi(hypersparc_setup_blockops), %o2
 402         or      %o2, %lo(hypersparc_setup_blockops), %o2
 403         ld      [%o1], %o4
 404 1:
 405         add     %o1, 4, %o1
 406         st      %o4, [%o0]
 407         add     %o0, 4, %o0
 408         cmp     %o1, %o2
 409         bne     1b
 410          ld     [%o1], %o4
 411         sta     %g0, [%g0] ASI_M_FLUSH_IWHOLE
 412         retl
 413          nop