drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include "drmP.h"
  28 #include "drm.h"
  29
  30 #include "nouveau_drv.h"
  31 #include "nouveau_ramht.h"
  32 #include "nouveau_dma.h"
  33
  34 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
  35 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
  36
  37 struct nouveau_fence {
  38         struct nouveau_channel *channel;
  39         struct kref refcount;
  40         struct list_head entry;
  41
  42         uint32_t sequence;
  43         bool signalled;
  44
  45         void (*work)(void *priv, bool signalled);
  46         void *priv;
  47 };
  48
  49 struct nouveau_semaphore {
  50         struct kref ref;
  51         struct drm_device *dev;
  52         struct drm_mm_node *mem;
  53 };
  54
  55 static inline struct nouveau_fence *
  56 nouveau_fence(void *sync_obj)
  57 {
  58         return (struct nouveau_fence *)sync_obj;
  59 }
  60
  61 static void
  62 nouveau_fence_del(struct kref *ref)
  63 {
  64         struct nouveau_fence *fence =
  65                 container_of(ref, struct nouveau_fence, refcount);
  66
  67         nouveau_channel_ref(NULL, &fence->channel);
  68         kfree(fence);
  69 }
  70
  71 void
  72 nouveau_fence_update(struct nouveau_channel *chan)
  73 {
  74         struct drm_device *dev = chan->dev;
  75         struct nouveau_fence *tmp, *fence;
  76         uint32_t sequence;
  77
  78         spin_lock(&chan->fence.lock);
  79
  80         if (USE_REFCNT(dev))
  81                 sequence = nvchan_rd32(chan, 0x48);
  82         else
  83                 sequence = atomic_read(&chan->fence.last_sequence_irq);
  84
  85         if (chan->fence.sequence_ack == sequence)
  86                 goto out;
  87         chan->fence.sequence_ack = sequence;
  88
  89         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
  90                 sequence = fence->sequence;
  91                 fence->signalled = true;
  92                 list_del(&fence->entry);
  93
  94                 if (unlikely(fence->work))
  95                         fence->work(fence->priv, true);
  96
  97                 kref_put(&fence->refcount, nouveau_fence_del);
  98
  99                 if (sequence == chan->fence.sequence_ack)
 100                         break;
 101         }
 102 out:
 103         spin_unlock(&chan->fence.lock);
 104 }
 105
 106 int
 107 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
 108                   bool emit)
 109 {
 110         struct nouveau_fence *fence;
 111         int ret = 0;
 112
 113         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 114         if (!fence)
 115                 return -ENOMEM;
 116         kref_init(&fence->refcount);
 117         nouveau_channel_ref(chan, &fence->channel);
 118
 119         if (emit)
 120                 ret = nouveau_fence_emit(fence);
 121
 122         if (ret)
 123                 nouveau_fence_unref((void *)&fence);
 124         *pfence = fence;
 125         return ret;
 126 }
 127
 128 struct nouveau_channel *
 129 nouveau_fence_channel(struct nouveau_fence *fence)
 130 {
 131         return fence ? fence->channel : NULL;
 132 }
 133
 134 int
 135 nouveau_fence_emit(struct nouveau_fence *fence)
 136 {
 137         struct nouveau_channel *chan = fence->channel;
 138         struct drm_device *dev = chan->dev;
 139         int ret;
 140
 141         ret = RING_SPACE(chan, 2);
 142         if (ret)
 143                 return ret;
 144
 145         if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
 146                 nouveau_fence_update(chan);
 147
 148                 BUG_ON(chan->fence.sequence ==
 149                        chan->fence.sequence_ack - 1);
 150         }
 151
 152         fence->sequence = ++chan->fence.sequence;
 153
 154         kref_get(&fence->refcount);
 155         spin_lock(&chan->fence.lock);
 156         list_add_tail(&fence->entry, &chan->fence.pending);
 157         spin_unlock(&chan->fence.lock);
 158
 159         BEGIN_RING(chan, NvSubSw, USE_REFCNT(dev) ? 0x0050 : 0x0150, 1);
 160         OUT_RING(chan, fence->sequence);
 161         FIRE_RING(chan);
 162
 163         return 0;
 164 }
 165
 166 void
 167 nouveau_fence_work(struct nouveau_fence *fence,
 168                    void (*work)(void *priv, bool signalled),
 169                    void *priv)
 170 {
 171         BUG_ON(fence->work);
 172
 173         spin_lock(&fence->channel->fence.lock);
 174
 175         if (fence->signalled) {
 176                 work(priv, true);
 177         } else {
 178                 fence->work = work;
 179                 fence->priv = priv;
 180         }
 181
 182         spin_unlock(&fence->channel->fence.lock);
 183 }
 184
 185 void
 186 nouveau_fence_unref(void **sync_obj)
 187 {
 188         struct nouveau_fence *fence = nouveau_fence(*sync_obj);
 189
 190         if (fence)
 191                 kref_put(&fence->refcount, nouveau_fence_del);
 192         *sync_obj = NULL;
 193 }
 194
 195 void *
 196 nouveau_fence_ref(void *sync_obj)
 197 {
 198         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 199
 200         kref_get(&fence->refcount);
 201         return sync_obj;
 202 }
 203
 204 bool
 205 nouveau_fence_signalled(void *sync_obj, void *sync_arg)
 206 {
 207         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 208         struct nouveau_channel *chan = fence->channel;
 209
 210         if (fence->signalled)
 211                 return true;
 212
 213         nouveau_fence_update(chan);
 214         return fence->signalled;
 215 }
 216
 217 int
 218 nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
 219 {
 220         unsigned long timeout = jiffies + (3 * DRM_HZ);
 221         int ret = 0;
 222
 223         while (1) {
 224                 if (nouveau_fence_signalled(sync_obj, sync_arg))
 225                         break;
 226
 227                 if (time_after_eq(jiffies, timeout)) {
 228                         ret = -EBUSY;
 229                         break;
 230                 }
 231
 232                 __set_current_state(intr ? TASK_INTERRUPTIBLE
 233                         : TASK_UNINTERRUPTIBLE);
 234                 if (lazy)
 235                         schedule_timeout(1);
 236
 237                 if (intr && signal_pending(current)) {
 238                         ret = -ERESTARTSYS;
 239                         break;
 240                 }
 241         }
 242
 243         __set_current_state(TASK_RUNNING);
 244
 245         return ret;
 246 }
 247
 248 static struct nouveau_semaphore *
 249 alloc_semaphore(struct drm_device *dev)
 250 {
 251         struct drm_nouveau_private *dev_priv = dev->dev_private;
 252         struct nouveau_semaphore *sema;
 253         int ret;
 254
 255         if (!USE_SEMA(dev))
 256                 return NULL;
 257
 258         sema = kmalloc(sizeof(*sema), GFP_KERNEL);
 259         if (!sema)
 260                 goto fail;
 261
 262         ret = drm_mm_pre_get(&dev_priv->fence.heap);
 263         if (ret)
 264                 goto fail;
 265
 266         spin_lock(&dev_priv->fence.lock);
 267         sema->mem = drm_mm_search_free(&dev_priv->fence.heap, 4, 0, 0);
 268         if (sema->mem)
 269                 sema->mem = drm_mm_get_block_atomic(sema->mem, 4, 0);
 270         spin_unlock(&dev_priv->fence.lock);
 271
 272         if (!sema->mem)
 273                 goto fail;
 274
 275         kref_init(&sema->ref);
 276         sema->dev = dev;
 277         nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 0);
 278
 279         return sema;
 280 fail:
 281         kfree(sema);
 282         return NULL;
 283 }
 284
 285 static void
 286 free_semaphore(struct kref *ref)
 287 {
 288         struct nouveau_semaphore *sema =
 289                 container_of(ref, struct nouveau_semaphore, ref);
 290         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 291
 292         spin_lock(&dev_priv->fence.lock);
 293         drm_mm_put_block(sema->mem);
 294         spin_unlock(&dev_priv->fence.lock);
 295
 296         kfree(sema);
 297 }
 298
 299 static void
 300 semaphore_work(void *priv, bool signalled)
 301 {
 302         struct nouveau_semaphore *sema = priv;
 303         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 304
 305         if (unlikely(!signalled))
 306                 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
 307
 308         kref_put(&sema->ref, free_semaphore);
 309 }
 310
 311 static int
 312 emit_semaphore(struct nouveau_channel *chan, int method,
 313                struct nouveau_semaphore *sema)
 314 {
 315         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 316         struct nouveau_fence *fence;
 317         bool smart = (dev_priv->card_type >= NV_50);
 318         int ret;
 319
 320         ret = RING_SPACE(chan, smart ? 8 : 4);
 321         if (ret)
 322                 return ret;
 323
 324         if (smart) {
 325                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 326                 OUT_RING(chan, NvSema);
 327         }
 328         BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 1);
 329         OUT_RING(chan, sema->mem->start);
 330
 331         if (smart && method == NV_SW_SEMAPHORE_ACQUIRE) {
 332                 /*
 333                  * NV50 tries to be too smart and context-switch
 334                  * between semaphores instead of doing a "first come,
 335                  * first served" strategy like previous cards
 336                  * do.
 337                  *
 338                  * That's bad because the ACQUIRE latency can get as
 339                  * large as the PFIFO context time slice in the
 340                  * typical DRI2 case where you have several
 341                  * outstanding semaphores at the same moment.
 342                  *
 343                  * If we're going to ACQUIRE, force the card to
 344                  * context switch before, just in case the matching
 345                  * RELEASE is already scheduled to be executed in
 346                  * another channel.
 347                  */
 348                 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
 349                 OUT_RING(chan, 0);
 350         }
 351
 352         BEGIN_RING(chan, NvSubSw, method, 1);
 353         OUT_RING(chan, 1);
 354
 355         if (smart && method == NV_SW_SEMAPHORE_RELEASE) {
 356                 /*
 357                  * Force the card to context switch, there may be
 358                  * another channel waiting for the semaphore we just
 359                  * released.
 360                  */
 361                 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
 362                 OUT_RING(chan, 0);
 363         }
 364
 365         /* Delay semaphore destruction until its work is done */
 366         ret = nouveau_fence_new(chan, &fence, true);
 367         if (ret)
 368                 return ret;
 369
 370         kref_get(&sema->ref);
 371         nouveau_fence_work(fence, semaphore_work, sema);
 372         nouveau_fence_unref((void *)&fence);
 373
 374         return 0;
 375 }
 376
 377 int
 378 nouveau_fence_sync(struct nouveau_fence *fence,
 379                    struct nouveau_channel *wchan)
 380 {
 381         struct nouveau_channel *chan = nouveau_fence_channel(fence);
 382         struct drm_device *dev = wchan->dev;
 383         struct nouveau_semaphore *sema;
 384         int ret;
 385
 386         if (likely(!fence || chan == wchan ||
 387                    nouveau_fence_signalled(fence, NULL)))
 388                 return 0;
 389
 390         sema = alloc_semaphore(dev);
 391         if (!sema) {
 392                 /* Early card or broken userspace, fall back to
 393                  * software sync. */
 394                 return nouveau_fence_wait(fence, NULL, true, false);
 395         }
 396
 397         /* try to take chan's mutex, if we can't take it right away
 398          * we have to fallback to software sync to prevent locking
 399          * order issues
 400          */
 401         if (!mutex_trylock(&chan->mutex)) {
 402                 free_semaphore(&sema->ref);
 403                 return nouveau_fence_wait(fence, NULL, true, false);
 404         }
 405
 406         /* Make wchan wait until it gets signalled */
 407         ret = emit_semaphore(wchan, NV_SW_SEMAPHORE_ACQUIRE, sema);
 408         if (ret)
 409                 goto out;
 410
 411         /* Signal the semaphore from chan */
 412         ret = emit_semaphore(chan, NV_SW_SEMAPHORE_RELEASE, sema);
 413         mutex_unlock(&chan->mutex);
 414 out:
 415         kref_put(&sema->ref, free_semaphore);
 416         return ret;
 417 }
 418
 419 int
 420 nouveau_fence_flush(void *sync_obj, void *sync_arg)
 421 {
 422         return 0;
 423 }
 424
 425 int
 426 nouveau_fence_channel_init(struct nouveau_channel *chan)
 427 {
 428         struct drm_device *dev = chan->dev;
 429         struct drm_nouveau_private *dev_priv = dev->dev_private;
 430         struct nouveau_gpuobj *obj = NULL;
 431         int ret;
 432
 433         /* Create an NV_SW object for various sync purposes */
 434         ret = nouveau_gpuobj_sw_new(chan, NV_SW, &obj);
 435         if (ret)
 436                 return ret;
 437
 438         ret = nouveau_ramht_insert(chan, NvSw, obj);
 439         nouveau_gpuobj_ref(NULL, &obj);
 440         if (ret)
 441                 return ret;
 442
 443         ret = RING_SPACE(chan, 2);
 444         if (ret)
 445                 return ret;
 446         BEGIN_RING(chan, NvSubSw, 0, 1);
 447         OUT_RING(chan, NvSw);
 448
 449         /* Create a DMA object for the shared cross-channel sync area. */
 450         if (USE_SEMA(dev)) {
 451                 struct drm_mm_node *mem = dev_priv->fence.bo->bo.mem.mm_node;
 452
 453                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 454                                              mem->start << PAGE_SHIFT,
 455                                              mem->size << PAGE_SHIFT,
 456                                              NV_DMA_ACCESS_RW,
 457                                              NV_DMA_TARGET_VIDMEM, &obj);
 458                 if (ret)
 459                         return ret;
 460
 461                 ret = nouveau_ramht_insert(chan, NvSema, obj);
 462                 nouveau_gpuobj_ref(NULL, &obj);
 463                 if (ret)
 464                         return ret;
 465
 466                 ret = RING_SPACE(chan, 2);
 467                 if (ret)
 468                         return ret;
 469                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 470                 OUT_RING(chan, NvSema);
 471         }
 472
 473         FIRE_RING(chan);
 474
 475         INIT_LIST_HEAD(&chan->fence.pending);
 476         spin_lock_init(&chan->fence.lock);
 477         atomic_set(&chan->fence.last_sequence_irq, 0);
 478
 479         return 0;
 480 }
 481
 482 void
 483 nouveau_fence_channel_fini(struct nouveau_channel *chan)
 484 {
 485         struct nouveau_fence *tmp, *fence;
 486
 487         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
 488                 fence->signalled = true;
 489                 list_del(&fence->entry);
 490
 491                 if (unlikely(fence->work))
 492                         fence->work(fence->priv, false);
 493
 494                 kref_put(&fence->refcount, nouveau_fence_del);
 495         }
 496 }
 497
 498 int
 499 nouveau_fence_init(struct drm_device *dev)
 500 {
 501         struct drm_nouveau_private *dev_priv = dev->dev_private;
 502         int ret;
 503
 504         /* Create a shared VRAM heap for cross-channel sync. */
 505         if (USE_SEMA(dev)) {
 506                 ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM,
 507                                      0, 0, false, true, &dev_priv->fence.bo);
 508                 if (ret)
 509                         return ret;
 510
 511                 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
 512                 if (ret)
 513                         goto fail;
 514
 515                 ret = nouveau_bo_map(dev_priv->fence.bo);
 516                 if (ret)
 517                         goto fail;
 518
 519                 ret = drm_mm_init(&dev_priv->fence.heap, 0,
 520                                   dev_priv->fence.bo->bo.mem.size);
 521                 if (ret)
 522                         goto fail;
 523
 524                 spin_lock_init(&dev_priv->fence.lock);
 525         }
 526
 527         return 0;
 528 fail:
 529         nouveau_bo_unmap(dev_priv->fence.bo);
 530         nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 531         return ret;
 532 }
 533
 534 void
 535 nouveau_fence_fini(struct drm_device *dev)
 536 {
 537         struct drm_nouveau_private *dev_priv = dev->dev_private;
 538
 539         if (USE_SEMA(dev)) {
 540                 drm_mm_takedown(&dev_priv->fence.heap);
 541                 nouveau_bo_unmap(dev_priv->fence.bo);
 542                 nouveau_bo_unpin(dev_priv->fence.bo);
 543                 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 544         }
 545 }