29eacfa29cfb6952c38b89d8e51883890d9125f2
2 * Copyright (C) 2013 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
18 #include <linux/spinlock.h>
19 #include <linux/shmem_fs.h>
26 /* called with dev->struct_mutex held */
27 static struct page
**get_pages(struct drm_gem_object
*obj
)
29 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
31 if (!msm_obj
->pages
) {
32 struct drm_device
*dev
= obj
->dev
;
33 struct page
**p
= drm_gem_get_pages(obj
, 0);
34 int npages
= obj
->size
>> PAGE_SHIFT
;
37 dev_err(dev
->dev
, "could not get pages: %ld\n",
42 msm_obj
->sgt
= drm_prime_pages_to_sg(p
, npages
);
43 if (IS_ERR(msm_obj
->sgt
)) {
44 dev_err(dev
->dev
, "failed to allocate sgt\n");
45 return ERR_CAST(msm_obj
->sgt
);
50 /* For non-cached buffers, ensure the new pages are clean
51 * because display controller, GPU, etc. are not coherent:
53 if (msm_obj
->flags
& (MSM_BO_WC
|MSM_BO_UNCACHED
))
54 dma_map_sg(dev
->dev
, msm_obj
->sgt
->sgl
,
55 msm_obj
->sgt
->nents
, DMA_BIDIRECTIONAL
);
58 return msm_obj
->pages
;
61 static void put_pages(struct drm_gem_object
*obj
)
63 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
66 /* For non-cached buffers, ensure the new pages are clean
67 * because display controller, GPU, etc. are not coherent:
69 if (msm_obj
->flags
& (MSM_BO_WC
|MSM_BO_UNCACHED
))
70 dma_unmap_sg(obj
->dev
->dev
, msm_obj
->sgt
->sgl
,
71 msm_obj
->sgt
->nents
, DMA_BIDIRECTIONAL
);
72 sg_free_table(msm_obj
->sgt
);
75 drm_gem_put_pages(obj
, msm_obj
->pages
, true, false);
76 msm_obj
->pages
= NULL
;
80 int msm_gem_mmap_obj(struct drm_gem_object
*obj
,
81 struct vm_area_struct
*vma
)
83 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
85 vma
->vm_flags
&= ~VM_PFNMAP
;
86 vma
->vm_flags
|= VM_MIXEDMAP
;
88 if (msm_obj
->flags
& MSM_BO_WC
) {
89 vma
->vm_page_prot
= pgprot_writecombine(vm_get_page_prot(vma
->vm_flags
));
90 } else if (msm_obj
->flags
& MSM_BO_UNCACHED
) {
91 vma
->vm_page_prot
= pgprot_noncached(vm_get_page_prot(vma
->vm_flags
));
94 * Shunt off cached objs to shmem file so they have their own
95 * address_space (so unmap_mapping_range does what we want,
96 * in particular in the case of mmap'd dmabufs)
101 vma
->vm_file
= obj
->filp
;
103 vma
->vm_page_prot
= vm_get_page_prot(vma
->vm_flags
);
109 int msm_gem_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
113 ret
= drm_gem_mmap(filp
, vma
);
115 DBG("mmap failed: %d", ret
);
119 return msm_gem_mmap_obj(vma
->vm_private_data
, vma
);
122 int msm_gem_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
124 struct drm_gem_object
*obj
= vma
->vm_private_data
;
125 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
126 struct drm_device
*dev
= obj
->dev
;
132 /* Make sure we don't parallel update on a fault, nor move or remove
133 * something from beneath our feet
135 ret
= mutex_lock_interruptible(&dev
->struct_mutex
);
139 /* make sure we have pages attached now */
140 pages
= get_pages(obj
);
142 ret
= PTR_ERR(pages
);
146 /* We don't use vmf->pgoff since that has the fake offset: */
147 pgoff
= ((unsigned long)vmf
->virtual_address
-
148 vma
->vm_start
) >> PAGE_SHIFT
;
150 pfn
= page_to_pfn(msm_obj
->pages
[pgoff
]);
152 VERB("Inserting %p pfn %lx, pa %lx", vmf
->virtual_address
,
153 pfn
, pfn
<< PAGE_SHIFT
);
155 ret
= vm_insert_mixed(vma
, (unsigned long)vmf
->virtual_address
, pfn
);
158 mutex_unlock(&dev
->struct_mutex
);
165 return VM_FAULT_NOPAGE
;
169 return VM_FAULT_SIGBUS
;
173 /** get mmap offset */
174 static uint64_t mmap_offset(struct drm_gem_object
*obj
)
176 struct drm_device
*dev
= obj
->dev
;
179 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
181 /* Make it mmapable */
182 ret
= drm_gem_create_mmap_offset(obj
);
185 dev_err(dev
->dev
, "could not allocate mmap offset\n");
189 return drm_vma_node_offset_addr(&obj
->vma_node
);
192 uint64_t msm_gem_mmap_offset(struct drm_gem_object
*obj
)
195 mutex_lock(&obj
->dev
->struct_mutex
);
196 offset
= mmap_offset(obj
);
197 mutex_unlock(&obj
->dev
->struct_mutex
);
201 /* helpers for dealing w/ iommu: */
202 static int map_range(struct iommu_domain
*domain
, unsigned int iova
,
203 struct sg_table
*sgt
, unsigned int len
, int prot
)
205 struct scatterlist
*sg
;
206 unsigned int da
= iova
;
213 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
214 u32 pa
= sg_phys(sg
) - sg
->offset
;
215 size_t bytes
= sg
->length
+ sg
->offset
;
217 VERB("map[%d]: %08x %08x(%x)", i
, iova
, pa
, bytes
);
219 ret
= iommu_map(domain
, da
, pa
, bytes
, prot
);
231 for_each_sg(sgt
->sgl
, sg
, i
, j
) {
232 size_t bytes
= sg
->length
+ sg
->offset
;
233 iommu_unmap(domain
, da
, bytes
);
239 static void unmap_range(struct iommu_domain
*domain
, unsigned int iova
,
240 struct sg_table
*sgt
, unsigned int len
)
242 struct scatterlist
*sg
;
243 unsigned int da
= iova
;
246 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
247 size_t bytes
= sg
->length
+ sg
->offset
;
250 unmapped
= iommu_unmap(domain
, da
, bytes
);
251 if (unmapped
< bytes
)
254 VERB("unmap[%d]: %08x(%x)", i
, iova
, bytes
);
256 BUG_ON(!IS_ALIGNED(bytes
, PAGE_SIZE
));
262 /* should be called under struct_mutex.. although it can be called
263 * from atomic context without struct_mutex to acquire an extra
264 * iova ref if you know one is already held.
266 * That means when I do eventually need to add support for unpinning
267 * the refcnt counter needs to be atomic_t.
269 int msm_gem_get_iova_locked(struct drm_gem_object
*obj
, int id
,
272 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
275 if (!msm_obj
->domain
[id
].iova
) {
276 struct msm_drm_private
*priv
= obj
->dev
->dev_private
;
277 uint32_t offset
= (uint32_t)mmap_offset(obj
);
279 pages
= get_pages(obj
);
281 return PTR_ERR(pages
);
282 // XXX ideally we would not map buffers writable when not needed...
283 ret
= map_range(priv
->iommus
[id
], offset
, msm_obj
->sgt
,
284 obj
->size
, IOMMU_READ
| IOMMU_WRITE
);
285 msm_obj
->domain
[id
].iova
= offset
;
289 *iova
= msm_obj
->domain
[id
].iova
;
294 int msm_gem_get_iova(struct drm_gem_object
*obj
, int id
, uint32_t *iova
)
297 mutex_lock(&obj
->dev
->struct_mutex
);
298 ret
= msm_gem_get_iova_locked(obj
, id
, iova
);
299 mutex_unlock(&obj
->dev
->struct_mutex
);
303 void msm_gem_put_iova(struct drm_gem_object
*obj
, int id
)
306 // NOTE: probably don't need a _locked() version.. we wouldn't
307 // normally unmap here, but instead just mark that it could be
308 // unmapped (if the iova refcnt drops to zero), but then later
309 // if another _get_iova_locked() fails we can start unmapping
310 // things that are no longer needed..
313 int msm_gem_dumb_create(struct drm_file
*file
, struct drm_device
*dev
,
314 struct drm_mode_create_dumb
*args
)
316 args
->pitch
= align_pitch(args
->width
, args
->bpp
);
317 args
->size
= PAGE_ALIGN(args
->pitch
* args
->height
);
318 return msm_gem_new_handle(dev
, file
, args
->size
,
319 MSM_BO_SCANOUT
| MSM_BO_WC
, &args
->handle
);
322 int msm_gem_dumb_destroy(struct drm_file
*file
, struct drm_device
*dev
,
325 /* No special work needed, drop the reference and see what falls out */
326 return drm_gem_handle_delete(file
, handle
);
329 int msm_gem_dumb_map_offset(struct drm_file
*file
, struct drm_device
*dev
,
330 uint32_t handle
, uint64_t *offset
)
332 struct drm_gem_object
*obj
;
335 /* GEM does all our handle to object mapping */
336 obj
= drm_gem_object_lookup(dev
, file
, handle
);
342 *offset
= msm_gem_mmap_offset(obj
);
344 drm_gem_object_unreference_unlocked(obj
);
350 void *msm_gem_vaddr_locked(struct drm_gem_object
*obj
)
352 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
353 WARN_ON(!mutex_is_locked(&obj
->dev
->struct_mutex
));
354 if (!msm_obj
->vaddr
) {
355 struct page
**pages
= get_pages(obj
);
357 return ERR_CAST(pages
);
358 msm_obj
->vaddr
= vmap(pages
, obj
->size
>> PAGE_SHIFT
,
359 VM_MAP
, pgprot_writecombine(PAGE_KERNEL
));
361 return msm_obj
->vaddr
;
364 void *msm_gem_vaddr(struct drm_gem_object
*obj
)
367 mutex_lock(&obj
->dev
->struct_mutex
);
368 ret
= msm_gem_vaddr_locked(obj
);
369 mutex_unlock(&obj
->dev
->struct_mutex
);
373 int msm_gem_queue_inactive_work(struct drm_gem_object
*obj
,
374 struct work_struct
*work
)
376 struct drm_device
*dev
= obj
->dev
;
377 struct msm_drm_private
*priv
= dev
->dev_private
;
378 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
381 mutex_lock(&dev
->struct_mutex
);
382 if (!list_empty(&work
->entry
)) {
384 } else if (is_active(msm_obj
)) {
385 list_add_tail(&work
->entry
, &msm_obj
->inactive_work
);
387 queue_work(priv
->wq
, work
);
389 mutex_unlock(&dev
->struct_mutex
);
394 void msm_gem_move_to_active(struct drm_gem_object
*obj
,
395 struct msm_gpu
*gpu
, bool write
, uint32_t fence
)
397 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
400 msm_obj
->write_fence
= fence
;
402 msm_obj
->read_fence
= fence
;
403 list_del_init(&msm_obj
->mm_list
);
404 list_add_tail(&msm_obj
->mm_list
, &gpu
->active_list
);
407 void msm_gem_move_to_inactive(struct drm_gem_object
*obj
)
409 struct drm_device
*dev
= obj
->dev
;
410 struct msm_drm_private
*priv
= dev
->dev_private
;
411 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
413 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
416 msm_obj
->read_fence
= 0;
417 msm_obj
->write_fence
= 0;
418 list_del_init(&msm_obj
->mm_list
);
419 list_add_tail(&msm_obj
->mm_list
, &priv
->inactive_list
);
421 while (!list_empty(&msm_obj
->inactive_work
)) {
422 struct work_struct
*work
;
424 work
= list_first_entry(&msm_obj
->inactive_work
,
425 struct work_struct
, entry
);
427 list_del_init(&work
->entry
);
428 queue_work(priv
->wq
, work
);
432 int msm_gem_cpu_prep(struct drm_gem_object
*obj
, uint32_t op
,
433 struct timespec
*timeout
)
435 struct drm_device
*dev
= obj
->dev
;
436 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
439 if (is_active(msm_obj
)) {
442 if (op
& MSM_PREP_READ
)
443 fence
= msm_obj
->write_fence
;
444 if (op
& MSM_PREP_WRITE
)
445 fence
= max(fence
, msm_obj
->read_fence
);
446 if (op
& MSM_PREP_NOSYNC
)
449 ret
= msm_wait_fence_interruptable(dev
, fence
, timeout
);
452 /* TODO cache maintenance */
457 int msm_gem_cpu_fini(struct drm_gem_object
*obj
)
459 /* TODO cache maintenance */
463 #ifdef CONFIG_DEBUG_FS
464 void msm_gem_describe(struct drm_gem_object
*obj
, struct seq_file
*m
)
466 struct drm_device
*dev
= obj
->dev
;
467 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
468 uint64_t off
= drm_vma_node_start(&obj
->vma_node
);
470 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
471 seq_printf(m
, "%08x: %c(r=%u,w=%u) %2d (%2d) %08llx %p %d\n",
472 msm_obj
->flags
, is_active(msm_obj
) ? 'A' : 'I',
473 msm_obj
->read_fence
, msm_obj
->write_fence
,
474 obj
->name
, obj
->refcount
.refcount
.counter
,
475 off
, msm_obj
->vaddr
, obj
->size
);
478 void msm_gem_describe_objects(struct list_head
*list
, struct seq_file
*m
)
480 struct msm_gem_object
*msm_obj
;
484 list_for_each_entry(msm_obj
, list
, mm_list
) {
485 struct drm_gem_object
*obj
= &msm_obj
->base
;
487 msm_gem_describe(obj
, m
);
492 seq_printf(m
, "Total %d objects, %zu bytes\n", count
, size
);
496 void msm_gem_free_object(struct drm_gem_object
*obj
)
498 struct drm_device
*dev
= obj
->dev
;
499 struct msm_gem_object
*msm_obj
= to_msm_bo(obj
);
502 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
504 /* object should not be on active list: */
505 WARN_ON(is_active(msm_obj
));
507 list_del(&msm_obj
->mm_list
);
509 for (id
= 0; id
< ARRAY_SIZE(msm_obj
->domain
); id
++) {
510 if (msm_obj
->domain
[id
].iova
) {
511 struct msm_drm_private
*priv
= obj
->dev
->dev_private
;
512 uint32_t offset
= (uint32_t)mmap_offset(obj
);
513 unmap_range(priv
->iommus
[id
], offset
,
514 msm_obj
->sgt
, obj
->size
);
518 drm_gem_free_mmap_offset(obj
);
521 vunmap(msm_obj
->vaddr
);
525 if (msm_obj
->resv
== &msm_obj
->_resv
)
526 reservation_object_fini(msm_obj
->resv
);
528 drm_gem_object_release(obj
);
533 /* convenience method to construct a GEM buffer object, and userspace handle */
534 int msm_gem_new_handle(struct drm_device
*dev
, struct drm_file
*file
,
535 uint32_t size
, uint32_t flags
, uint32_t *handle
)
537 struct drm_gem_object
*obj
;
540 ret
= mutex_lock_interruptible(&dev
->struct_mutex
);
544 obj
= msm_gem_new(dev
, size
, flags
);
546 mutex_unlock(&dev
->struct_mutex
);
551 ret
= drm_gem_handle_create(file
, obj
, handle
);
553 /* drop reference from allocate - handle holds it now */
554 drm_gem_object_unreference_unlocked(obj
);
559 struct drm_gem_object
*msm_gem_new(struct drm_device
*dev
,
560 uint32_t size
, uint32_t flags
)
562 struct msm_drm_private
*priv
= dev
->dev_private
;
563 struct msm_gem_object
*msm_obj
;
564 struct drm_gem_object
*obj
= NULL
;
567 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
569 size
= PAGE_ALIGN(size
);
571 switch (flags
& MSM_BO_CACHE_MASK
) {
572 case MSM_BO_UNCACHED
:
577 dev_err(dev
->dev
, "invalid cache flag: %x\n",
578 (flags
& MSM_BO_CACHE_MASK
));
583 msm_obj
= kzalloc(sizeof(*msm_obj
), GFP_KERNEL
);
589 obj
= &msm_obj
->base
;
591 ret
= drm_gem_object_init(dev
, obj
, size
);
595 msm_obj
->flags
= flags
;
597 msm_obj
->resv
= &msm_obj
->_resv
;
598 reservation_object_init(msm_obj
->resv
);
600 INIT_LIST_HEAD(&msm_obj
->submit_entry
);
601 INIT_LIST_HEAD(&msm_obj
->inactive_work
);
602 list_add_tail(&msm_obj
->mm_list
, &priv
->inactive_list
);
608 drm_gem_object_unreference_unlocked(obj
);
This page took 0.068346 seconds and 4 git commands to generate.