2 * Copyright 2008 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
25 * Jerome Glisse <glisse@freedesktop.org>
27 #include <linux/pagemap.h>
29 #include <drm/amdgpu_drm.h>
31 #include "amdgpu_trace.h"
33 int amdgpu_cs_get_ring(struct amdgpu_device
*adev
, u32 ip_type
,
34 u32 ip_instance
, u32 ring
,
35 struct amdgpu_ring
**out_ring
)
37 /* Right now all IPs have only one instance - multiple rings. */
38 if (ip_instance
!= 0) {
39 DRM_ERROR("invalid ip instance: %d\n", ip_instance
);
45 DRM_ERROR("unknown ip type: %d\n", ip_type
);
47 case AMDGPU_HW_IP_GFX
:
48 if (ring
< adev
->gfx
.num_gfx_rings
) {
49 *out_ring
= &adev
->gfx
.gfx_ring
[ring
];
51 DRM_ERROR("only %d gfx rings are supported now\n",
52 adev
->gfx
.num_gfx_rings
);
56 case AMDGPU_HW_IP_COMPUTE
:
57 if (ring
< adev
->gfx
.num_compute_rings
) {
58 *out_ring
= &adev
->gfx
.compute_ring
[ring
];
60 DRM_ERROR("only %d compute rings are supported now\n",
61 adev
->gfx
.num_compute_rings
);
65 case AMDGPU_HW_IP_DMA
:
66 if (ring
< adev
->sdma
.num_instances
) {
67 *out_ring
= &adev
->sdma
.instance
[ring
].ring
;
69 DRM_ERROR("only %d SDMA rings are supported\n",
70 adev
->sdma
.num_instances
);
74 case AMDGPU_HW_IP_UVD
:
75 *out_ring
= &adev
->uvd
.ring
;
77 case AMDGPU_HW_IP_VCE
:
79 *out_ring
= &adev
->vce
.ring
[ring
];
81 DRM_ERROR("only two VCE rings are supported\n");
89 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser
*p
,
90 struct drm_amdgpu_cs_chunk_fence
*data
,
93 struct drm_gem_object
*gobj
;
95 gobj
= drm_gem_object_lookup(p
->filp
, data
->handle
);
99 p
->uf_entry
.robj
= amdgpu_bo_ref(gem_to_amdgpu_bo(gobj
));
100 p
->uf_entry
.priority
= 0;
101 p
->uf_entry
.tv
.bo
= &p
->uf_entry
.robj
->tbo
;
102 p
->uf_entry
.tv
.shared
= true;
103 p
->uf_entry
.user_pages
= NULL
;
104 *offset
= data
->offset
;
106 drm_gem_object_unreference_unlocked(gobj
);
108 if (amdgpu_ttm_tt_get_usermm(p
->uf_entry
.robj
->tbo
.ttm
)) {
109 amdgpu_bo_unref(&p
->uf_entry
.robj
);
116 int amdgpu_cs_parser_init(struct amdgpu_cs_parser
*p
, void *data
)
118 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
119 struct amdgpu_vm
*vm
= &fpriv
->vm
;
120 union drm_amdgpu_cs
*cs
= data
;
121 uint64_t *chunk_array_user
;
122 uint64_t *chunk_array
;
123 unsigned size
, num_ibs
= 0;
124 uint32_t uf_offset
= 0;
128 if (cs
->in
.num_chunks
== 0)
131 chunk_array
= kmalloc_array(cs
->in
.num_chunks
, sizeof(uint64_t), GFP_KERNEL
);
135 p
->ctx
= amdgpu_ctx_get(fpriv
, cs
->in
.ctx_id
);
142 chunk_array_user
= (uint64_t __user
*)(unsigned long)(cs
->in
.chunks
);
143 if (copy_from_user(chunk_array
, chunk_array_user
,
144 sizeof(uint64_t)*cs
->in
.num_chunks
)) {
149 p
->nchunks
= cs
->in
.num_chunks
;
150 p
->chunks
= kmalloc_array(p
->nchunks
, sizeof(struct amdgpu_cs_chunk
),
157 for (i
= 0; i
< p
->nchunks
; i
++) {
158 struct drm_amdgpu_cs_chunk __user
**chunk_ptr
= NULL
;
159 struct drm_amdgpu_cs_chunk user_chunk
;
160 uint32_t __user
*cdata
;
162 chunk_ptr
= (void __user
*)(unsigned long)chunk_array
[i
];
163 if (copy_from_user(&user_chunk
, chunk_ptr
,
164 sizeof(struct drm_amdgpu_cs_chunk
))) {
167 goto free_partial_kdata
;
169 p
->chunks
[i
].chunk_id
= user_chunk
.chunk_id
;
170 p
->chunks
[i
].length_dw
= user_chunk
.length_dw
;
172 size
= p
->chunks
[i
].length_dw
;
173 cdata
= (void __user
*)(unsigned long)user_chunk
.chunk_data
;
175 p
->chunks
[i
].kdata
= drm_malloc_ab(size
, sizeof(uint32_t));
176 if (p
->chunks
[i
].kdata
== NULL
) {
179 goto free_partial_kdata
;
181 size
*= sizeof(uint32_t);
182 if (copy_from_user(p
->chunks
[i
].kdata
, cdata
, size
)) {
184 goto free_partial_kdata
;
187 switch (p
->chunks
[i
].chunk_id
) {
188 case AMDGPU_CHUNK_ID_IB
:
192 case AMDGPU_CHUNK_ID_FENCE
:
193 size
= sizeof(struct drm_amdgpu_cs_chunk_fence
);
194 if (p
->chunks
[i
].length_dw
* sizeof(uint32_t) < size
) {
196 goto free_partial_kdata
;
199 ret
= amdgpu_cs_user_fence_chunk(p
, p
->chunks
[i
].kdata
,
202 goto free_partial_kdata
;
206 case AMDGPU_CHUNK_ID_DEPENDENCIES
:
211 goto free_partial_kdata
;
215 ret
= amdgpu_job_alloc(p
->adev
, num_ibs
, &p
->job
, vm
);
219 if (p
->uf_entry
.robj
)
220 p
->job
->uf_addr
= uf_offset
;
228 drm_free_large(p
->chunks
[i
].kdata
);
231 amdgpu_ctx_put(p
->ctx
);
238 /* Returns how many bytes TTM can move per IB.
240 static u64
amdgpu_cs_get_threshold_for_moves(struct amdgpu_device
*adev
)
242 u64 real_vram_size
= adev
->mc
.real_vram_size
;
243 u64 vram_usage
= atomic64_read(&adev
->vram_usage
);
245 /* This function is based on the current VRAM usage.
247 * - If all of VRAM is free, allow relocating the number of bytes that
248 * is equal to 1/4 of the size of VRAM for this IB.
250 * - If more than one half of VRAM is occupied, only allow relocating
251 * 1 MB of data for this IB.
253 * - From 0 to one half of used VRAM, the threshold decreases
268 * Note: It's a threshold, not a limit. The threshold must be crossed
269 * for buffer relocations to stop, so any buffer of an arbitrary size
270 * can be moved as long as the threshold isn't crossed before
271 * the relocation takes place. We don't want to disable buffer
272 * relocations completely.
274 * The idea is that buffers should be placed in VRAM at creation time
275 * and TTM should only do a minimum number of relocations during
276 * command submission. In practice, you need to submit at least
277 * a dozen IBs to move all buffers to VRAM if they are in GTT.
279 * Also, things can get pretty crazy under memory pressure and actual
280 * VRAM usage can change a lot, so playing safe even at 50% does
281 * consistently increase performance.
284 u64 half_vram
= real_vram_size
>> 1;
285 u64 half_free_vram
= vram_usage
>= half_vram
? 0 : half_vram
- vram_usage
;
286 u64 bytes_moved_threshold
= half_free_vram
>> 1;
287 return max(bytes_moved_threshold
, 1024*1024ull);
290 static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser
*p
,
291 struct amdgpu_bo
*bo
)
293 u64 initial_bytes_moved
;
300 /* Avoid moving this one if we have moved too many buffers
301 * for this IB already.
303 * Note that this allows moving at least one buffer of
304 * any size, because it doesn't take the current "bo"
305 * into account. We don't want to disallow buffer moves
308 if (p
->bytes_moved
<= p
->bytes_moved_threshold
)
309 domain
= bo
->prefered_domains
;
311 domain
= bo
->allowed_domains
;
314 amdgpu_ttm_placement_from_domain(bo
, domain
);
315 initial_bytes_moved
= atomic64_read(&bo
->adev
->num_bytes_moved
);
316 r
= ttm_bo_validate(&bo
->tbo
, &bo
->placement
, true, false);
317 p
->bytes_moved
+= atomic64_read(&bo
->adev
->num_bytes_moved
) -
321 if (r
!= -ERESTARTSYS
&& domain
!= bo
->allowed_domains
) {
322 domain
= bo
->allowed_domains
;
330 int amdgpu_cs_list_validate(struct amdgpu_cs_parser
*p
,
331 struct list_head
*validated
)
333 struct amdgpu_bo_list_entry
*lobj
;
336 list_for_each_entry(lobj
, validated
, tv
.head
) {
337 struct amdgpu_bo
*bo
= lobj
->robj
;
338 bool binding_userptr
= false;
339 struct mm_struct
*usermm
;
341 usermm
= amdgpu_ttm_tt_get_usermm(bo
->tbo
.ttm
);
342 if (usermm
&& usermm
!= current
->mm
)
345 /* Check if we have user pages and nobody bound the BO already */
346 if (lobj
->user_pages
&& bo
->tbo
.ttm
->state
!= tt_bound
) {
347 size_t size
= sizeof(struct page
*);
349 size
*= bo
->tbo
.ttm
->num_pages
;
350 memcpy(bo
->tbo
.ttm
->pages
, lobj
->user_pages
, size
);
351 binding_userptr
= true;
354 r
= amdgpu_cs_bo_validate(p
, bo
);
358 r
= amdgpu_cs_bo_validate(p
, bo
);
363 if (binding_userptr
) {
364 drm_free_large(lobj
->user_pages
);
365 lobj
->user_pages
= NULL
;
371 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser
*p
,
372 union drm_amdgpu_cs
*cs
)
374 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
375 struct amdgpu_bo_list_entry
*e
;
376 struct list_head duplicates
;
377 bool need_mmap_lock
= false;
378 unsigned i
, tries
= 10;
381 INIT_LIST_HEAD(&p
->validated
);
383 p
->bo_list
= amdgpu_bo_list_get(fpriv
, cs
->in
.bo_list_handle
);
385 need_mmap_lock
= p
->bo_list
->first_userptr
!=
386 p
->bo_list
->num_entries
;
387 amdgpu_bo_list_get_list(p
->bo_list
, &p
->validated
);
390 INIT_LIST_HEAD(&duplicates
);
391 amdgpu_vm_get_pd_bo(&fpriv
->vm
, &p
->validated
, &p
->vm_pd
);
393 if (p
->uf_entry
.robj
)
394 list_add(&p
->uf_entry
.tv
.head
, &p
->validated
);
397 down_read(¤t
->mm
->mmap_sem
);
400 struct list_head need_pages
;
403 r
= ttm_eu_reserve_buffers(&p
->ticket
, &p
->validated
, true,
405 if (unlikely(r
!= 0)) {
406 DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
407 goto error_free_pages
;
410 /* Without a BO list we don't have userptr BOs */
414 INIT_LIST_HEAD(&need_pages
);
415 for (i
= p
->bo_list
->first_userptr
;
416 i
< p
->bo_list
->num_entries
; ++i
) {
418 e
= &p
->bo_list
->array
[i
];
420 if (amdgpu_ttm_tt_userptr_invalidated(e
->robj
->tbo
.ttm
,
421 &e
->user_invalidated
) && e
->user_pages
) {
423 /* We acquired a page array, but somebody
424 * invalidated it. Free it an try again
426 release_pages(e
->user_pages
,
427 e
->robj
->tbo
.ttm
->num_pages
,
429 drm_free_large(e
->user_pages
);
430 e
->user_pages
= NULL
;
433 if (e
->robj
->tbo
.ttm
->state
!= tt_bound
&&
435 list_del(&e
->tv
.head
);
436 list_add(&e
->tv
.head
, &need_pages
);
438 amdgpu_bo_unreserve(e
->robj
);
442 if (list_empty(&need_pages
))
445 /* Unreserve everything again. */
446 ttm_eu_backoff_reservation(&p
->ticket
, &p
->validated
);
448 /* We tried too many times, just abort */
451 DRM_ERROR("deadlock in %s\n", __func__
);
452 goto error_free_pages
;
455 /* Fill the page arrays for all useptrs. */
456 list_for_each_entry(e
, &need_pages
, tv
.head
) {
457 struct ttm_tt
*ttm
= e
->robj
->tbo
.ttm
;
459 e
->user_pages
= drm_calloc_large(ttm
->num_pages
,
460 sizeof(struct page
*));
461 if (!e
->user_pages
) {
463 DRM_ERROR("calloc failure in %s\n", __func__
);
464 goto error_free_pages
;
467 r
= amdgpu_ttm_tt_get_user_pages(ttm
, e
->user_pages
);
469 DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
470 drm_free_large(e
->user_pages
);
471 e
->user_pages
= NULL
;
472 goto error_free_pages
;
477 list_splice(&need_pages
, &p
->validated
);
480 amdgpu_vm_get_pt_bos(p
->adev
, &fpriv
->vm
, &duplicates
);
482 p
->bytes_moved_threshold
= amdgpu_cs_get_threshold_for_moves(p
->adev
);
485 r
= amdgpu_cs_list_validate(p
, &duplicates
);
487 DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
491 r
= amdgpu_cs_list_validate(p
, &p
->validated
);
493 DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
497 fpriv
->vm
.last_eviction_counter
=
498 atomic64_read(&p
->adev
->num_evictions
);
501 struct amdgpu_bo
*gds
= p
->bo_list
->gds_obj
;
502 struct amdgpu_bo
*gws
= p
->bo_list
->gws_obj
;
503 struct amdgpu_bo
*oa
= p
->bo_list
->oa_obj
;
504 struct amdgpu_vm
*vm
= &fpriv
->vm
;
507 for (i
= 0; i
< p
->bo_list
->num_entries
; i
++) {
508 struct amdgpu_bo
*bo
= p
->bo_list
->array
[i
].robj
;
510 p
->bo_list
->array
[i
].bo_va
= amdgpu_vm_bo_find(vm
, bo
);
514 p
->job
->gds_base
= amdgpu_bo_gpu_offset(gds
);
515 p
->job
->gds_size
= amdgpu_bo_size(gds
);
518 p
->job
->gws_base
= amdgpu_bo_gpu_offset(gws
);
519 p
->job
->gws_size
= amdgpu_bo_size(gws
);
522 p
->job
->oa_base
= amdgpu_bo_gpu_offset(oa
);
523 p
->job
->oa_size
= amdgpu_bo_size(oa
);
527 if (p
->uf_entry
.robj
)
528 p
->job
->uf_addr
+= amdgpu_bo_gpu_offset(p
->uf_entry
.robj
);
532 amdgpu_vm_move_pt_bos_in_lru(p
->adev
, &fpriv
->vm
);
533 ttm_eu_backoff_reservation(&p
->ticket
, &p
->validated
);
539 up_read(¤t
->mm
->mmap_sem
);
542 for (i
= p
->bo_list
->first_userptr
;
543 i
< p
->bo_list
->num_entries
; ++i
) {
544 e
= &p
->bo_list
->array
[i
];
549 release_pages(e
->user_pages
,
550 e
->robj
->tbo
.ttm
->num_pages
,
552 drm_free_large(e
->user_pages
);
559 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser
*p
)
561 struct amdgpu_bo_list_entry
*e
;
564 list_for_each_entry(e
, &p
->validated
, tv
.head
) {
565 struct reservation_object
*resv
= e
->robj
->tbo
.resv
;
566 r
= amdgpu_sync_resv(p
->adev
, &p
->job
->sync
, resv
, p
->filp
);
575 * cs_parser_fini() - clean parser states
576 * @parser: parser structure holding parsing context.
577 * @error: error number
579 * If error is set than unvalidate buffer, otherwise just free memory
580 * used by parsing context.
582 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser
*parser
, int error
, bool backoff
)
584 struct amdgpu_fpriv
*fpriv
= parser
->filp
->driver_priv
;
588 amdgpu_vm_move_pt_bos_in_lru(parser
->adev
, &fpriv
->vm
);
590 ttm_eu_fence_buffer_objects(&parser
->ticket
,
593 } else if (backoff
) {
594 ttm_eu_backoff_reservation(&parser
->ticket
,
597 fence_put(parser
->fence
);
600 amdgpu_ctx_put(parser
->ctx
);
602 amdgpu_bo_list_put(parser
->bo_list
);
604 for (i
= 0; i
< parser
->nchunks
; i
++)
605 drm_free_large(parser
->chunks
[i
].kdata
);
606 kfree(parser
->chunks
);
608 amdgpu_job_free(parser
->job
);
609 amdgpu_bo_unref(&parser
->uf_entry
.robj
);
612 static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser
*p
,
613 struct amdgpu_vm
*vm
)
615 struct amdgpu_device
*adev
= p
->adev
;
616 struct amdgpu_bo_va
*bo_va
;
617 struct amdgpu_bo
*bo
;
620 r
= amdgpu_vm_update_page_directory(adev
, vm
);
624 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, vm
->page_directory_fence
);
628 r
= amdgpu_vm_clear_freed(adev
, vm
);
633 for (i
= 0; i
< p
->bo_list
->num_entries
; i
++) {
636 /* ignore duplicates */
637 bo
= p
->bo_list
->array
[i
].robj
;
641 bo_va
= p
->bo_list
->array
[i
].bo_va
;
645 r
= amdgpu_vm_bo_update(adev
, bo_va
, false);
649 f
= bo_va
->last_pt_update
;
650 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
, f
);
657 r
= amdgpu_vm_clear_invalids(adev
, vm
, &p
->job
->sync
);
659 if (amdgpu_vm_debug
&& p
->bo_list
) {
660 /* Invalidate all BOs to test for userspace bugs */
661 for (i
= 0; i
< p
->bo_list
->num_entries
; i
++) {
662 /* ignore duplicates */
663 bo
= p
->bo_list
->array
[i
].robj
;
667 amdgpu_vm_bo_invalidate(adev
, bo
);
674 static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device
*adev
,
675 struct amdgpu_cs_parser
*p
)
677 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
678 struct amdgpu_vm
*vm
= &fpriv
->vm
;
679 struct amdgpu_ring
*ring
= p
->job
->ring
;
682 /* Only for UVD/VCE VM emulation */
683 if (ring
->funcs
->parse_cs
) {
685 for (i
= 0; i
< p
->job
->num_ibs
; i
++) {
686 r
= amdgpu_ring_parse_cs(ring
, p
, i
);
691 p
->job
->vm_pd_addr
= amdgpu_bo_gpu_offset(vm
->page_directory
);
693 r
= amdgpu_bo_vm_update_pte(p
, vm
);
698 return amdgpu_cs_sync_rings(p
);
701 static int amdgpu_cs_handle_lockup(struct amdgpu_device
*adev
, int r
)
704 r
= amdgpu_gpu_reset(adev
);
711 static int amdgpu_cs_ib_fill(struct amdgpu_device
*adev
,
712 struct amdgpu_cs_parser
*parser
)
714 struct amdgpu_fpriv
*fpriv
= parser
->filp
->driver_priv
;
715 struct amdgpu_vm
*vm
= &fpriv
->vm
;
719 for (i
= 0, j
= 0; i
< parser
->nchunks
&& j
< parser
->job
->num_ibs
; i
++) {
720 struct amdgpu_cs_chunk
*chunk
;
721 struct amdgpu_ib
*ib
;
722 struct drm_amdgpu_cs_chunk_ib
*chunk_ib
;
723 struct amdgpu_ring
*ring
;
725 chunk
= &parser
->chunks
[i
];
726 ib
= &parser
->job
->ibs
[j
];
727 chunk_ib
= (struct drm_amdgpu_cs_chunk_ib
*)chunk
->kdata
;
729 if (chunk
->chunk_id
!= AMDGPU_CHUNK_ID_IB
)
732 r
= amdgpu_cs_get_ring(adev
, chunk_ib
->ip_type
,
733 chunk_ib
->ip_instance
, chunk_ib
->ring
,
738 if (parser
->job
->ring
&& parser
->job
->ring
!= ring
)
741 parser
->job
->ring
= ring
;
743 if (ring
->funcs
->parse_cs
) {
744 struct amdgpu_bo_va_mapping
*m
;
745 struct amdgpu_bo
*aobj
= NULL
;
749 m
= amdgpu_cs_find_mapping(parser
, chunk_ib
->va_start
,
752 DRM_ERROR("IB va_start is invalid\n");
756 if ((chunk_ib
->va_start
+ chunk_ib
->ib_bytes
) >
757 (m
->it
.last
+ 1) * AMDGPU_GPU_PAGE_SIZE
) {
758 DRM_ERROR("IB va_start+ib_bytes is invalid\n");
762 /* the IB should be reserved at this point */
763 r
= amdgpu_bo_kmap(aobj
, (void **)&kptr
);
768 offset
= ((uint64_t)m
->it
.start
) * AMDGPU_GPU_PAGE_SIZE
;
769 kptr
+= chunk_ib
->va_start
- offset
;
771 r
= amdgpu_ib_get(adev
, NULL
, chunk_ib
->ib_bytes
, ib
);
773 DRM_ERROR("Failed to get ib !\n");
777 memcpy(ib
->ptr
, kptr
, chunk_ib
->ib_bytes
);
778 amdgpu_bo_kunmap(aobj
);
780 r
= amdgpu_ib_get(adev
, vm
, 0, ib
);
782 DRM_ERROR("Failed to get ib !\n");
786 ib
->gpu_addr
= chunk_ib
->va_start
;
789 ib
->length_dw
= chunk_ib
->ib_bytes
/ 4;
790 ib
->flags
= chunk_ib
->flags
;
794 /* UVD & VCE fw doesn't support user fences */
795 if (parser
->job
->uf_addr
&& (
796 parser
->job
->ring
->type
== AMDGPU_RING_TYPE_UVD
||
797 parser
->job
->ring
->type
== AMDGPU_RING_TYPE_VCE
))
803 static int amdgpu_cs_dependencies(struct amdgpu_device
*adev
,
804 struct amdgpu_cs_parser
*p
)
806 struct amdgpu_fpriv
*fpriv
= p
->filp
->driver_priv
;
809 for (i
= 0; i
< p
->nchunks
; ++i
) {
810 struct drm_amdgpu_cs_chunk_dep
*deps
;
811 struct amdgpu_cs_chunk
*chunk
;
814 chunk
= &p
->chunks
[i
];
816 if (chunk
->chunk_id
!= AMDGPU_CHUNK_ID_DEPENDENCIES
)
819 deps
= (struct drm_amdgpu_cs_chunk_dep
*)chunk
->kdata
;
820 num_deps
= chunk
->length_dw
* 4 /
821 sizeof(struct drm_amdgpu_cs_chunk_dep
);
823 for (j
= 0; j
< num_deps
; ++j
) {
824 struct amdgpu_ring
*ring
;
825 struct amdgpu_ctx
*ctx
;
828 r
= amdgpu_cs_get_ring(adev
, deps
[j
].ip_type
,
830 deps
[j
].ring
, &ring
);
834 ctx
= amdgpu_ctx_get(fpriv
, deps
[j
].ctx_id
);
838 fence
= amdgpu_ctx_get_fence(ctx
, ring
,
846 r
= amdgpu_sync_fence(adev
, &p
->job
->sync
,
859 static int amdgpu_cs_submit(struct amdgpu_cs_parser
*p
,
860 union drm_amdgpu_cs
*cs
)
862 struct amdgpu_ring
*ring
= p
->job
->ring
;
863 struct amd_sched_entity
*entity
= &p
->ctx
->rings
[ring
->idx
].entity
;
864 struct amdgpu_job
*job
;
870 r
= amd_sched_job_init(&job
->base
, &ring
->sched
, entity
, p
->filp
);
872 amdgpu_job_free(job
);
876 job
->owner
= p
->filp
;
877 job
->ctx
= entity
->fence_context
;
878 p
->fence
= fence_get(&job
->base
.s_fence
->finished
);
879 cs
->out
.handle
= amdgpu_ctx_add_fence(p
->ctx
, ring
, p
->fence
);
880 job
->uf_sequence
= cs
->out
.handle
;
881 amdgpu_job_free_resources(job
);
883 trace_amdgpu_cs_ioctl(job
);
884 amd_sched_entity_push_job(&job
->base
);
889 int amdgpu_cs_ioctl(struct drm_device
*dev
, void *data
, struct drm_file
*filp
)
891 struct amdgpu_device
*adev
= dev
->dev_private
;
892 union drm_amdgpu_cs
*cs
= data
;
893 struct amdgpu_cs_parser parser
= {};
894 bool reserved_buffers
= false;
897 if (!adev
->accel_working
)
903 r
= amdgpu_cs_parser_init(&parser
, data
);
905 DRM_ERROR("Failed to initialize parser !\n");
906 amdgpu_cs_parser_fini(&parser
, r
, false);
907 r
= amdgpu_cs_handle_lockup(adev
, r
);
910 r
= amdgpu_cs_parser_bos(&parser
, data
);
912 DRM_ERROR("Not enough memory for command submission!\n");
913 else if (r
&& r
!= -ERESTARTSYS
)
914 DRM_ERROR("Failed to process the buffer list %d!\n", r
);
916 reserved_buffers
= true;
917 r
= amdgpu_cs_ib_fill(adev
, &parser
);
921 r
= amdgpu_cs_dependencies(adev
, &parser
);
923 DRM_ERROR("Failed in the dependencies handling %d!\n", r
);
929 for (i
= 0; i
< parser
.job
->num_ibs
; i
++)
930 trace_amdgpu_cs(&parser
, i
);
932 r
= amdgpu_cs_ib_vm_chunk(adev
, &parser
);
936 r
= amdgpu_cs_submit(&parser
, cs
);
939 amdgpu_cs_parser_fini(&parser
, r
, reserved_buffers
);
940 r
= amdgpu_cs_handle_lockup(adev
, r
);
945 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
948 * @data: data from userspace
949 * @filp: file private
951 * Wait for the command submission identified by handle to finish.
953 int amdgpu_cs_wait_ioctl(struct drm_device
*dev
, void *data
,
954 struct drm_file
*filp
)
956 union drm_amdgpu_wait_cs
*wait
= data
;
957 struct amdgpu_device
*adev
= dev
->dev_private
;
958 unsigned long timeout
= amdgpu_gem_timeout(wait
->in
.timeout
);
959 struct amdgpu_ring
*ring
= NULL
;
960 struct amdgpu_ctx
*ctx
;
964 r
= amdgpu_cs_get_ring(adev
, wait
->in
.ip_type
, wait
->in
.ip_instance
,
965 wait
->in
.ring
, &ring
);
969 ctx
= amdgpu_ctx_get(filp
->driver_priv
, wait
->in
.ctx_id
);
973 fence
= amdgpu_ctx_get_fence(ctx
, ring
, wait
->in
.handle
);
977 r
= fence_wait_timeout(fence
, true, timeout
);
986 memset(wait
, 0, sizeof(*wait
));
987 wait
->out
.status
= (r
== 0);
993 * amdgpu_cs_find_bo_va - find bo_va for VM address
995 * @parser: command submission parser context
997 * @bo: resulting BO of the mapping found
999 * Search the buffer objects in the command submission context for a certain
1000 * virtual memory address. Returns allocation structure when found, NULL
1003 struct amdgpu_bo_va_mapping
*
1004 amdgpu_cs_find_mapping(struct amdgpu_cs_parser
*parser
,
1005 uint64_t addr
, struct amdgpu_bo
**bo
)
1007 struct amdgpu_bo_va_mapping
*mapping
;
1010 if (!parser
->bo_list
)
1013 addr
/= AMDGPU_GPU_PAGE_SIZE
;
1015 for (i
= 0; i
< parser
->bo_list
->num_entries
; i
++) {
1016 struct amdgpu_bo_list_entry
*lobj
;
1018 lobj
= &parser
->bo_list
->array
[i
];
1022 list_for_each_entry(mapping
, &lobj
->bo_va
->valids
, list
) {
1023 if (mapping
->it
.start
> addr
||
1024 addr
> mapping
->it
.last
)
1027 *bo
= lobj
->bo_va
->bo
;
1031 list_for_each_entry(mapping
, &lobj
->bo_va
->invalids
, list
) {
1032 if (mapping
->it
.start
> addr
||
1033 addr
> mapping
->it
.last
)
1036 *bo
= lobj
->bo_va
->bo
;