2 * Copyright (C) 2013 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
23 * Cmdstream submission:
26 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
27 #define BO_VALID 0x8000 /* is current addr in cmdstream correct/valid? */
28 #define BO_LOCKED 0x4000
29 #define BO_PINNED 0x2000
31 static struct msm_gem_submit
*submit_create(struct drm_device
*dev
,
32 struct msm_gpu
*gpu
, int nr_bos
, int nr_cmds
)
34 struct msm_gem_submit
*submit
;
35 int sz
= sizeof(*submit
) + (nr_bos
* sizeof(submit
->bos
[0])) +
36 (nr_cmds
* sizeof(*submit
->cmd
));
38 submit
= kmalloc(sz
, GFP_TEMPORARY
| __GFP_NOWARN
| __GFP_NORETRY
);
45 submit
->pid
= get_pid(task_pid(current
));
46 submit
->cmd
= (void *)&submit
->bos
[nr_bos
];
48 /* initially, until copy_from_user() and bo lookup succeeds: */
52 INIT_LIST_HEAD(&submit
->node
);
53 INIT_LIST_HEAD(&submit
->bo_list
);
54 ww_acquire_init(&submit
->ticket
, &reservation_ww_class
);
59 void msm_gem_submit_free(struct msm_gem_submit
*submit
)
61 fence_put(submit
->fence
);
62 list_del(&submit
->node
);
67 static int submit_lookup_objects(struct msm_gem_submit
*submit
,
68 struct drm_msm_gem_submit
*args
, struct drm_file
*file
)
73 spin_lock(&file
->table_lock
);
75 for (i
= 0; i
< args
->nr_bos
; i
++) {
76 struct drm_msm_gem_submit_bo submit_bo
;
77 struct drm_gem_object
*obj
;
78 struct msm_gem_object
*msm_obj
;
79 void __user
*userptr
=
80 u64_to_user_ptr(args
->bos
+ (i
* sizeof(submit_bo
)));
82 /* make sure we don't have garbage flags, in case we hit
83 * error path before flags is initialized:
85 submit
->bos
[i
].flags
= 0;
87 ret
= copy_from_user(&submit_bo
, userptr
, sizeof(submit_bo
));
93 if (submit_bo
.flags
& ~MSM_SUBMIT_BO_FLAGS
) {
94 DRM_ERROR("invalid flags: %x\n", submit_bo
.flags
);
99 submit
->bos
[i
].flags
= submit_bo
.flags
;
100 /* in validate_objects() we figure out if this is true: */
101 submit
->bos
[i
].iova
= submit_bo
.presumed
;
103 /* normally use drm_gem_object_lookup(), but for bulk lookup
104 * all under single table_lock just hit object_idr directly:
106 obj
= idr_find(&file
->object_idr
, submit_bo
.handle
);
108 DRM_ERROR("invalid handle %u at index %u\n", submit_bo
.handle
, i
);
113 msm_obj
= to_msm_bo(obj
);
115 if (!list_empty(&msm_obj
->submit_entry
)) {
116 DRM_ERROR("handle %u at index %u already on submit list\n",
117 submit_bo
.handle
, i
);
122 drm_gem_object_reference(obj
);
124 submit
->bos
[i
].obj
= msm_obj
;
126 list_add_tail(&msm_obj
->submit_entry
, &submit
->bo_list
);
131 spin_unlock(&file
->table_lock
);
136 static void submit_unlock_unpin_bo(struct msm_gem_submit
*submit
, int i
)
138 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
140 if (submit
->bos
[i
].flags
& BO_PINNED
)
141 msm_gem_put_iova(&msm_obj
->base
, submit
->gpu
->id
);
143 if (submit
->bos
[i
].flags
& BO_LOCKED
)
144 ww_mutex_unlock(&msm_obj
->resv
->lock
);
146 if (!(submit
->bos
[i
].flags
& BO_VALID
))
147 submit
->bos
[i
].iova
= 0;
149 submit
->bos
[i
].flags
&= ~(BO_LOCKED
| BO_PINNED
);
152 /* This is where we make sure all the bo's are reserved and pin'd: */
153 static int submit_lock_objects(struct msm_gem_submit
*submit
)
155 int contended
, slow_locked
= -1, i
, ret
= 0;
158 for (i
= 0; i
< submit
->nr_bos
; i
++) {
159 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
161 if (slow_locked
== i
)
166 if (!(submit
->bos
[i
].flags
& BO_LOCKED
)) {
167 ret
= ww_mutex_lock_interruptible(&msm_obj
->resv
->lock
,
171 submit
->bos
[i
].flags
|= BO_LOCKED
;
175 ww_acquire_done(&submit
->ticket
);
181 submit_unlock_unpin_bo(submit
, i
);
184 submit_unlock_unpin_bo(submit
, slow_locked
);
186 if (ret
== -EDEADLK
) {
187 struct msm_gem_object
*msm_obj
= submit
->bos
[contended
].obj
;
188 /* we lost out in a seqno race, lock and retry.. */
189 ret
= ww_mutex_lock_slow_interruptible(&msm_obj
->resv
->lock
,
192 submit
->bos
[contended
].flags
|= BO_LOCKED
;
193 slow_locked
= contended
;
201 static int submit_fence_sync(struct msm_gem_submit
*submit
)
205 for (i
= 0; i
< submit
->nr_bos
; i
++) {
206 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
207 bool write
= submit
->bos
[i
].flags
& MSM_SUBMIT_BO_WRITE
;
209 ret
= msm_gem_sync_object(&msm_obj
->base
, submit
->gpu
->fctx
, write
);
217 static int submit_pin_objects(struct msm_gem_submit
*submit
)
221 submit
->valid
= true;
223 for (i
= 0; i
< submit
->nr_bos
; i
++) {
224 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
227 /* if locking succeeded, pin bo: */
228 ret
= msm_gem_get_iova_locked(&msm_obj
->base
,
229 submit
->gpu
->id
, &iova
);
234 submit
->bos
[i
].flags
|= BO_PINNED
;
236 if (iova
== submit
->bos
[i
].iova
) {
237 submit
->bos
[i
].flags
|= BO_VALID
;
239 submit
->bos
[i
].iova
= iova
;
240 /* iova changed, so address in cmdstream is not valid: */
241 submit
->bos
[i
].flags
&= ~BO_VALID
;
242 submit
->valid
= false;
249 static int submit_bo(struct msm_gem_submit
*submit
, uint32_t idx
,
250 struct msm_gem_object
**obj
, uint32_t *iova
, bool *valid
)
252 if (idx
>= submit
->nr_bos
) {
253 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
254 idx
, submit
->nr_bos
);
259 *obj
= submit
->bos
[idx
].obj
;
261 *iova
= submit
->bos
[idx
].iova
;
263 *valid
= !!(submit
->bos
[idx
].flags
& BO_VALID
);
268 /* process the reloc's and patch up the cmdstream as needed: */
269 static int submit_reloc(struct msm_gem_submit
*submit
, struct msm_gem_object
*obj
,
270 uint32_t offset
, uint32_t nr_relocs
, uint64_t relocs
)
272 uint32_t i
, last_offset
= 0;
277 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset
);
281 /* For now, just map the entire thing. Eventually we probably
282 * to do it page-by-page, w/ kmap() if not vmap()d..
284 ptr
= msm_gem_get_vaddr_locked(&obj
->base
);
288 DBG("failed to map: %d", ret
);
292 for (i
= 0; i
< nr_relocs
; i
++) {
293 struct drm_msm_gem_submit_reloc submit_reloc
;
294 void __user
*userptr
=
295 u64_to_user_ptr(relocs
+ (i
* sizeof(submit_reloc
)));
299 ret
= copy_from_user(&submit_reloc
, userptr
, sizeof(submit_reloc
));
303 if (submit_reloc
.submit_offset
% 4) {
304 DRM_ERROR("non-aligned reloc offset: %u\n",
305 submit_reloc
.submit_offset
);
309 /* offset in dwords: */
310 off
= submit_reloc
.submit_offset
/ 4;
312 if ((off
>= (obj
->base
.size
/ 4)) ||
313 (off
< last_offset
)) {
314 DRM_ERROR("invalid offset %u at reloc %u\n", off
, i
);
318 ret
= submit_bo(submit
, submit_reloc
.reloc_idx
, NULL
, &iova
, &valid
);
325 iova
+= submit_reloc
.reloc_offset
;
327 if (submit_reloc
.shift
< 0)
328 iova
>>= -submit_reloc
.shift
;
330 iova
<<= submit_reloc
.shift
;
332 ptr
[off
] = iova
| submit_reloc
.or;
337 msm_gem_put_vaddr_locked(&obj
->base
);
342 static void submit_cleanup(struct msm_gem_submit
*submit
)
346 for (i
= 0; i
< submit
->nr_bos
; i
++) {
347 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
348 submit_unlock_unpin_bo(submit
, i
);
349 list_del_init(&msm_obj
->submit_entry
);
350 drm_gem_object_unreference(&msm_obj
->base
);
353 ww_acquire_fini(&submit
->ticket
);
356 int msm_ioctl_gem_submit(struct drm_device
*dev
, void *data
,
357 struct drm_file
*file
)
359 struct msm_drm_private
*priv
= dev
->dev_private
;
360 struct drm_msm_gem_submit
*args
= data
;
361 struct msm_file_private
*ctx
= file
->driver_priv
;
362 struct msm_gem_submit
*submit
;
363 struct msm_gpu
*gpu
= priv
->gpu
;
370 /* for now, we just have 3d pipe.. eventually this would need to
371 * be more clever to dispatch to appropriate gpu module:
373 if (args
->pipe
!= MSM_PIPE_3D0
)
376 ret
= mutex_lock_interruptible(&dev
->struct_mutex
);
380 submit
= submit_create(dev
, gpu
, args
->nr_bos
, args
->nr_cmds
);
386 ret
= submit_lookup_objects(submit
, args
, file
);
390 ret
= submit_lock_objects(submit
);
394 ret
= submit_fence_sync(submit
);
398 ret
= submit_pin_objects(submit
);
402 for (i
= 0; i
< args
->nr_cmds
; i
++) {
403 struct drm_msm_gem_submit_cmd submit_cmd
;
404 void __user
*userptr
=
405 u64_to_user_ptr(args
->cmds
+ (i
* sizeof(submit_cmd
)));
406 struct msm_gem_object
*msm_obj
;
409 ret
= copy_from_user(&submit_cmd
, userptr
, sizeof(submit_cmd
));
415 /* validate input from userspace: */
416 switch (submit_cmd
.type
) {
417 case MSM_SUBMIT_CMD_BUF
:
418 case MSM_SUBMIT_CMD_IB_TARGET_BUF
:
419 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF
:
422 DRM_ERROR("invalid type: %08x\n", submit_cmd
.type
);
427 ret
= submit_bo(submit
, submit_cmd
.submit_idx
,
428 &msm_obj
, &iova
, NULL
);
432 if (submit_cmd
.size
% 4) {
433 DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
439 if ((submit_cmd
.size
+ submit_cmd
.submit_offset
) >=
440 msm_obj
->base
.size
) {
441 DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd
.size
);
446 submit
->cmd
[i
].type
= submit_cmd
.type
;
447 submit
->cmd
[i
].size
= submit_cmd
.size
/ 4;
448 submit
->cmd
[i
].iova
= iova
+ submit_cmd
.submit_offset
;
449 submit
->cmd
[i
].idx
= submit_cmd
.submit_idx
;
454 ret
= submit_reloc(submit
, msm_obj
, submit_cmd
.submit_offset
,
455 submit_cmd
.nr_relocs
, submit_cmd
.relocs
);
462 ret
= msm_gpu_submit(gpu
, submit
, ctx
);
464 args
->fence
= submit
->fence
->seqno
;
467 submit_cleanup(submit
);
469 msm_gem_submit_free(submit
);
471 mutex_unlock(&dev
->struct_mutex
);