2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
31 #include <linux/seq_file.h>
32 #include <asm/atomic.h>
33 #include <linux/wait.h>
34 #include <linux/list.h>
35 #include <linux/kref.h>
36 #include <linux/slab.h>
39 #include "radeon_reg.h"
42 int radeon_fence_emit(struct radeon_device
*rdev
, struct radeon_fence
*fence
)
44 unsigned long irq_flags
;
46 write_lock_irqsave(&rdev
->fence_drv
.lock
, irq_flags
);
48 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
51 fence
->seq
= atomic_add_return(1, &rdev
->fence_drv
.seq
);
52 if (!rdev
->cp
.ready
) {
53 /* FIXME: cp is not running assume everythings is done right
56 WREG32(rdev
->fence_drv
.scratch_reg
, fence
->seq
);
58 radeon_fence_ring_emit(rdev
, fence
);
61 list_del(&fence
->list
);
62 list_add_tail(&fence
->list
, &rdev
->fence_drv
.emited
);
63 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
67 static bool radeon_fence_poll_locked(struct radeon_device
*rdev
)
69 struct radeon_fence
*fence
;
70 struct list_head
*i
, *n
;
73 unsigned long cjiffies
;
75 if (rdev
->wb
.enabled
) {
76 u32 scratch_index
= RADEON_WB_SCRATCH_OFFSET
+ rdev
->fence_drv
.scratch_reg
- rdev
->scratch
.reg_base
;
77 seq
= rdev
->wb
.wb
[scratch_index
/4];
79 seq
= RREG32(rdev
->fence_drv
.scratch_reg
);
80 if (seq
!= rdev
->fence_drv
.last_seq
) {
81 rdev
->fence_drv
.last_seq
= seq
;
82 rdev
->fence_drv
.last_jiffies
= jiffies
;
83 rdev
->fence_drv
.last_timeout
= RADEON_FENCE_JIFFIES_TIMEOUT
;
86 if (time_after(cjiffies
, rdev
->fence_drv
.last_jiffies
)) {
87 cjiffies
-= rdev
->fence_drv
.last_jiffies
;
88 if (time_after(rdev
->fence_drv
.last_timeout
, cjiffies
)) {
89 /* update the timeout */
90 rdev
->fence_drv
.last_timeout
-= cjiffies
;
92 /* the 500ms timeout is elapsed we should test
95 rdev
->fence_drv
.last_timeout
= 1;
98 /* wrap around update last jiffies, we will just wait
101 rdev
->fence_drv
.last_jiffies
= cjiffies
;
106 list_for_each(i
, &rdev
->fence_drv
.emited
) {
107 fence
= list_entry(i
, struct radeon_fence
, list
);
108 if (fence
->seq
== seq
) {
113 /* all fence previous to this one are considered as signaled */
119 list_add_tail(i
, &rdev
->fence_drv
.signaled
);
120 fence
= list_entry(i
, struct radeon_fence
, list
);
121 fence
->signaled
= true;
123 } while (i
!= &rdev
->fence_drv
.emited
);
129 static void radeon_fence_destroy(struct kref
*kref
)
131 unsigned long irq_flags
;
132 struct radeon_fence
*fence
;
134 fence
= container_of(kref
, struct radeon_fence
, kref
);
135 write_lock_irqsave(&fence
->rdev
->fence_drv
.lock
, irq_flags
);
136 list_del(&fence
->list
);
137 fence
->emited
= false;
138 write_unlock_irqrestore(&fence
->rdev
->fence_drv
.lock
, irq_flags
);
142 int radeon_fence_create(struct radeon_device
*rdev
, struct radeon_fence
**fence
)
144 unsigned long irq_flags
;
146 *fence
= kmalloc(sizeof(struct radeon_fence
), GFP_KERNEL
);
147 if ((*fence
) == NULL
) {
150 kref_init(&((*fence
)->kref
));
151 (*fence
)->rdev
= rdev
;
152 (*fence
)->emited
= false;
153 (*fence
)->signaled
= false;
155 INIT_LIST_HEAD(&(*fence
)->list
);
157 write_lock_irqsave(&rdev
->fence_drv
.lock
, irq_flags
);
158 list_add_tail(&(*fence
)->list
, &rdev
->fence_drv
.created
);
159 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
164 bool radeon_fence_signaled(struct radeon_fence
*fence
)
166 unsigned long irq_flags
;
167 bool signaled
= false;
172 if (fence
->rdev
->gpu_lockup
)
175 write_lock_irqsave(&fence
->rdev
->fence_drv
.lock
, irq_flags
);
176 signaled
= fence
->signaled
;
177 /* if we are shuting down report all fence as signaled */
178 if (fence
->rdev
->shutdown
) {
181 if (!fence
->emited
) {
182 WARN(1, "Querying an unemited fence : %p !\n", fence
);
186 radeon_fence_poll_locked(fence
->rdev
);
187 signaled
= fence
->signaled
;
189 write_unlock_irqrestore(&fence
->rdev
->fence_drv
.lock
, irq_flags
);
193 int radeon_fence_wait(struct radeon_fence
*fence
, bool intr
)
195 struct radeon_device
*rdev
;
196 unsigned long irq_flags
, timeout
;
201 WARN(1, "Querying an invalid fence : %p !\n", fence
);
205 if (radeon_fence_signaled(fence
)) {
208 timeout
= rdev
->fence_drv
.last_timeout
;
210 /* save current sequence used to check for GPU lockup */
211 seq
= rdev
->fence_drv
.last_seq
;
213 radeon_irq_kms_sw_irq_get(rdev
);
214 r
= wait_event_interruptible_timeout(rdev
->fence_drv
.queue
,
215 radeon_fence_signaled(fence
), timeout
);
216 radeon_irq_kms_sw_irq_put(rdev
);
217 if (unlikely(r
< 0)) {
221 radeon_irq_kms_sw_irq_get(rdev
);
222 r
= wait_event_timeout(rdev
->fence_drv
.queue
,
223 radeon_fence_signaled(fence
), timeout
);
224 radeon_irq_kms_sw_irq_put(rdev
);
226 if (unlikely(!radeon_fence_signaled(fence
))) {
227 /* we were interrupted for some reason and fence isn't
228 * isn't signaled yet, resume wait
234 /* don't protect read access to rdev->fence_drv.last_seq
235 * if we experiencing a lockup the value doesn't change
237 if (seq
== rdev
->fence_drv
.last_seq
&& radeon_gpu_is_lockup(rdev
)) {
238 /* good news we believe it's a lockup */
239 WARN(1, "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n", fence
->seq
, seq
);
240 /* FIXME: what should we do ? marking everyone
241 * as signaled for now
243 rdev
->gpu_lockup
= true;
244 r
= radeon_gpu_reset(rdev
);
247 WREG32(rdev
->fence_drv
.scratch_reg
, fence
->seq
);
248 rdev
->gpu_lockup
= false;
250 timeout
= RADEON_FENCE_JIFFIES_TIMEOUT
;
251 write_lock_irqsave(&rdev
->fence_drv
.lock
, irq_flags
);
252 rdev
->fence_drv
.last_timeout
= RADEON_FENCE_JIFFIES_TIMEOUT
;
253 rdev
->fence_drv
.last_jiffies
= jiffies
;
254 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
260 int radeon_fence_wait_next(struct radeon_device
*rdev
)
262 unsigned long irq_flags
;
263 struct radeon_fence
*fence
;
266 if (rdev
->gpu_lockup
) {
269 write_lock_irqsave(&rdev
->fence_drv
.lock
, irq_flags
);
270 if (list_empty(&rdev
->fence_drv
.emited
)) {
271 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
274 fence
= list_entry(rdev
->fence_drv
.emited
.next
,
275 struct radeon_fence
, list
);
276 radeon_fence_ref(fence
);
277 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
278 r
= radeon_fence_wait(fence
, false);
279 radeon_fence_unref(&fence
);
283 int radeon_fence_wait_last(struct radeon_device
*rdev
)
285 unsigned long irq_flags
;
286 struct radeon_fence
*fence
;
289 if (rdev
->gpu_lockup
) {
292 write_lock_irqsave(&rdev
->fence_drv
.lock
, irq_flags
);
293 if (list_empty(&rdev
->fence_drv
.emited
)) {
294 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
297 fence
= list_entry(rdev
->fence_drv
.emited
.prev
,
298 struct radeon_fence
, list
);
299 radeon_fence_ref(fence
);
300 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
301 r
= radeon_fence_wait(fence
, false);
302 radeon_fence_unref(&fence
);
306 struct radeon_fence
*radeon_fence_ref(struct radeon_fence
*fence
)
308 kref_get(&fence
->kref
);
312 void radeon_fence_unref(struct radeon_fence
**fence
)
314 struct radeon_fence
*tmp
= *fence
;
318 kref_put(&tmp
->kref
, &radeon_fence_destroy
);
322 void radeon_fence_process(struct radeon_device
*rdev
)
324 unsigned long irq_flags
;
327 write_lock_irqsave(&rdev
->fence_drv
.lock
, irq_flags
);
328 wake
= radeon_fence_poll_locked(rdev
);
329 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
331 wake_up_all(&rdev
->fence_drv
.queue
);
335 int radeon_fence_driver_init(struct radeon_device
*rdev
)
337 unsigned long irq_flags
;
340 write_lock_irqsave(&rdev
->fence_drv
.lock
, irq_flags
);
341 r
= radeon_scratch_get(rdev
, &rdev
->fence_drv
.scratch_reg
);
343 dev_err(rdev
->dev
, "fence failed to get scratch register\n");
344 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
347 WREG32(rdev
->fence_drv
.scratch_reg
, 0);
348 atomic_set(&rdev
->fence_drv
.seq
, 0);
349 INIT_LIST_HEAD(&rdev
->fence_drv
.created
);
350 INIT_LIST_HEAD(&rdev
->fence_drv
.emited
);
351 INIT_LIST_HEAD(&rdev
->fence_drv
.signaled
);
352 init_waitqueue_head(&rdev
->fence_drv
.queue
);
353 rdev
->fence_drv
.initialized
= true;
354 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
355 if (radeon_debugfs_fence_init(rdev
)) {
356 dev_err(rdev
->dev
, "fence debugfs file creation failed\n");
361 void radeon_fence_driver_fini(struct radeon_device
*rdev
)
363 unsigned long irq_flags
;
365 if (!rdev
->fence_drv
.initialized
)
367 wake_up_all(&rdev
->fence_drv
.queue
);
368 write_lock_irqsave(&rdev
->fence_drv
.lock
, irq_flags
);
369 radeon_scratch_free(rdev
, rdev
->fence_drv
.scratch_reg
);
370 write_unlock_irqrestore(&rdev
->fence_drv
.lock
, irq_flags
);
371 rdev
->fence_drv
.initialized
= false;
378 #if defined(CONFIG_DEBUG_FS)
379 static int radeon_debugfs_fence_info(struct seq_file
*m
, void *data
)
381 struct drm_info_node
*node
= (struct drm_info_node
*)m
->private;
382 struct drm_device
*dev
= node
->minor
->dev
;
383 struct radeon_device
*rdev
= dev
->dev_private
;
384 struct radeon_fence
*fence
;
386 seq_printf(m
, "Last signaled fence 0x%08X\n",
387 RREG32(rdev
->fence_drv
.scratch_reg
));
388 if (!list_empty(&rdev
->fence_drv
.emited
)) {
389 fence
= list_entry(rdev
->fence_drv
.emited
.prev
,
390 struct radeon_fence
, list
);
391 seq_printf(m
, "Last emited fence %p with 0x%08X\n",
397 static struct drm_info_list radeon_debugfs_fence_list
[] = {
398 {"radeon_fence_info", &radeon_debugfs_fence_info
, 0, NULL
},
402 int radeon_debugfs_fence_init(struct radeon_device
*rdev
)
404 #if defined(CONFIG_DEBUG_FS)
405 return radeon_debugfs_add_files(rdev
, radeon_debugfs_fence_list
, 1);