Commit | Line | Data |
---|---|---|
05235c53 CW |
1 | /* |
2 | * Copyright © 2008-2015 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #ifndef I915_GEM_REQUEST_H | |
26 | #define I915_GEM_REQUEST_H | |
27 | ||
04769652 CW |
28 | #include <linux/fence.h> |
29 | ||
30 | #include "i915_gem.h" | |
31 | ||
05235c53 CW |
32 | /** |
33 | * Request queue structure. | |
34 | * | |
35 | * The request queue allows us to note sequence numbers that have been emitted | |
36 | * and may be associated with active buffers to be retired. | |
37 | * | |
38 | * By keeping this list, we can avoid having to do questionable sequence | |
39 | * number comparisons on buffer last_read|write_seqno. It also allows an | |
40 | * emission time to be associated with the request for tracking how far ahead | |
41 | * of the GPU the submission is. | |
42 | * | |
04769652 | 43 | * The requests are reference counted. |
05235c53 CW |
44 | */ |
45 | struct drm_i915_gem_request { | |
04769652 CW |
46 | struct fence fence; |
47 | spinlock_t lock; | |
05235c53 CW |
48 | |
49 | /** On Which ring this request was generated */ | |
50 | struct drm_i915_private *i915; | |
51 | ||
52 | /** | |
53 | * Context and ring buffer related to this request | |
54 | * Contexts are refcounted, so when this request is associated with a | |
55 | * context, we must increment the context's refcount, to guarantee that | |
56 | * it persists while any request is linked to it. Requests themselves | |
57 | * are also refcounted, so the request will only be freed when the last | |
58 | * reference to it is dismissed, and the code in | |
59 | * i915_gem_request_free() will then decrement the refcount on the | |
60 | * context. | |
61 | */ | |
62 | struct i915_gem_context *ctx; | |
63 | struct intel_engine_cs *engine; | |
7e37f889 | 64 | struct intel_ring *ring; |
05235c53 CW |
65 | struct intel_signal_node signaling; |
66 | ||
67 | /** GEM sequence number associated with the previous request, | |
68 | * when the HWS breadcrumb is equal to this the GPU is processing | |
69 | * this request. | |
70 | */ | |
71 | u32 previous_seqno; | |
72 | ||
05235c53 CW |
73 | /** Position in the ringbuffer of the start of the request */ |
74 | u32 head; | |
75 | ||
76 | /** | |
77 | * Position in the ringbuffer of the start of the postfix. | |
78 | * This is required to calculate the maximum available ringbuffer | |
79 | * space without overwriting the postfix. | |
80 | */ | |
81 | u32 postfix; | |
82 | ||
83 | /** Position in the ringbuffer of the end of the whole request */ | |
84 | u32 tail; | |
85 | ||
86 | /** Preallocate space in the ringbuffer for the emitting the request */ | |
87 | u32 reserved_space; | |
88 | ||
89 | /** | |
90 | * Context related to the previous request. | |
91 | * As the contexts are accessed by the hardware until the switch is | |
92 | * completed to a new context, the hardware may still be writing | |
93 | * to the context object after the breadcrumb is visible. We must | |
94 | * not unpin/unbind/prune that object whilst still active and so | |
95 | * we keep the previous context pinned until the following (this) | |
96 | * request is retired. | |
97 | */ | |
98 | struct i915_gem_context *previous_context; | |
99 | ||
100 | /** Batch buffer related to this request if any (used for | |
101 | * error state dump only). | |
102 | */ | |
103 | struct drm_i915_gem_object *batch_obj; | |
fa545cbf | 104 | struct list_head active_list; |
05235c53 CW |
105 | |
106 | /** Time at which this request was emitted, in jiffies. */ | |
107 | unsigned long emitted_jiffies; | |
108 | ||
efdf7c06 CW |
109 | /** engine->request_list entry for this request */ |
110 | struct list_head link; | |
05235c53 | 111 | |
675d9ad7 CW |
112 | /** ring->request_list entry for this request */ |
113 | struct list_head ring_link; | |
114 | ||
05235c53 CW |
115 | struct drm_i915_file_private *file_priv; |
116 | /** file_priv list entry for this request */ | |
117 | struct list_head client_list; | |
118 | ||
119 | /** process identifier submitting this request */ | |
120 | struct pid *pid; | |
121 | ||
122 | /** | |
123 | * The ELSP only accepts two elements at a time, so we queue | |
124 | * context/tail pairs on a given queue (ring->execlist_queue) until the | |
125 | * hardware is available. The queue serves a double purpose: we also use | |
126 | * it to keep track of the up to 2 contexts currently in the hardware | |
127 | * (usually one in execution and the other queued up by the GPU): We | |
128 | * only remove elements from the head of the queue when the hardware | |
129 | * informs us that an element has been completed. | |
130 | * | |
131 | * All accesses to the queue are mediated by a spinlock | |
132 | * (ring->execlist_lock). | |
133 | */ | |
134 | ||
135 | /** Execlist link in the submission queue.*/ | |
136 | struct list_head execlist_link; | |
137 | ||
138 | /** Execlists no. of times this request has been sent to the ELSP */ | |
139 | int elsp_submitted; | |
140 | ||
141 | /** Execlists context hardware id. */ | |
142 | unsigned int ctx_hw_id; | |
143 | }; | |
144 | ||
04769652 CW |
145 | extern const struct fence_ops i915_fence_ops; |
146 | ||
147 | static inline bool fence_is_i915(struct fence *fence) | |
148 | { | |
149 | return fence->ops == &i915_fence_ops; | |
150 | } | |
151 | ||
05235c53 CW |
152 | struct drm_i915_gem_request * __must_check |
153 | i915_gem_request_alloc(struct intel_engine_cs *engine, | |
154 | struct i915_gem_context *ctx); | |
05235c53 CW |
155 | int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, |
156 | struct drm_file *file); | |
157 | void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); | |
158 | ||
159 | static inline u32 | |
160 | i915_gem_request_get_seqno(struct drm_i915_gem_request *req) | |
161 | { | |
04769652 | 162 | return req ? req->fence.seqno : 0; |
05235c53 CW |
163 | } |
164 | ||
165 | static inline struct intel_engine_cs * | |
166 | i915_gem_request_get_engine(struct drm_i915_gem_request *req) | |
167 | { | |
168 | return req ? req->engine : NULL; | |
169 | } | |
170 | ||
04769652 CW |
171 | static inline struct drm_i915_gem_request * |
172 | to_request(struct fence *fence) | |
173 | { | |
174 | /* We assume that NULL fence/request are interoperable */ | |
175 | BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0); | |
176 | GEM_BUG_ON(fence && !fence_is_i915(fence)); | |
177 | return container_of(fence, struct drm_i915_gem_request, fence); | |
178 | } | |
179 | ||
05235c53 | 180 | static inline struct drm_i915_gem_request * |
e8a261ea | 181 | i915_gem_request_get(struct drm_i915_gem_request *req) |
05235c53 | 182 | { |
04769652 | 183 | return to_request(fence_get(&req->fence)); |
05235c53 CW |
184 | } |
185 | ||
186 | static inline void | |
e8a261ea | 187 | i915_gem_request_put(struct drm_i915_gem_request *req) |
05235c53 | 188 | { |
04769652 | 189 | fence_put(&req->fence); |
05235c53 CW |
190 | } |
191 | ||
192 | static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst, | |
193 | struct drm_i915_gem_request *src) | |
194 | { | |
195 | if (src) | |
e8a261ea | 196 | i915_gem_request_get(src); |
05235c53 CW |
197 | |
198 | if (*pdst) | |
e8a261ea | 199 | i915_gem_request_put(*pdst); |
05235c53 CW |
200 | |
201 | *pdst = src; | |
202 | } | |
203 | ||
204 | void __i915_add_request(struct drm_i915_gem_request *req, | |
205 | struct drm_i915_gem_object *batch_obj, | |
206 | bool flush_caches); | |
207 | #define i915_add_request(req) \ | |
208 | __i915_add_request(req, NULL, true) | |
209 | #define i915_add_request_no_flush(req) \ | |
210 | __i915_add_request(req, NULL, false) | |
211 | ||
212 | struct intel_rps_client; | |
42df2714 CW |
213 | #define NO_WAITBOOST ERR_PTR(-1) |
214 | #define IS_RPS_CLIENT(p) (!IS_ERR(p)) | |
215 | #define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p)) | |
05235c53 | 216 | |
776f3236 CW |
217 | int i915_wait_request(struct drm_i915_gem_request *req, |
218 | bool interruptible, | |
219 | s64 *timeout, | |
220 | struct intel_rps_client *rps) | |
fa545cbf CW |
221 | __attribute__((nonnull(1))); |
222 | ||
05235c53 CW |
223 | static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine); |
224 | ||
225 | /** | |
226 | * Returns true if seq1 is later than seq2. | |
227 | */ | |
228 | static inline bool i915_seqno_passed(u32 seq1, u32 seq2) | |
229 | { | |
230 | return (s32)(seq1 - seq2) >= 0; | |
231 | } | |
232 | ||
233 | static inline bool | |
234 | i915_gem_request_started(const struct drm_i915_gem_request *req) | |
235 | { | |
236 | return i915_seqno_passed(intel_engine_get_seqno(req->engine), | |
237 | req->previous_seqno); | |
238 | } | |
239 | ||
240 | static inline bool | |
241 | i915_gem_request_completed(const struct drm_i915_gem_request *req) | |
242 | { | |
243 | return i915_seqno_passed(intel_engine_get_seqno(req->engine), | |
04769652 | 244 | req->fence.seqno); |
05235c53 CW |
245 | } |
246 | ||
247 | bool __i915_spin_request(const struct drm_i915_gem_request *request, | |
248 | int state, unsigned long timeout_us); | |
249 | static inline bool i915_spin_request(const struct drm_i915_gem_request *request, | |
250 | int state, unsigned long timeout_us) | |
251 | { | |
252 | return (i915_gem_request_started(request) && | |
253 | __i915_spin_request(request, state, timeout_us)); | |
254 | } | |
255 | ||
381f371b CW |
256 | /* We treat requests as fences. This is not be to confused with our |
257 | * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. | |
258 | * We use the fences to synchronize access from the CPU with activity on the | |
259 | * GPU, for example, we should not rewrite an object's PTE whilst the GPU | |
260 | * is reading them. We also track fences at a higher level to provide | |
261 | * implicit synchronisation around GEM objects, e.g. set-domain will wait | |
262 | * for outstanding GPU rendering before marking the object ready for CPU | |
263 | * access, or a pageflip will wait until the GPU is complete before showing | |
264 | * the frame on the scanout. | |
265 | * | |
266 | * In order to use a fence, the object must track the fence it needs to | |
267 | * serialise with. For example, GEM objects want to track both read and | |
268 | * write access so that we can perform concurrent read operations between | |
269 | * the CPU and GPU engines, as well as waiting for all rendering to | |
270 | * complete, or waiting for the last GPU user of a "fence register". The | |
271 | * object then embeds a #i915_gem_active to track the most recent (in | |
272 | * retirement order) request relevant for the desired mode of access. | |
273 | * The #i915_gem_active is updated with i915_gem_active_set() to track the | |
274 | * most recent fence request, typically this is done as part of | |
275 | * i915_vma_move_to_active(). | |
276 | * | |
277 | * When the #i915_gem_active completes (is retired), it will | |
278 | * signal its completion to the owner through a callback as well as mark | |
279 | * itself as idle (i915_gem_active.request == NULL). The owner | |
280 | * can then perform any action, such as delayed freeing of an active | |
281 | * resource including itself. | |
282 | */ | |
fa545cbf CW |
283 | struct i915_gem_active; |
284 | ||
285 | typedef void (*i915_gem_retire_fn)(struct i915_gem_active *, | |
286 | struct drm_i915_gem_request *); | |
287 | ||
381f371b CW |
288 | struct i915_gem_active { |
289 | struct drm_i915_gem_request *request; | |
fa545cbf CW |
290 | struct list_head link; |
291 | i915_gem_retire_fn retire; | |
381f371b CW |
292 | }; |
293 | ||
fa545cbf CW |
294 | void i915_gem_retire_noop(struct i915_gem_active *, |
295 | struct drm_i915_gem_request *request); | |
296 | ||
297 | /** | |
298 | * init_request_active - prepares the activity tracker for use | |
299 | * @active - the active tracker | |
300 | * @func - a callback when then the tracker is retired (becomes idle), | |
301 | * can be NULL | |
302 | * | |
303 | * init_request_active() prepares the embedded @active struct for use as | |
304 | * an activity tracker, that is for tracking the last known active request | |
305 | * associated with it. When the last request becomes idle, when it is retired | |
306 | * after completion, the optional callback @func is invoked. | |
307 | */ | |
308 | static inline void | |
309 | init_request_active(struct i915_gem_active *active, | |
310 | i915_gem_retire_fn retire) | |
311 | { | |
312 | INIT_LIST_HEAD(&active->link); | |
313 | active->retire = retire ?: i915_gem_retire_noop; | |
314 | } | |
315 | ||
27c01aae CW |
316 | /** |
317 | * i915_gem_active_set - updates the tracker to watch the current request | |
318 | * @active - the active tracker | |
319 | * @request - the request to watch | |
320 | * | |
321 | * i915_gem_active_set() watches the given @request for completion. Whilst | |
322 | * that @request is busy, the @active reports busy. When that @request is | |
323 | * retired, the @active tracker is updated to report idle. | |
324 | */ | |
381f371b CW |
325 | static inline void |
326 | i915_gem_active_set(struct i915_gem_active *active, | |
327 | struct drm_i915_gem_request *request) | |
328 | { | |
fa545cbf CW |
329 | list_move(&active->link, &request->active_list); |
330 | active->request = request; | |
381f371b CW |
331 | } |
332 | ||
d72d908b CW |
333 | static inline struct drm_i915_gem_request * |
334 | __i915_gem_active_peek(const struct i915_gem_active *active) | |
335 | { | |
336 | return active->request; | |
337 | } | |
338 | ||
27c01aae | 339 | /** |
fa545cbf | 340 | * i915_gem_active_peek - report the active request being monitored |
27c01aae CW |
341 | * @active - the active tracker |
342 | * | |
fa545cbf CW |
343 | * i915_gem_active_peek() returns the current request being tracked if |
344 | * still active, or NULL. It does not obtain a reference on the request | |
345 | * for the caller, so the caller must hold struct_mutex. | |
27c01aae CW |
346 | */ |
347 | static inline struct drm_i915_gem_request * | |
d72d908b | 348 | i915_gem_active_peek(const struct i915_gem_active *active, struct mutex *mutex) |
27c01aae | 349 | { |
fa545cbf CW |
350 | struct drm_i915_gem_request *request; |
351 | ||
352 | request = active->request; | |
353 | if (!request || i915_gem_request_completed(request)) | |
354 | return NULL; | |
355 | ||
356 | return request; | |
27c01aae CW |
357 | } |
358 | ||
359 | /** | |
360 | * i915_gem_active_get - return a reference to the active request | |
361 | * @active - the active tracker | |
362 | * | |
363 | * i915_gem_active_get() returns a reference to the active request, or NULL | |
364 | * if the active tracker is idle. The caller must hold struct_mutex. | |
365 | */ | |
366 | static inline struct drm_i915_gem_request * | |
d72d908b | 367 | i915_gem_active_get(const struct i915_gem_active *active, struct mutex *mutex) |
27c01aae | 368 | { |
fa545cbf | 369 | return i915_gem_request_get(i915_gem_active_peek(active, mutex)); |
27c01aae CW |
370 | } |
371 | ||
372 | /** | |
373 | * i915_gem_active_isset - report whether the active tracker is assigned | |
374 | * @active - the active tracker | |
375 | * | |
376 | * i915_gem_active_isset() returns true if the active tracker is currently | |
377 | * assigned to a request. Due to the lazy retiring, that request may be idle | |
378 | * and this may report stale information. | |
379 | */ | |
380 | static inline bool | |
381 | i915_gem_active_isset(const struct i915_gem_active *active) | |
382 | { | |
383 | return active->request; | |
384 | } | |
385 | ||
386 | /** | |
387 | * i915_gem_active_is_idle - report whether the active tracker is idle | |
388 | * @active - the active tracker | |
389 | * | |
390 | * i915_gem_active_is_idle() returns true if the active tracker is currently | |
391 | * unassigned or if the request is complete (but not yet retired). Requires | |
392 | * the caller to hold struct_mutex (but that can be relaxed if desired). | |
393 | */ | |
394 | static inline bool | |
d72d908b CW |
395 | i915_gem_active_is_idle(const struct i915_gem_active *active, |
396 | struct mutex *mutex) | |
27c01aae | 397 | { |
fa545cbf | 398 | return !i915_gem_active_peek(active, mutex); |
27c01aae CW |
399 | } |
400 | ||
401 | /** | |
402 | * i915_gem_active_wait - waits until the request is completed | |
403 | * @active - the active request on which to wait | |
404 | * | |
405 | * i915_gem_active_wait() waits until the request is completed before | |
406 | * returning. Note that it does not guarantee that the request is | |
407 | * retired first, see i915_gem_active_retire(). | |
fa545cbf CW |
408 | * |
409 | * i915_gem_active_wait() returns immediately if the active | |
410 | * request is already complete. | |
27c01aae CW |
411 | */ |
412 | static inline int __must_check | |
d72d908b | 413 | i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex) |
27c01aae CW |
414 | { |
415 | struct drm_i915_gem_request *request; | |
416 | ||
d72d908b | 417 | request = i915_gem_active_peek(active, mutex); |
27c01aae CW |
418 | if (!request) |
419 | return 0; | |
420 | ||
776f3236 | 421 | return i915_wait_request(request, true, NULL, NULL); |
27c01aae CW |
422 | } |
423 | ||
424 | /** | |
425 | * i915_gem_active_retire - waits until the request is retired | |
426 | * @active - the active request on which to wait | |
427 | * | |
428 | * i915_gem_active_retire() waits until the request is completed, | |
429 | * and then ensures that at least the retirement handler for this | |
430 | * @active tracker is called before returning. If the @active | |
431 | * tracker is idle, the function returns immediately. | |
432 | */ | |
433 | static inline int __must_check | |
fa545cbf | 434 | i915_gem_active_retire(struct i915_gem_active *active, |
d72d908b | 435 | struct mutex *mutex) |
27c01aae | 436 | { |
fa545cbf CW |
437 | struct drm_i915_gem_request *request; |
438 | int ret; | |
439 | ||
440 | request = active->request; | |
441 | if (!request) | |
442 | return 0; | |
443 | ||
776f3236 | 444 | ret = i915_wait_request(request, true, NULL, NULL); |
fa545cbf CW |
445 | if (ret) |
446 | return ret; | |
447 | ||
448 | list_del_init(&active->link); | |
449 | active->request = NULL; | |
450 | active->retire(active, request); | |
451 | ||
452 | return 0; | |
27c01aae CW |
453 | } |
454 | ||
455 | /* Convenience functions for peeking at state inside active's request whilst | |
456 | * guarded by the struct_mutex. | |
457 | */ | |
458 | ||
459 | static inline uint32_t | |
d72d908b CW |
460 | i915_gem_active_get_seqno(const struct i915_gem_active *active, |
461 | struct mutex *mutex) | |
27c01aae | 462 | { |
d72d908b | 463 | return i915_gem_request_get_seqno(i915_gem_active_peek(active, mutex)); |
27c01aae CW |
464 | } |
465 | ||
466 | static inline struct intel_engine_cs * | |
d72d908b CW |
467 | i915_gem_active_get_engine(const struct i915_gem_active *active, |
468 | struct mutex *mutex) | |
27c01aae | 469 | { |
d72d908b | 470 | return i915_gem_request_get_engine(i915_gem_active_peek(active, mutex)); |
27c01aae CW |
471 | } |
472 | ||
381f371b CW |
473 | #define for_each_active(mask, idx) \ |
474 | for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx)) | |
475 | ||
05235c53 | 476 | #endif /* I915_GEM_REQUEST_H */ |