Commit | Line | Data |
---|---|---|
05235c53 CW |
1 | /* |
2 | * Copyright © 2008-2015 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #ifndef I915_GEM_REQUEST_H | |
26 | #define I915_GEM_REQUEST_H | |
27 | ||
04769652 CW |
28 | #include <linux/fence.h> |
29 | ||
30 | #include "i915_gem.h" | |
31 | ||
05235c53 CW |
32 | /** |
33 | * Request queue structure. | |
34 | * | |
35 | * The request queue allows us to note sequence numbers that have been emitted | |
36 | * and may be associated with active buffers to be retired. | |
37 | * | |
38 | * By keeping this list, we can avoid having to do questionable sequence | |
39 | * number comparisons on buffer last_read|write_seqno. It also allows an | |
40 | * emission time to be associated with the request for tracking how far ahead | |
41 | * of the GPU the submission is. | |
42 | * | |
04769652 | 43 | * The requests are reference counted. |
05235c53 CW |
44 | */ |
45 | struct drm_i915_gem_request { | |
04769652 CW |
46 | struct fence fence; |
47 | spinlock_t lock; | |
05235c53 CW |
48 | |
49 | /** On Which ring this request was generated */ | |
50 | struct drm_i915_private *i915; | |
51 | ||
52 | /** | |
53 | * Context and ring buffer related to this request | |
54 | * Contexts are refcounted, so when this request is associated with a | |
55 | * context, we must increment the context's refcount, to guarantee that | |
56 | * it persists while any request is linked to it. Requests themselves | |
57 | * are also refcounted, so the request will only be freed when the last | |
58 | * reference to it is dismissed, and the code in | |
59 | * i915_gem_request_free() will then decrement the refcount on the | |
60 | * context. | |
61 | */ | |
62 | struct i915_gem_context *ctx; | |
63 | struct intel_engine_cs *engine; | |
7e37f889 | 64 | struct intel_ring *ring; |
05235c53 CW |
65 | struct intel_signal_node signaling; |
66 | ||
67 | /** GEM sequence number associated with the previous request, | |
68 | * when the HWS breadcrumb is equal to this the GPU is processing | |
69 | * this request. | |
70 | */ | |
71 | u32 previous_seqno; | |
72 | ||
05235c53 CW |
73 | /** Position in the ringbuffer of the start of the request */ |
74 | u32 head; | |
75 | ||
76 | /** | |
77 | * Position in the ringbuffer of the start of the postfix. | |
78 | * This is required to calculate the maximum available ringbuffer | |
79 | * space without overwriting the postfix. | |
80 | */ | |
81 | u32 postfix; | |
82 | ||
83 | /** Position in the ringbuffer of the end of the whole request */ | |
84 | u32 tail; | |
85 | ||
86 | /** Preallocate space in the ringbuffer for the emitting the request */ | |
87 | u32 reserved_space; | |
88 | ||
89 | /** | |
90 | * Context related to the previous request. | |
91 | * As the contexts are accessed by the hardware until the switch is | |
92 | * completed to a new context, the hardware may still be writing | |
93 | * to the context object after the breadcrumb is visible. We must | |
94 | * not unpin/unbind/prune that object whilst still active and so | |
95 | * we keep the previous context pinned until the following (this) | |
96 | * request is retired. | |
97 | */ | |
98 | struct i915_gem_context *previous_context; | |
99 | ||
100 | /** Batch buffer related to this request if any (used for | |
101 | * error state dump only). | |
102 | */ | |
103 | struct drm_i915_gem_object *batch_obj; | |
104 | ||
105 | /** Time at which this request was emitted, in jiffies. */ | |
106 | unsigned long emitted_jiffies; | |
107 | ||
108 | /** global list entry for this request */ | |
109 | struct list_head list; | |
110 | ||
111 | struct drm_i915_file_private *file_priv; | |
112 | /** file_priv list entry for this request */ | |
113 | struct list_head client_list; | |
114 | ||
115 | /** process identifier submitting this request */ | |
116 | struct pid *pid; | |
117 | ||
118 | /** | |
119 | * The ELSP only accepts two elements at a time, so we queue | |
120 | * context/tail pairs on a given queue (ring->execlist_queue) until the | |
121 | * hardware is available. The queue serves a double purpose: we also use | |
122 | * it to keep track of the up to 2 contexts currently in the hardware | |
123 | * (usually one in execution and the other queued up by the GPU): We | |
124 | * only remove elements from the head of the queue when the hardware | |
125 | * informs us that an element has been completed. | |
126 | * | |
127 | * All accesses to the queue are mediated by a spinlock | |
128 | * (ring->execlist_lock). | |
129 | */ | |
130 | ||
131 | /** Execlist link in the submission queue.*/ | |
132 | struct list_head execlist_link; | |
133 | ||
134 | /** Execlists no. of times this request has been sent to the ELSP */ | |
135 | int elsp_submitted; | |
136 | ||
137 | /** Execlists context hardware id. */ | |
138 | unsigned int ctx_hw_id; | |
139 | }; | |
140 | ||
04769652 CW |
141 | extern const struct fence_ops i915_fence_ops; |
142 | ||
143 | static inline bool fence_is_i915(struct fence *fence) | |
144 | { | |
145 | return fence->ops == &i915_fence_ops; | |
146 | } | |
147 | ||
05235c53 CW |
148 | struct drm_i915_gem_request * __must_check |
149 | i915_gem_request_alloc(struct intel_engine_cs *engine, | |
150 | struct i915_gem_context *ctx); | |
05235c53 CW |
151 | int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, |
152 | struct drm_file *file); | |
153 | void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); | |
154 | ||
155 | static inline u32 | |
156 | i915_gem_request_get_seqno(struct drm_i915_gem_request *req) | |
157 | { | |
04769652 | 158 | return req ? req->fence.seqno : 0; |
05235c53 CW |
159 | } |
160 | ||
161 | static inline struct intel_engine_cs * | |
162 | i915_gem_request_get_engine(struct drm_i915_gem_request *req) | |
163 | { | |
164 | return req ? req->engine : NULL; | |
165 | } | |
166 | ||
04769652 CW |
167 | static inline struct drm_i915_gem_request * |
168 | to_request(struct fence *fence) | |
169 | { | |
170 | /* We assume that NULL fence/request are interoperable */ | |
171 | BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0); | |
172 | GEM_BUG_ON(fence && !fence_is_i915(fence)); | |
173 | return container_of(fence, struct drm_i915_gem_request, fence); | |
174 | } | |
175 | ||
05235c53 | 176 | static inline struct drm_i915_gem_request * |
e8a261ea | 177 | i915_gem_request_get(struct drm_i915_gem_request *req) |
05235c53 | 178 | { |
04769652 | 179 | return to_request(fence_get(&req->fence)); |
05235c53 CW |
180 | } |
181 | ||
182 | static inline void | |
e8a261ea | 183 | i915_gem_request_put(struct drm_i915_gem_request *req) |
05235c53 | 184 | { |
04769652 | 185 | fence_put(&req->fence); |
05235c53 CW |
186 | } |
187 | ||
188 | static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst, | |
189 | struct drm_i915_gem_request *src) | |
190 | { | |
191 | if (src) | |
e8a261ea | 192 | i915_gem_request_get(src); |
05235c53 CW |
193 | |
194 | if (*pdst) | |
e8a261ea | 195 | i915_gem_request_put(*pdst); |
05235c53 CW |
196 | |
197 | *pdst = src; | |
198 | } | |
199 | ||
200 | void __i915_add_request(struct drm_i915_gem_request *req, | |
201 | struct drm_i915_gem_object *batch_obj, | |
202 | bool flush_caches); | |
203 | #define i915_add_request(req) \ | |
204 | __i915_add_request(req, NULL, true) | |
205 | #define i915_add_request_no_flush(req) \ | |
206 | __i915_add_request(req, NULL, false) | |
207 | ||
208 | struct intel_rps_client; | |
42df2714 CW |
209 | #define NO_WAITBOOST ERR_PTR(-1) |
210 | #define IS_RPS_CLIENT(p) (!IS_ERR(p)) | |
211 | #define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p)) | |
05235c53 CW |
212 | |
213 | int __i915_wait_request(struct drm_i915_gem_request *req, | |
214 | bool interruptible, | |
215 | s64 *timeout, | |
216 | struct intel_rps_client *rps); | |
217 | int __must_check i915_wait_request(struct drm_i915_gem_request *req); | |
218 | ||
219 | static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine); | |
220 | ||
221 | /** | |
222 | * Returns true if seq1 is later than seq2. | |
223 | */ | |
224 | static inline bool i915_seqno_passed(u32 seq1, u32 seq2) | |
225 | { | |
226 | return (s32)(seq1 - seq2) >= 0; | |
227 | } | |
228 | ||
229 | static inline bool | |
230 | i915_gem_request_started(const struct drm_i915_gem_request *req) | |
231 | { | |
232 | return i915_seqno_passed(intel_engine_get_seqno(req->engine), | |
233 | req->previous_seqno); | |
234 | } | |
235 | ||
236 | static inline bool | |
237 | i915_gem_request_completed(const struct drm_i915_gem_request *req) | |
238 | { | |
239 | return i915_seqno_passed(intel_engine_get_seqno(req->engine), | |
04769652 | 240 | req->fence.seqno); |
05235c53 CW |
241 | } |
242 | ||
243 | bool __i915_spin_request(const struct drm_i915_gem_request *request, | |
244 | int state, unsigned long timeout_us); | |
245 | static inline bool i915_spin_request(const struct drm_i915_gem_request *request, | |
246 | int state, unsigned long timeout_us) | |
247 | { | |
248 | return (i915_gem_request_started(request) && | |
249 | __i915_spin_request(request, state, timeout_us)); | |
250 | } | |
251 | ||
381f371b CW |
252 | /* We treat requests as fences. This is not be to confused with our |
253 | * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. | |
254 | * We use the fences to synchronize access from the CPU with activity on the | |
255 | * GPU, for example, we should not rewrite an object's PTE whilst the GPU | |
256 | * is reading them. We also track fences at a higher level to provide | |
257 | * implicit synchronisation around GEM objects, e.g. set-domain will wait | |
258 | * for outstanding GPU rendering before marking the object ready for CPU | |
259 | * access, or a pageflip will wait until the GPU is complete before showing | |
260 | * the frame on the scanout. | |
261 | * | |
262 | * In order to use a fence, the object must track the fence it needs to | |
263 | * serialise with. For example, GEM objects want to track both read and | |
264 | * write access so that we can perform concurrent read operations between | |
265 | * the CPU and GPU engines, as well as waiting for all rendering to | |
266 | * complete, or waiting for the last GPU user of a "fence register". The | |
267 | * object then embeds a #i915_gem_active to track the most recent (in | |
268 | * retirement order) request relevant for the desired mode of access. | |
269 | * The #i915_gem_active is updated with i915_gem_active_set() to track the | |
270 | * most recent fence request, typically this is done as part of | |
271 | * i915_vma_move_to_active(). | |
272 | * | |
273 | * When the #i915_gem_active completes (is retired), it will | |
274 | * signal its completion to the owner through a callback as well as mark | |
275 | * itself as idle (i915_gem_active.request == NULL). The owner | |
276 | * can then perform any action, such as delayed freeing of an active | |
277 | * resource including itself. | |
278 | */ | |
279 | struct i915_gem_active { | |
280 | struct drm_i915_gem_request *request; | |
281 | }; | |
282 | ||
27c01aae CW |
283 | /** |
284 | * i915_gem_active_set - updates the tracker to watch the current request | |
285 | * @active - the active tracker | |
286 | * @request - the request to watch | |
287 | * | |
288 | * i915_gem_active_set() watches the given @request for completion. Whilst | |
289 | * that @request is busy, the @active reports busy. When that @request is | |
290 | * retired, the @active tracker is updated to report idle. | |
291 | */ | |
381f371b CW |
292 | static inline void |
293 | i915_gem_active_set(struct i915_gem_active *active, | |
294 | struct drm_i915_gem_request *request) | |
295 | { | |
296 | i915_gem_request_assign(&active->request, request); | |
297 | } | |
298 | ||
27c01aae CW |
299 | /** |
300 | * i915_gem_active_peek - report the request being monitored | |
301 | * @active - the active tracker | |
302 | * | |
303 | * i915_gem_active_peek() returns the current request being tracked, or NULL. | |
304 | * It does not obtain a reference on the request for the caller, so the | |
305 | * caller must hold struct_mutex. | |
306 | */ | |
307 | static inline struct drm_i915_gem_request * | |
308 | i915_gem_active_peek(const struct i915_gem_active *active) | |
309 | { | |
310 | return active->request; | |
311 | } | |
312 | ||
313 | /** | |
314 | * i915_gem_active_get - return a reference to the active request | |
315 | * @active - the active tracker | |
316 | * | |
317 | * i915_gem_active_get() returns a reference to the active request, or NULL | |
318 | * if the active tracker is idle. The caller must hold struct_mutex. | |
319 | */ | |
320 | static inline struct drm_i915_gem_request * | |
321 | i915_gem_active_get(const struct i915_gem_active *active) | |
322 | { | |
323 | struct drm_i915_gem_request *request; | |
324 | ||
325 | request = i915_gem_active_peek(active); | |
326 | if (!request || i915_gem_request_completed(request)) | |
327 | return NULL; | |
328 | ||
329 | return i915_gem_request_get(request); | |
330 | } | |
331 | ||
332 | /** | |
333 | * i915_gem_active_isset - report whether the active tracker is assigned | |
334 | * @active - the active tracker | |
335 | * | |
336 | * i915_gem_active_isset() returns true if the active tracker is currently | |
337 | * assigned to a request. Due to the lazy retiring, that request may be idle | |
338 | * and this may report stale information. | |
339 | */ | |
340 | static inline bool | |
341 | i915_gem_active_isset(const struct i915_gem_active *active) | |
342 | { | |
343 | return active->request; | |
344 | } | |
345 | ||
346 | /** | |
347 | * i915_gem_active_is_idle - report whether the active tracker is idle | |
348 | * @active - the active tracker | |
349 | * | |
350 | * i915_gem_active_is_idle() returns true if the active tracker is currently | |
351 | * unassigned or if the request is complete (but not yet retired). Requires | |
352 | * the caller to hold struct_mutex (but that can be relaxed if desired). | |
353 | */ | |
354 | static inline bool | |
355 | i915_gem_active_is_idle(const struct i915_gem_active *active) | |
356 | { | |
357 | struct drm_i915_gem_request *request; | |
358 | ||
359 | request = i915_gem_active_peek(active); | |
360 | if (!request || i915_gem_request_completed(request)) | |
361 | return true; | |
362 | ||
363 | return false; | |
364 | } | |
365 | ||
366 | /** | |
367 | * i915_gem_active_wait - waits until the request is completed | |
368 | * @active - the active request on which to wait | |
369 | * | |
370 | * i915_gem_active_wait() waits until the request is completed before | |
371 | * returning. Note that it does not guarantee that the request is | |
372 | * retired first, see i915_gem_active_retire(). | |
373 | */ | |
374 | static inline int __must_check | |
375 | i915_gem_active_wait(const struct i915_gem_active *active) | |
376 | { | |
377 | struct drm_i915_gem_request *request; | |
378 | ||
379 | request = i915_gem_active_peek(active); | |
380 | if (!request) | |
381 | return 0; | |
382 | ||
383 | return i915_wait_request(request); | |
384 | } | |
385 | ||
386 | /** | |
387 | * i915_gem_active_retire - waits until the request is retired | |
388 | * @active - the active request on which to wait | |
389 | * | |
390 | * i915_gem_active_retire() waits until the request is completed, | |
391 | * and then ensures that at least the retirement handler for this | |
392 | * @active tracker is called before returning. If the @active | |
393 | * tracker is idle, the function returns immediately. | |
394 | */ | |
395 | static inline int __must_check | |
396 | i915_gem_active_retire(const struct i915_gem_active *active) | |
397 | { | |
398 | return i915_gem_active_wait(active); | |
399 | } | |
400 | ||
401 | /* Convenience functions for peeking at state inside active's request whilst | |
402 | * guarded by the struct_mutex. | |
403 | */ | |
404 | ||
405 | static inline uint32_t | |
406 | i915_gem_active_get_seqno(const struct i915_gem_active *active) | |
407 | { | |
408 | return i915_gem_request_get_seqno(i915_gem_active_peek(active)); | |
409 | } | |
410 | ||
411 | static inline struct intel_engine_cs * | |
412 | i915_gem_active_get_engine(const struct i915_gem_active *active) | |
413 | { | |
414 | return i915_gem_request_get_engine(i915_gem_active_peek(active)); | |
415 | } | |
416 | ||
381f371b CW |
417 | #define for_each_active(mask, idx) \ |
418 | for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx)) | |
419 | ||
05235c53 | 420 | #endif /* I915_GEM_REQUEST_H */ |