Commit | Line | Data |
---|---|---|
c8b75bca EA |
1 | /* |
2 | * Copyright (C) 2015 Broadcom | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | ||
9 | #include "drmP.h" | |
10 | #include "drm_gem_cma_helper.h" | |
11 | ||
12 | struct vc4_dev { | |
13 | struct drm_device *dev; | |
14 | ||
15 | struct vc4_hdmi *hdmi; | |
16 | struct vc4_hvs *hvs; | |
17 | struct vc4_crtc *crtc[3]; | |
d3f5168a | 18 | struct vc4_v3d *v3d; |
48666d56 DF |
19 | |
20 | struct drm_fbdev_cma *fbdev; | |
c826a6e1 | 21 | |
21461365 EA |
22 | struct vc4_hang_state *hang_state; |
23 | ||
c826a6e1 EA |
24 | /* The kernel-space BO cache. Tracks buffers that have been |
25 | * unreferenced by all other users (refcounts of 0!) but not | |
26 | * yet freed, so we can do cheap allocations. | |
27 | */ | |
28 | struct vc4_bo_cache { | |
29 | /* Array of list heads for entries in the BO cache, | |
30 | * based on number of pages, so we can do O(1) lookups | |
31 | * in the cache when allocating. | |
32 | */ | |
33 | struct list_head *size_list; | |
34 | uint32_t size_list_size; | |
35 | ||
36 | /* List of all BOs in the cache, ordered by age, so we | |
37 | * can do O(1) lookups when trying to free old | |
38 | * buffers. | |
39 | */ | |
40 | struct list_head time_list; | |
41 | struct work_struct time_work; | |
42 | struct timer_list time_timer; | |
43 | } bo_cache; | |
44 | ||
45 | struct vc4_bo_stats { | |
46 | u32 num_allocated; | |
47 | u32 size_allocated; | |
48 | u32 num_cached; | |
49 | u32 size_cached; | |
50 | } bo_stats; | |
51 | ||
52 | /* Protects bo_cache and the BO stats. */ | |
53 | struct mutex bo_lock; | |
d5b1a78a EA |
54 | |
55 | /* Sequence number for the last job queued in job_list. | |
56 | * Starts at 0 (no jobs emitted). | |
57 | */ | |
58 | uint64_t emit_seqno; | |
59 | ||
60 | /* Sequence number for the last completed job on the GPU. | |
61 | * Starts at 0 (no jobs completed). | |
62 | */ | |
63 | uint64_t finished_seqno; | |
64 | ||
65 | /* List of all struct vc4_exec_info for jobs to be executed. | |
66 | * The first job in the list is the one currently programmed | |
67 | * into ct0ca/ct1ca for execution. | |
68 | */ | |
69 | struct list_head job_list; | |
70 | /* List of the finished vc4_exec_infos waiting to be freed by | |
71 | * job_done_work. | |
72 | */ | |
73 | struct list_head job_done_list; | |
74 | /* Spinlock used to synchronize the job_list and seqno | |
75 | * accesses between the IRQ handler and GEM ioctls. | |
76 | */ | |
77 | spinlock_t job_lock; | |
78 | wait_queue_head_t job_wait_queue; | |
79 | struct work_struct job_done_work; | |
80 | ||
b501bacc EA |
81 | /* List of struct vc4_seqno_cb for callbacks to be made from a |
82 | * workqueue when the given seqno is passed. | |
83 | */ | |
84 | struct list_head seqno_cb_list; | |
85 | ||
d5b1a78a EA |
86 | /* The binner overflow memory that's currently set up in |
87 | * BPOA/BPOS registers. When overflow occurs and a new one is | |
88 | * allocated, the previous one will be moved to | |
89 | * vc4->current_exec's free list. | |
90 | */ | |
91 | struct vc4_bo *overflow_mem; | |
92 | struct work_struct overflow_mem_work; | |
93 | ||
94 | struct { | |
95 | uint32_t last_ct0ca, last_ct1ca; | |
96 | struct timer_list timer; | |
97 | struct work_struct reset_work; | |
98 | } hangcheck; | |
99 | ||
100 | struct semaphore async_modeset; | |
c8b75bca EA |
101 | }; |
102 | ||
103 | static inline struct vc4_dev * | |
104 | to_vc4_dev(struct drm_device *dev) | |
105 | { | |
106 | return (struct vc4_dev *)dev->dev_private; | |
107 | } | |
108 | ||
109 | struct vc4_bo { | |
110 | struct drm_gem_cma_object base; | |
c826a6e1 | 111 | |
d5b1a78a EA |
112 | /* seqno of the last job to render to this BO. */ |
113 | uint64_t seqno; | |
114 | ||
c826a6e1 EA |
115 | /* List entry for the BO's position in either |
116 | * vc4_exec_info->unref_list or vc4_dev->bo_cache.time_list | |
117 | */ | |
118 | struct list_head unref_head; | |
119 | ||
120 | /* Time in jiffies when the BO was put in vc4->bo_cache. */ | |
121 | unsigned long free_time; | |
122 | ||
123 | /* List entry for the BO's position in vc4_dev->bo_cache.size_list */ | |
124 | struct list_head size_head; | |
463873d5 EA |
125 | |
126 | /* Struct for shader validation state, if created by | |
127 | * DRM_IOCTL_VC4_CREATE_SHADER_BO. | |
128 | */ | |
129 | struct vc4_validated_shader_info *validated_shader; | |
c8b75bca EA |
130 | }; |
131 | ||
132 | static inline struct vc4_bo * | |
133 | to_vc4_bo(struct drm_gem_object *bo) | |
134 | { | |
135 | return (struct vc4_bo *)bo; | |
136 | } | |
137 | ||
b501bacc EA |
138 | struct vc4_seqno_cb { |
139 | struct work_struct work; | |
140 | uint64_t seqno; | |
141 | void (*func)(struct vc4_seqno_cb *cb); | |
142 | }; | |
143 | ||
d3f5168a EA |
144 | struct vc4_v3d { |
145 | struct platform_device *pdev; | |
146 | void __iomem *regs; | |
147 | }; | |
148 | ||
c8b75bca EA |
149 | struct vc4_hvs { |
150 | struct platform_device *pdev; | |
151 | void __iomem *regs; | |
152 | void __iomem *dlist; | |
153 | }; | |
154 | ||
155 | struct vc4_plane { | |
156 | struct drm_plane base; | |
157 | }; | |
158 | ||
159 | static inline struct vc4_plane * | |
160 | to_vc4_plane(struct drm_plane *plane) | |
161 | { | |
162 | return (struct vc4_plane *)plane; | |
163 | } | |
164 | ||
165 | enum vc4_encoder_type { | |
166 | VC4_ENCODER_TYPE_HDMI, | |
167 | VC4_ENCODER_TYPE_VEC, | |
168 | VC4_ENCODER_TYPE_DSI0, | |
169 | VC4_ENCODER_TYPE_DSI1, | |
170 | VC4_ENCODER_TYPE_SMI, | |
171 | VC4_ENCODER_TYPE_DPI, | |
172 | }; | |
173 | ||
174 | struct vc4_encoder { | |
175 | struct drm_encoder base; | |
176 | enum vc4_encoder_type type; | |
177 | u32 clock_select; | |
178 | }; | |
179 | ||
180 | static inline struct vc4_encoder * | |
181 | to_vc4_encoder(struct drm_encoder *encoder) | |
182 | { | |
183 | return container_of(encoder, struct vc4_encoder, base); | |
184 | } | |
185 | ||
d3f5168a EA |
186 | #define V3D_READ(offset) readl(vc4->v3d->regs + offset) |
187 | #define V3D_WRITE(offset, val) writel(val, vc4->v3d->regs + offset) | |
c8b75bca EA |
188 | #define HVS_READ(offset) readl(vc4->hvs->regs + offset) |
189 | #define HVS_WRITE(offset, val) writel(val, vc4->hvs->regs + offset) | |
190 | ||
d5b1a78a EA |
191 | struct vc4_exec_info { |
192 | /* Sequence number for this bin/render job. */ | |
193 | uint64_t seqno; | |
194 | ||
195 | /* Kernel-space copy of the ioctl arguments */ | |
196 | struct drm_vc4_submit_cl *args; | |
197 | ||
198 | /* This is the array of BOs that were looked up at the start of exec. | |
199 | * Command validation will use indices into this array. | |
200 | */ | |
201 | struct drm_gem_cma_object **bo; | |
202 | uint32_t bo_count; | |
203 | ||
204 | /* Pointers for our position in vc4->job_list */ | |
205 | struct list_head head; | |
206 | ||
207 | /* List of other BOs used in the job that need to be released | |
208 | * once the job is complete. | |
209 | */ | |
210 | struct list_head unref_list; | |
211 | ||
212 | /* Current unvalidated indices into @bo loaded by the non-hardware | |
213 | * VC4_PACKET_GEM_HANDLES. | |
214 | */ | |
215 | uint32_t bo_index[2]; | |
216 | ||
217 | /* This is the BO where we store the validated command lists, shader | |
218 | * records, and uniforms. | |
219 | */ | |
220 | struct drm_gem_cma_object *exec_bo; | |
221 | ||
222 | /** | |
223 | * This tracks the per-shader-record state (packet 64) that | |
224 | * determines the length of the shader record and the offset | |
225 | * it's expected to be found at. It gets read in from the | |
226 | * command lists. | |
227 | */ | |
228 | struct vc4_shader_state { | |
229 | uint32_t addr; | |
230 | /* Maximum vertex index referenced by any primitive using this | |
231 | * shader state. | |
232 | */ | |
233 | uint32_t max_index; | |
234 | } *shader_state; | |
235 | ||
236 | /** How many shader states the user declared they were using. */ | |
237 | uint32_t shader_state_size; | |
238 | /** How many shader state records the validator has seen. */ | |
239 | uint32_t shader_state_count; | |
240 | ||
241 | bool found_tile_binning_mode_config_packet; | |
242 | bool found_start_tile_binning_packet; | |
243 | bool found_increment_semaphore_packet; | |
244 | bool found_flush; | |
245 | uint8_t bin_tiles_x, bin_tiles_y; | |
246 | struct drm_gem_cma_object *tile_bo; | |
247 | uint32_t tile_alloc_offset; | |
248 | ||
249 | /** | |
250 | * Computed addresses pointing into exec_bo where we start the | |
251 | * bin thread (ct0) and render thread (ct1). | |
252 | */ | |
253 | uint32_t ct0ca, ct0ea; | |
254 | uint32_t ct1ca, ct1ea; | |
255 | ||
256 | /* Pointer to the unvalidated bin CL (if present). */ | |
257 | void *bin_u; | |
258 | ||
259 | /* Pointers to the shader recs. These paddr gets incremented as CL | |
260 | * packets are relocated in validate_gl_shader_state, and the vaddrs | |
261 | * (u and v) get incremented and size decremented as the shader recs | |
262 | * themselves are validated. | |
263 | */ | |
264 | void *shader_rec_u; | |
265 | void *shader_rec_v; | |
266 | uint32_t shader_rec_p; | |
267 | uint32_t shader_rec_size; | |
268 | ||
269 | /* Pointers to the uniform data. These pointers are incremented, and | |
270 | * size decremented, as each batch of uniforms is uploaded. | |
271 | */ | |
272 | void *uniforms_u; | |
273 | void *uniforms_v; | |
274 | uint32_t uniforms_p; | |
275 | uint32_t uniforms_size; | |
276 | }; | |
277 | ||
278 | static inline struct vc4_exec_info * | |
279 | vc4_first_job(struct vc4_dev *vc4) | |
280 | { | |
281 | if (list_empty(&vc4->job_list)) | |
282 | return NULL; | |
283 | return list_first_entry(&vc4->job_list, struct vc4_exec_info, head); | |
284 | } | |
285 | ||
463873d5 EA |
286 | /** |
287 | * struct vc4_texture_sample_info - saves the offsets into the UBO for texture | |
288 | * setup parameters. | |
289 | * | |
290 | * This will be used at draw time to relocate the reference to the texture | |
291 | * contents in p0, and validate that the offset combined with | |
292 | * width/height/stride/etc. from p1 and p2/p3 doesn't sample outside the BO. | |
293 | * Note that the hardware treats unprovided config parameters as 0, so not all | |
294 | * of them need to be set up for every texure sample, and we'll store ~0 as | |
295 | * the offset to mark the unused ones. | |
296 | * | |
297 | * See the VC4 3D architecture guide page 41 ("Texture and Memory Lookup Unit | |
298 | * Setup") for definitions of the texture parameters. | |
299 | */ | |
300 | struct vc4_texture_sample_info { | |
301 | bool is_direct; | |
302 | uint32_t p_offset[4]; | |
303 | }; | |
304 | ||
305 | /** | |
306 | * struct vc4_validated_shader_info - information about validated shaders that | |
307 | * needs to be used from command list validation. | |
308 | * | |
309 | * For a given shader, each time a shader state record references it, we need | |
310 | * to verify that the shader doesn't read more uniforms than the shader state | |
311 | * record's uniform BO pointer can provide, and we need to apply relocations | |
312 | * and validate the shader state record's uniforms that define the texture | |
313 | * samples. | |
314 | */ | |
315 | struct vc4_validated_shader_info { | |
316 | uint32_t uniforms_size; | |
317 | uint32_t uniforms_src_size; | |
318 | uint32_t num_texture_samples; | |
319 | struct vc4_texture_sample_info *texture_samples; | |
320 | }; | |
321 | ||
c8b75bca EA |
322 | /** |
323 | * _wait_for - magic (register) wait macro | |
324 | * | |
325 | * Does the right thing for modeset paths when run under kdgb or similar atomic | |
326 | * contexts. Note that it's important that we check the condition again after | |
327 | * having timed out, since the timeout could be due to preemption or similar and | |
328 | * we've never had a chance to check the condition before the timeout. | |
329 | */ | |
330 | #define _wait_for(COND, MS, W) ({ \ | |
331 | unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ | |
332 | int ret__ = 0; \ | |
333 | while (!(COND)) { \ | |
334 | if (time_after(jiffies, timeout__)) { \ | |
335 | if (!(COND)) \ | |
336 | ret__ = -ETIMEDOUT; \ | |
337 | break; \ | |
338 | } \ | |
339 | if (W && drm_can_sleep()) { \ | |
340 | msleep(W); \ | |
341 | } else { \ | |
342 | cpu_relax(); \ | |
343 | } \ | |
344 | } \ | |
345 | ret__; \ | |
346 | }) | |
347 | ||
348 | #define wait_for(COND, MS) _wait_for(COND, MS, 1) | |
349 | ||
350 | /* vc4_bo.c */ | |
c826a6e1 | 351 | struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size); |
c8b75bca | 352 | void vc4_free_object(struct drm_gem_object *gem_obj); |
c826a6e1 EA |
353 | struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size, |
354 | bool from_cache); | |
c8b75bca EA |
355 | int vc4_dumb_create(struct drm_file *file_priv, |
356 | struct drm_device *dev, | |
357 | struct drm_mode_create_dumb *args); | |
358 | struct dma_buf *vc4_prime_export(struct drm_device *dev, | |
359 | struct drm_gem_object *obj, int flags); | |
d5bc60f6 EA |
360 | int vc4_create_bo_ioctl(struct drm_device *dev, void *data, |
361 | struct drm_file *file_priv); | |
463873d5 EA |
362 | int vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data, |
363 | struct drm_file *file_priv); | |
d5bc60f6 EA |
364 | int vc4_mmap_bo_ioctl(struct drm_device *dev, void *data, |
365 | struct drm_file *file_priv); | |
21461365 EA |
366 | int vc4_get_hang_state_ioctl(struct drm_device *dev, void *data, |
367 | struct drm_file *file_priv); | |
463873d5 EA |
368 | int vc4_mmap(struct file *filp, struct vm_area_struct *vma); |
369 | int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma); | |
370 | void *vc4_prime_vmap(struct drm_gem_object *obj); | |
c826a6e1 EA |
371 | void vc4_bo_cache_init(struct drm_device *dev); |
372 | void vc4_bo_cache_destroy(struct drm_device *dev); | |
373 | int vc4_bo_stats_debugfs(struct seq_file *m, void *arg); | |
c8b75bca EA |
374 | |
375 | /* vc4_crtc.c */ | |
376 | extern struct platform_driver vc4_crtc_driver; | |
1f43710a DA |
377 | int vc4_enable_vblank(struct drm_device *dev, unsigned int crtc_id); |
378 | void vc4_disable_vblank(struct drm_device *dev, unsigned int crtc_id); | |
c8b75bca EA |
379 | void vc4_cancel_page_flip(struct drm_crtc *crtc, struct drm_file *file); |
380 | int vc4_crtc_debugfs_regs(struct seq_file *m, void *arg); | |
381 | ||
382 | /* vc4_debugfs.c */ | |
383 | int vc4_debugfs_init(struct drm_minor *minor); | |
384 | void vc4_debugfs_cleanup(struct drm_minor *minor); | |
385 | ||
386 | /* vc4_drv.c */ | |
387 | void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index); | |
388 | ||
d5b1a78a EA |
389 | /* vc4_gem.c */ |
390 | void vc4_gem_init(struct drm_device *dev); | |
391 | void vc4_gem_destroy(struct drm_device *dev); | |
392 | int vc4_submit_cl_ioctl(struct drm_device *dev, void *data, | |
393 | struct drm_file *file_priv); | |
394 | int vc4_wait_seqno_ioctl(struct drm_device *dev, void *data, | |
395 | struct drm_file *file_priv); | |
396 | int vc4_wait_bo_ioctl(struct drm_device *dev, void *data, | |
397 | struct drm_file *file_priv); | |
398 | void vc4_submit_next_job(struct drm_device *dev); | |
399 | int vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno, | |
400 | uint64_t timeout_ns, bool interruptible); | |
401 | void vc4_job_handle_completed(struct vc4_dev *vc4); | |
b501bacc EA |
402 | int vc4_queue_seqno_cb(struct drm_device *dev, |
403 | struct vc4_seqno_cb *cb, uint64_t seqno, | |
404 | void (*func)(struct vc4_seqno_cb *cb)); | |
d5b1a78a | 405 | |
c8b75bca EA |
406 | /* vc4_hdmi.c */ |
407 | extern struct platform_driver vc4_hdmi_driver; | |
408 | int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused); | |
409 | ||
d5b1a78a EA |
410 | /* vc4_irq.c */ |
411 | irqreturn_t vc4_irq(int irq, void *arg); | |
412 | void vc4_irq_preinstall(struct drm_device *dev); | |
413 | int vc4_irq_postinstall(struct drm_device *dev); | |
414 | void vc4_irq_uninstall(struct drm_device *dev); | |
415 | void vc4_irq_reset(struct drm_device *dev); | |
416 | ||
c8b75bca EA |
417 | /* vc4_hvs.c */ |
418 | extern struct platform_driver vc4_hvs_driver; | |
419 | void vc4_hvs_dump_state(struct drm_device *dev); | |
420 | int vc4_hvs_debugfs_regs(struct seq_file *m, void *unused); | |
421 | ||
422 | /* vc4_kms.c */ | |
423 | int vc4_kms_load(struct drm_device *dev); | |
424 | ||
425 | /* vc4_plane.c */ | |
426 | struct drm_plane *vc4_plane_init(struct drm_device *dev, | |
427 | enum drm_plane_type type); | |
428 | u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist); | |
429 | u32 vc4_plane_dlist_size(struct drm_plane_state *state); | |
b501bacc EA |
430 | void vc4_plane_async_set_fb(struct drm_plane *plane, |
431 | struct drm_framebuffer *fb); | |
463873d5 | 432 | |
d3f5168a EA |
433 | /* vc4_v3d.c */ |
434 | extern struct platform_driver vc4_v3d_driver; | |
435 | int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused); | |
436 | int vc4_v3d_debugfs_regs(struct seq_file *m, void *unused); | |
d5b1a78a EA |
437 | int vc4_v3d_set_power(struct vc4_dev *vc4, bool on); |
438 | ||
439 | /* vc4_validate.c */ | |
440 | int | |
441 | vc4_validate_bin_cl(struct drm_device *dev, | |
442 | void *validated, | |
443 | void *unvalidated, | |
444 | struct vc4_exec_info *exec); | |
445 | ||
446 | int | |
447 | vc4_validate_shader_recs(struct drm_device *dev, struct vc4_exec_info *exec); | |
448 | ||
449 | struct drm_gem_cma_object *vc4_use_bo(struct vc4_exec_info *exec, | |
450 | uint32_t hindex); | |
451 | ||
452 | int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec); | |
453 | ||
454 | bool vc4_check_tex_size(struct vc4_exec_info *exec, | |
455 | struct drm_gem_cma_object *fbo, | |
456 | uint32_t offset, uint8_t tiling_format, | |
457 | uint32_t width, uint32_t height, uint8_t cpp); | |
d3f5168a | 458 | |
463873d5 EA |
459 | /* vc4_validate_shader.c */ |
460 | struct vc4_validated_shader_info * | |
461 | vc4_validate_shader(struct drm_gem_cma_object *shader_obj); |