projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
radeon/audio: set_avi_packet() function cleanup
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
1 /*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
31 #include "cikd.h"
32 #include "atom.h"
33 #include "cik_blit_shaders.h"
34 #include "radeon_ucode.h"
35 #include "clearstate_ci.h"
36 #include "radeon_kfd.h"
37
38 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
45 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
46 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
47
48 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
49 MODULE_FIRMWARE("radeon/bonaire_me.bin");
50 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
51 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
52 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
53 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
54 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
55 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
56
57 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
58 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
59 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
60 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
61 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
62 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
63 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
64 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
65 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
66
67 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
68 MODULE_FIRMWARE("radeon/hawaii_me.bin");
69 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
70 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
71 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
72 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
73 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
74 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
75
76 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
77 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
78 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
79 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
80 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
81 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
82
83 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
84 MODULE_FIRMWARE("radeon/kaveri_me.bin");
85 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
86 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
87 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
88 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
89 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
90
91 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
92 MODULE_FIRMWARE("radeon/KABINI_me.bin");
93 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
94 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
95 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
96 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
97
98 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
99 MODULE_FIRMWARE("radeon/kabini_me.bin");
100 MODULE_FIRMWARE("radeon/kabini_ce.bin");
101 MODULE_FIRMWARE("radeon/kabini_mec.bin");
102 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
103 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
104
105 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
106 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
107 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
108 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
109 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
110 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
111
112 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
113 MODULE_FIRMWARE("radeon/mullins_me.bin");
114 MODULE_FIRMWARE("radeon/mullins_ce.bin");
115 MODULE_FIRMWARE("radeon/mullins_mec.bin");
116 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
117 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
118
119 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120 extern void r600_ih_ring_fini(struct radeon_device *rdev);
121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
123 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
124 extern void sumo_rlc_fini(struct radeon_device *rdev);
125 extern int sumo_rlc_init(struct radeon_device *rdev);
126 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
127 extern void si_rlc_reset(struct radeon_device *rdev);
128 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
129 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
130 extern int cik_sdma_resume(struct radeon_device *rdev);
131 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
132 extern void cik_sdma_fini(struct radeon_device *rdev);
133 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
134 static void cik_rlc_stop(struct radeon_device *rdev);
135 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
136 static void cik_program_aspm(struct radeon_device *rdev);
137 static void cik_init_pg(struct radeon_device *rdev);
138 static void cik_init_cg(struct radeon_device *rdev);
139 static void cik_fini_pg(struct radeon_device *rdev);
140 static void cik_fini_cg(struct radeon_device *rdev);
141 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
142 bool enable);
143
144 /* get temperature in millidegrees */
145 int ci_get_temp(struct radeon_device *rdev)
146 {
147 u32 temp;
148 int actual_temp = 0;
149
150 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
151 CTF_TEMP_SHIFT;
152
153 if (temp & 0x200)
154 actual_temp = 255;
155 else
156 actual_temp = temp & 0x1ff;
157
158 actual_temp = actual_temp * 1000;
159
160 return actual_temp;
161 }
162
163 /* get temperature in millidegrees */
164 int kv_get_temp(struct radeon_device *rdev)
165 {
166 u32 temp;
167 int actual_temp = 0;
168
169 temp = RREG32_SMC(0xC0300E0C);
170
171 if (temp)
172 actual_temp = (temp / 8) - 49;
173 else
174 actual_temp = 0;
175
176 actual_temp = actual_temp * 1000;
177
178 return actual_temp;
179 }
180
181 /*
182 * Indirect registers accessor
183 */
184 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
185 {
186 unsigned long flags;
187 u32 r;
188
189 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
190 WREG32(PCIE_INDEX, reg);
191 (void)RREG32(PCIE_INDEX);
192 r = RREG32(PCIE_DATA);
193 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
194 return r;
195 }
196
197 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
198 {
199 unsigned long flags;
200
201 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
202 WREG32(PCIE_INDEX, reg);
203 (void)RREG32(PCIE_INDEX);
204 WREG32(PCIE_DATA, v);
205 (void)RREG32(PCIE_DATA);
206 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
207 }
208
209 static const u32 spectre_rlc_save_restore_register_list[] =
210 {
211 (0x0e00 << 16) | (0xc12c >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0xc140 >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0xc150 >> 2),
216 0x00000000,
217 (0x0e00 << 16) | (0xc15c >> 2),
218 0x00000000,
219 (0x0e00 << 16) | (0xc168 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0xc170 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0xc178 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0xc204 >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc2b4 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc2b8 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc2bc >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0xc2c0 >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0x8228 >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0x829c >> 2),
238 0x00000000,
239 (0x0e00 << 16) | (0x869c >> 2),
240 0x00000000,
241 (0x0600 << 16) | (0x98f4 >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0x98f8 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0x9900 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0xc260 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0x90e8 >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0x3c000 >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0x3c00c >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0x8c1c >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0x9700 >> 2),
258 0x00000000,
259 (0x0e00 << 16) | (0xcd20 >> 2),
260 0x00000000,
261 (0x4e00 << 16) | (0xcd20 >> 2),
262 0x00000000,
263 (0x5e00 << 16) | (0xcd20 >> 2),
264 0x00000000,
265 (0x6e00 << 16) | (0xcd20 >> 2),
266 0x00000000,
267 (0x7e00 << 16) | (0xcd20 >> 2),
268 0x00000000,
269 (0x8e00 << 16) | (0xcd20 >> 2),
270 0x00000000,
271 (0x9e00 << 16) | (0xcd20 >> 2),
272 0x00000000,
273 (0xae00 << 16) | (0xcd20 >> 2),
274 0x00000000,
275 (0xbe00 << 16) | (0xcd20 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x89bc >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0x8900 >> 2),
280 0x00000000,
281 0x3,
282 (0x0e00 << 16) | (0xc130 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0xc134 >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0xc1fc >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0xc208 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0xc264 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xc268 >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0xc26c >> 2),
295 0x00000000,
296 (0x0e00 << 16) | (0xc270 >> 2),
297 0x00000000,
298 (0x0e00 << 16) | (0xc274 >> 2),
299 0x00000000,
300 (0x0e00 << 16) | (0xc278 >> 2),
301 0x00000000,
302 (0x0e00 << 16) | (0xc27c >> 2),
303 0x00000000,
304 (0x0e00 << 16) | (0xc280 >> 2),
305 0x00000000,
306 (0x0e00 << 16) | (0xc284 >> 2),
307 0x00000000,
308 (0x0e00 << 16) | (0xc288 >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0xc28c >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0xc290 >> 2),
313 0x00000000,
314 (0x0e00 << 16) | (0xc294 >> 2),
315 0x00000000,
316 (0x0e00 << 16) | (0xc298 >> 2),
317 0x00000000,
318 (0x0e00 << 16) | (0xc29c >> 2),
319 0x00000000,
320 (0x0e00 << 16) | (0xc2a0 >> 2),
321 0x00000000,
322 (0x0e00 << 16) | (0xc2a4 >> 2),
323 0x00000000,
324 (0x0e00 << 16) | (0xc2a8 >> 2),
325 0x00000000,
326 (0x0e00 << 16) | (0xc2ac >> 2),
327 0x00000000,
328 (0x0e00 << 16) | (0xc2b0 >> 2),
329 0x00000000,
330 (0x0e00 << 16) | (0x301d0 >> 2),
331 0x00000000,
332 (0x0e00 << 16) | (0x30238 >> 2),
333 0x00000000,
334 (0x0e00 << 16) | (0x30250 >> 2),
335 0x00000000,
336 (0x0e00 << 16) | (0x30254 >> 2),
337 0x00000000,
338 (0x0e00 << 16) | (0x30258 >> 2),
339 0x00000000,
340 (0x0e00 << 16) | (0x3025c >> 2),
341 0x00000000,
342 (0x4e00 << 16) | (0xc900 >> 2),
343 0x00000000,
344 (0x5e00 << 16) | (0xc900 >> 2),
345 0x00000000,
346 (0x6e00 << 16) | (0xc900 >> 2),
347 0x00000000,
348 (0x7e00 << 16) | (0xc900 >> 2),
349 0x00000000,
350 (0x8e00 << 16) | (0xc900 >> 2),
351 0x00000000,
352 (0x9e00 << 16) | (0xc900 >> 2),
353 0x00000000,
354 (0xae00 << 16) | (0xc900 >> 2),
355 0x00000000,
356 (0xbe00 << 16) | (0xc900 >> 2),
357 0x00000000,
358 (0x4e00 << 16) | (0xc904 >> 2),
359 0x00000000,
360 (0x5e00 << 16) | (0xc904 >> 2),
361 0x00000000,
362 (0x6e00 << 16) | (0xc904 >> 2),
363 0x00000000,
364 (0x7e00 << 16) | (0xc904 >> 2),
365 0x00000000,
366 (0x8e00 << 16) | (0xc904 >> 2),
367 0x00000000,
368 (0x9e00 << 16) | (0xc904 >> 2),
369 0x00000000,
370 (0xae00 << 16) | (0xc904 >> 2),
371 0x00000000,
372 (0xbe00 << 16) | (0xc904 >> 2),
373 0x00000000,
374 (0x4e00 << 16) | (0xc908 >> 2),
375 0x00000000,
376 (0x5e00 << 16) | (0xc908 >> 2),
377 0x00000000,
378 (0x6e00 << 16) | (0xc908 >> 2),
379 0x00000000,
380 (0x7e00 << 16) | (0xc908 >> 2),
381 0x00000000,
382 (0x8e00 << 16) | (0xc908 >> 2),
383 0x00000000,
384 (0x9e00 << 16) | (0xc908 >> 2),
385 0x00000000,
386 (0xae00 << 16) | (0xc908 >> 2),
387 0x00000000,
388 (0xbe00 << 16) | (0xc908 >> 2),
389 0x00000000,
390 (0x4e00 << 16) | (0xc90c >> 2),
391 0x00000000,
392 (0x5e00 << 16) | (0xc90c >> 2),
393 0x00000000,
394 (0x6e00 << 16) | (0xc90c >> 2),
395 0x00000000,
396 (0x7e00 << 16) | (0xc90c >> 2),
397 0x00000000,
398 (0x8e00 << 16) | (0xc90c >> 2),
399 0x00000000,
400 (0x9e00 << 16) | (0xc90c >> 2),
401 0x00000000,
402 (0xae00 << 16) | (0xc90c >> 2),
403 0x00000000,
404 (0xbe00 << 16) | (0xc90c >> 2),
405 0x00000000,
406 (0x4e00 << 16) | (0xc910 >> 2),
407 0x00000000,
408 (0x5e00 << 16) | (0xc910 >> 2),
409 0x00000000,
410 (0x6e00 << 16) | (0xc910 >> 2),
411 0x00000000,
412 (0x7e00 << 16) | (0xc910 >> 2),
413 0x00000000,
414 (0x8e00 << 16) | (0xc910 >> 2),
415 0x00000000,
416 (0x9e00 << 16) | (0xc910 >> 2),
417 0x00000000,
418 (0xae00 << 16) | (0xc910 >> 2),
419 0x00000000,
420 (0xbe00 << 16) | (0xc910 >> 2),
421 0x00000000,
422 (0x0e00 << 16) | (0xc99c >> 2),
423 0x00000000,
424 (0x0e00 << 16) | (0x9834 >> 2),
425 0x00000000,
426 (0x0000 << 16) | (0x30f00 >> 2),
427 0x00000000,
428 (0x0001 << 16) | (0x30f00 >> 2),
429 0x00000000,
430 (0x0000 << 16) | (0x30f04 >> 2),
431 0x00000000,
432 (0x0001 << 16) | (0x30f04 >> 2),
433 0x00000000,
434 (0x0000 << 16) | (0x30f08 >> 2),
435 0x00000000,
436 (0x0001 << 16) | (0x30f08 >> 2),
437 0x00000000,
438 (0x0000 << 16) | (0x30f0c >> 2),
439 0x00000000,
440 (0x0001 << 16) | (0x30f0c >> 2),
441 0x00000000,
442 (0x0600 << 16) | (0x9b7c >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x8a14 >> 2),
445 0x00000000,
446 (0x0e00 << 16) | (0x8a18 >> 2),
447 0x00000000,
448 (0x0600 << 16) | (0x30a00 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x8bf0 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x8bcc >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x8b24 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x30a04 >> 2),
457 0x00000000,
458 (0x0600 << 16) | (0x30a10 >> 2),
459 0x00000000,
460 (0x0600 << 16) | (0x30a14 >> 2),
461 0x00000000,
462 (0x0600 << 16) | (0x30a18 >> 2),
463 0x00000000,
464 (0x0600 << 16) | (0x30a2c >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0xc700 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0xc704 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0xc708 >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0xc768 >> 2),
473 0x00000000,
474 (0x0400 << 16) | (0xc770 >> 2),
475 0x00000000,
476 (0x0400 << 16) | (0xc774 >> 2),
477 0x00000000,
478 (0x0400 << 16) | (0xc778 >> 2),
479 0x00000000,
480 (0x0400 << 16) | (0xc77c >> 2),
481 0x00000000,
482 (0x0400 << 16) | (0xc780 >> 2),
483 0x00000000,
484 (0x0400 << 16) | (0xc784 >> 2),
485 0x00000000,
486 (0x0400 << 16) | (0xc788 >> 2),
487 0x00000000,
488 (0x0400 << 16) | (0xc78c >> 2),
489 0x00000000,
490 (0x0400 << 16) | (0xc798 >> 2),
491 0x00000000,
492 (0x0400 << 16) | (0xc79c >> 2),
493 0x00000000,
494 (0x0400 << 16) | (0xc7a0 >> 2),
495 0x00000000,
496 (0x0400 << 16) | (0xc7a4 >> 2),
497 0x00000000,
498 (0x0400 << 16) | (0xc7a8 >> 2),
499 0x00000000,
500 (0x0400 << 16) | (0xc7ac >> 2),
501 0x00000000,
502 (0x0400 << 16) | (0xc7b0 >> 2),
503 0x00000000,
504 (0x0400 << 16) | (0xc7b4 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x9100 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x3c010 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x92a8 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x92ac >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x92b4 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x92b8 >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x92bc >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x92c0 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x92c4 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x92c8 >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x92cc >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0x92d0 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0x8c00 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0x8c04 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0x8c20 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0x8c38 >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0x8c3c >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0xae00 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0x9604 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xac08 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xac0c >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0xac10 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0xac14 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0xac58 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0xac68 >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0xac6c >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0xac70 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0xac74 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0xac78 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0xac7c >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0xac80 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0xac84 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0xac88 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0xac8c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x970c >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x9714 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x9718 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x971c >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x31068 >> 2),
583 0x00000000,
584 (0x4e00 << 16) | (0x31068 >> 2),
585 0x00000000,
586 (0x5e00 << 16) | (0x31068 >> 2),
587 0x00000000,
588 (0x6e00 << 16) | (0x31068 >> 2),
589 0x00000000,
590 (0x7e00 << 16) | (0x31068 >> 2),
591 0x00000000,
592 (0x8e00 << 16) | (0x31068 >> 2),
593 0x00000000,
594 (0x9e00 << 16) | (0x31068 >> 2),
595 0x00000000,
596 (0xae00 << 16) | (0x31068 >> 2),
597 0x00000000,
598 (0xbe00 << 16) | (0x31068 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xcd10 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xcd14 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x88b0 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0x88b4 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0x88b8 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0x88bc >> 2),
611 0x00000000,
612 (0x0400 << 16) | (0x89c0 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x88c4 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x88c8 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x88d0 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x88d4 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0x88d8 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8980 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x30938 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x3093c >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x30940 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x89a0 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x30900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x30904 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x89b4 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c210 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c214 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x3c218 >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x8904 >> 2),
647 0x00000000,
648 0x5,
649 (0x0e00 << 16) | (0x8c28 >> 2),
650 (0x0e00 << 16) | (0x8c2c >> 2),
651 (0x0e00 << 16) | (0x8c30 >> 2),
652 (0x0e00 << 16) | (0x8c34 >> 2),
653 (0x0e00 << 16) | (0x9600 >> 2),
654 };
655
656 static const u32 kalindi_rlc_save_restore_register_list[] =
657 {
658 (0x0e00 << 16) | (0xc12c >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xc140 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xc150 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xc15c >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc168 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc170 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc204 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc2b4 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc2b8 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc2bc >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0xc2c0 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x8228 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x829c >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x869c >> 2),
685 0x00000000,
686 (0x0600 << 16) | (0x98f4 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x98f8 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x9900 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0xc260 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x90e8 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x3c000 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x3c00c >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x8c1c >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x9700 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0xcd20 >> 2),
705 0x00000000,
706 (0x4e00 << 16) | (0xcd20 >> 2),
707 0x00000000,
708 (0x5e00 << 16) | (0xcd20 >> 2),
709 0x00000000,
710 (0x6e00 << 16) | (0xcd20 >> 2),
711 0x00000000,
712 (0x7e00 << 16) | (0xcd20 >> 2),
713 0x00000000,
714 (0x0e00 << 16) | (0x89bc >> 2),
715 0x00000000,
716 (0x0e00 << 16) | (0x8900 >> 2),
717 0x00000000,
718 0x3,
719 (0x0e00 << 16) | (0xc130 >> 2),
720 0x00000000,
721 (0x0e00 << 16) | (0xc134 >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0xc1fc >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc208 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0xc264 >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0xc268 >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0xc26c >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0xc270 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0xc274 >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0xc28c >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0xc290 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc294 >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0xc298 >> 2),
744 0x00000000,
745 (0x0e00 << 16) | (0xc2a0 >> 2),
746 0x00000000,
747 (0x0e00 << 16) | (0xc2a4 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc2a8 >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0xc2ac >> 2),
752 0x00000000,
753 (0x0e00 << 16) | (0x301d0 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x30238 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x30250 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x30254 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x30258 >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x3025c >> 2),
764 0x00000000,
765 (0x4e00 << 16) | (0xc900 >> 2),
766 0x00000000,
767 (0x5e00 << 16) | (0xc900 >> 2),
768 0x00000000,
769 (0x6e00 << 16) | (0xc900 >> 2),
770 0x00000000,
771 (0x7e00 << 16) | (0xc900 >> 2),
772 0x00000000,
773 (0x4e00 << 16) | (0xc904 >> 2),
774 0x00000000,
775 (0x5e00 << 16) | (0xc904 >> 2),
776 0x00000000,
777 (0x6e00 << 16) | (0xc904 >> 2),
778 0x00000000,
779 (0x7e00 << 16) | (0xc904 >> 2),
780 0x00000000,
781 (0x4e00 << 16) | (0xc908 >> 2),
782 0x00000000,
783 (0x5e00 << 16) | (0xc908 >> 2),
784 0x00000000,
785 (0x6e00 << 16) | (0xc908 >> 2),
786 0x00000000,
787 (0x7e00 << 16) | (0xc908 >> 2),
788 0x00000000,
789 (0x4e00 << 16) | (0xc90c >> 2),
790 0x00000000,
791 (0x5e00 << 16) | (0xc90c >> 2),
792 0x00000000,
793 (0x6e00 << 16) | (0xc90c >> 2),
794 0x00000000,
795 (0x7e00 << 16) | (0xc90c >> 2),
796 0x00000000,
797 (0x4e00 << 16) | (0xc910 >> 2),
798 0x00000000,
799 (0x5e00 << 16) | (0xc910 >> 2),
800 0x00000000,
801 (0x6e00 << 16) | (0xc910 >> 2),
802 0x00000000,
803 (0x7e00 << 16) | (0xc910 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc99c >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x9834 >> 2),
808 0x00000000,
809 (0x0000 << 16) | (0x30f00 >> 2),
810 0x00000000,
811 (0x0000 << 16) | (0x30f04 >> 2),
812 0x00000000,
813 (0x0000 << 16) | (0x30f08 >> 2),
814 0x00000000,
815 (0x0000 << 16) | (0x30f0c >> 2),
816 0x00000000,
817 (0x0600 << 16) | (0x9b7c >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x8a14 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x8a18 >> 2),
822 0x00000000,
823 (0x0600 << 16) | (0x30a00 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x8bf0 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0x8bcc >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0x8b24 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0x30a04 >> 2),
832 0x00000000,
833 (0x0600 << 16) | (0x30a10 >> 2),
834 0x00000000,
835 (0x0600 << 16) | (0x30a14 >> 2),
836 0x00000000,
837 (0x0600 << 16) | (0x30a18 >> 2),
838 0x00000000,
839 (0x0600 << 16) | (0x30a2c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xc700 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xc704 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xc708 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xc768 >> 2),
848 0x00000000,
849 (0x0400 << 16) | (0xc770 >> 2),
850 0x00000000,
851 (0x0400 << 16) | (0xc774 >> 2),
852 0x00000000,
853 (0x0400 << 16) | (0xc798 >> 2),
854 0x00000000,
855 (0x0400 << 16) | (0xc79c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x9100 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0x3c010 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x8c00 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x8c04 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x8c20 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x8c38 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0x8c3c >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0xae00 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x9604 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0xac08 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0xac0c >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0xac10 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0xac14 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0xac58 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0xac68 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0xac6c >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0xac70 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0xac74 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0xac78 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0xac7c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0xac80 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0xac84 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xac88 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xac8c >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x970c >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x9714 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x9718 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x971c >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x31068 >> 2),
914 0x00000000,
915 (0x4e00 << 16) | (0x31068 >> 2),
916 0x00000000,
917 (0x5e00 << 16) | (0x31068 >> 2),
918 0x00000000,
919 (0x6e00 << 16) | (0x31068 >> 2),
920 0x00000000,
921 (0x7e00 << 16) | (0x31068 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0xcd10 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0xcd14 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x88b0 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x88b4 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x88b8 >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0x88bc >> 2),
934 0x00000000,
935 (0x0400 << 16) | (0x89c0 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0x88c4 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0x88c8 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0x88d0 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0x88d4 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0x88d8 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0x8980 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0x30938 >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0x3093c >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0x30940 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0x89a0 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0x30900 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0x30904 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0x89b4 >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0x3e1fc >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x3c210 >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x3c214 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x3c218 >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x8904 >> 2),
972 0x00000000,
973 0x5,
974 (0x0e00 << 16) | (0x8c28 >> 2),
975 (0x0e00 << 16) | (0x8c2c >> 2),
976 (0x0e00 << 16) | (0x8c30 >> 2),
977 (0x0e00 << 16) | (0x8c34 >> 2),
978 (0x0e00 << 16) | (0x9600 >> 2),
979 };
980
981 static const u32 bonaire_golden_spm_registers[] =
982 {
983 0x30800, 0xe0ffffff, 0xe0000000
984 };
985
986 static const u32 bonaire_golden_common_registers[] =
987 {
988 0xc770, 0xffffffff, 0x00000800,
989 0xc774, 0xffffffff, 0x00000800,
990 0xc798, 0xffffffff, 0x00007fbf,
991 0xc79c, 0xffffffff, 0x00007faf
992 };
993
994 static const u32 bonaire_golden_registers[] =
995 {
996 0x3354, 0x00000333, 0x00000333,
997 0x3350, 0x000c0fc0, 0x00040200,
998 0x9a10, 0x00010000, 0x00058208,
999 0x3c000, 0xffff1fff, 0x00140000,
1000 0x3c200, 0xfdfc0fff, 0x00000100,
1001 0x3c234, 0x40000000, 0x40000200,
1002 0x9830, 0xffffffff, 0x00000000,
1003 0x9834, 0xf00fffff, 0x00000400,
1004 0x9838, 0x0002021c, 0x00020200,
1005 0xc78, 0x00000080, 0x00000000,
1006 0x5bb0, 0x000000f0, 0x00000070,
1007 0x5bc0, 0xf0311fff, 0x80300000,
1008 0x98f8, 0x73773777, 0x12010001,
1009 0x350c, 0x00810000, 0x408af000,
1010 0x7030, 0x31000111, 0x00000011,
1011 0x2f48, 0x73773777, 0x12010001,
1012 0x220c, 0x00007fb6, 0x0021a1b1,
1013 0x2210, 0x00007fb6, 0x002021b1,
1014 0x2180, 0x00007fb6, 0x00002191,
1015 0x2218, 0x00007fb6, 0x002121b1,
1016 0x221c, 0x00007fb6, 0x002021b1,
1017 0x21dc, 0x00007fb6, 0x00002191,
1018 0x21e0, 0x00007fb6, 0x00002191,
1019 0x3628, 0x0000003f, 0x0000000a,
1020 0x362c, 0x0000003f, 0x0000000a,
1021 0x2ae4, 0x00073ffe, 0x000022a2,
1022 0x240c, 0x000007ff, 0x00000000,
1023 0x8a14, 0xf000003f, 0x00000007,
1024 0x8bf0, 0x00002001, 0x00000001,
1025 0x8b24, 0xffffffff, 0x00ffffff,
1026 0x30a04, 0x0000ff0f, 0x00000000,
1027 0x28a4c, 0x07ffffff, 0x06000000,
1028 0x4d8, 0x00000fff, 0x00000100,
1029 0x3e78, 0x00000001, 0x00000002,
1030 0x9100, 0x03000000, 0x0362c688,
1031 0x8c00, 0x000000ff, 0x00000001,
1032 0xe40, 0x00001fff, 0x00001fff,
1033 0x9060, 0x0000007f, 0x00000020,
1034 0x9508, 0x00010000, 0x00010000,
1035 0xac14, 0x000003ff, 0x000000f3,
1036 0xac0c, 0xffffffff, 0x00001032
1037 };
1038
1039 static const u32 bonaire_mgcg_cgcg_init[] =
1040 {
1041 0xc420, 0xffffffff, 0xfffffffc,
1042 0x30800, 0xffffffff, 0xe0000000,
1043 0x3c2a0, 0xffffffff, 0x00000100,
1044 0x3c208, 0xffffffff, 0x00000100,
1045 0x3c2c0, 0xffffffff, 0xc0000100,
1046 0x3c2c8, 0xffffffff, 0xc0000100,
1047 0x3c2c4, 0xffffffff, 0xc0000100,
1048 0x55e4, 0xffffffff, 0x00600100,
1049 0x3c280, 0xffffffff, 0x00000100,
1050 0x3c214, 0xffffffff, 0x06000100,
1051 0x3c220, 0xffffffff, 0x00000100,
1052 0x3c218, 0xffffffff, 0x06000100,
1053 0x3c204, 0xffffffff, 0x00000100,
1054 0x3c2e0, 0xffffffff, 0x00000100,
1055 0x3c224, 0xffffffff, 0x00000100,
1056 0x3c200, 0xffffffff, 0x00000100,
1057 0x3c230, 0xffffffff, 0x00000100,
1058 0x3c234, 0xffffffff, 0x00000100,
1059 0x3c250, 0xffffffff, 0x00000100,
1060 0x3c254, 0xffffffff, 0x00000100,
1061 0x3c258, 0xffffffff, 0x00000100,
1062 0x3c25c, 0xffffffff, 0x00000100,
1063 0x3c260, 0xffffffff, 0x00000100,
1064 0x3c27c, 0xffffffff, 0x00000100,
1065 0x3c278, 0xffffffff, 0x00000100,
1066 0x3c210, 0xffffffff, 0x06000100,
1067 0x3c290, 0xffffffff, 0x00000100,
1068 0x3c274, 0xffffffff, 0x00000100,
1069 0x3c2b4, 0xffffffff, 0x00000100,
1070 0x3c2b0, 0xffffffff, 0x00000100,
1071 0x3c270, 0xffffffff, 0x00000100,
1072 0x30800, 0xffffffff, 0xe0000000,
1073 0x3c020, 0xffffffff, 0x00010000,
1074 0x3c024, 0xffffffff, 0x00030002,
1075 0x3c028, 0xffffffff, 0x00040007,
1076 0x3c02c, 0xffffffff, 0x00060005,
1077 0x3c030, 0xffffffff, 0x00090008,
1078 0x3c034, 0xffffffff, 0x00010000,
1079 0x3c038, 0xffffffff, 0x00030002,
1080 0x3c03c, 0xffffffff, 0x00040007,
1081 0x3c040, 0xffffffff, 0x00060005,
1082 0x3c044, 0xffffffff, 0x00090008,
1083 0x3c048, 0xffffffff, 0x00010000,
1084 0x3c04c, 0xffffffff, 0x00030002,
1085 0x3c050, 0xffffffff, 0x00040007,
1086 0x3c054, 0xffffffff, 0x00060005,
1087 0x3c058, 0xffffffff, 0x00090008,
1088 0x3c05c, 0xffffffff, 0x00010000,
1089 0x3c060, 0xffffffff, 0x00030002,
1090 0x3c064, 0xffffffff, 0x00040007,
1091 0x3c068, 0xffffffff, 0x00060005,
1092 0x3c06c, 0xffffffff, 0x00090008,
1093 0x3c070, 0xffffffff, 0x00010000,
1094 0x3c074, 0xffffffff, 0x00030002,
1095 0x3c078, 0xffffffff, 0x00040007,
1096 0x3c07c, 0xffffffff, 0x00060005,
1097 0x3c080, 0xffffffff, 0x00090008,
1098 0x3c084, 0xffffffff, 0x00010000,
1099 0x3c088, 0xffffffff, 0x00030002,
1100 0x3c08c, 0xffffffff, 0x00040007,
1101 0x3c090, 0xffffffff, 0x00060005,
1102 0x3c094, 0xffffffff, 0x00090008,
1103 0x3c098, 0xffffffff, 0x00010000,
1104 0x3c09c, 0xffffffff, 0x00030002,
1105 0x3c0a0, 0xffffffff, 0x00040007,
1106 0x3c0a4, 0xffffffff, 0x00060005,
1107 0x3c0a8, 0xffffffff, 0x00090008,
1108 0x3c000, 0xffffffff, 0x96e00200,
1109 0x8708, 0xffffffff, 0x00900100,
1110 0xc424, 0xffffffff, 0x0020003f,
1111 0x38, 0xffffffff, 0x0140001c,
1112 0x3c, 0x000f0000, 0x000f0000,
1113 0x220, 0xffffffff, 0xC060000C,
1114 0x224, 0xc0000fff, 0x00000100,
1115 0xf90, 0xffffffff, 0x00000100,
1116 0xf98, 0x00000101, 0x00000000,
1117 0x20a8, 0xffffffff, 0x00000104,
1118 0x55e4, 0xff000fff, 0x00000100,
1119 0x30cc, 0xc0000fff, 0x00000104,
1120 0xc1e4, 0x00000001, 0x00000001,
1121 0xd00c, 0xff000ff0, 0x00000100,
1122 0xd80c, 0xff000ff0, 0x00000100
1123 };
1124
1125 static const u32 spectre_golden_spm_registers[] =
1126 {
1127 0x30800, 0xe0ffffff, 0xe0000000
1128 };
1129
1130 static const u32 spectre_golden_common_registers[] =
1131 {
1132 0xc770, 0xffffffff, 0x00000800,
1133 0xc774, 0xffffffff, 0x00000800,
1134 0xc798, 0xffffffff, 0x00007fbf,
1135 0xc79c, 0xffffffff, 0x00007faf
1136 };
1137
1138 static const u32 spectre_golden_registers[] =
1139 {
1140 0x3c000, 0xffff1fff, 0x96940200,
1141 0x3c00c, 0xffff0001, 0xff000000,
1142 0x3c200, 0xfffc0fff, 0x00000100,
1143 0x6ed8, 0x00010101, 0x00010000,
1144 0x9834, 0xf00fffff, 0x00000400,
1145 0x9838, 0xfffffffc, 0x00020200,
1146 0x5bb0, 0x000000f0, 0x00000070,
1147 0x5bc0, 0xf0311fff, 0x80300000,
1148 0x98f8, 0x73773777, 0x12010001,
1149 0x9b7c, 0x00ff0000, 0x00fc0000,
1150 0x2f48, 0x73773777, 0x12010001,
1151 0x8a14, 0xf000003f, 0x00000007,
1152 0x8b24, 0xffffffff, 0x00ffffff,
1153 0x28350, 0x3f3f3fff, 0x00000082,
1154 0x28354, 0x0000003f, 0x00000000,
1155 0x3e78, 0x00000001, 0x00000002,
1156 0x913c, 0xffff03df, 0x00000004,
1157 0xc768, 0x00000008, 0x00000008,
1158 0x8c00, 0x000008ff, 0x00000800,
1159 0x9508, 0x00010000, 0x00010000,
1160 0xac0c, 0xffffffff, 0x54763210,
1161 0x214f8, 0x01ff01ff, 0x00000002,
1162 0x21498, 0x007ff800, 0x00200000,
1163 0x2015c, 0xffffffff, 0x00000f40,
1164 0x30934, 0xffffffff, 0x00000001
1165 };
1166
1167 static const u32 spectre_mgcg_cgcg_init[] =
1168 {
1169 0xc420, 0xffffffff, 0xfffffffc,
1170 0x30800, 0xffffffff, 0xe0000000,
1171 0x3c2a0, 0xffffffff, 0x00000100,
1172 0x3c208, 0xffffffff, 0x00000100,
1173 0x3c2c0, 0xffffffff, 0x00000100,
1174 0x3c2c8, 0xffffffff, 0x00000100,
1175 0x3c2c4, 0xffffffff, 0x00000100,
1176 0x55e4, 0xffffffff, 0x00600100,
1177 0x3c280, 0xffffffff, 0x00000100,
1178 0x3c214, 0xffffffff, 0x06000100,
1179 0x3c220, 0xffffffff, 0x00000100,
1180 0x3c218, 0xffffffff, 0x06000100,
1181 0x3c204, 0xffffffff, 0x00000100,
1182 0x3c2e0, 0xffffffff, 0x00000100,
1183 0x3c224, 0xffffffff, 0x00000100,
1184 0x3c200, 0xffffffff, 0x00000100,
1185 0x3c230, 0xffffffff, 0x00000100,
1186 0x3c234, 0xffffffff, 0x00000100,
1187 0x3c250, 0xffffffff, 0x00000100,
1188 0x3c254, 0xffffffff, 0x00000100,
1189 0x3c258, 0xffffffff, 0x00000100,
1190 0x3c25c, 0xffffffff, 0x00000100,
1191 0x3c260, 0xffffffff, 0x00000100,
1192 0x3c27c, 0xffffffff, 0x00000100,
1193 0x3c278, 0xffffffff, 0x00000100,
1194 0x3c210, 0xffffffff, 0x06000100,
1195 0x3c290, 0xffffffff, 0x00000100,
1196 0x3c274, 0xffffffff, 0x00000100,
1197 0x3c2b4, 0xffffffff, 0x00000100,
1198 0x3c2b0, 0xffffffff, 0x00000100,
1199 0x3c270, 0xffffffff, 0x00000100,
1200 0x30800, 0xffffffff, 0xe0000000,
1201 0x3c020, 0xffffffff, 0x00010000,
1202 0x3c024, 0xffffffff, 0x00030002,
1203 0x3c028, 0xffffffff, 0x00040007,
1204 0x3c02c, 0xffffffff, 0x00060005,
1205 0x3c030, 0xffffffff, 0x00090008,
1206 0x3c034, 0xffffffff, 0x00010000,
1207 0x3c038, 0xffffffff, 0x00030002,
1208 0x3c03c, 0xffffffff, 0x00040007,
1209 0x3c040, 0xffffffff, 0x00060005,
1210 0x3c044, 0xffffffff, 0x00090008,
1211 0x3c048, 0xffffffff, 0x00010000,
1212 0x3c04c, 0xffffffff, 0x00030002,
1213 0x3c050, 0xffffffff, 0x00040007,
1214 0x3c054, 0xffffffff, 0x00060005,
1215 0x3c058, 0xffffffff, 0x00090008,
1216 0x3c05c, 0xffffffff, 0x00010000,
1217 0x3c060, 0xffffffff, 0x00030002,
1218 0x3c064, 0xffffffff, 0x00040007,
1219 0x3c068, 0xffffffff, 0x00060005,
1220 0x3c06c, 0xffffffff, 0x00090008,
1221 0x3c070, 0xffffffff, 0x00010000,
1222 0x3c074, 0xffffffff, 0x00030002,
1223 0x3c078, 0xffffffff, 0x00040007,
1224 0x3c07c, 0xffffffff, 0x00060005,
1225 0x3c080, 0xffffffff, 0x00090008,
1226 0x3c084, 0xffffffff, 0x00010000,
1227 0x3c088, 0xffffffff, 0x00030002,
1228 0x3c08c, 0xffffffff, 0x00040007,
1229 0x3c090, 0xffffffff, 0x00060005,
1230 0x3c094, 0xffffffff, 0x00090008,
1231 0x3c098, 0xffffffff, 0x00010000,
1232 0x3c09c, 0xffffffff, 0x00030002,
1233 0x3c0a0, 0xffffffff, 0x00040007,
1234 0x3c0a4, 0xffffffff, 0x00060005,
1235 0x3c0a8, 0xffffffff, 0x00090008,
1236 0x3c0ac, 0xffffffff, 0x00010000,
1237 0x3c0b0, 0xffffffff, 0x00030002,
1238 0x3c0b4, 0xffffffff, 0x00040007,
1239 0x3c0b8, 0xffffffff, 0x00060005,
1240 0x3c0bc, 0xffffffff, 0x00090008,
1241 0x3c000, 0xffffffff, 0x96e00200,
1242 0x8708, 0xffffffff, 0x00900100,
1243 0xc424, 0xffffffff, 0x0020003f,
1244 0x38, 0xffffffff, 0x0140001c,
1245 0x3c, 0x000f0000, 0x000f0000,
1246 0x220, 0xffffffff, 0xC060000C,
1247 0x224, 0xc0000fff, 0x00000100,
1248 0xf90, 0xffffffff, 0x00000100,
1249 0xf98, 0x00000101, 0x00000000,
1250 0x20a8, 0xffffffff, 0x00000104,
1251 0x55e4, 0xff000fff, 0x00000100,
1252 0x30cc, 0xc0000fff, 0x00000104,
1253 0xc1e4, 0x00000001, 0x00000001,
1254 0xd00c, 0xff000ff0, 0x00000100,
1255 0xd80c, 0xff000ff0, 0x00000100
1256 };
1257
1258 static const u32 kalindi_golden_spm_registers[] =
1259 {
1260 0x30800, 0xe0ffffff, 0xe0000000
1261 };
1262
1263 static const u32 kalindi_golden_common_registers[] =
1264 {
1265 0xc770, 0xffffffff, 0x00000800,
1266 0xc774, 0xffffffff, 0x00000800,
1267 0xc798, 0xffffffff, 0x00007fbf,
1268 0xc79c, 0xffffffff, 0x00007faf
1269 };
1270
1271 static const u32 kalindi_golden_registers[] =
1272 {
1273 0x3c000, 0xffffdfff, 0x6e944040,
1274 0x55e4, 0xff607fff, 0xfc000100,
1275 0x3c220, 0xff000fff, 0x00000100,
1276 0x3c224, 0xff000fff, 0x00000100,
1277 0x3c200, 0xfffc0fff, 0x00000100,
1278 0x6ed8, 0x00010101, 0x00010000,
1279 0x9830, 0xffffffff, 0x00000000,
1280 0x9834, 0xf00fffff, 0x00000400,
1281 0x5bb0, 0x000000f0, 0x00000070,
1282 0x5bc0, 0xf0311fff, 0x80300000,
1283 0x98f8, 0x73773777, 0x12010001,
1284 0x98fc, 0xffffffff, 0x00000010,
1285 0x9b7c, 0x00ff0000, 0x00fc0000,
1286 0x8030, 0x00001f0f, 0x0000100a,
1287 0x2f48, 0x73773777, 0x12010001,
1288 0x2408, 0x000fffff, 0x000c007f,
1289 0x8a14, 0xf000003f, 0x00000007,
1290 0x8b24, 0x3fff3fff, 0x00ffcfff,
1291 0x30a04, 0x0000ff0f, 0x00000000,
1292 0x28a4c, 0x07ffffff, 0x06000000,
1293 0x4d8, 0x00000fff, 0x00000100,
1294 0x3e78, 0x00000001, 0x00000002,
1295 0xc768, 0x00000008, 0x00000008,
1296 0x8c00, 0x000000ff, 0x00000003,
1297 0x214f8, 0x01ff01ff, 0x00000002,
1298 0x21498, 0x007ff800, 0x00200000,
1299 0x2015c, 0xffffffff, 0x00000f40,
1300 0x88c4, 0x001f3ae3, 0x00000082,
1301 0x88d4, 0x0000001f, 0x00000010,
1302 0x30934, 0xffffffff, 0x00000000
1303 };
1304
1305 static const u32 kalindi_mgcg_cgcg_init[] =
1306 {
1307 0xc420, 0xffffffff, 0xfffffffc,
1308 0x30800, 0xffffffff, 0xe0000000,
1309 0x3c2a0, 0xffffffff, 0x00000100,
1310 0x3c208, 0xffffffff, 0x00000100,
1311 0x3c2c0, 0xffffffff, 0x00000100,
1312 0x3c2c8, 0xffffffff, 0x00000100,
1313 0x3c2c4, 0xffffffff, 0x00000100,
1314 0x55e4, 0xffffffff, 0x00600100,
1315 0x3c280, 0xffffffff, 0x00000100,
1316 0x3c214, 0xffffffff, 0x06000100,
1317 0x3c220, 0xffffffff, 0x00000100,
1318 0x3c218, 0xffffffff, 0x06000100,
1319 0x3c204, 0xffffffff, 0x00000100,
1320 0x3c2e0, 0xffffffff, 0x00000100,
1321 0x3c224, 0xffffffff, 0x00000100,
1322 0x3c200, 0xffffffff, 0x00000100,
1323 0x3c230, 0xffffffff, 0x00000100,
1324 0x3c234, 0xffffffff, 0x00000100,
1325 0x3c250, 0xffffffff, 0x00000100,
1326 0x3c254, 0xffffffff, 0x00000100,
1327 0x3c258, 0xffffffff, 0x00000100,
1328 0x3c25c, 0xffffffff, 0x00000100,
1329 0x3c260, 0xffffffff, 0x00000100,
1330 0x3c27c, 0xffffffff, 0x00000100,
1331 0x3c278, 0xffffffff, 0x00000100,
1332 0x3c210, 0xffffffff, 0x06000100,
1333 0x3c290, 0xffffffff, 0x00000100,
1334 0x3c274, 0xffffffff, 0x00000100,
1335 0x3c2b4, 0xffffffff, 0x00000100,
1336 0x3c2b0, 0xffffffff, 0x00000100,
1337 0x3c270, 0xffffffff, 0x00000100,
1338 0x30800, 0xffffffff, 0xe0000000,
1339 0x3c020, 0xffffffff, 0x00010000,
1340 0x3c024, 0xffffffff, 0x00030002,
1341 0x3c028, 0xffffffff, 0x00040007,
1342 0x3c02c, 0xffffffff, 0x00060005,
1343 0x3c030, 0xffffffff, 0x00090008,
1344 0x3c034, 0xffffffff, 0x00010000,
1345 0x3c038, 0xffffffff, 0x00030002,
1346 0x3c03c, 0xffffffff, 0x00040007,
1347 0x3c040, 0xffffffff, 0x00060005,
1348 0x3c044, 0xffffffff, 0x00090008,
1349 0x3c000, 0xffffffff, 0x96e00200,
1350 0x8708, 0xffffffff, 0x00900100,
1351 0xc424, 0xffffffff, 0x0020003f,
1352 0x38, 0xffffffff, 0x0140001c,
1353 0x3c, 0x000f0000, 0x000f0000,
1354 0x220, 0xffffffff, 0xC060000C,
1355 0x224, 0xc0000fff, 0x00000100,
1356 0x20a8, 0xffffffff, 0x00000104,
1357 0x55e4, 0xff000fff, 0x00000100,
1358 0x30cc, 0xc0000fff, 0x00000104,
1359 0xc1e4, 0x00000001, 0x00000001,
1360 0xd00c, 0xff000ff0, 0x00000100,
1361 0xd80c, 0xff000ff0, 0x00000100
1362 };
1363
1364 static const u32 hawaii_golden_spm_registers[] =
1365 {
1366 0x30800, 0xe0ffffff, 0xe0000000
1367 };
1368
1369 static const u32 hawaii_golden_common_registers[] =
1370 {
1371 0x30800, 0xffffffff, 0xe0000000,
1372 0x28350, 0xffffffff, 0x3a00161a,
1373 0x28354, 0xffffffff, 0x0000002e,
1374 0x9a10, 0xffffffff, 0x00018208,
1375 0x98f8, 0xffffffff, 0x12011003
1376 };
1377
1378 static const u32 hawaii_golden_registers[] =
1379 {
1380 0x3354, 0x00000333, 0x00000333,
1381 0x9a10, 0x00010000, 0x00058208,
1382 0x9830, 0xffffffff, 0x00000000,
1383 0x9834, 0xf00fffff, 0x00000400,
1384 0x9838, 0x0002021c, 0x00020200,
1385 0xc78, 0x00000080, 0x00000000,
1386 0x5bb0, 0x000000f0, 0x00000070,
1387 0x5bc0, 0xf0311fff, 0x80300000,
1388 0x350c, 0x00810000, 0x408af000,
1389 0x7030, 0x31000111, 0x00000011,
1390 0x2f48, 0x73773777, 0x12010001,
1391 0x2120, 0x0000007f, 0x0000001b,
1392 0x21dc, 0x00007fb6, 0x00002191,
1393 0x3628, 0x0000003f, 0x0000000a,
1394 0x362c, 0x0000003f, 0x0000000a,
1395 0x2ae4, 0x00073ffe, 0x000022a2,
1396 0x240c, 0x000007ff, 0x00000000,
1397 0x8bf0, 0x00002001, 0x00000001,
1398 0x8b24, 0xffffffff, 0x00ffffff,
1399 0x30a04, 0x0000ff0f, 0x00000000,
1400 0x28a4c, 0x07ffffff, 0x06000000,
1401 0x3e78, 0x00000001, 0x00000002,
1402 0xc768, 0x00000008, 0x00000008,
1403 0xc770, 0x00000f00, 0x00000800,
1404 0xc774, 0x00000f00, 0x00000800,
1405 0xc798, 0x00ffffff, 0x00ff7fbf,
1406 0xc79c, 0x00ffffff, 0x00ff7faf,
1407 0x8c00, 0x000000ff, 0x00000800,
1408 0xe40, 0x00001fff, 0x00001fff,
1409 0x9060, 0x0000007f, 0x00000020,
1410 0x9508, 0x00010000, 0x00010000,
1411 0xae00, 0x00100000, 0x000ff07c,
1412 0xac14, 0x000003ff, 0x0000000f,
1413 0xac10, 0xffffffff, 0x7564fdec,
1414 0xac0c, 0xffffffff, 0x3120b9a8,
1415 0xac08, 0x20000000, 0x0f9c0000
1416 };
1417
1418 static const u32 hawaii_mgcg_cgcg_init[] =
1419 {
1420 0xc420, 0xffffffff, 0xfffffffd,
1421 0x30800, 0xffffffff, 0xe0000000,
1422 0x3c2a0, 0xffffffff, 0x00000100,
1423 0x3c208, 0xffffffff, 0x00000100,
1424 0x3c2c0, 0xffffffff, 0x00000100,
1425 0x3c2c8, 0xffffffff, 0x00000100,
1426 0x3c2c4, 0xffffffff, 0x00000100,
1427 0x55e4, 0xffffffff, 0x00200100,
1428 0x3c280, 0xffffffff, 0x00000100,
1429 0x3c214, 0xffffffff, 0x06000100,
1430 0x3c220, 0xffffffff, 0x00000100,
1431 0x3c218, 0xffffffff, 0x06000100,
1432 0x3c204, 0xffffffff, 0x00000100,
1433 0x3c2e0, 0xffffffff, 0x00000100,
1434 0x3c224, 0xffffffff, 0x00000100,
1435 0x3c200, 0xffffffff, 0x00000100,
1436 0x3c230, 0xffffffff, 0x00000100,
1437 0x3c234, 0xffffffff, 0x00000100,
1438 0x3c250, 0xffffffff, 0x00000100,
1439 0x3c254, 0xffffffff, 0x00000100,
1440 0x3c258, 0xffffffff, 0x00000100,
1441 0x3c25c, 0xffffffff, 0x00000100,
1442 0x3c260, 0xffffffff, 0x00000100,
1443 0x3c27c, 0xffffffff, 0x00000100,
1444 0x3c278, 0xffffffff, 0x00000100,
1445 0x3c210, 0xffffffff, 0x06000100,
1446 0x3c290, 0xffffffff, 0x00000100,
1447 0x3c274, 0xffffffff, 0x00000100,
1448 0x3c2b4, 0xffffffff, 0x00000100,
1449 0x3c2b0, 0xffffffff, 0x00000100,
1450 0x3c270, 0xffffffff, 0x00000100,
1451 0x30800, 0xffffffff, 0xe0000000,
1452 0x3c020, 0xffffffff, 0x00010000,
1453 0x3c024, 0xffffffff, 0x00030002,
1454 0x3c028, 0xffffffff, 0x00040007,
1455 0x3c02c, 0xffffffff, 0x00060005,
1456 0x3c030, 0xffffffff, 0x00090008,
1457 0x3c034, 0xffffffff, 0x00010000,
1458 0x3c038, 0xffffffff, 0x00030002,
1459 0x3c03c, 0xffffffff, 0x00040007,
1460 0x3c040, 0xffffffff, 0x00060005,
1461 0x3c044, 0xffffffff, 0x00090008,
1462 0x3c048, 0xffffffff, 0x00010000,
1463 0x3c04c, 0xffffffff, 0x00030002,
1464 0x3c050, 0xffffffff, 0x00040007,
1465 0x3c054, 0xffffffff, 0x00060005,
1466 0x3c058, 0xffffffff, 0x00090008,
1467 0x3c05c, 0xffffffff, 0x00010000,
1468 0x3c060, 0xffffffff, 0x00030002,
1469 0x3c064, 0xffffffff, 0x00040007,
1470 0x3c068, 0xffffffff, 0x00060005,
1471 0x3c06c, 0xffffffff, 0x00090008,
1472 0x3c070, 0xffffffff, 0x00010000,
1473 0x3c074, 0xffffffff, 0x00030002,
1474 0x3c078, 0xffffffff, 0x00040007,
1475 0x3c07c, 0xffffffff, 0x00060005,
1476 0x3c080, 0xffffffff, 0x00090008,
1477 0x3c084, 0xffffffff, 0x00010000,
1478 0x3c088, 0xffffffff, 0x00030002,
1479 0x3c08c, 0xffffffff, 0x00040007,
1480 0x3c090, 0xffffffff, 0x00060005,
1481 0x3c094, 0xffffffff, 0x00090008,
1482 0x3c098, 0xffffffff, 0x00010000,
1483 0x3c09c, 0xffffffff, 0x00030002,
1484 0x3c0a0, 0xffffffff, 0x00040007,
1485 0x3c0a4, 0xffffffff, 0x00060005,
1486 0x3c0a8, 0xffffffff, 0x00090008,
1487 0x3c0ac, 0xffffffff, 0x00010000,
1488 0x3c0b0, 0xffffffff, 0x00030002,
1489 0x3c0b4, 0xffffffff, 0x00040007,
1490 0x3c0b8, 0xffffffff, 0x00060005,
1491 0x3c0bc, 0xffffffff, 0x00090008,
1492 0x3c0c0, 0xffffffff, 0x00010000,
1493 0x3c0c4, 0xffffffff, 0x00030002,
1494 0x3c0c8, 0xffffffff, 0x00040007,
1495 0x3c0cc, 0xffffffff, 0x00060005,
1496 0x3c0d0, 0xffffffff, 0x00090008,
1497 0x3c0d4, 0xffffffff, 0x00010000,
1498 0x3c0d8, 0xffffffff, 0x00030002,
1499 0x3c0dc, 0xffffffff, 0x00040007,
1500 0x3c0e0, 0xffffffff, 0x00060005,
1501 0x3c0e4, 0xffffffff, 0x00090008,
1502 0x3c0e8, 0xffffffff, 0x00010000,
1503 0x3c0ec, 0xffffffff, 0x00030002,
1504 0x3c0f0, 0xffffffff, 0x00040007,
1505 0x3c0f4, 0xffffffff, 0x00060005,
1506 0x3c0f8, 0xffffffff, 0x00090008,
1507 0xc318, 0xffffffff, 0x00020200,
1508 0x3350, 0xffffffff, 0x00000200,
1509 0x15c0, 0xffffffff, 0x00000400,
1510 0x55e8, 0xffffffff, 0x00000000,
1511 0x2f50, 0xffffffff, 0x00000902,
1512 0x3c000, 0xffffffff, 0x96940200,
1513 0x8708, 0xffffffff, 0x00900100,
1514 0xc424, 0xffffffff, 0x0020003f,
1515 0x38, 0xffffffff, 0x0140001c,
1516 0x3c, 0x000f0000, 0x000f0000,
1517 0x220, 0xffffffff, 0xc060000c,
1518 0x224, 0xc0000fff, 0x00000100,
1519 0xf90, 0xffffffff, 0x00000100,
1520 0xf98, 0x00000101, 0x00000000,
1521 0x20a8, 0xffffffff, 0x00000104,
1522 0x55e4, 0xff000fff, 0x00000100,
1523 0x30cc, 0xc0000fff, 0x00000104,
1524 0xc1e4, 0x00000001, 0x00000001,
1525 0xd00c, 0xff000ff0, 0x00000100,
1526 0xd80c, 0xff000ff0, 0x00000100
1527 };
1528
1529 static const u32 godavari_golden_registers[] =
1530 {
1531 0x55e4, 0xff607fff, 0xfc000100,
1532 0x6ed8, 0x00010101, 0x00010000,
1533 0x9830, 0xffffffff, 0x00000000,
1534 0x98302, 0xf00fffff, 0x00000400,
1535 0x6130, 0xffffffff, 0x00010000,
1536 0x5bb0, 0x000000f0, 0x00000070,
1537 0x5bc0, 0xf0311fff, 0x80300000,
1538 0x98f8, 0x73773777, 0x12010001,
1539 0x98fc, 0xffffffff, 0x00000010,
1540 0x8030, 0x00001f0f, 0x0000100a,
1541 0x2f48, 0x73773777, 0x12010001,
1542 0x2408, 0x000fffff, 0x000c007f,
1543 0x8a14, 0xf000003f, 0x00000007,
1544 0x8b24, 0xffffffff, 0x00ff0fff,
1545 0x30a04, 0x0000ff0f, 0x00000000,
1546 0x28a4c, 0x07ffffff, 0x06000000,
1547 0x4d8, 0x00000fff, 0x00000100,
1548 0xd014, 0x00010000, 0x00810001,
1549 0xd814, 0x00010000, 0x00810001,
1550 0x3e78, 0x00000001, 0x00000002,
1551 0xc768, 0x00000008, 0x00000008,
1552 0xc770, 0x00000f00, 0x00000800,
1553 0xc774, 0x00000f00, 0x00000800,
1554 0xc798, 0x00ffffff, 0x00ff7fbf,
1555 0xc79c, 0x00ffffff, 0x00ff7faf,
1556 0x8c00, 0x000000ff, 0x00000001,
1557 0x214f8, 0x01ff01ff, 0x00000002,
1558 0x21498, 0x007ff800, 0x00200000,
1559 0x2015c, 0xffffffff, 0x00000f40,
1560 0x88c4, 0x001f3ae3, 0x00000082,
1561 0x88d4, 0x0000001f, 0x00000010,
1562 0x30934, 0xffffffff, 0x00000000
1563 };
1564
1565
1566 static void cik_init_golden_registers(struct radeon_device *rdev)
1567 {
1568 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1569 mutex_lock(&rdev->grbm_idx_mutex);
1570 switch (rdev->family) {
1571 case CHIP_BONAIRE:
1572 radeon_program_register_sequence(rdev,
1573 bonaire_mgcg_cgcg_init,
1574 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1575 radeon_program_register_sequence(rdev,
1576 bonaire_golden_registers,
1577 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1578 radeon_program_register_sequence(rdev,
1579 bonaire_golden_common_registers,
1580 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1581 radeon_program_register_sequence(rdev,
1582 bonaire_golden_spm_registers,
1583 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1584 break;
1585 case CHIP_KABINI:
1586 radeon_program_register_sequence(rdev,
1587 kalindi_mgcg_cgcg_init,
1588 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1589 radeon_program_register_sequence(rdev,
1590 kalindi_golden_registers,
1591 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1592 radeon_program_register_sequence(rdev,
1593 kalindi_golden_common_registers,
1594 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1595 radeon_program_register_sequence(rdev,
1596 kalindi_golden_spm_registers,
1597 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1598 break;
1599 case CHIP_MULLINS:
1600 radeon_program_register_sequence(rdev,
1601 kalindi_mgcg_cgcg_init,
1602 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1603 radeon_program_register_sequence(rdev,
1604 godavari_golden_registers,
1605 (const u32)ARRAY_SIZE(godavari_golden_registers));
1606 radeon_program_register_sequence(rdev,
1607 kalindi_golden_common_registers,
1608 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1609 radeon_program_register_sequence(rdev,
1610 kalindi_golden_spm_registers,
1611 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1612 break;
1613 case CHIP_KAVERI:
1614 radeon_program_register_sequence(rdev,
1615 spectre_mgcg_cgcg_init,
1616 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1617 radeon_program_register_sequence(rdev,
1618 spectre_golden_registers,
1619 (const u32)ARRAY_SIZE(spectre_golden_registers));
1620 radeon_program_register_sequence(rdev,
1621 spectre_golden_common_registers,
1622 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1623 radeon_program_register_sequence(rdev,
1624 spectre_golden_spm_registers,
1625 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1626 break;
1627 case CHIP_HAWAII:
1628 radeon_program_register_sequence(rdev,
1629 hawaii_mgcg_cgcg_init,
1630 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1631 radeon_program_register_sequence(rdev,
1632 hawaii_golden_registers,
1633 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1634 radeon_program_register_sequence(rdev,
1635 hawaii_golden_common_registers,
1636 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1637 radeon_program_register_sequence(rdev,
1638 hawaii_golden_spm_registers,
1639 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1640 break;
1641 default:
1642 break;
1643 }
1644 mutex_unlock(&rdev->grbm_idx_mutex);
1645 }
1646
1647 /**
1648 * cik_get_xclk - get the xclk
1649 *
1650 * @rdev: radeon_device pointer
1651 *
1652 * Returns the reference clock used by the gfx engine
1653 * (CIK).
1654 */
1655 u32 cik_get_xclk(struct radeon_device *rdev)
1656 {
1657 u32 reference_clock = rdev->clock.spll.reference_freq;
1658
1659 if (rdev->flags & RADEON_IS_IGP) {
1660 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1661 return reference_clock / 2;
1662 } else {
1663 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1664 return reference_clock / 4;
1665 }
1666 return reference_clock;
1667 }
1668
1669 /**
1670 * cik_mm_rdoorbell - read a doorbell dword
1671 *
1672 * @rdev: radeon_device pointer
1673 * @index: doorbell index
1674 *
1675 * Returns the value in the doorbell aperture at the
1676 * requested doorbell index (CIK).
1677 */
1678 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1679 {
1680 if (index < rdev->doorbell.num_doorbells) {
1681 return readl(rdev->doorbell.ptr + index);
1682 } else {
1683 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1684 return 0;
1685 }
1686 }
1687
1688 /**
1689 * cik_mm_wdoorbell - write a doorbell dword
1690 *
1691 * @rdev: radeon_device pointer
1692 * @index: doorbell index
1693 * @v: value to write
1694 *
1695 * Writes @v to the doorbell aperture at the
1696 * requested doorbell index (CIK).
1697 */
1698 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1699 {
1700 if (index < rdev->doorbell.num_doorbells) {
1701 writel(v, rdev->doorbell.ptr + index);
1702 } else {
1703 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1704 }
1705 }
1706
1707 #define BONAIRE_IO_MC_REGS_SIZE 36
1708
1709 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1710 {
1711 {0x00000070, 0x04400000},
1712 {0x00000071, 0x80c01803},
1713 {0x00000072, 0x00004004},
1714 {0x00000073, 0x00000100},
1715 {0x00000074, 0x00ff0000},
1716 {0x00000075, 0x34000000},
1717 {0x00000076, 0x08000014},
1718 {0x00000077, 0x00cc08ec},
1719 {0x00000078, 0x00000400},
1720 {0x00000079, 0x00000000},
1721 {0x0000007a, 0x04090000},
1722 {0x0000007c, 0x00000000},
1723 {0x0000007e, 0x4408a8e8},
1724 {0x0000007f, 0x00000304},
1725 {0x00000080, 0x00000000},
1726 {0x00000082, 0x00000001},
1727 {0x00000083, 0x00000002},
1728 {0x00000084, 0xf3e4f400},
1729 {0x00000085, 0x052024e3},
1730 {0x00000087, 0x00000000},
1731 {0x00000088, 0x01000000},
1732 {0x0000008a, 0x1c0a0000},
1733 {0x0000008b, 0xff010000},
1734 {0x0000008d, 0xffffefff},
1735 {0x0000008e, 0xfff3efff},
1736 {0x0000008f, 0xfff3efbf},
1737 {0x00000092, 0xf7ffffff},
1738 {0x00000093, 0xffffff7f},
1739 {0x00000095, 0x00101101},
1740 {0x00000096, 0x00000fff},
1741 {0x00000097, 0x00116fff},
1742 {0x00000098, 0x60010000},
1743 {0x00000099, 0x10010000},
1744 {0x0000009a, 0x00006000},
1745 {0x0000009b, 0x00001000},
1746 {0x0000009f, 0x00b48000}
1747 };
1748
1749 #define HAWAII_IO_MC_REGS_SIZE 22
1750
1751 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1752 {
1753 {0x0000007d, 0x40000000},
1754 {0x0000007e, 0x40180304},
1755 {0x0000007f, 0x0000ff00},
1756 {0x00000081, 0x00000000},
1757 {0x00000083, 0x00000800},
1758 {0x00000086, 0x00000000},
1759 {0x00000087, 0x00000100},
1760 {0x00000088, 0x00020100},
1761 {0x00000089, 0x00000000},
1762 {0x0000008b, 0x00040000},
1763 {0x0000008c, 0x00000100},
1764 {0x0000008e, 0xff010000},
1765 {0x00000090, 0xffffefff},
1766 {0x00000091, 0xfff3efff},
1767 {0x00000092, 0xfff3efbf},
1768 {0x00000093, 0xf7ffffff},
1769 {0x00000094, 0xffffff7f},
1770 {0x00000095, 0x00000fff},
1771 {0x00000096, 0x00116fff},
1772 {0x00000097, 0x60010000},
1773 {0x00000098, 0x10010000},
1774 {0x0000009f, 0x00c79000}
1775 };
1776
1777
1778 /**
1779 * cik_srbm_select - select specific register instances
1780 *
1781 * @rdev: radeon_device pointer
1782 * @me: selected ME (micro engine)
1783 * @pipe: pipe
1784 * @queue: queue
1785 * @vmid: VMID
1786 *
1787 * Switches the currently active registers instances. Some
1788 * registers are instanced per VMID, others are instanced per
1789 * me/pipe/queue combination.
1790 */
1791 static void cik_srbm_select(struct radeon_device *rdev,
1792 u32 me, u32 pipe, u32 queue, u32 vmid)
1793 {
1794 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1795 MEID(me & 0x3) |
1796 VMID(vmid & 0xf) |
1797 QUEUEID(queue & 0x7));
1798 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1799 }
1800
1801 /* ucode loading */
1802 /**
1803 * ci_mc_load_microcode - load MC ucode into the hw
1804 *
1805 * @rdev: radeon_device pointer
1806 *
1807 * Load the GDDR MC ucode into the hw (CIK).
1808 * Returns 0 on success, error on failure.
1809 */
1810 int ci_mc_load_microcode(struct radeon_device *rdev)
1811 {
1812 const __be32 *fw_data = NULL;
1813 const __le32 *new_fw_data = NULL;
1814 u32 running, blackout = 0, tmp;
1815 u32 *io_mc_regs = NULL;
1816 const __le32 *new_io_mc_regs = NULL;
1817 int i, regs_size, ucode_size;
1818
1819 if (!rdev->mc_fw)
1820 return -EINVAL;
1821
1822 if (rdev->new_fw) {
1823 const struct mc_firmware_header_v1_0 *hdr =
1824 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1825
1826 radeon_ucode_print_mc_hdr(&hdr->header);
1827
1828 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1829 new_io_mc_regs = (const __le32 *)
1830 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1831 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1832 new_fw_data = (const __le32 *)
1833 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1834 } else {
1835 ucode_size = rdev->mc_fw->size / 4;
1836
1837 switch (rdev->family) {
1838 case CHIP_BONAIRE:
1839 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1840 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1841 break;
1842 case CHIP_HAWAII:
1843 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1844 regs_size = HAWAII_IO_MC_REGS_SIZE;
1845 break;
1846 default:
1847 return -EINVAL;
1848 }
1849 fw_data = (const __be32 *)rdev->mc_fw->data;
1850 }
1851
1852 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1853
1854 if (running == 0) {
1855 if (running) {
1856 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1857 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1858 }
1859
1860 /* reset the engine and set to writable */
1861 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1862 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1863
1864 /* load mc io regs */
1865 for (i = 0; i < regs_size; i++) {
1866 if (rdev->new_fw) {
1867 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1868 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1869 } else {
1870 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1871 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1872 }
1873 }
1874
1875 tmp = RREG32(MC_SEQ_MISC0);
1876 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1877 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1878 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1879 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1880 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1881 }
1882
1883 /* load the MC ucode */
1884 for (i = 0; i < ucode_size; i++) {
1885 if (rdev->new_fw)
1886 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1887 else
1888 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1889 }
1890
1891 /* put the engine back into the active state */
1892 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1893 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1894 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1895
1896 /* wait for training to complete */
1897 for (i = 0; i < rdev->usec_timeout; i++) {
1898 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1899 break;
1900 udelay(1);
1901 }
1902 for (i = 0; i < rdev->usec_timeout; i++) {
1903 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1904 break;
1905 udelay(1);
1906 }
1907
1908 if (running)
1909 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1910 }
1911
1912 return 0;
1913 }
1914
1915 /**
1916 * cik_init_microcode - load ucode images from disk
1917 *
1918 * @rdev: radeon_device pointer
1919 *
1920 * Use the firmware interface to load the ucode images into
1921 * the driver (not loaded into hw).
1922 * Returns 0 on success, error on failure.
1923 */
1924 static int cik_init_microcode(struct radeon_device *rdev)
1925 {
1926 const char *chip_name;
1927 const char *new_chip_name;
1928 size_t pfp_req_size, me_req_size, ce_req_size,
1929 mec_req_size, rlc_req_size, mc_req_size = 0,
1930 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1931 char fw_name[30];
1932 int new_fw = 0;
1933 int err;
1934 int num_fw;
1935
1936 DRM_DEBUG("\n");
1937
1938 switch (rdev->family) {
1939 case CHIP_BONAIRE:
1940 chip_name = "BONAIRE";
1941 new_chip_name = "bonaire";
1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1948 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1951 num_fw = 8;
1952 break;
1953 case CHIP_HAWAII:
1954 chip_name = "HAWAII";
1955 new_chip_name = "hawaii";
1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1961 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1962 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1963 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1964 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1965 num_fw = 8;
1966 break;
1967 case CHIP_KAVERI:
1968 chip_name = "KAVERI";
1969 new_chip_name = "kaveri";
1970 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1971 me_req_size = CIK_ME_UCODE_SIZE * 4;
1972 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1973 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1974 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1975 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1976 num_fw = 7;
1977 break;
1978 case CHIP_KABINI:
1979 chip_name = "KABINI";
1980 new_chip_name = "kabini";
1981 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1982 me_req_size = CIK_ME_UCODE_SIZE * 4;
1983 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1984 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1985 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1986 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1987 num_fw = 6;
1988 break;
1989 case CHIP_MULLINS:
1990 chip_name = "MULLINS";
1991 new_chip_name = "mullins";
1992 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1993 me_req_size = CIK_ME_UCODE_SIZE * 4;
1994 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1995 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1996 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1997 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1998 num_fw = 6;
1999 break;
2000 default: BUG();
2001 }
2002
2003 DRM_INFO("Loading %s Microcode\n", new_chip_name);
2004
2005 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2006 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2007 if (err) {
2008 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2009 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2010 if (err)
2011 goto out;
2012 if (rdev->pfp_fw->size != pfp_req_size) {
2013 printk(KERN_ERR
2014 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2015 rdev->pfp_fw->size, fw_name);
2016 err = -EINVAL;
2017 goto out;
2018 }
2019 } else {
2020 err = radeon_ucode_validate(rdev->pfp_fw);
2021 if (err) {
2022 printk(KERN_ERR
2023 "cik_fw: validation failed for firmware \"%s\"\n",
2024 fw_name);
2025 goto out;
2026 } else {
2027 new_fw++;
2028 }
2029 }
2030
2031 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2032 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2033 if (err) {
2034 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2035 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2036 if (err)
2037 goto out;
2038 if (rdev->me_fw->size != me_req_size) {
2039 printk(KERN_ERR
2040 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2041 rdev->me_fw->size, fw_name);
2042 err = -EINVAL;
2043 }
2044 } else {
2045 err = radeon_ucode_validate(rdev->me_fw);
2046 if (err) {
2047 printk(KERN_ERR
2048 "cik_fw: validation failed for firmware \"%s\"\n",
2049 fw_name);
2050 goto out;
2051 } else {
2052 new_fw++;
2053 }
2054 }
2055
2056 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2057 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2058 if (err) {
2059 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2060 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2061 if (err)
2062 goto out;
2063 if (rdev->ce_fw->size != ce_req_size) {
2064 printk(KERN_ERR
2065 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2066 rdev->ce_fw->size, fw_name);
2067 err = -EINVAL;
2068 }
2069 } else {
2070 err = radeon_ucode_validate(rdev->ce_fw);
2071 if (err) {
2072 printk(KERN_ERR
2073 "cik_fw: validation failed for firmware \"%s\"\n",
2074 fw_name);
2075 goto out;
2076 } else {
2077 new_fw++;
2078 }
2079 }
2080
2081 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2082 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2083 if (err) {
2084 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2085 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2086 if (err)
2087 goto out;
2088 if (rdev->mec_fw->size != mec_req_size) {
2089 printk(KERN_ERR
2090 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2091 rdev->mec_fw->size, fw_name);
2092 err = -EINVAL;
2093 }
2094 } else {
2095 err = radeon_ucode_validate(rdev->mec_fw);
2096 if (err) {
2097 printk(KERN_ERR
2098 "cik_fw: validation failed for firmware \"%s\"\n",
2099 fw_name);
2100 goto out;
2101 } else {
2102 new_fw++;
2103 }
2104 }
2105
2106 if (rdev->family == CHIP_KAVERI) {
2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2108 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2109 if (err) {
2110 goto out;
2111 } else {
2112 err = radeon_ucode_validate(rdev->mec2_fw);
2113 if (err) {
2114 goto out;
2115 } else {
2116 new_fw++;
2117 }
2118 }
2119 }
2120
2121 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2122 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2123 if (err) {
2124 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2125 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2126 if (err)
2127 goto out;
2128 if (rdev->rlc_fw->size != rlc_req_size) {
2129 printk(KERN_ERR
2130 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2131 rdev->rlc_fw->size, fw_name);
2132 err = -EINVAL;
2133 }
2134 } else {
2135 err = radeon_ucode_validate(rdev->rlc_fw);
2136 if (err) {
2137 printk(KERN_ERR
2138 "cik_fw: validation failed for firmware \"%s\"\n",
2139 fw_name);
2140 goto out;
2141 } else {
2142 new_fw++;
2143 }
2144 }
2145
2146 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2147 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2148 if (err) {
2149 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2150 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2151 if (err)
2152 goto out;
2153 if (rdev->sdma_fw->size != sdma_req_size) {
2154 printk(KERN_ERR
2155 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2156 rdev->sdma_fw->size, fw_name);
2157 err = -EINVAL;
2158 }
2159 } else {
2160 err = radeon_ucode_validate(rdev->sdma_fw);
2161 if (err) {
2162 printk(KERN_ERR
2163 "cik_fw: validation failed for firmware \"%s\"\n",
2164 fw_name);
2165 goto out;
2166 } else {
2167 new_fw++;
2168 }
2169 }
2170
2171 /* No SMC, MC ucode on APUs */
2172 if (!(rdev->flags & RADEON_IS_IGP)) {
2173 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2174 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2175 if (err) {
2176 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2177 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2178 if (err) {
2179 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2180 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2181 if (err)
2182 goto out;
2183 }
2184 if ((rdev->mc_fw->size != mc_req_size) &&
2185 (rdev->mc_fw->size != mc2_req_size)){
2186 printk(KERN_ERR
2187 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2188 rdev->mc_fw->size, fw_name);
2189 err = -EINVAL;
2190 }
2191 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2192 } else {
2193 err = radeon_ucode_validate(rdev->mc_fw);
2194 if (err) {
2195 printk(KERN_ERR
2196 "cik_fw: validation failed for firmware \"%s\"\n",
2197 fw_name);
2198 goto out;
2199 } else {
2200 new_fw++;
2201 }
2202 }
2203
2204 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2205 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2206 if (err) {
2207 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2208 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2209 if (err) {
2210 printk(KERN_ERR
2211 "smc: error loading firmware \"%s\"\n",
2212 fw_name);
2213 release_firmware(rdev->smc_fw);
2214 rdev->smc_fw = NULL;
2215 err = 0;
2216 } else if (rdev->smc_fw->size != smc_req_size) {
2217 printk(KERN_ERR
2218 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2219 rdev->smc_fw->size, fw_name);
2220 err = -EINVAL;
2221 }
2222 } else {
2223 err = radeon_ucode_validate(rdev->smc_fw);
2224 if (err) {
2225 printk(KERN_ERR
2226 "cik_fw: validation failed for firmware \"%s\"\n",
2227 fw_name);
2228 goto out;
2229 } else {
2230 new_fw++;
2231 }
2232 }
2233 }
2234
2235 if (new_fw == 0) {
2236 rdev->new_fw = false;
2237 } else if (new_fw < num_fw) {
2238 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2239 err = -EINVAL;
2240 } else {
2241 rdev->new_fw = true;
2242 }
2243
2244 out:
2245 if (err) {
2246 if (err != -EINVAL)
2247 printk(KERN_ERR
2248 "cik_cp: Failed to load firmware \"%s\"\n",
2249 fw_name);
2250 release_firmware(rdev->pfp_fw);
2251 rdev->pfp_fw = NULL;
2252 release_firmware(rdev->me_fw);
2253 rdev->me_fw = NULL;
2254 release_firmware(rdev->ce_fw);
2255 rdev->ce_fw = NULL;
2256 release_firmware(rdev->mec_fw);
2257 rdev->mec_fw = NULL;
2258 release_firmware(rdev->mec2_fw);
2259 rdev->mec2_fw = NULL;
2260 release_firmware(rdev->rlc_fw);
2261 rdev->rlc_fw = NULL;
2262 release_firmware(rdev->sdma_fw);
2263 rdev->sdma_fw = NULL;
2264 release_firmware(rdev->mc_fw);
2265 rdev->mc_fw = NULL;
2266 release_firmware(rdev->smc_fw);
2267 rdev->smc_fw = NULL;
2268 }
2269 return err;
2270 }
2271
2272 /*
2273 * Core functions
2274 */
2275 /**
2276 * cik_tiling_mode_table_init - init the hw tiling table
2277 *
2278 * @rdev: radeon_device pointer
2279 *
2280 * Starting with SI, the tiling setup is done globally in a
2281 * set of 32 tiling modes. Rather than selecting each set of
2282 * parameters per surface as on older asics, we just select
2283 * which index in the tiling table we want to use, and the
2284 * surface uses those parameters (CIK).
2285 */
2286 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2287 {
2288 const u32 num_tile_mode_states = 32;
2289 const u32 num_secondary_tile_mode_states = 16;
2290 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2291 u32 num_pipe_configs;
2292 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2293 rdev->config.cik.max_shader_engines;
2294
2295 switch (rdev->config.cik.mem_row_size_in_kb) {
2296 case 1:
2297 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2298 break;
2299 case 2:
2300 default:
2301 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2302 break;
2303 case 4:
2304 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2305 break;
2306 }
2307
2308 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2309 if (num_pipe_configs > 8)
2310 num_pipe_configs = 16;
2311
2312 if (num_pipe_configs == 16) {
2313 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2314 switch (reg_offset) {
2315 case 0:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2320 break;
2321 case 1:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2325 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2326 break;
2327 case 2:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2332 break;
2333 case 3:
2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2335 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2336 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2337 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2338 break;
2339 case 4:
2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2343 TILE_SPLIT(split_equal_to_row_size));
2344 break;
2345 case 5:
2346 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2347 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2348 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2349 break;
2350 case 6:
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2355 break;
2356 case 7:
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2360 TILE_SPLIT(split_equal_to_row_size));
2361 break;
2362 case 8:
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2365 break;
2366 case 9:
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2369 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2370 break;
2371 case 10:
2372 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2373 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2374 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2375 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2376 break;
2377 case 11:
2378 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2381 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2382 break;
2383 case 12:
2384 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2385 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2386 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2387 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2388 break;
2389 case 13:
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2393 break;
2394 case 14:
2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2396 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2397 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2398 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2399 break;
2400 case 16:
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2402 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2404 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2405 break;
2406 case 17:
2407 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2408 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2409 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2410 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2411 break;
2412 case 27:
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2414 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2416 break;
2417 case 28:
2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2419 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2420 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2421 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2422 break;
2423 case 29:
2424 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2425 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2428 break;
2429 case 30:
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2434 break;
2435 default:
2436 gb_tile_moden = 0;
2437 break;
2438 }
2439 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2440 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2441 }
2442 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2443 switch (reg_offset) {
2444 case 0:
2445 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2448 NUM_BANKS(ADDR_SURF_16_BANK));
2449 break;
2450 case 1:
2451 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2454 NUM_BANKS(ADDR_SURF_16_BANK));
2455 break;
2456 case 2:
2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2460 NUM_BANKS(ADDR_SURF_16_BANK));
2461 break;
2462 case 3:
2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2466 NUM_BANKS(ADDR_SURF_16_BANK));
2467 break;
2468 case 4:
2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2472 NUM_BANKS(ADDR_SURF_8_BANK));
2473 break;
2474 case 5:
2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2478 NUM_BANKS(ADDR_SURF_4_BANK));
2479 break;
2480 case 6:
2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2484 NUM_BANKS(ADDR_SURF_2_BANK));
2485 break;
2486 case 8:
2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2490 NUM_BANKS(ADDR_SURF_16_BANK));
2491 break;
2492 case 9:
2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2496 NUM_BANKS(ADDR_SURF_16_BANK));
2497 break;
2498 case 10:
2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2502 NUM_BANKS(ADDR_SURF_16_BANK));
2503 break;
2504 case 11:
2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2508 NUM_BANKS(ADDR_SURF_8_BANK));
2509 break;
2510 case 12:
2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2514 NUM_BANKS(ADDR_SURF_4_BANK));
2515 break;
2516 case 13:
2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2520 NUM_BANKS(ADDR_SURF_2_BANK));
2521 break;
2522 case 14:
2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2526 NUM_BANKS(ADDR_SURF_2_BANK));
2527 break;
2528 default:
2529 gb_tile_moden = 0;
2530 break;
2531 }
2532 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2533 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2534 }
2535 } else if (num_pipe_configs == 8) {
2536 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2537 switch (reg_offset) {
2538 case 0:
2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2543 break;
2544 case 1:
2545 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2548 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2549 break;
2550 case 2:
2551 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2553 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2554 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2555 break;
2556 case 3:
2557 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2558 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2559 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2560 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2561 break;
2562 case 4:
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 TILE_SPLIT(split_equal_to_row_size));
2567 break;
2568 case 5:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2572 break;
2573 case 6:
2574 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2575 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2576 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2577 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2578 break;
2579 case 7:
2580 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2583 TILE_SPLIT(split_equal_to_row_size));
2584 break;
2585 case 8:
2586 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2587 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2588 break;
2589 case 9:
2590 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2592 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2593 break;
2594 case 10:
2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2598 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2599 break;
2600 case 11:
2601 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2602 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2603 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2604 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2605 break;
2606 case 12:
2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2608 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2611 break;
2612 case 13:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2616 break;
2617 case 14:
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 break;
2623 case 16:
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2628 break;
2629 case 17:
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2634 break;
2635 case 27:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2638 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2639 break;
2640 case 28:
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2645 break;
2646 case 29:
2647 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2649 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2651 break;
2652 case 30:
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2657 break;
2658 default:
2659 gb_tile_moden = 0;
2660 break;
2661 }
2662 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2664 }
2665 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2666 switch (reg_offset) {
2667 case 0:
2668 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2671 NUM_BANKS(ADDR_SURF_16_BANK));
2672 break;
2673 case 1:
2674 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2675 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2676 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2677 NUM_BANKS(ADDR_SURF_16_BANK));
2678 break;
2679 case 2:
2680 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2683 NUM_BANKS(ADDR_SURF_16_BANK));
2684 break;
2685 case 3:
2686 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2689 NUM_BANKS(ADDR_SURF_16_BANK));
2690 break;
2691 case 4:
2692 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2695 NUM_BANKS(ADDR_SURF_8_BANK));
2696 break;
2697 case 5:
2698 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2701 NUM_BANKS(ADDR_SURF_4_BANK));
2702 break;
2703 case 6:
2704 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2705 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2706 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2707 NUM_BANKS(ADDR_SURF_2_BANK));
2708 break;
2709 case 8:
2710 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2713 NUM_BANKS(ADDR_SURF_16_BANK));
2714 break;
2715 case 9:
2716 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2719 NUM_BANKS(ADDR_SURF_16_BANK));
2720 break;
2721 case 10:
2722 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2723 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2724 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2725 NUM_BANKS(ADDR_SURF_16_BANK));
2726 break;
2727 case 11:
2728 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2731 NUM_BANKS(ADDR_SURF_16_BANK));
2732 break;
2733 case 12:
2734 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2737 NUM_BANKS(ADDR_SURF_8_BANK));
2738 break;
2739 case 13:
2740 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2741 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2742 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2743 NUM_BANKS(ADDR_SURF_4_BANK));
2744 break;
2745 case 14:
2746 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2749 NUM_BANKS(ADDR_SURF_2_BANK));
2750 break;
2751 default:
2752 gb_tile_moden = 0;
2753 break;
2754 }
2755 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2756 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2757 }
2758 } else if (num_pipe_configs == 4) {
2759 if (num_rbs == 4) {
2760 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2761 switch (reg_offset) {
2762 case 0:
2763 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2765 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2766 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2767 break;
2768 case 1:
2769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2771 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2773 break;
2774 case 2:
2775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2777 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2779 break;
2780 case 3:
2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2783 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2785 break;
2786 case 4:
2787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2790 TILE_SPLIT(split_equal_to_row_size));
2791 break;
2792 case 5:
2793 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2794 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2796 break;
2797 case 6:
2798 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2799 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2800 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2801 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2802 break;
2803 case 7:
2804 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2805 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2806 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2807 TILE_SPLIT(split_equal_to_row_size));
2808 break;
2809 case 8:
2810 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2811 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2812 break;
2813 case 9:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2817 break;
2818 case 10:
2819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2823 break;
2824 case 11:
2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2827 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2829 break;
2830 case 12:
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2835 break;
2836 case 13:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2838 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2839 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2840 break;
2841 case 14:
2842 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2843 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2844 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2845 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2846 break;
2847 case 16:
2848 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2849 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2850 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2851 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2852 break;
2853 case 17:
2854 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2855 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2856 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2857 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2858 break;
2859 case 27:
2860 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2861 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2862 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2863 break;
2864 case 28:
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2869 break;
2870 case 29:
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2875 break;
2876 case 30:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2879 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2881 break;
2882 default:
2883 gb_tile_moden = 0;
2884 break;
2885 }
2886 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2887 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2888 }
2889 } else if (num_rbs < 4) {
2890 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2891 switch (reg_offset) {
2892 case 0:
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2896 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2897 break;
2898 case 1:
2899 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2900 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2901 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2903 break;
2904 case 2:
2905 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2906 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2907 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2908 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2909 break;
2910 case 3:
2911 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2912 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2913 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2914 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2915 break;
2916 case 4:
2917 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2919 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2920 TILE_SPLIT(split_equal_to_row_size));
2921 break;
2922 case 5:
2923 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2924 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2925 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2926 break;
2927 case 6:
2928 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2929 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2930 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2931 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2932 break;
2933 case 7:
2934 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2935 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2936 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2937 TILE_SPLIT(split_equal_to_row_size));
2938 break;
2939 case 8:
2940 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2941 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2942 break;
2943 case 9:
2944 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2945 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2946 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2947 break;
2948 case 10:
2949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2950 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2951 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2953 break;
2954 case 11:
2955 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2956 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2957 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2958 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2959 break;
2960 case 12:
2961 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2962 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2963 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2964 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2965 break;
2966 case 13:
2967 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2968 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2969 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2970 break;
2971 case 14:
2972 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2973 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2974 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2975 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2976 break;
2977 case 16:
2978 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2979 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2980 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2981 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2982 break;
2983 case 17:
2984 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2985 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2986 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2987 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2988 break;
2989 case 27:
2990 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2991 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2992 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2993 break;
2994 case 28:
2995 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2996 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2997 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2998 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2999 break;
3000 case 29:
3001 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3002 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3003 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3004 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3005 break;
3006 case 30:
3007 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3008 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3009 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3010 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3011 break;
3012 default:
3013 gb_tile_moden = 0;
3014 break;
3015 }
3016 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3017 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3018 }
3019 }
3020 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3021 switch (reg_offset) {
3022 case 0:
3023 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3026 NUM_BANKS(ADDR_SURF_16_BANK));
3027 break;
3028 case 1:
3029 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3030 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3031 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3032 NUM_BANKS(ADDR_SURF_16_BANK));
3033 break;
3034 case 2:
3035 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3038 NUM_BANKS(ADDR_SURF_16_BANK));
3039 break;
3040 case 3:
3041 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3042 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3043 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3044 NUM_BANKS(ADDR_SURF_16_BANK));
3045 break;
3046 case 4:
3047 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3048 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3049 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3050 NUM_BANKS(ADDR_SURF_16_BANK));
3051 break;
3052 case 5:
3053 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3054 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3055 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3056 NUM_BANKS(ADDR_SURF_8_BANK));
3057 break;
3058 case 6:
3059 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3060 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3061 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3062 NUM_BANKS(ADDR_SURF_4_BANK));
3063 break;
3064 case 8:
3065 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3066 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3067 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3068 NUM_BANKS(ADDR_SURF_16_BANK));
3069 break;
3070 case 9:
3071 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3074 NUM_BANKS(ADDR_SURF_16_BANK));
3075 break;
3076 case 10:
3077 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3078 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3079 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3080 NUM_BANKS(ADDR_SURF_16_BANK));
3081 break;
3082 case 11:
3083 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3084 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3085 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3086 NUM_BANKS(ADDR_SURF_16_BANK));
3087 break;
3088 case 12:
3089 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3090 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3091 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3092 NUM_BANKS(ADDR_SURF_16_BANK));
3093 break;
3094 case 13:
3095 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3098 NUM_BANKS(ADDR_SURF_8_BANK));
3099 break;
3100 case 14:
3101 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3104 NUM_BANKS(ADDR_SURF_4_BANK));
3105 break;
3106 default:
3107 gb_tile_moden = 0;
3108 break;
3109 }
3110 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3111 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3112 }
3113 } else if (num_pipe_configs == 2) {
3114 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3115 switch (reg_offset) {
3116 case 0:
3117 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3118 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3119 PIPE_CONFIG(ADDR_SURF_P2) |
3120 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3121 break;
3122 case 1:
3123 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3124 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3125 PIPE_CONFIG(ADDR_SURF_P2) |
3126 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3127 break;
3128 case 2:
3129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3130 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3131 PIPE_CONFIG(ADDR_SURF_P2) |
3132 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3133 break;
3134 case 3:
3135 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3136 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3137 PIPE_CONFIG(ADDR_SURF_P2) |
3138 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3139 break;
3140 case 4:
3141 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3142 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3143 PIPE_CONFIG(ADDR_SURF_P2) |
3144 TILE_SPLIT(split_equal_to_row_size));
3145 break;
3146 case 5:
3147 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3148 PIPE_CONFIG(ADDR_SURF_P2) |
3149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3150 break;
3151 case 6:
3152 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3153 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3154 PIPE_CONFIG(ADDR_SURF_P2) |
3155 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3156 break;
3157 case 7:
3158 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3159 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3160 PIPE_CONFIG(ADDR_SURF_P2) |
3161 TILE_SPLIT(split_equal_to_row_size));
3162 break;
3163 case 8:
3164 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3165 PIPE_CONFIG(ADDR_SURF_P2);
3166 break;
3167 case 9:
3168 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3170 PIPE_CONFIG(ADDR_SURF_P2));
3171 break;
3172 case 10:
3173 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3174 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3175 PIPE_CONFIG(ADDR_SURF_P2) |
3176 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3177 break;
3178 case 11:
3179 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3180 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3181 PIPE_CONFIG(ADDR_SURF_P2) |
3182 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3183 break;
3184 case 12:
3185 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3186 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3187 PIPE_CONFIG(ADDR_SURF_P2) |
3188 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3189 break;
3190 case 13:
3191 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3192 PIPE_CONFIG(ADDR_SURF_P2) |
3193 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3194 break;
3195 case 14:
3196 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3198 PIPE_CONFIG(ADDR_SURF_P2) |
3199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3200 break;
3201 case 16:
3202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3204 PIPE_CONFIG(ADDR_SURF_P2) |
3205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3206 break;
3207 case 17:
3208 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3209 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3210 PIPE_CONFIG(ADDR_SURF_P2) |
3211 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3212 break;
3213 case 27:
3214 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3215 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3216 PIPE_CONFIG(ADDR_SURF_P2));
3217 break;
3218 case 28:
3219 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3220 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3221 PIPE_CONFIG(ADDR_SURF_P2) |
3222 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3223 break;
3224 case 29:
3225 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3226 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3227 PIPE_CONFIG(ADDR_SURF_P2) |
3228 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3229 break;
3230 case 30:
3231 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3232 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3233 PIPE_CONFIG(ADDR_SURF_P2) |
3234 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3235 break;
3236 default:
3237 gb_tile_moden = 0;
3238 break;
3239 }
3240 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3241 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3242 }
3243 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3244 switch (reg_offset) {
3245 case 0:
3246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3249 NUM_BANKS(ADDR_SURF_16_BANK));
3250 break;
3251 case 1:
3252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3255 NUM_BANKS(ADDR_SURF_16_BANK));
3256 break;
3257 case 2:
3258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3261 NUM_BANKS(ADDR_SURF_16_BANK));
3262 break;
3263 case 3:
3264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3267 NUM_BANKS(ADDR_SURF_16_BANK));
3268 break;
3269 case 4:
3270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3273 NUM_BANKS(ADDR_SURF_16_BANK));
3274 break;
3275 case 5:
3276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3279 NUM_BANKS(ADDR_SURF_16_BANK));
3280 break;
3281 case 6:
3282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3285 NUM_BANKS(ADDR_SURF_8_BANK));
3286 break;
3287 case 8:
3288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3291 NUM_BANKS(ADDR_SURF_16_BANK));
3292 break;
3293 case 9:
3294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3297 NUM_BANKS(ADDR_SURF_16_BANK));
3298 break;
3299 case 10:
3300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3303 NUM_BANKS(ADDR_SURF_16_BANK));
3304 break;
3305 case 11:
3306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3309 NUM_BANKS(ADDR_SURF_16_BANK));
3310 break;
3311 case 12:
3312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3315 NUM_BANKS(ADDR_SURF_16_BANK));
3316 break;
3317 case 13:
3318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3321 NUM_BANKS(ADDR_SURF_16_BANK));
3322 break;
3323 case 14:
3324 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3325 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3326 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3327 NUM_BANKS(ADDR_SURF_8_BANK));
3328 break;
3329 default:
3330 gb_tile_moden = 0;
3331 break;
3332 }
3333 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3334 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3335 }
3336 } else
3337 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3338 }
3339
3340 /**
3341 * cik_select_se_sh - select which SE, SH to address
3342 *
3343 * @rdev: radeon_device pointer
3344 * @se_num: shader engine to address
3345 * @sh_num: sh block to address
3346 *
3347 * Select which SE, SH combinations to address. Certain
3348 * registers are instanced per SE or SH. 0xffffffff means
3349 * broadcast to all SEs or SHs (CIK).
3350 */
3351 static void cik_select_se_sh(struct radeon_device *rdev,
3352 u32 se_num, u32 sh_num)
3353 {
3354 u32 data = INSTANCE_BROADCAST_WRITES;
3355
3356 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3357 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3358 else if (se_num == 0xffffffff)
3359 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3360 else if (sh_num == 0xffffffff)
3361 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3362 else
3363 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3364 WREG32(GRBM_GFX_INDEX, data);
3365 }
3366
3367 /**
3368 * cik_create_bitmask - create a bitmask
3369 *
3370 * @bit_width: length of the mask
3371 *
3372 * create a variable length bit mask (CIK).
3373 * Returns the bitmask.
3374 */
3375 static u32 cik_create_bitmask(u32 bit_width)
3376 {
3377 u32 i, mask = 0;
3378
3379 for (i = 0; i < bit_width; i++) {
3380 mask <<= 1;
3381 mask |= 1;
3382 }
3383 return mask;
3384 }
3385
3386 /**
3387 * cik_get_rb_disabled - computes the mask of disabled RBs
3388 *
3389 * @rdev: radeon_device pointer
3390 * @max_rb_num: max RBs (render backends) for the asic
3391 * @se_num: number of SEs (shader engines) for the asic
3392 * @sh_per_se: number of SH blocks per SE for the asic
3393 *
3394 * Calculates the bitmask of disabled RBs (CIK).
3395 * Returns the disabled RB bitmask.
3396 */
3397 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3398 u32 max_rb_num_per_se,
3399 u32 sh_per_se)
3400 {
3401 u32 data, mask;
3402
3403 data = RREG32(CC_RB_BACKEND_DISABLE);
3404 if (data & 1)
3405 data &= BACKEND_DISABLE_MASK;
3406 else
3407 data = 0;
3408 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3409
3410 data >>= BACKEND_DISABLE_SHIFT;
3411
3412 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3413
3414 return data & mask;
3415 }
3416
3417 /**
3418 * cik_setup_rb - setup the RBs on the asic
3419 *
3420 * @rdev: radeon_device pointer
3421 * @se_num: number of SEs (shader engines) for the asic
3422 * @sh_per_se: number of SH blocks per SE for the asic
3423 * @max_rb_num: max RBs (render backends) for the asic
3424 *
3425 * Configures per-SE/SH RB registers (CIK).
3426 */
3427 static void cik_setup_rb(struct radeon_device *rdev,
3428 u32 se_num, u32 sh_per_se,
3429 u32 max_rb_num_per_se)
3430 {
3431 int i, j;
3432 u32 data, mask;
3433 u32 disabled_rbs = 0;
3434 u32 enabled_rbs = 0;
3435
3436 mutex_lock(&rdev->grbm_idx_mutex);
3437 for (i = 0; i < se_num; i++) {
3438 for (j = 0; j < sh_per_se; j++) {
3439 cik_select_se_sh(rdev, i, j);
3440 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3441 if (rdev->family == CHIP_HAWAII)
3442 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3443 else
3444 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3445 }
3446 }
3447 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3448 mutex_unlock(&rdev->grbm_idx_mutex);
3449
3450 mask = 1;
3451 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3452 if (!(disabled_rbs & mask))
3453 enabled_rbs |= mask;
3454 mask <<= 1;
3455 }
3456
3457 rdev->config.cik.backend_enable_mask = enabled_rbs;
3458
3459 mutex_lock(&rdev->grbm_idx_mutex);
3460 for (i = 0; i < se_num; i++) {
3461 cik_select_se_sh(rdev, i, 0xffffffff);
3462 data = 0;
3463 for (j = 0; j < sh_per_se; j++) {
3464 switch (enabled_rbs & 3) {
3465 case 0:
3466 if (j == 0)
3467 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3468 else
3469 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3470 break;
3471 case 1:
3472 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3473 break;
3474 case 2:
3475 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3476 break;
3477 case 3:
3478 default:
3479 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3480 break;
3481 }
3482 enabled_rbs >>= 2;
3483 }
3484 WREG32(PA_SC_RASTER_CONFIG, data);
3485 }
3486 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3487 mutex_unlock(&rdev->grbm_idx_mutex);
3488 }
3489
3490 /**
3491 * cik_gpu_init - setup the 3D engine
3492 *
3493 * @rdev: radeon_device pointer
3494 *
3495 * Configures the 3D engine and tiling configuration
3496 * registers so that the 3D engine is usable.
3497 */
3498 static void cik_gpu_init(struct radeon_device *rdev)
3499 {
3500 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3501 u32 mc_shared_chmap, mc_arb_ramcfg;
3502 u32 hdp_host_path_cntl;
3503 u32 tmp;
3504 int i, j;
3505
3506 switch (rdev->family) {
3507 case CHIP_BONAIRE:
3508 rdev->config.cik.max_shader_engines = 2;
3509 rdev->config.cik.max_tile_pipes = 4;
3510 rdev->config.cik.max_cu_per_sh = 7;
3511 rdev->config.cik.max_sh_per_se = 1;
3512 rdev->config.cik.max_backends_per_se = 2;
3513 rdev->config.cik.max_texture_channel_caches = 4;
3514 rdev->config.cik.max_gprs = 256;
3515 rdev->config.cik.max_gs_threads = 32;
3516 rdev->config.cik.max_hw_contexts = 8;
3517
3518 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3519 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3520 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3521 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3522 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3523 break;
3524 case CHIP_HAWAII:
3525 rdev->config.cik.max_shader_engines = 4;
3526 rdev->config.cik.max_tile_pipes = 16;
3527 rdev->config.cik.max_cu_per_sh = 11;
3528 rdev->config.cik.max_sh_per_se = 1;
3529 rdev->config.cik.max_backends_per_se = 4;
3530 rdev->config.cik.max_texture_channel_caches = 16;
3531 rdev->config.cik.max_gprs = 256;
3532 rdev->config.cik.max_gs_threads = 32;
3533 rdev->config.cik.max_hw_contexts = 8;
3534
3535 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3536 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3537 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3538 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3539 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3540 break;
3541 case CHIP_KAVERI:
3542 rdev->config.cik.max_shader_engines = 1;
3543 rdev->config.cik.max_tile_pipes = 4;
3544 if ((rdev->pdev->device == 0x1304) ||
3545 (rdev->pdev->device == 0x1305) ||
3546 (rdev->pdev->device == 0x130C) ||
3547 (rdev->pdev->device == 0x130F) ||
3548 (rdev->pdev->device == 0x1310) ||
3549 (rdev->pdev->device == 0x1311) ||
3550 (rdev->pdev->device == 0x131C)) {
3551 rdev->config.cik.max_cu_per_sh = 8;
3552 rdev->config.cik.max_backends_per_se = 2;
3553 } else if ((rdev->pdev->device == 0x1309) ||
3554 (rdev->pdev->device == 0x130A) ||
3555 (rdev->pdev->device == 0x130D) ||
3556 (rdev->pdev->device == 0x1313) ||
3557 (rdev->pdev->device == 0x131D)) {
3558 rdev->config.cik.max_cu_per_sh = 6;
3559 rdev->config.cik.max_backends_per_se = 2;
3560 } else if ((rdev->pdev->device == 0x1306) ||
3561 (rdev->pdev->device == 0x1307) ||
3562 (rdev->pdev->device == 0x130B) ||
3563 (rdev->pdev->device == 0x130E) ||
3564 (rdev->pdev->device == 0x1315) ||
3565 (rdev->pdev->device == 0x1318) ||
3566 (rdev->pdev->device == 0x131B)) {
3567 rdev->config.cik.max_cu_per_sh = 4;
3568 rdev->config.cik.max_backends_per_se = 1;
3569 } else {
3570 rdev->config.cik.max_cu_per_sh = 3;
3571 rdev->config.cik.max_backends_per_se = 1;
3572 }
3573 rdev->config.cik.max_sh_per_se = 1;
3574 rdev->config.cik.max_texture_channel_caches = 4;
3575 rdev->config.cik.max_gprs = 256;
3576 rdev->config.cik.max_gs_threads = 16;
3577 rdev->config.cik.max_hw_contexts = 8;
3578
3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3584 break;
3585 case CHIP_KABINI:
3586 case CHIP_MULLINS:
3587 default:
3588 rdev->config.cik.max_shader_engines = 1;
3589 rdev->config.cik.max_tile_pipes = 2;
3590 rdev->config.cik.max_cu_per_sh = 2;
3591 rdev->config.cik.max_sh_per_se = 1;
3592 rdev->config.cik.max_backends_per_se = 1;
3593 rdev->config.cik.max_texture_channel_caches = 2;
3594 rdev->config.cik.max_gprs = 256;
3595 rdev->config.cik.max_gs_threads = 16;
3596 rdev->config.cik.max_hw_contexts = 8;
3597
3598 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3599 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3600 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3601 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3602 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3603 break;
3604 }
3605
3606 /* Initialize HDP */
3607 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3608 WREG32((0x2c14 + j), 0x00000000);
3609 WREG32((0x2c18 + j), 0x00000000);
3610 WREG32((0x2c1c + j), 0x00000000);
3611 WREG32((0x2c20 + j), 0x00000000);
3612 WREG32((0x2c24 + j), 0x00000000);
3613 }
3614
3615 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3616
3617 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3618
3619 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3620 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3621
3622 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3623 rdev->config.cik.mem_max_burst_length_bytes = 256;
3624 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3625 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3626 if (rdev->config.cik.mem_row_size_in_kb > 4)
3627 rdev->config.cik.mem_row_size_in_kb = 4;
3628 /* XXX use MC settings? */
3629 rdev->config.cik.shader_engine_tile_size = 32;
3630 rdev->config.cik.num_gpus = 1;
3631 rdev->config.cik.multi_gpu_tile_size = 64;
3632
3633 /* fix up row size */
3634 gb_addr_config &= ~ROW_SIZE_MASK;
3635 switch (rdev->config.cik.mem_row_size_in_kb) {
3636 case 1:
3637 default:
3638 gb_addr_config |= ROW_SIZE(0);
3639 break;
3640 case 2:
3641 gb_addr_config |= ROW_SIZE(1);
3642 break;
3643 case 4:
3644 gb_addr_config |= ROW_SIZE(2);
3645 break;
3646 }
3647
3648 /* setup tiling info dword. gb_addr_config is not adequate since it does
3649 * not have bank info, so create a custom tiling dword.
3650 * bits 3:0 num_pipes
3651 * bits 7:4 num_banks
3652 * bits 11:8 group_size
3653 * bits 15:12 row_size
3654 */
3655 rdev->config.cik.tile_config = 0;
3656 switch (rdev->config.cik.num_tile_pipes) {
3657 case 1:
3658 rdev->config.cik.tile_config |= (0 << 0);
3659 break;
3660 case 2:
3661 rdev->config.cik.tile_config |= (1 << 0);
3662 break;
3663 case 4:
3664 rdev->config.cik.tile_config |= (2 << 0);
3665 break;
3666 case 8:
3667 default:
3668 /* XXX what about 12? */
3669 rdev->config.cik.tile_config |= (3 << 0);
3670 break;
3671 }
3672 rdev->config.cik.tile_config |=
3673 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3674 rdev->config.cik.tile_config |=
3675 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3676 rdev->config.cik.tile_config |=
3677 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3678
3679 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3680 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3681 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3682 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3683 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3684 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3685 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3686 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3687
3688 cik_tiling_mode_table_init(rdev);
3689
3690 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3691 rdev->config.cik.max_sh_per_se,
3692 rdev->config.cik.max_backends_per_se);
3693
3694 rdev->config.cik.active_cus = 0;
3695 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3696 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3697 rdev->config.cik.active_cus +=
3698 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3699 }
3700 }
3701
3702 /* set HW defaults for 3D engine */
3703 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3704
3705 mutex_lock(&rdev->grbm_idx_mutex);
3706 /*
3707 * making sure that the following register writes will be broadcasted
3708 * to all the shaders
3709 */
3710 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3711 WREG32(SX_DEBUG_1, 0x20);
3712
3713 WREG32(TA_CNTL_AUX, 0x00010000);
3714
3715 tmp = RREG32(SPI_CONFIG_CNTL);
3716 tmp |= 0x03000000;
3717 WREG32(SPI_CONFIG_CNTL, tmp);
3718
3719 WREG32(SQ_CONFIG, 1);
3720
3721 WREG32(DB_DEBUG, 0);
3722
3723 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3724 tmp |= 0x00000400;
3725 WREG32(DB_DEBUG2, tmp);
3726
3727 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3728 tmp |= 0x00020200;
3729 WREG32(DB_DEBUG3, tmp);
3730
3731 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3732 tmp |= 0x00018208;
3733 WREG32(CB_HW_CONTROL, tmp);
3734
3735 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3736
3737 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3738 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3739 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3740 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3741
3742 WREG32(VGT_NUM_INSTANCES, 1);
3743
3744 WREG32(CP_PERFMON_CNTL, 0);
3745
3746 WREG32(SQ_CONFIG, 0);
3747
3748 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3749 FORCE_EOV_MAX_REZ_CNT(255)));
3750
3751 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3752 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3753
3754 WREG32(VGT_GS_VERTEX_REUSE, 16);
3755 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3756
3757 tmp = RREG32(HDP_MISC_CNTL);
3758 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3759 WREG32(HDP_MISC_CNTL, tmp);
3760
3761 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3762 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3763
3764 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3765 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3766 mutex_unlock(&rdev->grbm_idx_mutex);
3767
3768 udelay(50);
3769 }
3770
3771 /*
3772 * GPU scratch registers helpers function.
3773 */
3774 /**
3775 * cik_scratch_init - setup driver info for CP scratch regs
3776 *
3777 * @rdev: radeon_device pointer
3778 *
3779 * Set up the number and offset of the CP scratch registers.
3780 * NOTE: use of CP scratch registers is a legacy inferface and
3781 * is not used by default on newer asics (r6xx+). On newer asics,
3782 * memory buffers are used for fences rather than scratch regs.
3783 */
3784 static void cik_scratch_init(struct radeon_device *rdev)
3785 {
3786 int i;
3787
3788 rdev->scratch.num_reg = 7;
3789 rdev->scratch.reg_base = SCRATCH_REG0;
3790 for (i = 0; i < rdev->scratch.num_reg; i++) {
3791 rdev->scratch.free[i] = true;
3792 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3793 }
3794 }
3795
3796 /**
3797 * cik_ring_test - basic gfx ring test
3798 *
3799 * @rdev: radeon_device pointer
3800 * @ring: radeon_ring structure holding ring information
3801 *
3802 * Allocate a scratch register and write to it using the gfx ring (CIK).
3803 * Provides a basic gfx ring test to verify that the ring is working.
3804 * Used by cik_cp_gfx_resume();
3805 * Returns 0 on success, error on failure.
3806 */
3807 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3808 {
3809 uint32_t scratch;
3810 uint32_t tmp = 0;
3811 unsigned i;
3812 int r;
3813
3814 r = radeon_scratch_get(rdev, &scratch);
3815 if (r) {
3816 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3817 return r;
3818 }
3819 WREG32(scratch, 0xCAFEDEAD);
3820 r = radeon_ring_lock(rdev, ring, 3);
3821 if (r) {
3822 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3823 radeon_scratch_free(rdev, scratch);
3824 return r;
3825 }
3826 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3827 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3828 radeon_ring_write(ring, 0xDEADBEEF);
3829 radeon_ring_unlock_commit(rdev, ring, false);
3830
3831 for (i = 0; i < rdev->usec_timeout; i++) {
3832 tmp = RREG32(scratch);
3833 if (tmp == 0xDEADBEEF)
3834 break;
3835 DRM_UDELAY(1);
3836 }
3837 if (i < rdev->usec_timeout) {
3838 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3839 } else {
3840 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3841 ring->idx, scratch, tmp);
3842 r = -EINVAL;
3843 }
3844 radeon_scratch_free(rdev, scratch);
3845 return r;
3846 }
3847
3848 /**
3849 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3850 *
3851 * @rdev: radeon_device pointer
3852 * @ridx: radeon ring index
3853 *
3854 * Emits an hdp flush on the cp.
3855 */
3856 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3857 int ridx)
3858 {
3859 struct radeon_ring *ring = &rdev->ring[ridx];
3860 u32 ref_and_mask;
3861
3862 switch (ring->idx) {
3863 case CAYMAN_RING_TYPE_CP1_INDEX:
3864 case CAYMAN_RING_TYPE_CP2_INDEX:
3865 default:
3866 switch (ring->me) {
3867 case 0:
3868 ref_and_mask = CP2 << ring->pipe;
3869 break;
3870 case 1:
3871 ref_and_mask = CP6 << ring->pipe;
3872 break;
3873 default:
3874 return;
3875 }
3876 break;
3877 case RADEON_RING_TYPE_GFX_INDEX:
3878 ref_and_mask = CP0;
3879 break;
3880 }
3881
3882 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3883 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3884 WAIT_REG_MEM_FUNCTION(3) | /* == */
3885 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3886 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3887 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3888 radeon_ring_write(ring, ref_and_mask);
3889 radeon_ring_write(ring, ref_and_mask);
3890 radeon_ring_write(ring, 0x20); /* poll interval */
3891 }
3892
3893 /**
3894 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3895 *
3896 * @rdev: radeon_device pointer
3897 * @fence: radeon fence object
3898 *
3899 * Emits a fence sequnce number on the gfx ring and flushes
3900 * GPU caches.
3901 */
3902 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3903 struct radeon_fence *fence)
3904 {
3905 struct radeon_ring *ring = &rdev->ring[fence->ring];
3906 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3907
3908 /* EVENT_WRITE_EOP - flush caches, send int */
3909 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3910 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3911 EOP_TC_ACTION_EN |
3912 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3913 EVENT_INDEX(5)));
3914 radeon_ring_write(ring, addr & 0xfffffffc);
3915 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3916 radeon_ring_write(ring, fence->seq);
3917 radeon_ring_write(ring, 0);
3918 }
3919
3920 /**
3921 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3922 *
3923 * @rdev: radeon_device pointer
3924 * @fence: radeon fence object
3925 *
3926 * Emits a fence sequnce number on the compute ring and flushes
3927 * GPU caches.
3928 */
3929 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3930 struct radeon_fence *fence)
3931 {
3932 struct radeon_ring *ring = &rdev->ring[fence->ring];
3933 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3934
3935 /* RELEASE_MEM - flush caches, send int */
3936 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3937 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3938 EOP_TC_ACTION_EN |
3939 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3940 EVENT_INDEX(5)));
3941 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3942 radeon_ring_write(ring, addr & 0xfffffffc);
3943 radeon_ring_write(ring, upper_32_bits(addr));
3944 radeon_ring_write(ring, fence->seq);
3945 radeon_ring_write(ring, 0);
3946 }
3947
3948 /**
3949 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3950 *
3951 * @rdev: radeon_device pointer
3952 * @ring: radeon ring buffer object
3953 * @semaphore: radeon semaphore object
3954 * @emit_wait: Is this a sempahore wait?
3955 *
3956 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3957 * from running ahead of semaphore waits.
3958 */
3959 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3960 struct radeon_ring *ring,
3961 struct radeon_semaphore *semaphore,
3962 bool emit_wait)
3963 {
3964 uint64_t addr = semaphore->gpu_addr;
3965 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3966
3967 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3968 radeon_ring_write(ring, lower_32_bits(addr));
3969 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3970
3971 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3972 /* Prevent the PFP from running ahead of the semaphore wait */
3973 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3974 radeon_ring_write(ring, 0x0);
3975 }
3976
3977 return true;
3978 }
3979
3980 /**
3981 * cik_copy_cpdma - copy pages using the CP DMA engine
3982 *
3983 * @rdev: radeon_device pointer
3984 * @src_offset: src GPU address
3985 * @dst_offset: dst GPU address
3986 * @num_gpu_pages: number of GPU pages to xfer
3987 * @resv: reservation object to sync to
3988 *
3989 * Copy GPU paging using the CP DMA engine (CIK+).
3990 * Used by the radeon ttm implementation to move pages if
3991 * registered as the asic copy callback.
3992 */
3993 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3994 uint64_t src_offset, uint64_t dst_offset,
3995 unsigned num_gpu_pages,
3996 struct reservation_object *resv)
3997 {
3998 struct radeon_fence *fence;
3999 struct radeon_sync sync;
4000 int ring_index = rdev->asic->copy.blit_ring_index;
4001 struct radeon_ring *ring = &rdev->ring[ring_index];
4002 u32 size_in_bytes, cur_size_in_bytes, control;
4003 int i, num_loops;
4004 int r = 0;
4005
4006 radeon_sync_create(&sync);
4007
4008 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
4009 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
4010 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
4011 if (r) {
4012 DRM_ERROR("radeon: moving bo (%d).\n", r);
4013 radeon_sync_free(rdev, &sync, NULL);
4014 return ERR_PTR(r);
4015 }
4016
4017 radeon_sync_resv(rdev, &sync, resv, false);
4018 radeon_sync_rings(rdev, &sync, ring->idx);
4019
4020 for (i = 0; i < num_loops; i++) {
4021 cur_size_in_bytes = size_in_bytes;
4022 if (cur_size_in_bytes > 0x1fffff)
4023 cur_size_in_bytes = 0x1fffff;
4024 size_in_bytes -= cur_size_in_bytes;
4025 control = 0;
4026 if (size_in_bytes == 0)
4027 control |= PACKET3_DMA_DATA_CP_SYNC;
4028 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4029 radeon_ring_write(ring, control);
4030 radeon_ring_write(ring, lower_32_bits(src_offset));
4031 radeon_ring_write(ring, upper_32_bits(src_offset));
4032 radeon_ring_write(ring, lower_32_bits(dst_offset));
4033 radeon_ring_write(ring, upper_32_bits(dst_offset));
4034 radeon_ring_write(ring, cur_size_in_bytes);
4035 src_offset += cur_size_in_bytes;
4036 dst_offset += cur_size_in_bytes;
4037 }
4038
4039 r = radeon_fence_emit(rdev, &fence, ring->idx);
4040 if (r) {
4041 radeon_ring_unlock_undo(rdev, ring);
4042 radeon_sync_free(rdev, &sync, NULL);
4043 return ERR_PTR(r);
4044 }
4045
4046 radeon_ring_unlock_commit(rdev, ring, false);
4047 radeon_sync_free(rdev, &sync, fence);
4048
4049 return fence;
4050 }
4051
4052 /*
4053 * IB stuff
4054 */
4055 /**
4056 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4057 *
4058 * @rdev: radeon_device pointer
4059 * @ib: radeon indirect buffer object
4060 *
4061 * Emits an DE (drawing engine) or CE (constant engine) IB
4062 * on the gfx ring. IBs are usually generated by userspace
4063 * acceleration drivers and submitted to the kernel for
4064 * sheduling on the ring. This function schedules the IB
4065 * on the gfx ring for execution by the GPU.
4066 */
4067 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4068 {
4069 struct radeon_ring *ring = &rdev->ring[ib->ring];
4070 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
4071 u32 header, control = INDIRECT_BUFFER_VALID;
4072
4073 if (ib->is_const_ib) {
4074 /* set switch buffer packet before const IB */
4075 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4076 radeon_ring_write(ring, 0);
4077
4078 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4079 } else {
4080 u32 next_rptr;
4081 if (ring->rptr_save_reg) {
4082 next_rptr = ring->wptr + 3 + 4;
4083 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4084 radeon_ring_write(ring, ((ring->rptr_save_reg -
4085 PACKET3_SET_UCONFIG_REG_START) >> 2));
4086 radeon_ring_write(ring, next_rptr);
4087 } else if (rdev->wb.enabled) {
4088 next_rptr = ring->wptr + 5 + 4;
4089 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4090 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4091 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4092 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4093 radeon_ring_write(ring, next_rptr);
4094 }
4095
4096 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4097 }
4098
4099 control |= ib->length_dw | (vm_id << 24);
4100
4101 radeon_ring_write(ring, header);
4102 radeon_ring_write(ring,
4103 #ifdef __BIG_ENDIAN
4104 (2 << 0) |
4105 #endif
4106 (ib->gpu_addr & 0xFFFFFFFC));
4107 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4108 radeon_ring_write(ring, control);
4109 }
4110
4111 /**
4112 * cik_ib_test - basic gfx ring IB test
4113 *
4114 * @rdev: radeon_device pointer
4115 * @ring: radeon_ring structure holding ring information
4116 *
4117 * Allocate an IB and execute it on the gfx ring (CIK).
4118 * Provides a basic gfx ring test to verify that IBs are working.
4119 * Returns 0 on success, error on failure.
4120 */
4121 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4122 {
4123 struct radeon_ib ib;
4124 uint32_t scratch;
4125 uint32_t tmp = 0;
4126 unsigned i;
4127 int r;
4128
4129 r = radeon_scratch_get(rdev, &scratch);
4130 if (r) {
4131 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4132 return r;
4133 }
4134 WREG32(scratch, 0xCAFEDEAD);
4135 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4136 if (r) {
4137 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4138 radeon_scratch_free(rdev, scratch);
4139 return r;
4140 }
4141 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4142 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4143 ib.ptr[2] = 0xDEADBEEF;
4144 ib.length_dw = 3;
4145 r = radeon_ib_schedule(rdev, &ib, NULL, false);
4146 if (r) {
4147 radeon_scratch_free(rdev, scratch);
4148 radeon_ib_free(rdev, &ib);
4149 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4150 return r;
4151 }
4152 r = radeon_fence_wait(ib.fence, false);
4153 if (r) {
4154 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4155 radeon_scratch_free(rdev, scratch);
4156 radeon_ib_free(rdev, &ib);
4157 return r;
4158 }
4159 for (i = 0; i < rdev->usec_timeout; i++) {
4160 tmp = RREG32(scratch);
4161 if (tmp == 0xDEADBEEF)
4162 break;
4163 DRM_UDELAY(1);
4164 }
4165 if (i < rdev->usec_timeout) {
4166 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4167 } else {
4168 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4169 scratch, tmp);
4170 r = -EINVAL;
4171 }
4172 radeon_scratch_free(rdev, scratch);
4173 radeon_ib_free(rdev, &ib);
4174 return r;
4175 }
4176
4177 /*
4178 * CP.
4179 * On CIK, gfx and compute now have independant command processors.
4180 *
4181 * GFX
4182 * Gfx consists of a single ring and can process both gfx jobs and
4183 * compute jobs. The gfx CP consists of three microengines (ME):
4184 * PFP - Pre-Fetch Parser
4185 * ME - Micro Engine
4186 * CE - Constant Engine
4187 * The PFP and ME make up what is considered the Drawing Engine (DE).
4188 * The CE is an asynchronous engine used for updating buffer desciptors
4189 * used by the DE so that they can be loaded into cache in parallel
4190 * while the DE is processing state update packets.
4191 *
4192 * Compute
4193 * The compute CP consists of two microengines (ME):
4194 * MEC1 - Compute MicroEngine 1
4195 * MEC2 - Compute MicroEngine 2
4196 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4197 * The queues are exposed to userspace and are programmed directly
4198 * by the compute runtime.
4199 */
4200 /**
4201 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4202 *
4203 * @rdev: radeon_device pointer
4204 * @enable: enable or disable the MEs
4205 *
4206 * Halts or unhalts the gfx MEs.
4207 */
4208 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4209 {
4210 if (enable)
4211 WREG32(CP_ME_CNTL, 0);
4212 else {
4213 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4214 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4215 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4216 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4217 }
4218 udelay(50);
4219 }
4220
4221 /**
4222 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4223 *
4224 * @rdev: radeon_device pointer
4225 *
4226 * Loads the gfx PFP, ME, and CE ucode.
4227 * Returns 0 for success, -EINVAL if the ucode is not available.
4228 */
4229 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4230 {
4231 int i;
4232
4233 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4234 return -EINVAL;
4235
4236 cik_cp_gfx_enable(rdev, false);
4237
4238 if (rdev->new_fw) {
4239 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4240 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4241 const struct gfx_firmware_header_v1_0 *ce_hdr =
4242 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4243 const struct gfx_firmware_header_v1_0 *me_hdr =
4244 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4245 const __le32 *fw_data;
4246 u32 fw_size;
4247
4248 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4249 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4250 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4251
4252 /* PFP */
4253 fw_data = (const __le32 *)
4254 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4255 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4256 WREG32(CP_PFP_UCODE_ADDR, 0);
4257 for (i = 0; i < fw_size; i++)
4258 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4259 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
4260
4261 /* CE */
4262 fw_data = (const __le32 *)
4263 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4264 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4265 WREG32(CP_CE_UCODE_ADDR, 0);
4266 for (i = 0; i < fw_size; i++)
4267 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4268 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
4269
4270 /* ME */
4271 fw_data = (const __be32 *)
4272 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4273 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4274 WREG32(CP_ME_RAM_WADDR, 0);
4275 for (i = 0; i < fw_size; i++)
4276 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4277 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4278 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
4279 } else {
4280 const __be32 *fw_data;
4281
4282 /* PFP */
4283 fw_data = (const __be32 *)rdev->pfp_fw->data;
4284 WREG32(CP_PFP_UCODE_ADDR, 0);
4285 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4286 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4287 WREG32(CP_PFP_UCODE_ADDR, 0);
4288
4289 /* CE */
4290 fw_data = (const __be32 *)rdev->ce_fw->data;
4291 WREG32(CP_CE_UCODE_ADDR, 0);
4292 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4293 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4294 WREG32(CP_CE_UCODE_ADDR, 0);
4295
4296 /* ME */
4297 fw_data = (const __be32 *)rdev->me_fw->data;
4298 WREG32(CP_ME_RAM_WADDR, 0);
4299 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4300 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4301 WREG32(CP_ME_RAM_WADDR, 0);
4302 }
4303
4304 return 0;
4305 }
4306
4307 /**
4308 * cik_cp_gfx_start - start the gfx ring
4309 *
4310 * @rdev: radeon_device pointer
4311 *
4312 * Enables the ring and loads the clear state context and other
4313 * packets required to init the ring.
4314 * Returns 0 for success, error for failure.
4315 */
4316 static int cik_cp_gfx_start(struct radeon_device *rdev)
4317 {
4318 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4319 int r, i;
4320
4321 /* init the CP */
4322 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4323 WREG32(CP_ENDIAN_SWAP, 0);
4324 WREG32(CP_DEVICE_ID, 1);
4325
4326 cik_cp_gfx_enable(rdev, true);
4327
4328 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4329 if (r) {
4330 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4331 return r;
4332 }
4333
4334 /* init the CE partitions. CE only used for gfx on CIK */
4335 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4336 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4337 radeon_ring_write(ring, 0x8000);
4338 radeon_ring_write(ring, 0x8000);
4339
4340 /* setup clear context state */
4341 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4342 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4343
4344 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4345 radeon_ring_write(ring, 0x80000000);
4346 radeon_ring_write(ring, 0x80000000);
4347
4348 for (i = 0; i < cik_default_size; i++)
4349 radeon_ring_write(ring, cik_default_state[i]);
4350
4351 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4352 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4353
4354 /* set clear context state */
4355 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4356 radeon_ring_write(ring, 0);
4357
4358 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4359 radeon_ring_write(ring, 0x00000316);
4360 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4361 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4362
4363 radeon_ring_unlock_commit(rdev, ring, false);
4364
4365 return 0;
4366 }
4367
4368 /**
4369 * cik_cp_gfx_fini - stop the gfx ring
4370 *
4371 * @rdev: radeon_device pointer
4372 *
4373 * Stop the gfx ring and tear down the driver ring
4374 * info.
4375 */
4376 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4377 {
4378 cik_cp_gfx_enable(rdev, false);
4379 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4380 }
4381
4382 /**
4383 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4384 *
4385 * @rdev: radeon_device pointer
4386 *
4387 * Program the location and size of the gfx ring buffer
4388 * and test it to make sure it's working.
4389 * Returns 0 for success, error for failure.
4390 */
4391 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4392 {
4393 struct radeon_ring *ring;
4394 u32 tmp;
4395 u32 rb_bufsz;
4396 u64 rb_addr;
4397 int r;
4398
4399 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4400 if (rdev->family != CHIP_HAWAII)
4401 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4402
4403 /* Set the write pointer delay */
4404 WREG32(CP_RB_WPTR_DELAY, 0);
4405
4406 /* set the RB to use vmid 0 */
4407 WREG32(CP_RB_VMID, 0);
4408
4409 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4410
4411 /* ring 0 - compute and gfx */
4412 /* Set ring buffer size */
4413 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4414 rb_bufsz = order_base_2(ring->ring_size / 8);
4415 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4416 #ifdef __BIG_ENDIAN
4417 tmp |= BUF_SWAP_32BIT;
4418 #endif
4419 WREG32(CP_RB0_CNTL, tmp);
4420
4421 /* Initialize the ring buffer's read and write pointers */
4422 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4423 ring->wptr = 0;
4424 WREG32(CP_RB0_WPTR, ring->wptr);
4425
4426 /* set the wb address wether it's enabled or not */
4427 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4428 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4429
4430 /* scratch register shadowing is no longer supported */
4431 WREG32(SCRATCH_UMSK, 0);
4432
4433 if (!rdev->wb.enabled)
4434 tmp |= RB_NO_UPDATE;
4435
4436 mdelay(1);
4437 WREG32(CP_RB0_CNTL, tmp);
4438
4439 rb_addr = ring->gpu_addr >> 8;
4440 WREG32(CP_RB0_BASE, rb_addr);
4441 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4442
4443 /* start the ring */
4444 cik_cp_gfx_start(rdev);
4445 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4446 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4447 if (r) {
4448 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4449 return r;
4450 }
4451
4452 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4453 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4454
4455 return 0;
4456 }
4457
4458 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4459 struct radeon_ring *ring)
4460 {
4461 u32 rptr;
4462
4463 if (rdev->wb.enabled)
4464 rptr = rdev->wb.wb[ring->rptr_offs/4];
4465 else
4466 rptr = RREG32(CP_RB0_RPTR);
4467
4468 return rptr;
4469 }
4470
4471 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4472 struct radeon_ring *ring)
4473 {
4474 u32 wptr;
4475
4476 wptr = RREG32(CP_RB0_WPTR);
4477
4478 return wptr;
4479 }
4480
4481 void cik_gfx_set_wptr(struct radeon_device *rdev,
4482 struct radeon_ring *ring)
4483 {
4484 WREG32(CP_RB0_WPTR, ring->wptr);
4485 (void)RREG32(CP_RB0_WPTR);
4486 }
4487
4488 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4489 struct radeon_ring *ring)
4490 {
4491 u32 rptr;
4492
4493 if (rdev->wb.enabled) {
4494 rptr = rdev->wb.wb[ring->rptr_offs/4];
4495 } else {
4496 mutex_lock(&rdev->srbm_mutex);
4497 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4498 rptr = RREG32(CP_HQD_PQ_RPTR);
4499 cik_srbm_select(rdev, 0, 0, 0, 0);
4500 mutex_unlock(&rdev->srbm_mutex);
4501 }
4502
4503 return rptr;
4504 }
4505
4506 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4507 struct radeon_ring *ring)
4508 {
4509 u32 wptr;
4510
4511 if (rdev->wb.enabled) {
4512 /* XXX check if swapping is necessary on BE */
4513 wptr = rdev->wb.wb[ring->wptr_offs/4];
4514 } else {
4515 mutex_lock(&rdev->srbm_mutex);
4516 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4517 wptr = RREG32(CP_HQD_PQ_WPTR);
4518 cik_srbm_select(rdev, 0, 0, 0, 0);
4519 mutex_unlock(&rdev->srbm_mutex);
4520 }
4521
4522 return wptr;
4523 }
4524
4525 void cik_compute_set_wptr(struct radeon_device *rdev,
4526 struct radeon_ring *ring)
4527 {
4528 /* XXX check if swapping is necessary on BE */
4529 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4530 WDOORBELL32(ring->doorbell_index, ring->wptr);
4531 }
4532
4533 /**
4534 * cik_cp_compute_enable - enable/disable the compute CP MEs
4535 *
4536 * @rdev: radeon_device pointer
4537 * @enable: enable or disable the MEs
4538 *
4539 * Halts or unhalts the compute MEs.
4540 */
4541 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4542 {
4543 if (enable)
4544 WREG32(CP_MEC_CNTL, 0);
4545 else {
4546 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4547 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4548 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4549 }
4550 udelay(50);
4551 }
4552
4553 /**
4554 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4555 *
4556 * @rdev: radeon_device pointer
4557 *
4558 * Loads the compute MEC1&2 ucode.
4559 * Returns 0 for success, -EINVAL if the ucode is not available.
4560 */
4561 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4562 {
4563 int i;
4564
4565 if (!rdev->mec_fw)
4566 return -EINVAL;
4567
4568 cik_cp_compute_enable(rdev, false);
4569
4570 if (rdev->new_fw) {
4571 const struct gfx_firmware_header_v1_0 *mec_hdr =
4572 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4573 const __le32 *fw_data;
4574 u32 fw_size;
4575
4576 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4577
4578 /* MEC1 */
4579 fw_data = (const __le32 *)
4580 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4581 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4582 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4583 for (i = 0; i < fw_size; i++)
4584 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4585 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4586
4587 /* MEC2 */
4588 if (rdev->family == CHIP_KAVERI) {
4589 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4590 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4591
4592 fw_data = (const __le32 *)
4593 (rdev->mec2_fw->data +
4594 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4595 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4596 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4597 for (i = 0; i < fw_size; i++)
4598 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4599 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4600 }
4601 } else {
4602 const __be32 *fw_data;
4603
4604 /* MEC1 */
4605 fw_data = (const __be32 *)rdev->mec_fw->data;
4606 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4607 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4608 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4609 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4610
4611 if (rdev->family == CHIP_KAVERI) {
4612 /* MEC2 */
4613 fw_data = (const __be32 *)rdev->mec_fw->data;
4614 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4615 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4616 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4617 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4618 }
4619 }
4620
4621 return 0;
4622 }
4623
4624 /**
4625 * cik_cp_compute_start - start the compute queues
4626 *
4627 * @rdev: radeon_device pointer
4628 *
4629 * Enable the compute queues.
4630 * Returns 0 for success, error for failure.
4631 */
4632 static int cik_cp_compute_start(struct radeon_device *rdev)
4633 {
4634 cik_cp_compute_enable(rdev, true);
4635
4636 return 0;
4637 }
4638
4639 /**
4640 * cik_cp_compute_fini - stop the compute queues
4641 *
4642 * @rdev: radeon_device pointer
4643 *
4644 * Stop the compute queues and tear down the driver queue
4645 * info.
4646 */
4647 static void cik_cp_compute_fini(struct radeon_device *rdev)
4648 {
4649 int i, idx, r;
4650
4651 cik_cp_compute_enable(rdev, false);
4652
4653 for (i = 0; i < 2; i++) {
4654 if (i == 0)
4655 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4656 else
4657 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4658
4659 if (rdev->ring[idx].mqd_obj) {
4660 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4661 if (unlikely(r != 0))
4662 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4663
4664 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4665 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4666
4667 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4668 rdev->ring[idx].mqd_obj = NULL;
4669 }
4670 }
4671 }
4672
4673 static void cik_mec_fini(struct radeon_device *rdev)
4674 {
4675 int r;
4676
4677 if (rdev->mec.hpd_eop_obj) {
4678 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4679 if (unlikely(r != 0))
4680 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4681 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4682 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4683
4684 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4685 rdev->mec.hpd_eop_obj = NULL;
4686 }
4687 }
4688
4689 #define MEC_HPD_SIZE 2048
4690
4691 static int cik_mec_init(struct radeon_device *rdev)
4692 {
4693 int r;
4694 u32 *hpd;
4695
4696 /*
4697 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4698 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4699 * Nonetheless, we assign only 1 pipe because all other pipes will
4700 * be handled by KFD
4701 */
4702 rdev->mec.num_mec = 1;
4703 rdev->mec.num_pipe = 1;
4704 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4705
4706 if (rdev->mec.hpd_eop_obj == NULL) {
4707 r = radeon_bo_create(rdev,
4708 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4709 PAGE_SIZE, true,
4710 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4711 &rdev->mec.hpd_eop_obj);
4712 if (r) {
4713 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4714 return r;
4715 }
4716 }
4717
4718 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4719 if (unlikely(r != 0)) {
4720 cik_mec_fini(rdev);
4721 return r;
4722 }
4723 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4724 &rdev->mec.hpd_eop_gpu_addr);
4725 if (r) {
4726 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4727 cik_mec_fini(rdev);
4728 return r;
4729 }
4730 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4731 if (r) {
4732 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4733 cik_mec_fini(rdev);
4734 return r;
4735 }
4736
4737 /* clear memory. Not sure if this is required or not */
4738 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4739
4740 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4741 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4742
4743 return 0;
4744 }
4745
4746 struct hqd_registers
4747 {
4748 u32 cp_mqd_base_addr;
4749 u32 cp_mqd_base_addr_hi;
4750 u32 cp_hqd_active;
4751 u32 cp_hqd_vmid;
4752 u32 cp_hqd_persistent_state;
4753 u32 cp_hqd_pipe_priority;
4754 u32 cp_hqd_queue_priority;
4755 u32 cp_hqd_quantum;
4756 u32 cp_hqd_pq_base;
4757 u32 cp_hqd_pq_base_hi;
4758 u32 cp_hqd_pq_rptr;
4759 u32 cp_hqd_pq_rptr_report_addr;
4760 u32 cp_hqd_pq_rptr_report_addr_hi;
4761 u32 cp_hqd_pq_wptr_poll_addr;
4762 u32 cp_hqd_pq_wptr_poll_addr_hi;
4763 u32 cp_hqd_pq_doorbell_control;
4764 u32 cp_hqd_pq_wptr;
4765 u32 cp_hqd_pq_control;
4766 u32 cp_hqd_ib_base_addr;
4767 u32 cp_hqd_ib_base_addr_hi;
4768 u32 cp_hqd_ib_rptr;
4769 u32 cp_hqd_ib_control;
4770 u32 cp_hqd_iq_timer;
4771 u32 cp_hqd_iq_rptr;
4772 u32 cp_hqd_dequeue_request;
4773 u32 cp_hqd_dma_offload;
4774 u32 cp_hqd_sema_cmd;
4775 u32 cp_hqd_msg_type;
4776 u32 cp_hqd_atomic0_preop_lo;
4777 u32 cp_hqd_atomic0_preop_hi;
4778 u32 cp_hqd_atomic1_preop_lo;
4779 u32 cp_hqd_atomic1_preop_hi;
4780 u32 cp_hqd_hq_scheduler0;
4781 u32 cp_hqd_hq_scheduler1;
4782 u32 cp_mqd_control;
4783 };
4784
4785 struct bonaire_mqd
4786 {
4787 u32 header;
4788 u32 dispatch_initiator;
4789 u32 dimensions[3];
4790 u32 start_idx[3];
4791 u32 num_threads[3];
4792 u32 pipeline_stat_enable;
4793 u32 perf_counter_enable;
4794 u32 pgm[2];
4795 u32 tba[2];
4796 u32 tma[2];
4797 u32 pgm_rsrc[2];
4798 u32 vmid;
4799 u32 resource_limits;
4800 u32 static_thread_mgmt01[2];
4801 u32 tmp_ring_size;
4802 u32 static_thread_mgmt23[2];
4803 u32 restart[3];
4804 u32 thread_trace_enable;
4805 u32 reserved1;
4806 u32 user_data[16];
4807 u32 vgtcs_invoke_count[2];
4808 struct hqd_registers queue_state;
4809 u32 dequeue_cntr;
4810 u32 interrupt_queue[64];
4811 };
4812
4813 /**
4814 * cik_cp_compute_resume - setup the compute queue registers
4815 *
4816 * @rdev: radeon_device pointer
4817 *
4818 * Program the compute queues and test them to make sure they
4819 * are working.
4820 * Returns 0 for success, error for failure.
4821 */
4822 static int cik_cp_compute_resume(struct radeon_device *rdev)
4823 {
4824 int r, i, j, idx;
4825 u32 tmp;
4826 bool use_doorbell = true;
4827 u64 hqd_gpu_addr;
4828 u64 mqd_gpu_addr;
4829 u64 eop_gpu_addr;
4830 u64 wb_gpu_addr;
4831 u32 *buf;
4832 struct bonaire_mqd *mqd;
4833
4834 r = cik_cp_compute_start(rdev);
4835 if (r)
4836 return r;
4837
4838 /* fix up chicken bits */
4839 tmp = RREG32(CP_CPF_DEBUG);
4840 tmp |= (1 << 23);
4841 WREG32(CP_CPF_DEBUG, tmp);
4842
4843 /* init the pipes */
4844 mutex_lock(&rdev->srbm_mutex);
4845
4846 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
4847
4848 cik_srbm_select(rdev, 0, 0, 0, 0);
4849
4850 /* write the EOP addr */
4851 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4852 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4853
4854 /* set the VMID assigned */
4855 WREG32(CP_HPD_EOP_VMID, 0);
4856
4857 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4858 tmp = RREG32(CP_HPD_EOP_CONTROL);
4859 tmp &= ~EOP_SIZE_MASK;
4860 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4861 WREG32(CP_HPD_EOP_CONTROL, tmp);
4862
4863 mutex_unlock(&rdev->srbm_mutex);
4864
4865 /* init the queues. Just two for now. */
4866 for (i = 0; i < 2; i++) {
4867 if (i == 0)
4868 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4869 else
4870 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4871
4872 if (rdev->ring[idx].mqd_obj == NULL) {
4873 r = radeon_bo_create(rdev,
4874 sizeof(struct bonaire_mqd),
4875 PAGE_SIZE, true,
4876 RADEON_GEM_DOMAIN_GTT, 0, NULL,
4877 NULL, &rdev->ring[idx].mqd_obj);
4878 if (r) {
4879 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4880 return r;
4881 }
4882 }
4883
4884 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4885 if (unlikely(r != 0)) {
4886 cik_cp_compute_fini(rdev);
4887 return r;
4888 }
4889 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4890 &mqd_gpu_addr);
4891 if (r) {
4892 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4893 cik_cp_compute_fini(rdev);
4894 return r;
4895 }
4896 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4897 if (r) {
4898 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4899 cik_cp_compute_fini(rdev);
4900 return r;
4901 }
4902
4903 /* init the mqd struct */
4904 memset(buf, 0, sizeof(struct bonaire_mqd));
4905
4906 mqd = (struct bonaire_mqd *)buf;
4907 mqd->header = 0xC0310800;
4908 mqd->static_thread_mgmt01[0] = 0xffffffff;
4909 mqd->static_thread_mgmt01[1] = 0xffffffff;
4910 mqd->static_thread_mgmt23[0] = 0xffffffff;
4911 mqd->static_thread_mgmt23[1] = 0xffffffff;
4912
4913 mutex_lock(&rdev->srbm_mutex);
4914 cik_srbm_select(rdev, rdev->ring[idx].me,
4915 rdev->ring[idx].pipe,
4916 rdev->ring[idx].queue, 0);
4917
4918 /* disable wptr polling */
4919 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4920 tmp &= ~WPTR_POLL_EN;
4921 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4922
4923 /* enable doorbell? */
4924 mqd->queue_state.cp_hqd_pq_doorbell_control =
4925 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4926 if (use_doorbell)
4927 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4928 else
4929 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4930 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4931 mqd->queue_state.cp_hqd_pq_doorbell_control);
4932
4933 /* disable the queue if it's active */
4934 mqd->queue_state.cp_hqd_dequeue_request = 0;
4935 mqd->queue_state.cp_hqd_pq_rptr = 0;
4936 mqd->queue_state.cp_hqd_pq_wptr= 0;
4937 if (RREG32(CP_HQD_ACTIVE) & 1) {
4938 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4939 for (j = 0; j < rdev->usec_timeout; j++) {
4940 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4941 break;
4942 udelay(1);
4943 }
4944 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4945 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4946 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4947 }
4948
4949 /* set the pointer to the MQD */
4950 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4951 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4952 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4953 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4954 /* set MQD vmid to 0 */
4955 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4956 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4957 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4958
4959 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4960 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4961 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4962 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4963 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4964 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4965
4966 /* set up the HQD, this is similar to CP_RB0_CNTL */
4967 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4968 mqd->queue_state.cp_hqd_pq_control &=
4969 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4970
4971 mqd->queue_state.cp_hqd_pq_control |=
4972 order_base_2(rdev->ring[idx].ring_size / 8);
4973 mqd->queue_state.cp_hqd_pq_control |=
4974 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4975 #ifdef __BIG_ENDIAN
4976 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4977 #endif
4978 mqd->queue_state.cp_hqd_pq_control &=
4979 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4980 mqd->queue_state.cp_hqd_pq_control |=
4981 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4982 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4983
4984 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4985 if (i == 0)
4986 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4987 else
4988 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4989 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4990 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4991 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4992 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4993 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4994
4995 /* set the wb address wether it's enabled or not */
4996 if (i == 0)
4997 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4998 else
4999 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
5000 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
5001 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
5002 upper_32_bits(wb_gpu_addr) & 0xffff;
5003 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
5004 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
5005 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5006 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
5007
5008 /* enable the doorbell if requested */
5009 if (use_doorbell) {
5010 mqd->queue_state.cp_hqd_pq_doorbell_control =
5011 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5012 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5013 mqd->queue_state.cp_hqd_pq_doorbell_control |=
5014 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
5015 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5016 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5017 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5018
5019 } else {
5020 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5021 }
5022 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5023 mqd->queue_state.cp_hqd_pq_doorbell_control);
5024
5025 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5026 rdev->ring[idx].wptr = 0;
5027 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5028 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
5029 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
5030
5031 /* set the vmid for the queue */
5032 mqd->queue_state.cp_hqd_vmid = 0;
5033 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5034
5035 /* activate the queue */
5036 mqd->queue_state.cp_hqd_active = 1;
5037 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5038
5039 cik_srbm_select(rdev, 0, 0, 0, 0);
5040 mutex_unlock(&rdev->srbm_mutex);
5041
5042 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5043 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5044
5045 rdev->ring[idx].ready = true;
5046 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5047 if (r)
5048 rdev->ring[idx].ready = false;
5049 }
5050
5051 return 0;
5052 }
5053
5054 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5055 {
5056 cik_cp_gfx_enable(rdev, enable);
5057 cik_cp_compute_enable(rdev, enable);
5058 }
5059
5060 static int cik_cp_load_microcode(struct radeon_device *rdev)
5061 {
5062 int r;
5063
5064 r = cik_cp_gfx_load_microcode(rdev);
5065 if (r)
5066 return r;
5067 r = cik_cp_compute_load_microcode(rdev);
5068 if (r)
5069 return r;
5070
5071 return 0;
5072 }
5073
5074 static void cik_cp_fini(struct radeon_device *rdev)
5075 {
5076 cik_cp_gfx_fini(rdev);
5077 cik_cp_compute_fini(rdev);
5078 }
5079
5080 static int cik_cp_resume(struct radeon_device *rdev)
5081 {
5082 int r;
5083
5084 cik_enable_gui_idle_interrupt(rdev, false);
5085
5086 r = cik_cp_load_microcode(rdev);
5087 if (r)
5088 return r;
5089
5090 r = cik_cp_gfx_resume(rdev);
5091 if (r)
5092 return r;
5093 r = cik_cp_compute_resume(rdev);
5094 if (r)
5095 return r;
5096
5097 cik_enable_gui_idle_interrupt(rdev, true);
5098
5099 return 0;
5100 }
5101
5102 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5103 {
5104 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5105 RREG32(GRBM_STATUS));
5106 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5107 RREG32(GRBM_STATUS2));
5108 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5109 RREG32(GRBM_STATUS_SE0));
5110 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5111 RREG32(GRBM_STATUS_SE1));
5112 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5113 RREG32(GRBM_STATUS_SE2));
5114 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5115 RREG32(GRBM_STATUS_SE3));
5116 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5117 RREG32(SRBM_STATUS));
5118 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5119 RREG32(SRBM_STATUS2));
5120 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5121 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5122 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5123 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5124 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5125 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5126 RREG32(CP_STALLED_STAT1));
5127 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5128 RREG32(CP_STALLED_STAT2));
5129 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5130 RREG32(CP_STALLED_STAT3));
5131 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5132 RREG32(CP_CPF_BUSY_STAT));
5133 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5134 RREG32(CP_CPF_STALLED_STAT1));
5135 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5136 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5137 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5138 RREG32(CP_CPC_STALLED_STAT1));
5139 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5140 }
5141
5142 /**
5143 * cik_gpu_check_soft_reset - check which blocks are busy
5144 *
5145 * @rdev: radeon_device pointer
5146 *
5147 * Check which blocks are busy and return the relevant reset
5148 * mask to be used by cik_gpu_soft_reset().
5149 * Returns a mask of the blocks to be reset.
5150 */
5151 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5152 {
5153 u32 reset_mask = 0;
5154 u32 tmp;
5155
5156 /* GRBM_STATUS */
5157 tmp = RREG32(GRBM_STATUS);
5158 if (tmp & (PA_BUSY | SC_BUSY |
5159 BCI_BUSY | SX_BUSY |
5160 TA_BUSY | VGT_BUSY |
5161 DB_BUSY | CB_BUSY |
5162 GDS_BUSY | SPI_BUSY |
5163 IA_BUSY | IA_BUSY_NO_DMA))
5164 reset_mask |= RADEON_RESET_GFX;
5165
5166 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5167 reset_mask |= RADEON_RESET_CP;
5168
5169 /* GRBM_STATUS2 */
5170 tmp = RREG32(GRBM_STATUS2);
5171 if (tmp & RLC_BUSY)
5172 reset_mask |= RADEON_RESET_RLC;
5173
5174 /* SDMA0_STATUS_REG */
5175 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5176 if (!(tmp & SDMA_IDLE))
5177 reset_mask |= RADEON_RESET_DMA;
5178
5179 /* SDMA1_STATUS_REG */
5180 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5181 if (!(tmp & SDMA_IDLE))
5182 reset_mask |= RADEON_RESET_DMA1;
5183
5184 /* SRBM_STATUS2 */
5185 tmp = RREG32(SRBM_STATUS2);
5186 if (tmp & SDMA_BUSY)
5187 reset_mask |= RADEON_RESET_DMA;
5188
5189 if (tmp & SDMA1_BUSY)
5190 reset_mask |= RADEON_RESET_DMA1;
5191
5192 /* SRBM_STATUS */
5193 tmp = RREG32(SRBM_STATUS);
5194
5195 if (tmp & IH_BUSY)
5196 reset_mask |= RADEON_RESET_IH;
5197
5198 if (tmp & SEM_BUSY)
5199 reset_mask |= RADEON_RESET_SEM;
5200
5201 if (tmp & GRBM_RQ_PENDING)
5202 reset_mask |= RADEON_RESET_GRBM;
5203
5204 if (tmp & VMC_BUSY)
5205 reset_mask |= RADEON_RESET_VMC;
5206
5207 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5208 MCC_BUSY | MCD_BUSY))
5209 reset_mask |= RADEON_RESET_MC;
5210
5211 if (evergreen_is_display_hung(rdev))
5212 reset_mask |= RADEON_RESET_DISPLAY;
5213
5214 /* Skip MC reset as it's mostly likely not hung, just busy */
5215 if (reset_mask & RADEON_RESET_MC) {
5216 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5217 reset_mask &= ~RADEON_RESET_MC;
5218 }
5219
5220 return reset_mask;
5221 }
5222
5223 /**
5224 * cik_gpu_soft_reset - soft reset GPU
5225 *
5226 * @rdev: radeon_device pointer
5227 * @reset_mask: mask of which blocks to reset
5228 *
5229 * Soft reset the blocks specified in @reset_mask.
5230 */
5231 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5232 {
5233 struct evergreen_mc_save save;
5234 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5235 u32 tmp;
5236
5237 if (reset_mask == 0)
5238 return;
5239
5240 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5241
5242 cik_print_gpu_status_regs(rdev);
5243 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5244 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5245 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5246 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5247
5248 /* disable CG/PG */
5249 cik_fini_pg(rdev);
5250 cik_fini_cg(rdev);
5251
5252 /* stop the rlc */
5253 cik_rlc_stop(rdev);
5254
5255 /* Disable GFX parsing/prefetching */
5256 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5257
5258 /* Disable MEC parsing/prefetching */
5259 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5260
5261 if (reset_mask & RADEON_RESET_DMA) {
5262 /* sdma0 */
5263 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5264 tmp |= SDMA_HALT;
5265 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5266 }
5267 if (reset_mask & RADEON_RESET_DMA1) {
5268 /* sdma1 */
5269 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5270 tmp |= SDMA_HALT;
5271 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5272 }
5273
5274 evergreen_mc_stop(rdev, &save);
5275 if (evergreen_mc_wait_for_idle(rdev)) {
5276 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5277 }
5278
5279 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5280 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5281
5282 if (reset_mask & RADEON_RESET_CP) {
5283 grbm_soft_reset |= SOFT_RESET_CP;
5284
5285 srbm_soft_reset |= SOFT_RESET_GRBM;
5286 }
5287
5288 if (reset_mask & RADEON_RESET_DMA)
5289 srbm_soft_reset |= SOFT_RESET_SDMA;
5290
5291 if (reset_mask & RADEON_RESET_DMA1)
5292 srbm_soft_reset |= SOFT_RESET_SDMA1;
5293
5294 if (reset_mask & RADEON_RESET_DISPLAY)
5295 srbm_soft_reset |= SOFT_RESET_DC;
5296
5297 if (reset_mask & RADEON_RESET_RLC)
5298 grbm_soft_reset |= SOFT_RESET_RLC;
5299
5300 if (reset_mask & RADEON_RESET_SEM)
5301 srbm_soft_reset |= SOFT_RESET_SEM;
5302
5303 if (reset_mask & RADEON_RESET_IH)
5304 srbm_soft_reset |= SOFT_RESET_IH;
5305
5306 if (reset_mask & RADEON_RESET_GRBM)
5307 srbm_soft_reset |= SOFT_RESET_GRBM;
5308
5309 if (reset_mask & RADEON_RESET_VMC)
5310 srbm_soft_reset |= SOFT_RESET_VMC;
5311
5312 if (!(rdev->flags & RADEON_IS_IGP)) {
5313 if (reset_mask & RADEON_RESET_MC)
5314 srbm_soft_reset |= SOFT_RESET_MC;
5315 }
5316
5317 if (grbm_soft_reset) {
5318 tmp = RREG32(GRBM_SOFT_RESET);
5319 tmp |= grbm_soft_reset;
5320 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5321 WREG32(GRBM_SOFT_RESET, tmp);
5322 tmp = RREG32(GRBM_SOFT_RESET);
5323
5324 udelay(50);
5325
5326 tmp &= ~grbm_soft_reset;
5327 WREG32(GRBM_SOFT_RESET, tmp);
5328 tmp = RREG32(GRBM_SOFT_RESET);
5329 }
5330
5331 if (srbm_soft_reset) {
5332 tmp = RREG32(SRBM_SOFT_RESET);
5333 tmp |= srbm_soft_reset;
5334 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5335 WREG32(SRBM_SOFT_RESET, tmp);
5336 tmp = RREG32(SRBM_SOFT_RESET);
5337
5338 udelay(50);
5339
5340 tmp &= ~srbm_soft_reset;
5341 WREG32(SRBM_SOFT_RESET, tmp);
5342 tmp = RREG32(SRBM_SOFT_RESET);
5343 }
5344
5345 /* Wait a little for things to settle down */
5346 udelay(50);
5347
5348 evergreen_mc_resume(rdev, &save);
5349 udelay(50);
5350
5351 cik_print_gpu_status_regs(rdev);
5352 }
5353
5354 struct kv_reset_save_regs {
5355 u32 gmcon_reng_execute;
5356 u32 gmcon_misc;
5357 u32 gmcon_misc3;
5358 };
5359
5360 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5361 struct kv_reset_save_regs *save)
5362 {
5363 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5364 save->gmcon_misc = RREG32(GMCON_MISC);
5365 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5366
5367 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5368 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5369 STCTRL_STUTTER_EN));
5370 }
5371
5372 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5373 struct kv_reset_save_regs *save)
5374 {
5375 int i;
5376
5377 WREG32(GMCON_PGFSM_WRITE, 0);
5378 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5379
5380 for (i = 0; i < 5; i++)
5381 WREG32(GMCON_PGFSM_WRITE, 0);
5382
5383 WREG32(GMCON_PGFSM_WRITE, 0);
5384 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5385
5386 for (i = 0; i < 5; i++)
5387 WREG32(GMCON_PGFSM_WRITE, 0);
5388
5389 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5390 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5391
5392 for (i = 0; i < 5; i++)
5393 WREG32(GMCON_PGFSM_WRITE, 0);
5394
5395 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5396 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5397
5398 for (i = 0; i < 5; i++)
5399 WREG32(GMCON_PGFSM_WRITE, 0);
5400
5401 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5402 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5403
5404 for (i = 0; i < 5; i++)
5405 WREG32(GMCON_PGFSM_WRITE, 0);
5406
5407 WREG32(GMCON_PGFSM_WRITE, 0);
5408 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5409
5410 for (i = 0; i < 5; i++)
5411 WREG32(GMCON_PGFSM_WRITE, 0);
5412
5413 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5414 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5415
5416 for (i = 0; i < 5; i++)
5417 WREG32(GMCON_PGFSM_WRITE, 0);
5418
5419 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5420 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5421
5422 for (i = 0; i < 5; i++)
5423 WREG32(GMCON_PGFSM_WRITE, 0);
5424
5425 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5426 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5427
5428 for (i = 0; i < 5; i++)
5429 WREG32(GMCON_PGFSM_WRITE, 0);
5430
5431 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5432 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5433
5434 for (i = 0; i < 5; i++)
5435 WREG32(GMCON_PGFSM_WRITE, 0);
5436
5437 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5438 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5439
5440 WREG32(GMCON_MISC3, save->gmcon_misc3);
5441 WREG32(GMCON_MISC, save->gmcon_misc);
5442 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5443 }
5444
5445 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5446 {
5447 struct evergreen_mc_save save;
5448 struct kv_reset_save_regs kv_save = { 0 };
5449 u32 tmp, i;
5450
5451 dev_info(rdev->dev, "GPU pci config reset\n");
5452
5453 /* disable dpm? */
5454
5455 /* disable cg/pg */
5456 cik_fini_pg(rdev);
5457 cik_fini_cg(rdev);
5458
5459 /* Disable GFX parsing/prefetching */
5460 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5461
5462 /* Disable MEC parsing/prefetching */
5463 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5464
5465 /* sdma0 */
5466 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5467 tmp |= SDMA_HALT;
5468 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5469 /* sdma1 */
5470 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5471 tmp |= SDMA_HALT;
5472 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5473 /* XXX other engines? */
5474
5475 /* halt the rlc, disable cp internal ints */
5476 cik_rlc_stop(rdev);
5477
5478 udelay(50);
5479
5480 /* disable mem access */
5481 evergreen_mc_stop(rdev, &save);
5482 if (evergreen_mc_wait_for_idle(rdev)) {
5483 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5484 }
5485
5486 if (rdev->flags & RADEON_IS_IGP)
5487 kv_save_regs_for_reset(rdev, &kv_save);
5488
5489 /* disable BM */
5490 pci_clear_master(rdev->pdev);
5491 /* reset */
5492 radeon_pci_config_reset(rdev);
5493
5494 udelay(100);
5495
5496 /* wait for asic to come out of reset */
5497 for (i = 0; i < rdev->usec_timeout; i++) {
5498 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5499 break;
5500 udelay(1);
5501 }
5502
5503 /* does asic init need to be run first??? */
5504 if (rdev->flags & RADEON_IS_IGP)
5505 kv_restore_regs_for_reset(rdev, &kv_save);
5506 }
5507
5508 /**
5509 * cik_asic_reset - soft reset GPU
5510 *
5511 * @rdev: radeon_device pointer
5512 *
5513 * Look up which blocks are hung and attempt
5514 * to reset them.
5515 * Returns 0 for success.
5516 */
5517 int cik_asic_reset(struct radeon_device *rdev)
5518 {
5519 u32 reset_mask;
5520
5521 reset_mask = cik_gpu_check_soft_reset(rdev);
5522
5523 if (reset_mask)
5524 r600_set_bios_scratch_engine_hung(rdev, true);
5525
5526 /* try soft reset */
5527 cik_gpu_soft_reset(rdev, reset_mask);
5528
5529 reset_mask = cik_gpu_check_soft_reset(rdev);
5530
5531 /* try pci config reset */
5532 if (reset_mask && radeon_hard_reset)
5533 cik_gpu_pci_config_reset(rdev);
5534
5535 reset_mask = cik_gpu_check_soft_reset(rdev);
5536
5537 if (!reset_mask)
5538 r600_set_bios_scratch_engine_hung(rdev, false);
5539
5540 return 0;
5541 }
5542
5543 /**
5544 * cik_gfx_is_lockup - check if the 3D engine is locked up
5545 *
5546 * @rdev: radeon_device pointer
5547 * @ring: radeon_ring structure holding ring information
5548 *
5549 * Check if the 3D engine is locked up (CIK).
5550 * Returns true if the engine is locked, false if not.
5551 */
5552 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5553 {
5554 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5555
5556 if (!(reset_mask & (RADEON_RESET_GFX |
5557 RADEON_RESET_COMPUTE |
5558 RADEON_RESET_CP))) {
5559 radeon_ring_lockup_update(rdev, ring);
5560 return false;
5561 }
5562 return radeon_ring_test_lockup(rdev, ring);
5563 }
5564
5565 /* MC */
5566 /**
5567 * cik_mc_program - program the GPU memory controller
5568 *
5569 * @rdev: radeon_device pointer
5570 *
5571 * Set the location of vram, gart, and AGP in the GPU's
5572 * physical address space (CIK).
5573 */
5574 static void cik_mc_program(struct radeon_device *rdev)
5575 {
5576 struct evergreen_mc_save save;
5577 u32 tmp;
5578 int i, j;
5579
5580 /* Initialize HDP */
5581 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5582 WREG32((0x2c14 + j), 0x00000000);
5583 WREG32((0x2c18 + j), 0x00000000);
5584 WREG32((0x2c1c + j), 0x00000000);
5585 WREG32((0x2c20 + j), 0x00000000);
5586 WREG32((0x2c24 + j), 0x00000000);
5587 }
5588 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5589
5590 evergreen_mc_stop(rdev, &save);
5591 if (radeon_mc_wait_for_idle(rdev)) {
5592 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5593 }
5594 /* Lockout access through VGA aperture*/
5595 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5596 /* Update configuration */
5597 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5598 rdev->mc.vram_start >> 12);
5599 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5600 rdev->mc.vram_end >> 12);
5601 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5602 rdev->vram_scratch.gpu_addr >> 12);
5603 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5604 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5605 WREG32(MC_VM_FB_LOCATION, tmp);
5606 /* XXX double check these! */
5607 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5608 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5609 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5610 WREG32(MC_VM_AGP_BASE, 0);
5611 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5612 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5613 if (radeon_mc_wait_for_idle(rdev)) {
5614 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5615 }
5616 evergreen_mc_resume(rdev, &save);
5617 /* we need to own VRAM, so turn off the VGA renderer here
5618 * to stop it overwriting our objects */
5619 rv515_vga_render_disable(rdev);
5620 }
5621
5622 /**
5623 * cik_mc_init - initialize the memory controller driver params
5624 *
5625 * @rdev: radeon_device pointer
5626 *
5627 * Look up the amount of vram, vram width, and decide how to place
5628 * vram and gart within the GPU's physical address space (CIK).
5629 * Returns 0 for success.
5630 */
5631 static int cik_mc_init(struct radeon_device *rdev)
5632 {
5633 u32 tmp;
5634 int chansize, numchan;
5635
5636 /* Get VRAM informations */
5637 rdev->mc.vram_is_ddr = true;
5638 tmp = RREG32(MC_ARB_RAMCFG);
5639 if (tmp & CHANSIZE_MASK) {
5640 chansize = 64;
5641 } else {
5642 chansize = 32;
5643 }
5644 tmp = RREG32(MC_SHARED_CHMAP);
5645 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5646 case 0:
5647 default:
5648 numchan = 1;
5649 break;
5650 case 1:
5651 numchan = 2;
5652 break;
5653 case 2:
5654 numchan = 4;
5655 break;
5656 case 3:
5657 numchan = 8;
5658 break;
5659 case 4:
5660 numchan = 3;
5661 break;
5662 case 5:
5663 numchan = 6;
5664 break;
5665 case 6:
5666 numchan = 10;
5667 break;
5668 case 7:
5669 numchan = 12;
5670 break;
5671 case 8:
5672 numchan = 16;
5673 break;
5674 }
5675 rdev->mc.vram_width = numchan * chansize;
5676 /* Could aper size report 0 ? */
5677 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5678 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5679 /* size in MB on si */
5680 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5681 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5682 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5683 si_vram_gtt_location(rdev, &rdev->mc);
5684 radeon_update_bandwidth_info(rdev);
5685
5686 return 0;
5687 }
5688
5689 /*
5690 * GART
5691 * VMID 0 is the physical GPU addresses as used by the kernel.
5692 * VMIDs 1-15 are used for userspace clients and are handled
5693 * by the radeon vm/hsa code.
5694 */
5695 /**
5696 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5697 *
5698 * @rdev: radeon_device pointer
5699 *
5700 * Flush the TLB for the VMID 0 page table (CIK).
5701 */
5702 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5703 {
5704 /* flush hdp cache */
5705 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5706
5707 /* bits 0-15 are the VM contexts0-15 */
5708 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5709 }
5710
5711 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5712 {
5713 int i;
5714 uint32_t sh_mem_bases, sh_mem_config;
5715
5716 sh_mem_bases = 0x6000 | 0x6000 << 16;
5717 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5718 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5719
5720 mutex_lock(&rdev->srbm_mutex);
5721 for (i = 8; i < 16; i++) {
5722 cik_srbm_select(rdev, 0, 0, 0, i);
5723 /* CP and shaders */
5724 WREG32(SH_MEM_CONFIG, sh_mem_config);
5725 WREG32(SH_MEM_APE1_BASE, 1);
5726 WREG32(SH_MEM_APE1_LIMIT, 0);
5727 WREG32(SH_MEM_BASES, sh_mem_bases);
5728 }
5729 cik_srbm_select(rdev, 0, 0, 0, 0);
5730 mutex_unlock(&rdev->srbm_mutex);
5731 }
5732
5733 /**
5734 * cik_pcie_gart_enable - gart enable
5735 *
5736 * @rdev: radeon_device pointer
5737 *
5738 * This sets up the TLBs, programs the page tables for VMID0,
5739 * sets up the hw for VMIDs 1-15 which are allocated on
5740 * demand, and sets up the global locations for the LDS, GDS,
5741 * and GPUVM for FSA64 clients (CIK).
5742 * Returns 0 for success, errors for failure.
5743 */
5744 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5745 {
5746 int r, i;
5747
5748 if (rdev->gart.robj == NULL) {
5749 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5750 return -EINVAL;
5751 }
5752 r = radeon_gart_table_vram_pin(rdev);
5753 if (r)
5754 return r;
5755 /* Setup TLB control */
5756 WREG32(MC_VM_MX_L1_TLB_CNTL,
5757 (0xA << 7) |
5758 ENABLE_L1_TLB |
5759 ENABLE_L1_FRAGMENT_PROCESSING |
5760 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5761 ENABLE_ADVANCED_DRIVER_MODEL |
5762 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5763 /* Setup L2 cache */
5764 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5765 ENABLE_L2_FRAGMENT_PROCESSING |
5766 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5767 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5768 EFFECTIVE_L2_QUEUE_SIZE(7) |
5769 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5770 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5771 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5772 BANK_SELECT(4) |
5773 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5774 /* setup context0 */
5775 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5776 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5777 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5778 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5779 (u32)(rdev->dummy_page.addr >> 12));
5780 WREG32(VM_CONTEXT0_CNTL2, 0);
5781 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5782 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5783
5784 WREG32(0x15D4, 0);
5785 WREG32(0x15D8, 0);
5786 WREG32(0x15DC, 0);
5787
5788 /* restore context1-15 */
5789 /* set vm size, must be a multiple of 4 */
5790 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5791 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5792 for (i = 1; i < 16; i++) {
5793 if (i < 8)
5794 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5795 rdev->vm_manager.saved_table_addr[i]);
5796 else
5797 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5798 rdev->vm_manager.saved_table_addr[i]);
5799 }
5800
5801 /* enable context1-15 */
5802 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5803 (u32)(rdev->dummy_page.addr >> 12));
5804 WREG32(VM_CONTEXT1_CNTL2, 4);
5805 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5806 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5807 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5808 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5809 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5810 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5811 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5812 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5813 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5814 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5815 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5816 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5817 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5818 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5819
5820 if (rdev->family == CHIP_KAVERI) {
5821 u32 tmp = RREG32(CHUB_CONTROL);
5822 tmp &= ~BYPASS_VM;
5823 WREG32(CHUB_CONTROL, tmp);
5824 }
5825
5826 /* XXX SH_MEM regs */
5827 /* where to put LDS, scratch, GPUVM in FSA64 space */
5828 mutex_lock(&rdev->srbm_mutex);
5829 for (i = 0; i < 16; i++) {
5830 cik_srbm_select(rdev, 0, 0, 0, i);
5831 /* CP and shaders */
5832 WREG32(SH_MEM_CONFIG, 0);
5833 WREG32(SH_MEM_APE1_BASE, 1);
5834 WREG32(SH_MEM_APE1_LIMIT, 0);
5835 WREG32(SH_MEM_BASES, 0);
5836 /* SDMA GFX */
5837 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5838 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5839 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5840 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5841 /* XXX SDMA RLC - todo */
5842 }
5843 cik_srbm_select(rdev, 0, 0, 0, 0);
5844 mutex_unlock(&rdev->srbm_mutex);
5845
5846 cik_pcie_init_compute_vmid(rdev);
5847
5848 cik_pcie_gart_tlb_flush(rdev);
5849 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5850 (unsigned)(rdev->mc.gtt_size >> 20),
5851 (unsigned long long)rdev->gart.table_addr);
5852 rdev->gart.ready = true;
5853 return 0;
5854 }
5855
5856 /**
5857 * cik_pcie_gart_disable - gart disable
5858 *
5859 * @rdev: radeon_device pointer
5860 *
5861 * This disables all VM page table (CIK).
5862 */
5863 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5864 {
5865 unsigned i;
5866
5867 for (i = 1; i < 16; ++i) {
5868 uint32_t reg;
5869 if (i < 8)
5870 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5871 else
5872 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5873 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5874 }
5875
5876 /* Disable all tables */
5877 WREG32(VM_CONTEXT0_CNTL, 0);
5878 WREG32(VM_CONTEXT1_CNTL, 0);
5879 /* Setup TLB control */
5880 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5881 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5882 /* Setup L2 cache */
5883 WREG32(VM_L2_CNTL,
5884 ENABLE_L2_FRAGMENT_PROCESSING |
5885 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5886 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5887 EFFECTIVE_L2_QUEUE_SIZE(7) |
5888 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5889 WREG32(VM_L2_CNTL2, 0);
5890 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5891 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5892 radeon_gart_table_vram_unpin(rdev);
5893 }
5894
5895 /**
5896 * cik_pcie_gart_fini - vm fini callback
5897 *
5898 * @rdev: radeon_device pointer
5899 *
5900 * Tears down the driver GART/VM setup (CIK).
5901 */
5902 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5903 {
5904 cik_pcie_gart_disable(rdev);
5905 radeon_gart_table_vram_free(rdev);
5906 radeon_gart_fini(rdev);
5907 }
5908
5909 /* vm parser */
5910 /**
5911 * cik_ib_parse - vm ib_parse callback
5912 *
5913 * @rdev: radeon_device pointer
5914 * @ib: indirect buffer pointer
5915 *
5916 * CIK uses hw IB checking so this is a nop (CIK).
5917 */
5918 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5919 {
5920 return 0;
5921 }
5922
5923 /*
5924 * vm
5925 * VMID 0 is the physical GPU addresses as used by the kernel.
5926 * VMIDs 1-15 are used for userspace clients and are handled
5927 * by the radeon vm/hsa code.
5928 */
5929 /**
5930 * cik_vm_init - cik vm init callback
5931 *
5932 * @rdev: radeon_device pointer
5933 *
5934 * Inits cik specific vm parameters (number of VMs, base of vram for
5935 * VMIDs 1-15) (CIK).
5936 * Returns 0 for success.
5937 */
5938 int cik_vm_init(struct radeon_device *rdev)
5939 {
5940 /*
5941 * number of VMs
5942 * VMID 0 is reserved for System
5943 * radeon graphics/compute will use VMIDs 1-7
5944 * amdkfd will use VMIDs 8-15
5945 */
5946 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
5947 /* base offset of vram pages */
5948 if (rdev->flags & RADEON_IS_IGP) {
5949 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5950 tmp <<= 22;
5951 rdev->vm_manager.vram_base_offset = tmp;
5952 } else
5953 rdev->vm_manager.vram_base_offset = 0;
5954
5955 return 0;
5956 }
5957
5958 /**
5959 * cik_vm_fini - cik vm fini callback
5960 *
5961 * @rdev: radeon_device pointer
5962 *
5963 * Tear down any asic specific VM setup (CIK).
5964 */
5965 void cik_vm_fini(struct radeon_device *rdev)
5966 {
5967 }
5968
5969 /**
5970 * cik_vm_decode_fault - print human readable fault info
5971 *
5972 * @rdev: radeon_device pointer
5973 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5974 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5975 *
5976 * Print human readable fault information (CIK).
5977 */
5978 static void cik_vm_decode_fault(struct radeon_device *rdev,
5979 u32 status, u32 addr, u32 mc_client)
5980 {
5981 u32 mc_id;
5982 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5983 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5984 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5985 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5986
5987 if (rdev->family == CHIP_HAWAII)
5988 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5989 else
5990 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5991
5992 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5993 protections, vmid, addr,
5994 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5995 block, mc_client, mc_id);
5996 }
5997
5998 /**
5999 * cik_vm_flush - cik vm flush using the CP
6000 *
6001 * @rdev: radeon_device pointer
6002 *
6003 * Update the page table base and flush the VM TLB
6004 * using the CP (CIK).
6005 */
6006 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
6007 unsigned vm_id, uint64_t pd_addr)
6008 {
6009 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
6010
6011 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6012 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6013 WRITE_DATA_DST_SEL(0)));
6014 if (vm_id < 8) {
6015 radeon_ring_write(ring,
6016 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
6017 } else {
6018 radeon_ring_write(ring,
6019 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
6020 }
6021 radeon_ring_write(ring, 0);
6022 radeon_ring_write(ring, pd_addr >> 12);
6023
6024 /* update SH_MEM_* regs */
6025 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6026 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6027 WRITE_DATA_DST_SEL(0)));
6028 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6029 radeon_ring_write(ring, 0);
6030 radeon_ring_write(ring, VMID(vm_id));
6031
6032 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
6033 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6034 WRITE_DATA_DST_SEL(0)));
6035 radeon_ring_write(ring, SH_MEM_BASES >> 2);
6036 radeon_ring_write(ring, 0);
6037
6038 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6039 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6040 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6041 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
6042
6043 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6044 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6045 WRITE_DATA_DST_SEL(0)));
6046 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6047 radeon_ring_write(ring, 0);
6048 radeon_ring_write(ring, VMID(0));
6049
6050 /* HDP flush */
6051 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
6052
6053 /* bits 0-15 are the VM contexts0-15 */
6054 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6055 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
6056 WRITE_DATA_DST_SEL(0)));
6057 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6058 radeon_ring_write(ring, 0);
6059 radeon_ring_write(ring, 1 << vm_id);
6060
6061 /* wait for the invalidate to complete */
6062 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6063 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6064 WAIT_REG_MEM_FUNCTION(0) | /* always */
6065 WAIT_REG_MEM_ENGINE(0))); /* me */
6066 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6067 radeon_ring_write(ring, 0);
6068 radeon_ring_write(ring, 0); /* ref */
6069 radeon_ring_write(ring, 0); /* mask */
6070 radeon_ring_write(ring, 0x20); /* poll interval */
6071
6072 /* compute doesn't have PFP */
6073 if (usepfp) {
6074 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6075 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6076 radeon_ring_write(ring, 0x0);
6077 }
6078 }
6079
6080 /*
6081 * RLC
6082 * The RLC is a multi-purpose microengine that handles a
6083 * variety of functions, the most important of which is
6084 * the interrupt controller.
6085 */
6086 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6087 bool enable)
6088 {
6089 u32 tmp = RREG32(CP_INT_CNTL_RING0);
6090
6091 if (enable)
6092 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6093 else
6094 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6095 WREG32(CP_INT_CNTL_RING0, tmp);
6096 }
6097
6098 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6099 {
6100 u32 tmp;
6101
6102 tmp = RREG32(RLC_LB_CNTL);
6103 if (enable)
6104 tmp |= LOAD_BALANCE_ENABLE;
6105 else
6106 tmp &= ~LOAD_BALANCE_ENABLE;
6107 WREG32(RLC_LB_CNTL, tmp);
6108 }
6109
6110 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6111 {
6112 u32 i, j, k;
6113 u32 mask;
6114
6115 mutex_lock(&rdev->grbm_idx_mutex);
6116 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6117 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6118 cik_select_se_sh(rdev, i, j);
6119 for (k = 0; k < rdev->usec_timeout; k++) {
6120 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6121 break;
6122 udelay(1);
6123 }
6124 }
6125 }
6126 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6127 mutex_unlock(&rdev->grbm_idx_mutex);
6128
6129 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6130 for (k = 0; k < rdev->usec_timeout; k++) {
6131 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6132 break;
6133 udelay(1);
6134 }
6135 }
6136
6137 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6138 {
6139 u32 tmp;
6140
6141 tmp = RREG32(RLC_CNTL);
6142 if (tmp != rlc)
6143 WREG32(RLC_CNTL, rlc);
6144 }
6145
6146 static u32 cik_halt_rlc(struct radeon_device *rdev)
6147 {
6148 u32 data, orig;
6149
6150 orig = data = RREG32(RLC_CNTL);
6151
6152 if (data & RLC_ENABLE) {
6153 u32 i;
6154
6155 data &= ~RLC_ENABLE;
6156 WREG32(RLC_CNTL, data);
6157
6158 for (i = 0; i < rdev->usec_timeout; i++) {
6159 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6160 break;
6161 udelay(1);
6162 }
6163
6164 cik_wait_for_rlc_serdes(rdev);
6165 }
6166
6167 return orig;
6168 }
6169
6170 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6171 {
6172 u32 tmp, i, mask;
6173
6174 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6175 WREG32(RLC_GPR_REG2, tmp);
6176
6177 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6178 for (i = 0; i < rdev->usec_timeout; i++) {
6179 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6180 break;
6181 udelay(1);
6182 }
6183
6184 for (i = 0; i < rdev->usec_timeout; i++) {
6185 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6186 break;
6187 udelay(1);
6188 }
6189 }
6190
6191 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6192 {
6193 u32 tmp;
6194
6195 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6196 WREG32(RLC_GPR_REG2, tmp);
6197 }
6198
6199 /**
6200 * cik_rlc_stop - stop the RLC ME
6201 *
6202 * @rdev: radeon_device pointer
6203 *
6204 * Halt the RLC ME (MicroEngine) (CIK).
6205 */
6206 static void cik_rlc_stop(struct radeon_device *rdev)
6207 {
6208 WREG32(RLC_CNTL, 0);
6209
6210 cik_enable_gui_idle_interrupt(rdev, false);
6211
6212 cik_wait_for_rlc_serdes(rdev);
6213 }
6214
6215 /**
6216 * cik_rlc_start - start the RLC ME
6217 *
6218 * @rdev: radeon_device pointer
6219 *
6220 * Unhalt the RLC ME (MicroEngine) (CIK).
6221 */
6222 static void cik_rlc_start(struct radeon_device *rdev)
6223 {
6224 WREG32(RLC_CNTL, RLC_ENABLE);
6225
6226 cik_enable_gui_idle_interrupt(rdev, true);
6227
6228 udelay(50);
6229 }
6230
6231 /**
6232 * cik_rlc_resume - setup the RLC hw
6233 *
6234 * @rdev: radeon_device pointer
6235 *
6236 * Initialize the RLC registers, load the ucode,
6237 * and start the RLC (CIK).
6238 * Returns 0 for success, -EINVAL if the ucode is not available.
6239 */
6240 static int cik_rlc_resume(struct radeon_device *rdev)
6241 {
6242 u32 i, size, tmp;
6243
6244 if (!rdev->rlc_fw)
6245 return -EINVAL;
6246
6247 cik_rlc_stop(rdev);
6248
6249 /* disable CG */
6250 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6251 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6252
6253 si_rlc_reset(rdev);
6254
6255 cik_init_pg(rdev);
6256
6257 cik_init_cg(rdev);
6258
6259 WREG32(RLC_LB_CNTR_INIT, 0);
6260 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6261
6262 mutex_lock(&rdev->grbm_idx_mutex);
6263 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6264 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6265 WREG32(RLC_LB_PARAMS, 0x00600408);
6266 WREG32(RLC_LB_CNTL, 0x80000004);
6267 mutex_unlock(&rdev->grbm_idx_mutex);
6268
6269 WREG32(RLC_MC_CNTL, 0);
6270 WREG32(RLC_UCODE_CNTL, 0);
6271
6272 if (rdev->new_fw) {
6273 const struct rlc_firmware_header_v1_0 *hdr =
6274 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6275 const __le32 *fw_data = (const __le32 *)
6276 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6277
6278 radeon_ucode_print_rlc_hdr(&hdr->header);
6279
6280 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6281 WREG32(RLC_GPM_UCODE_ADDR, 0);
6282 for (i = 0; i < size; i++)
6283 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6284 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
6285 } else {
6286 const __be32 *fw_data;
6287
6288 switch (rdev->family) {
6289 case CHIP_BONAIRE:
6290 case CHIP_HAWAII:
6291 default:
6292 size = BONAIRE_RLC_UCODE_SIZE;
6293 break;
6294 case CHIP_KAVERI:
6295 size = KV_RLC_UCODE_SIZE;
6296 break;
6297 case CHIP_KABINI:
6298 size = KB_RLC_UCODE_SIZE;
6299 break;
6300 case CHIP_MULLINS:
6301 size = ML_RLC_UCODE_SIZE;
6302 break;
6303 }
6304
6305 fw_data = (const __be32 *)rdev->rlc_fw->data;
6306 WREG32(RLC_GPM_UCODE_ADDR, 0);
6307 for (i = 0; i < size; i++)
6308 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6309 WREG32(RLC_GPM_UCODE_ADDR, 0);
6310 }
6311
6312 /* XXX - find out what chips support lbpw */
6313 cik_enable_lbpw(rdev, false);
6314
6315 if (rdev->family == CHIP_BONAIRE)
6316 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6317
6318 cik_rlc_start(rdev);
6319
6320 return 0;
6321 }
6322
6323 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6324 {
6325 u32 data, orig, tmp, tmp2;
6326
6327 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6328
6329 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6330 cik_enable_gui_idle_interrupt(rdev, true);
6331
6332 tmp = cik_halt_rlc(rdev);
6333
6334 mutex_lock(&rdev->grbm_idx_mutex);
6335 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6336 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6337 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6338 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6339 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6340 mutex_unlock(&rdev->grbm_idx_mutex);
6341
6342 cik_update_rlc(rdev, tmp);
6343
6344 data |= CGCG_EN | CGLS_EN;
6345 } else {
6346 cik_enable_gui_idle_interrupt(rdev, false);
6347
6348 RREG32(CB_CGTT_SCLK_CTRL);
6349 RREG32(CB_CGTT_SCLK_CTRL);
6350 RREG32(CB_CGTT_SCLK_CTRL);
6351 RREG32(CB_CGTT_SCLK_CTRL);
6352
6353 data &= ~(CGCG_EN | CGLS_EN);
6354 }
6355
6356 if (orig != data)
6357 WREG32(RLC_CGCG_CGLS_CTRL, data);
6358
6359 }
6360
6361 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6362 {
6363 u32 data, orig, tmp = 0;
6364
6365 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6366 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6367 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6368 orig = data = RREG32(CP_MEM_SLP_CNTL);
6369 data |= CP_MEM_LS_EN;
6370 if (orig != data)
6371 WREG32(CP_MEM_SLP_CNTL, data);
6372 }
6373 }
6374
6375 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6376 data |= 0x00000001;
6377 data &= 0xfffffffd;
6378 if (orig != data)
6379 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6380
6381 tmp = cik_halt_rlc(rdev);
6382
6383 mutex_lock(&rdev->grbm_idx_mutex);
6384 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6385 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6386 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6387 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6388 WREG32(RLC_SERDES_WR_CTRL, data);
6389 mutex_unlock(&rdev->grbm_idx_mutex);
6390
6391 cik_update_rlc(rdev, tmp);
6392
6393 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6394 orig = data = RREG32(CGTS_SM_CTRL_REG);
6395 data &= ~SM_MODE_MASK;
6396 data |= SM_MODE(0x2);
6397 data |= SM_MODE_ENABLE;
6398 data &= ~CGTS_OVERRIDE;
6399 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6400 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6401 data &= ~CGTS_LS_OVERRIDE;
6402 data &= ~ON_MONITOR_ADD_MASK;
6403 data |= ON_MONITOR_ADD_EN;
6404 data |= ON_MONITOR_ADD(0x96);
6405 if (orig != data)
6406 WREG32(CGTS_SM_CTRL_REG, data);
6407 }
6408 } else {
6409 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6410 data |= 0x00000003;
6411 if (orig != data)
6412 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6413
6414 data = RREG32(RLC_MEM_SLP_CNTL);
6415 if (data & RLC_MEM_LS_EN) {
6416 data &= ~RLC_MEM_LS_EN;
6417 WREG32(RLC_MEM_SLP_CNTL, data);
6418 }
6419
6420 data = RREG32(CP_MEM_SLP_CNTL);
6421 if (data & CP_MEM_LS_EN) {
6422 data &= ~CP_MEM_LS_EN;
6423 WREG32(CP_MEM_SLP_CNTL, data);
6424 }
6425
6426 orig = data = RREG32(CGTS_SM_CTRL_REG);
6427 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6428 if (orig != data)
6429 WREG32(CGTS_SM_CTRL_REG, data);
6430
6431 tmp = cik_halt_rlc(rdev);
6432
6433 mutex_lock(&rdev->grbm_idx_mutex);
6434 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6435 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6436 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6437 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6438 WREG32(RLC_SERDES_WR_CTRL, data);
6439 mutex_unlock(&rdev->grbm_idx_mutex);
6440
6441 cik_update_rlc(rdev, tmp);
6442 }
6443 }
6444
6445 static const u32 mc_cg_registers[] =
6446 {
6447 MC_HUB_MISC_HUB_CG,
6448 MC_HUB_MISC_SIP_CG,
6449 MC_HUB_MISC_VM_CG,
6450 MC_XPB_CLK_GAT,
6451 ATC_MISC_CG,
6452 MC_CITF_MISC_WR_CG,
6453 MC_CITF_MISC_RD_CG,
6454 MC_CITF_MISC_VM_CG,
6455 VM_L2_CG,
6456 };
6457
6458 static void cik_enable_mc_ls(struct radeon_device *rdev,
6459 bool enable)
6460 {
6461 int i;
6462 u32 orig, data;
6463
6464 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6465 orig = data = RREG32(mc_cg_registers[i]);
6466 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6467 data |= MC_LS_ENABLE;
6468 else
6469 data &= ~MC_LS_ENABLE;
6470 if (data != orig)
6471 WREG32(mc_cg_registers[i], data);
6472 }
6473 }
6474
6475 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6476 bool enable)
6477 {
6478 int i;
6479 u32 orig, data;
6480
6481 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6482 orig = data = RREG32(mc_cg_registers[i]);
6483 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6484 data |= MC_CG_ENABLE;
6485 else
6486 data &= ~MC_CG_ENABLE;
6487 if (data != orig)
6488 WREG32(mc_cg_registers[i], data);
6489 }
6490 }
6491
6492 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6493 bool enable)
6494 {
6495 u32 orig, data;
6496
6497 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6498 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6499 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6500 } else {
6501 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6502 data |= 0xff000000;
6503 if (data != orig)
6504 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6505
6506 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6507 data |= 0xff000000;
6508 if (data != orig)
6509 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6510 }
6511 }
6512
6513 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6514 bool enable)
6515 {
6516 u32 orig, data;
6517
6518 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6519 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6520 data |= 0x100;
6521 if (orig != data)
6522 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6523
6524 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6525 data |= 0x100;
6526 if (orig != data)
6527 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6528 } else {
6529 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6530 data &= ~0x100;
6531 if (orig != data)
6532 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6533
6534 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6535 data &= ~0x100;
6536 if (orig != data)
6537 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6538 }
6539 }
6540
6541 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6542 bool enable)
6543 {
6544 u32 orig, data;
6545
6546 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6547 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6548 data = 0xfff;
6549 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6550
6551 orig = data = RREG32(UVD_CGC_CTRL);
6552 data |= DCM;
6553 if (orig != data)
6554 WREG32(UVD_CGC_CTRL, data);
6555 } else {
6556 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6557 data &= ~0xfff;
6558 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6559
6560 orig = data = RREG32(UVD_CGC_CTRL);
6561 data &= ~DCM;
6562 if (orig != data)
6563 WREG32(UVD_CGC_CTRL, data);
6564 }
6565 }
6566
6567 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6568 bool enable)
6569 {
6570 u32 orig, data;
6571
6572 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6573
6574 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6575 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6576 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6577 else
6578 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6579 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6580
6581 if (orig != data)
6582 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6583 }
6584
6585 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6586 bool enable)
6587 {
6588 u32 orig, data;
6589
6590 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6591
6592 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6593 data &= ~CLOCK_GATING_DIS;
6594 else
6595 data |= CLOCK_GATING_DIS;
6596
6597 if (orig != data)
6598 WREG32(HDP_HOST_PATH_CNTL, data);
6599 }
6600
6601 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6602 bool enable)
6603 {
6604 u32 orig, data;
6605
6606 orig = data = RREG32(HDP_MEM_POWER_LS);
6607
6608 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6609 data |= HDP_LS_ENABLE;
6610 else
6611 data &= ~HDP_LS_ENABLE;
6612
6613 if (orig != data)
6614 WREG32(HDP_MEM_POWER_LS, data);
6615 }
6616
6617 void cik_update_cg(struct radeon_device *rdev,
6618 u32 block, bool enable)
6619 {
6620
6621 if (block & RADEON_CG_BLOCK_GFX) {
6622 cik_enable_gui_idle_interrupt(rdev, false);
6623 /* order matters! */
6624 if (enable) {
6625 cik_enable_mgcg(rdev, true);
6626 cik_enable_cgcg(rdev, true);
6627 } else {
6628 cik_enable_cgcg(rdev, false);
6629 cik_enable_mgcg(rdev, false);
6630 }
6631 cik_enable_gui_idle_interrupt(rdev, true);
6632 }
6633
6634 if (block & RADEON_CG_BLOCK_MC) {
6635 if (!(rdev->flags & RADEON_IS_IGP)) {
6636 cik_enable_mc_mgcg(rdev, enable);
6637 cik_enable_mc_ls(rdev, enable);
6638 }
6639 }
6640
6641 if (block & RADEON_CG_BLOCK_SDMA) {
6642 cik_enable_sdma_mgcg(rdev, enable);
6643 cik_enable_sdma_mgls(rdev, enable);
6644 }
6645
6646 if (block & RADEON_CG_BLOCK_BIF) {
6647 cik_enable_bif_mgls(rdev, enable);
6648 }
6649
6650 if (block & RADEON_CG_BLOCK_UVD) {
6651 if (rdev->has_uvd)
6652 cik_enable_uvd_mgcg(rdev, enable);
6653 }
6654
6655 if (block & RADEON_CG_BLOCK_HDP) {
6656 cik_enable_hdp_mgcg(rdev, enable);
6657 cik_enable_hdp_ls(rdev, enable);
6658 }
6659
6660 if (block & RADEON_CG_BLOCK_VCE) {
6661 vce_v2_0_enable_mgcg(rdev, enable);
6662 }
6663 }
6664
6665 static void cik_init_cg(struct radeon_device *rdev)
6666 {
6667
6668 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6669
6670 if (rdev->has_uvd)
6671 si_init_uvd_internal_cg(rdev);
6672
6673 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6674 RADEON_CG_BLOCK_SDMA |
6675 RADEON_CG_BLOCK_BIF |
6676 RADEON_CG_BLOCK_UVD |
6677 RADEON_CG_BLOCK_HDP), true);
6678 }
6679
6680 static void cik_fini_cg(struct radeon_device *rdev)
6681 {
6682 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6683 RADEON_CG_BLOCK_SDMA |
6684 RADEON_CG_BLOCK_BIF |
6685 RADEON_CG_BLOCK_UVD |
6686 RADEON_CG_BLOCK_HDP), false);
6687
6688 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6689 }
6690
6691 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6692 bool enable)
6693 {
6694 u32 data, orig;
6695
6696 orig = data = RREG32(RLC_PG_CNTL);
6697 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6698 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6699 else
6700 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6701 if (orig != data)
6702 WREG32(RLC_PG_CNTL, data);
6703 }
6704
6705 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6706 bool enable)
6707 {
6708 u32 data, orig;
6709
6710 orig = data = RREG32(RLC_PG_CNTL);
6711 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6712 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6713 else
6714 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6715 if (orig != data)
6716 WREG32(RLC_PG_CNTL, data);
6717 }
6718
6719 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6720 {
6721 u32 data, orig;
6722
6723 orig = data = RREG32(RLC_PG_CNTL);
6724 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6725 data &= ~DISABLE_CP_PG;
6726 else
6727 data |= DISABLE_CP_PG;
6728 if (orig != data)
6729 WREG32(RLC_PG_CNTL, data);
6730 }
6731
6732 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6733 {
6734 u32 data, orig;
6735
6736 orig = data = RREG32(RLC_PG_CNTL);
6737 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6738 data &= ~DISABLE_GDS_PG;
6739 else
6740 data |= DISABLE_GDS_PG;
6741 if (orig != data)
6742 WREG32(RLC_PG_CNTL, data);
6743 }
6744
6745 #define CP_ME_TABLE_SIZE 96
6746 #define CP_ME_TABLE_OFFSET 2048
6747 #define CP_MEC_TABLE_OFFSET 4096
6748
6749 void cik_init_cp_pg_table(struct radeon_device *rdev)
6750 {
6751 volatile u32 *dst_ptr;
6752 int me, i, max_me = 4;
6753 u32 bo_offset = 0;
6754 u32 table_offset, table_size;
6755
6756 if (rdev->family == CHIP_KAVERI)
6757 max_me = 5;
6758
6759 if (rdev->rlc.cp_table_ptr == NULL)
6760 return;
6761
6762 /* write the cp table buffer */
6763 dst_ptr = rdev->rlc.cp_table_ptr;
6764 for (me = 0; me < max_me; me++) {
6765 if (rdev->new_fw) {
6766 const __le32 *fw_data;
6767 const struct gfx_firmware_header_v1_0 *hdr;
6768
6769 if (me == 0) {
6770 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6771 fw_data = (const __le32 *)
6772 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6773 table_offset = le32_to_cpu(hdr->jt_offset);
6774 table_size = le32_to_cpu(hdr->jt_size);
6775 } else if (me == 1) {
6776 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6777 fw_data = (const __le32 *)
6778 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6779 table_offset = le32_to_cpu(hdr->jt_offset);
6780 table_size = le32_to_cpu(hdr->jt_size);
6781 } else if (me == 2) {
6782 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6783 fw_data = (const __le32 *)
6784 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6785 table_offset = le32_to_cpu(hdr->jt_offset);
6786 table_size = le32_to_cpu(hdr->jt_size);
6787 } else if (me == 3) {
6788 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6789 fw_data = (const __le32 *)
6790 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6791 table_offset = le32_to_cpu(hdr->jt_offset);
6792 table_size = le32_to_cpu(hdr->jt_size);
6793 } else {
6794 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6795 fw_data = (const __le32 *)
6796 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6797 table_offset = le32_to_cpu(hdr->jt_offset);
6798 table_size = le32_to_cpu(hdr->jt_size);
6799 }
6800
6801 for (i = 0; i < table_size; i ++) {
6802 dst_ptr[bo_offset + i] =
6803 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6804 }
6805 bo_offset += table_size;
6806 } else {
6807 const __be32 *fw_data;
6808 table_size = CP_ME_TABLE_SIZE;
6809
6810 if (me == 0) {
6811 fw_data = (const __be32 *)rdev->ce_fw->data;
6812 table_offset = CP_ME_TABLE_OFFSET;
6813 } else if (me == 1) {
6814 fw_data = (const __be32 *)rdev->pfp_fw->data;
6815 table_offset = CP_ME_TABLE_OFFSET;
6816 } else if (me == 2) {
6817 fw_data = (const __be32 *)rdev->me_fw->data;
6818 table_offset = CP_ME_TABLE_OFFSET;
6819 } else {
6820 fw_data = (const __be32 *)rdev->mec_fw->data;
6821 table_offset = CP_MEC_TABLE_OFFSET;
6822 }
6823
6824 for (i = 0; i < table_size; i ++) {
6825 dst_ptr[bo_offset + i] =
6826 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6827 }
6828 bo_offset += table_size;
6829 }
6830 }
6831 }
6832
6833 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6834 bool enable)
6835 {
6836 u32 data, orig;
6837
6838 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6839 orig = data = RREG32(RLC_PG_CNTL);
6840 data |= GFX_PG_ENABLE;
6841 if (orig != data)
6842 WREG32(RLC_PG_CNTL, data);
6843
6844 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6845 data |= AUTO_PG_EN;
6846 if (orig != data)
6847 WREG32(RLC_AUTO_PG_CTRL, data);
6848 } else {
6849 orig = data = RREG32(RLC_PG_CNTL);
6850 data &= ~GFX_PG_ENABLE;
6851 if (orig != data)
6852 WREG32(RLC_PG_CNTL, data);
6853
6854 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6855 data &= ~AUTO_PG_EN;
6856 if (orig != data)
6857 WREG32(RLC_AUTO_PG_CTRL, data);
6858
6859 data = RREG32(DB_RENDER_CONTROL);
6860 }
6861 }
6862
6863 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6864 {
6865 u32 mask = 0, tmp, tmp1;
6866 int i;
6867
6868 mutex_lock(&rdev->grbm_idx_mutex);
6869 cik_select_se_sh(rdev, se, sh);
6870 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6871 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6872 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6873 mutex_unlock(&rdev->grbm_idx_mutex);
6874
6875 tmp &= 0xffff0000;
6876
6877 tmp |= tmp1;
6878 tmp >>= 16;
6879
6880 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6881 mask <<= 1;
6882 mask |= 1;
6883 }
6884
6885 return (~tmp) & mask;
6886 }
6887
6888 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6889 {
6890 u32 i, j, k, active_cu_number = 0;
6891 u32 mask, counter, cu_bitmap;
6892 u32 tmp = 0;
6893
6894 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6895 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6896 mask = 1;
6897 cu_bitmap = 0;
6898 counter = 0;
6899 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6900 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6901 if (counter < 2)
6902 cu_bitmap |= mask;
6903 counter ++;
6904 }
6905 mask <<= 1;
6906 }
6907
6908 active_cu_number += counter;
6909 tmp |= (cu_bitmap << (i * 16 + j * 8));
6910 }
6911 }
6912
6913 WREG32(RLC_PG_AO_CU_MASK, tmp);
6914
6915 tmp = RREG32(RLC_MAX_PG_CU);
6916 tmp &= ~MAX_PU_CU_MASK;
6917 tmp |= MAX_PU_CU(active_cu_number);
6918 WREG32(RLC_MAX_PG_CU, tmp);
6919 }
6920
6921 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6922 bool enable)
6923 {
6924 u32 data, orig;
6925
6926 orig = data = RREG32(RLC_PG_CNTL);
6927 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6928 data |= STATIC_PER_CU_PG_ENABLE;
6929 else
6930 data &= ~STATIC_PER_CU_PG_ENABLE;
6931 if (orig != data)
6932 WREG32(RLC_PG_CNTL, data);
6933 }
6934
6935 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6936 bool enable)
6937 {
6938 u32 data, orig;
6939
6940 orig = data = RREG32(RLC_PG_CNTL);
6941 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6942 data |= DYN_PER_CU_PG_ENABLE;
6943 else
6944 data &= ~DYN_PER_CU_PG_ENABLE;
6945 if (orig != data)
6946 WREG32(RLC_PG_CNTL, data);
6947 }
6948
6949 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6950 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6951
6952 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6953 {
6954 u32 data, orig;
6955 u32 i;
6956
6957 if (rdev->rlc.cs_data) {
6958 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6959 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6960 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6961 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6962 } else {
6963 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6964 for (i = 0; i < 3; i++)
6965 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6966 }
6967 if (rdev->rlc.reg_list) {
6968 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6969 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6970 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6971 }
6972
6973 orig = data = RREG32(RLC_PG_CNTL);
6974 data |= GFX_PG_SRC;
6975 if (orig != data)
6976 WREG32(RLC_PG_CNTL, data);
6977
6978 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6979 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6980
6981 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6982 data &= ~IDLE_POLL_COUNT_MASK;
6983 data |= IDLE_POLL_COUNT(0x60);
6984 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6985
6986 data = 0x10101010;
6987 WREG32(RLC_PG_DELAY, data);
6988
6989 data = RREG32(RLC_PG_DELAY_2);
6990 data &= ~0xff;
6991 data |= 0x3;
6992 WREG32(RLC_PG_DELAY_2, data);
6993
6994 data = RREG32(RLC_AUTO_PG_CTRL);
6995 data &= ~GRBM_REG_SGIT_MASK;
6996 data |= GRBM_REG_SGIT(0x700);
6997 WREG32(RLC_AUTO_PG_CTRL, data);
6998
6999 }
7000
7001 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
7002 {
7003 cik_enable_gfx_cgpg(rdev, enable);
7004 cik_enable_gfx_static_mgpg(rdev, enable);
7005 cik_enable_gfx_dynamic_mgpg(rdev, enable);
7006 }
7007
7008 u32 cik_get_csb_size(struct radeon_device *rdev)
7009 {
7010 u32 count = 0;
7011 const struct cs_section_def *sect = NULL;
7012 const struct cs_extent_def *ext = NULL;
7013
7014 if (rdev->rlc.cs_data == NULL)
7015 return 0;
7016
7017 /* begin clear state */
7018 count += 2;
7019 /* context control state */
7020 count += 3;
7021
7022 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7023 for (ext = sect->section; ext->extent != NULL; ++ext) {
7024 if (sect->id == SECT_CONTEXT)
7025 count += 2 + ext->reg_count;
7026 else
7027 return 0;
7028 }
7029 }
7030 /* pa_sc_raster_config/pa_sc_raster_config1 */
7031 count += 4;
7032 /* end clear state */
7033 count += 2;
7034 /* clear state */
7035 count += 2;
7036
7037 return count;
7038 }
7039
7040 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
7041 {
7042 u32 count = 0, i;
7043 const struct cs_section_def *sect = NULL;
7044 const struct cs_extent_def *ext = NULL;
7045
7046 if (rdev->rlc.cs_data == NULL)
7047 return;
7048 if (buffer == NULL)
7049 return;
7050
7051 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7052 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
7053
7054 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
7055 buffer[count++] = cpu_to_le32(0x80000000);
7056 buffer[count++] = cpu_to_le32(0x80000000);
7057
7058 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7059 for (ext = sect->section; ext->extent != NULL; ++ext) {
7060 if (sect->id == SECT_CONTEXT) {
7061 buffer[count++] =
7062 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7063 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
7064 for (i = 0; i < ext->reg_count; i++)
7065 buffer[count++] = cpu_to_le32(ext->extent[i]);
7066 } else {
7067 return;
7068 }
7069 }
7070 }
7071
7072 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7073 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
7074 switch (rdev->family) {
7075 case CHIP_BONAIRE:
7076 buffer[count++] = cpu_to_le32(0x16000012);
7077 buffer[count++] = cpu_to_le32(0x00000000);
7078 break;
7079 case CHIP_KAVERI:
7080 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7081 buffer[count++] = cpu_to_le32(0x00000000);
7082 break;
7083 case CHIP_KABINI:
7084 case CHIP_MULLINS:
7085 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7086 buffer[count++] = cpu_to_le32(0x00000000);
7087 break;
7088 case CHIP_HAWAII:
7089 buffer[count++] = cpu_to_le32(0x3a00161a);
7090 buffer[count++] = cpu_to_le32(0x0000002e);
7091 break;
7092 default:
7093 buffer[count++] = cpu_to_le32(0x00000000);
7094 buffer[count++] = cpu_to_le32(0x00000000);
7095 break;
7096 }
7097
7098 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7099 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7100
7101 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7102 buffer[count++] = cpu_to_le32(0);
7103 }
7104
7105 static void cik_init_pg(struct radeon_device *rdev)
7106 {
7107 if (rdev->pg_flags) {
7108 cik_enable_sck_slowdown_on_pu(rdev, true);
7109 cik_enable_sck_slowdown_on_pd(rdev, true);
7110 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7111 cik_init_gfx_cgpg(rdev);
7112 cik_enable_cp_pg(rdev, true);
7113 cik_enable_gds_pg(rdev, true);
7114 }
7115 cik_init_ao_cu_mask(rdev);
7116 cik_update_gfx_pg(rdev, true);
7117 }
7118 }
7119
7120 static void cik_fini_pg(struct radeon_device *rdev)
7121 {
7122 if (rdev->pg_flags) {
7123 cik_update_gfx_pg(rdev, false);
7124 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7125 cik_enable_cp_pg(rdev, false);
7126 cik_enable_gds_pg(rdev, false);
7127 }
7128 }
7129 }
7130
7131 /*
7132 * Interrupts
7133 * Starting with r6xx, interrupts are handled via a ring buffer.
7134 * Ring buffers are areas of GPU accessible memory that the GPU
7135 * writes interrupt vectors into and the host reads vectors out of.
7136 * There is a rptr (read pointer) that determines where the
7137 * host is currently reading, and a wptr (write pointer)
7138 * which determines where the GPU has written. When the
7139 * pointers are equal, the ring is idle. When the GPU
7140 * writes vectors to the ring buffer, it increments the
7141 * wptr. When there is an interrupt, the host then starts
7142 * fetching commands and processing them until the pointers are
7143 * equal again at which point it updates the rptr.
7144 */
7145
7146 /**
7147 * cik_enable_interrupts - Enable the interrupt ring buffer
7148 *
7149 * @rdev: radeon_device pointer
7150 *
7151 * Enable the interrupt ring buffer (CIK).
7152 */
7153 static void cik_enable_interrupts(struct radeon_device *rdev)
7154 {
7155 u32 ih_cntl = RREG32(IH_CNTL);
7156 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7157
7158 ih_cntl |= ENABLE_INTR;
7159 ih_rb_cntl |= IH_RB_ENABLE;
7160 WREG32(IH_CNTL, ih_cntl);
7161 WREG32(IH_RB_CNTL, ih_rb_cntl);
7162 rdev->ih.enabled = true;
7163 }
7164
7165 /**
7166 * cik_disable_interrupts - Disable the interrupt ring buffer
7167 *
7168 * @rdev: radeon_device pointer
7169 *
7170 * Disable the interrupt ring buffer (CIK).
7171 */
7172 static void cik_disable_interrupts(struct radeon_device *rdev)
7173 {
7174 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7175 u32 ih_cntl = RREG32(IH_CNTL);
7176
7177 ih_rb_cntl &= ~IH_RB_ENABLE;
7178 ih_cntl &= ~ENABLE_INTR;
7179 WREG32(IH_RB_CNTL, ih_rb_cntl);
7180 WREG32(IH_CNTL, ih_cntl);
7181 /* set rptr, wptr to 0 */
7182 WREG32(IH_RB_RPTR, 0);
7183 WREG32(IH_RB_WPTR, 0);
7184 rdev->ih.enabled = false;
7185 rdev->ih.rptr = 0;
7186 }
7187
7188 /**
7189 * cik_disable_interrupt_state - Disable all interrupt sources
7190 *
7191 * @rdev: radeon_device pointer
7192 *
7193 * Clear all interrupt enable bits used by the driver (CIK).
7194 */
7195 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7196 {
7197 u32 tmp;
7198
7199 /* gfx ring */
7200 tmp = RREG32(CP_INT_CNTL_RING0) &
7201 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7202 WREG32(CP_INT_CNTL_RING0, tmp);
7203 /* sdma */
7204 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7205 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7206 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7207 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7208 /* compute queues */
7209 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7210 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7211 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7212 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7213 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7214 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7215 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7216 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7217 /* grbm */
7218 WREG32(GRBM_INT_CNTL, 0);
7219 /* vline/vblank, etc. */
7220 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7221 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7222 if (rdev->num_crtc >= 4) {
7223 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7224 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7225 }
7226 if (rdev->num_crtc >= 6) {
7227 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7228 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7229 }
7230 /* pflip */
7231 if (rdev->num_crtc >= 2) {
7232 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7233 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7234 }
7235 if (rdev->num_crtc >= 4) {
7236 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7237 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7238 }
7239 if (rdev->num_crtc >= 6) {
7240 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7241 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7242 }
7243
7244 /* dac hotplug */
7245 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7246
7247 /* digital hotplug */
7248 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7249 WREG32(DC_HPD1_INT_CONTROL, tmp);
7250 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7251 WREG32(DC_HPD2_INT_CONTROL, tmp);
7252 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7253 WREG32(DC_HPD3_INT_CONTROL, tmp);
7254 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7255 WREG32(DC_HPD4_INT_CONTROL, tmp);
7256 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7257 WREG32(DC_HPD5_INT_CONTROL, tmp);
7258 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7259 WREG32(DC_HPD6_INT_CONTROL, tmp);
7260
7261 }
7262
7263 /**
7264 * cik_irq_init - init and enable the interrupt ring
7265 *
7266 * @rdev: radeon_device pointer
7267 *
7268 * Allocate a ring buffer for the interrupt controller,
7269 * enable the RLC, disable interrupts, enable the IH
7270 * ring buffer and enable it (CIK).
7271 * Called at device load and reume.
7272 * Returns 0 for success, errors for failure.
7273 */
7274 static int cik_irq_init(struct radeon_device *rdev)
7275 {
7276 int ret = 0;
7277 int rb_bufsz;
7278 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7279
7280 /* allocate ring */
7281 ret = r600_ih_ring_alloc(rdev);
7282 if (ret)
7283 return ret;
7284
7285 /* disable irqs */
7286 cik_disable_interrupts(rdev);
7287
7288 /* init rlc */
7289 ret = cik_rlc_resume(rdev);
7290 if (ret) {
7291 r600_ih_ring_fini(rdev);
7292 return ret;
7293 }
7294
7295 /* setup interrupt control */
7296 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7297 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7298 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7299 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7300 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7301 */
7302 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7303 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7304 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7305 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7306
7307 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7308 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7309
7310 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7311 IH_WPTR_OVERFLOW_CLEAR |
7312 (rb_bufsz << 1));
7313
7314 if (rdev->wb.enabled)
7315 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7316
7317 /* set the writeback address whether it's enabled or not */
7318 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7319 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7320
7321 WREG32(IH_RB_CNTL, ih_rb_cntl);
7322
7323 /* set rptr, wptr to 0 */
7324 WREG32(IH_RB_RPTR, 0);
7325 WREG32(IH_RB_WPTR, 0);
7326
7327 /* Default settings for IH_CNTL (disabled at first) */
7328 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7329 /* RPTR_REARM only works if msi's are enabled */
7330 if (rdev->msi_enabled)
7331 ih_cntl |= RPTR_REARM;
7332 WREG32(IH_CNTL, ih_cntl);
7333
7334 /* force the active interrupt state to all disabled */
7335 cik_disable_interrupt_state(rdev);
7336
7337 pci_set_master(rdev->pdev);
7338
7339 /* enable irqs */
7340 cik_enable_interrupts(rdev);
7341
7342 return ret;
7343 }
7344
7345 /**
7346 * cik_irq_set - enable/disable interrupt sources
7347 *
7348 * @rdev: radeon_device pointer
7349 *
7350 * Enable interrupt sources on the GPU (vblanks, hpd,
7351 * etc.) (CIK).
7352 * Returns 0 for success, errors for failure.
7353 */
7354 int cik_irq_set(struct radeon_device *rdev)
7355 {
7356 u32 cp_int_cntl;
7357 u32 cp_m1p0;
7358 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7359 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7360 u32 grbm_int_cntl = 0;
7361 u32 dma_cntl, dma_cntl1;
7362 u32 thermal_int;
7363
7364 if (!rdev->irq.installed) {
7365 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7366 return -EINVAL;
7367 }
7368 /* don't enable anything if the ih is disabled */
7369 if (!rdev->ih.enabled) {
7370 cik_disable_interrupts(rdev);
7371 /* force the active interrupt state to all disabled */
7372 cik_disable_interrupt_state(rdev);
7373 return 0;
7374 }
7375
7376 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7377 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7378 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7379
7380 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7381 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7382 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7383 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7384 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7385 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7386
7387 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7388 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7389
7390 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7391
7392 if (rdev->flags & RADEON_IS_IGP)
7393 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7394 ~(THERM_INTH_MASK | THERM_INTL_MASK);
7395 else
7396 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7397 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
7398
7399 /* enable CP interrupts on all rings */
7400 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7401 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7402 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7403 }
7404 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7405 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7406 DRM_DEBUG("si_irq_set: sw int cp1\n");
7407 if (ring->me == 1) {
7408 switch (ring->pipe) {
7409 case 0:
7410 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7411 break;
7412 default:
7413 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7414 break;
7415 }
7416 } else {
7417 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7418 }
7419 }
7420 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7421 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7422 DRM_DEBUG("si_irq_set: sw int cp2\n");
7423 if (ring->me == 1) {
7424 switch (ring->pipe) {
7425 case 0:
7426 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7427 break;
7428 default:
7429 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7430 break;
7431 }
7432 } else {
7433 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7434 }
7435 }
7436
7437 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7438 DRM_DEBUG("cik_irq_set: sw int dma\n");
7439 dma_cntl |= TRAP_ENABLE;
7440 }
7441
7442 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7443 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7444 dma_cntl1 |= TRAP_ENABLE;
7445 }
7446
7447 if (rdev->irq.crtc_vblank_int[0] ||
7448 atomic_read(&rdev->irq.pflip[0])) {
7449 DRM_DEBUG("cik_irq_set: vblank 0\n");
7450 crtc1 |= VBLANK_INTERRUPT_MASK;
7451 }
7452 if (rdev->irq.crtc_vblank_int[1] ||
7453 atomic_read(&rdev->irq.pflip[1])) {
7454 DRM_DEBUG("cik_irq_set: vblank 1\n");
7455 crtc2 |= VBLANK_INTERRUPT_MASK;
7456 }
7457 if (rdev->irq.crtc_vblank_int[2] ||
7458 atomic_read(&rdev->irq.pflip[2])) {
7459 DRM_DEBUG("cik_irq_set: vblank 2\n");
7460 crtc3 |= VBLANK_INTERRUPT_MASK;
7461 }
7462 if (rdev->irq.crtc_vblank_int[3] ||
7463 atomic_read(&rdev->irq.pflip[3])) {
7464 DRM_DEBUG("cik_irq_set: vblank 3\n");
7465 crtc4 |= VBLANK_INTERRUPT_MASK;
7466 }
7467 if (rdev->irq.crtc_vblank_int[4] ||
7468 atomic_read(&rdev->irq.pflip[4])) {
7469 DRM_DEBUG("cik_irq_set: vblank 4\n");
7470 crtc5 |= VBLANK_INTERRUPT_MASK;
7471 }
7472 if (rdev->irq.crtc_vblank_int[5] ||
7473 atomic_read(&rdev->irq.pflip[5])) {
7474 DRM_DEBUG("cik_irq_set: vblank 5\n");
7475 crtc6 |= VBLANK_INTERRUPT_MASK;
7476 }
7477 if (rdev->irq.hpd[0]) {
7478 DRM_DEBUG("cik_irq_set: hpd 1\n");
7479 hpd1 |= DC_HPDx_INT_EN;
7480 }
7481 if (rdev->irq.hpd[1]) {
7482 DRM_DEBUG("cik_irq_set: hpd 2\n");
7483 hpd2 |= DC_HPDx_INT_EN;
7484 }
7485 if (rdev->irq.hpd[2]) {
7486 DRM_DEBUG("cik_irq_set: hpd 3\n");
7487 hpd3 |= DC_HPDx_INT_EN;
7488 }
7489 if (rdev->irq.hpd[3]) {
7490 DRM_DEBUG("cik_irq_set: hpd 4\n");
7491 hpd4 |= DC_HPDx_INT_EN;
7492 }
7493 if (rdev->irq.hpd[4]) {
7494 DRM_DEBUG("cik_irq_set: hpd 5\n");
7495 hpd5 |= DC_HPDx_INT_EN;
7496 }
7497 if (rdev->irq.hpd[5]) {
7498 DRM_DEBUG("cik_irq_set: hpd 6\n");
7499 hpd6 |= DC_HPDx_INT_EN;
7500 }
7501
7502 if (rdev->irq.dpm_thermal) {
7503 DRM_DEBUG("dpm thermal\n");
7504 if (rdev->flags & RADEON_IS_IGP)
7505 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7506 else
7507 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7508 }
7509
7510 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7511
7512 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7513 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7514
7515 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7516
7517 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7518
7519 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7520 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7521 if (rdev->num_crtc >= 4) {
7522 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7523 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7524 }
7525 if (rdev->num_crtc >= 6) {
7526 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7527 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7528 }
7529
7530 if (rdev->num_crtc >= 2) {
7531 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7532 GRPH_PFLIP_INT_MASK);
7533 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7534 GRPH_PFLIP_INT_MASK);
7535 }
7536 if (rdev->num_crtc >= 4) {
7537 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7538 GRPH_PFLIP_INT_MASK);
7539 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7540 GRPH_PFLIP_INT_MASK);
7541 }
7542 if (rdev->num_crtc >= 6) {
7543 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7544 GRPH_PFLIP_INT_MASK);
7545 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7546 GRPH_PFLIP_INT_MASK);
7547 }
7548
7549 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7550 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7551 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7552 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7553 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7554 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7555
7556 if (rdev->flags & RADEON_IS_IGP)
7557 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7558 else
7559 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7560
7561 return 0;
7562 }
7563
7564 /**
7565 * cik_irq_ack - ack interrupt sources
7566 *
7567 * @rdev: radeon_device pointer
7568 *
7569 * Ack interrupt sources on the GPU (vblanks, hpd,
7570 * etc.) (CIK). Certain interrupts sources are sw
7571 * generated and do not require an explicit ack.
7572 */
7573 static inline void cik_irq_ack(struct radeon_device *rdev)
7574 {
7575 u32 tmp;
7576
7577 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7578 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7579 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7580 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7581 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7582 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7583 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7584
7585 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7586 EVERGREEN_CRTC0_REGISTER_OFFSET);
7587 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7588 EVERGREEN_CRTC1_REGISTER_OFFSET);
7589 if (rdev->num_crtc >= 4) {
7590 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7591 EVERGREEN_CRTC2_REGISTER_OFFSET);
7592 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7593 EVERGREEN_CRTC3_REGISTER_OFFSET);
7594 }
7595 if (rdev->num_crtc >= 6) {
7596 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7597 EVERGREEN_CRTC4_REGISTER_OFFSET);
7598 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7599 EVERGREEN_CRTC5_REGISTER_OFFSET);
7600 }
7601
7602 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7603 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7604 GRPH_PFLIP_INT_CLEAR);
7605 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7606 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7607 GRPH_PFLIP_INT_CLEAR);
7608 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7609 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7610 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7611 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7612 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7613 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7614 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7615 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7616
7617 if (rdev->num_crtc >= 4) {
7618 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7619 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7620 GRPH_PFLIP_INT_CLEAR);
7621 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7622 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7623 GRPH_PFLIP_INT_CLEAR);
7624 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7625 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7626 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7627 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7628 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7629 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7630 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7631 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7632 }
7633
7634 if (rdev->num_crtc >= 6) {
7635 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7636 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7637 GRPH_PFLIP_INT_CLEAR);
7638 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7639 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7640 GRPH_PFLIP_INT_CLEAR);
7641 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7642 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7643 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7644 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7645 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7646 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7647 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7648 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7649 }
7650
7651 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7652 tmp = RREG32(DC_HPD1_INT_CONTROL);
7653 tmp |= DC_HPDx_INT_ACK;
7654 WREG32(DC_HPD1_INT_CONTROL, tmp);
7655 }
7656 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7657 tmp = RREG32(DC_HPD2_INT_CONTROL);
7658 tmp |= DC_HPDx_INT_ACK;
7659 WREG32(DC_HPD2_INT_CONTROL, tmp);
7660 }
7661 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7662 tmp = RREG32(DC_HPD3_INT_CONTROL);
7663 tmp |= DC_HPDx_INT_ACK;
7664 WREG32(DC_HPD3_INT_CONTROL, tmp);
7665 }
7666 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7667 tmp = RREG32(DC_HPD4_INT_CONTROL);
7668 tmp |= DC_HPDx_INT_ACK;
7669 WREG32(DC_HPD4_INT_CONTROL, tmp);
7670 }
7671 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7672 tmp = RREG32(DC_HPD5_INT_CONTROL);
7673 tmp |= DC_HPDx_INT_ACK;
7674 WREG32(DC_HPD5_INT_CONTROL, tmp);
7675 }
7676 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7677 tmp = RREG32(DC_HPD5_INT_CONTROL);
7678 tmp |= DC_HPDx_INT_ACK;
7679 WREG32(DC_HPD6_INT_CONTROL, tmp);
7680 }
7681 }
7682
7683 /**
7684 * cik_irq_disable - disable interrupts
7685 *
7686 * @rdev: radeon_device pointer
7687 *
7688 * Disable interrupts on the hw (CIK).
7689 */
7690 static void cik_irq_disable(struct radeon_device *rdev)
7691 {
7692 cik_disable_interrupts(rdev);
7693 /* Wait and acknowledge irq */
7694 mdelay(1);
7695 cik_irq_ack(rdev);
7696 cik_disable_interrupt_state(rdev);
7697 }
7698
7699 /**
7700 * cik_irq_disable - disable interrupts for suspend
7701 *
7702 * @rdev: radeon_device pointer
7703 *
7704 * Disable interrupts and stop the RLC (CIK).
7705 * Used for suspend.
7706 */
7707 static void cik_irq_suspend(struct radeon_device *rdev)
7708 {
7709 cik_irq_disable(rdev);
7710 cik_rlc_stop(rdev);
7711 }
7712
7713 /**
7714 * cik_irq_fini - tear down interrupt support
7715 *
7716 * @rdev: radeon_device pointer
7717 *
7718 * Disable interrupts on the hw and free the IH ring
7719 * buffer (CIK).
7720 * Used for driver unload.
7721 */
7722 static void cik_irq_fini(struct radeon_device *rdev)
7723 {
7724 cik_irq_suspend(rdev);
7725 r600_ih_ring_fini(rdev);
7726 }
7727
7728 /**
7729 * cik_get_ih_wptr - get the IH ring buffer wptr
7730 *
7731 * @rdev: radeon_device pointer
7732 *
7733 * Get the IH ring buffer wptr from either the register
7734 * or the writeback memory buffer (CIK). Also check for
7735 * ring buffer overflow and deal with it.
7736 * Used by cik_irq_process().
7737 * Returns the value of the wptr.
7738 */
7739 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7740 {
7741 u32 wptr, tmp;
7742
7743 if (rdev->wb.enabled)
7744 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7745 else
7746 wptr = RREG32(IH_RB_WPTR);
7747
7748 if (wptr & RB_OVERFLOW) {
7749 wptr &= ~RB_OVERFLOW;
7750 /* When a ring buffer overflow happen start parsing interrupt
7751 * from the last not overwritten vector (wptr + 16). Hopefully
7752 * this should allow us to catchup.
7753 */
7754 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7755 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7756 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7757 tmp = RREG32(IH_RB_CNTL);
7758 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7759 WREG32(IH_RB_CNTL, tmp);
7760 }
7761 return (wptr & rdev->ih.ptr_mask);
7762 }
7763
7764 /* CIK IV Ring
7765 * Each IV ring entry is 128 bits:
7766 * [7:0] - interrupt source id
7767 * [31:8] - reserved
7768 * [59:32] - interrupt source data
7769 * [63:60] - reserved
7770 * [71:64] - RINGID
7771 * CP:
7772 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7773 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7774 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7775 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7776 * PIPE_ID - ME0 0=3D
7777 * - ME1&2 compute dispatcher (4 pipes each)
7778 * SDMA:
7779 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7780 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7781 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7782 * [79:72] - VMID
7783 * [95:80] - PASID
7784 * [127:96] - reserved
7785 */
7786 /**
7787 * cik_irq_process - interrupt handler
7788 *
7789 * @rdev: radeon_device pointer
7790 *
7791 * Interrupt hander (CIK). Walk the IH ring,
7792 * ack interrupts and schedule work to handle
7793 * interrupt events.
7794 * Returns irq process return code.
7795 */
7796 int cik_irq_process(struct radeon_device *rdev)
7797 {
7798 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7799 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7800 u32 wptr;
7801 u32 rptr;
7802 u32 src_id, src_data, ring_id;
7803 u8 me_id, pipe_id, queue_id;
7804 u32 ring_index;
7805 bool queue_hotplug = false;
7806 bool queue_reset = false;
7807 u32 addr, status, mc_client;
7808 bool queue_thermal = false;
7809
7810 if (!rdev->ih.enabled || rdev->shutdown)
7811 return IRQ_NONE;
7812
7813 wptr = cik_get_ih_wptr(rdev);
7814
7815 restart_ih:
7816 /* is somebody else already processing irqs? */
7817 if (atomic_xchg(&rdev->ih.lock, 1))
7818 return IRQ_NONE;
7819
7820 rptr = rdev->ih.rptr;
7821 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7822
7823 /* Order reading of wptr vs. reading of IH ring data */
7824 rmb();
7825
7826 /* display interrupts */
7827 cik_irq_ack(rdev);
7828
7829 while (rptr != wptr) {
7830 /* wptr/rptr are in bytes! */
7831 ring_index = rptr / 4;
7832
7833 radeon_kfd_interrupt(rdev,
7834 (const void *) &rdev->ih.ring[ring_index]);
7835
7836 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7837 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7838 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7839
7840 switch (src_id) {
7841 case 1: /* D1 vblank/vline */
7842 switch (src_data) {
7843 case 0: /* D1 vblank */
7844 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7845 if (rdev->irq.crtc_vblank_int[0]) {
7846 drm_handle_vblank(rdev->ddev, 0);
7847 rdev->pm.vblank_sync = true;
7848 wake_up(&rdev->irq.vblank_queue);
7849 }
7850 if (atomic_read(&rdev->irq.pflip[0]))
7851 radeon_crtc_handle_vblank(rdev, 0);
7852 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7853 DRM_DEBUG("IH: D1 vblank\n");
7854 }
7855 break;
7856 case 1: /* D1 vline */
7857 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7858 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7859 DRM_DEBUG("IH: D1 vline\n");
7860 }
7861 break;
7862 default:
7863 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7864 break;
7865 }
7866 break;
7867 case 2: /* D2 vblank/vline */
7868 switch (src_data) {
7869 case 0: /* D2 vblank */
7870 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7871 if (rdev->irq.crtc_vblank_int[1]) {
7872 drm_handle_vblank(rdev->ddev, 1);
7873 rdev->pm.vblank_sync = true;
7874 wake_up(&rdev->irq.vblank_queue);
7875 }
7876 if (atomic_read(&rdev->irq.pflip[1]))
7877 radeon_crtc_handle_vblank(rdev, 1);
7878 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7879 DRM_DEBUG("IH: D2 vblank\n");
7880 }
7881 break;
7882 case 1: /* D2 vline */
7883 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7884 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7885 DRM_DEBUG("IH: D2 vline\n");
7886 }
7887 break;
7888 default:
7889 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7890 break;
7891 }
7892 break;
7893 case 3: /* D3 vblank/vline */
7894 switch (src_data) {
7895 case 0: /* D3 vblank */
7896 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7897 if (rdev->irq.crtc_vblank_int[2]) {
7898 drm_handle_vblank(rdev->ddev, 2);
7899 rdev->pm.vblank_sync = true;
7900 wake_up(&rdev->irq.vblank_queue);
7901 }
7902 if (atomic_read(&rdev->irq.pflip[2]))
7903 radeon_crtc_handle_vblank(rdev, 2);
7904 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7905 DRM_DEBUG("IH: D3 vblank\n");
7906 }
7907 break;
7908 case 1: /* D3 vline */
7909 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7910 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7911 DRM_DEBUG("IH: D3 vline\n");
7912 }
7913 break;
7914 default:
7915 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7916 break;
7917 }
7918 break;
7919 case 4: /* D4 vblank/vline */
7920 switch (src_data) {
7921 case 0: /* D4 vblank */
7922 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7923 if (rdev->irq.crtc_vblank_int[3]) {
7924 drm_handle_vblank(rdev->ddev, 3);
7925 rdev->pm.vblank_sync = true;
7926 wake_up(&rdev->irq.vblank_queue);
7927 }
7928 if (atomic_read(&rdev->irq.pflip[3]))
7929 radeon_crtc_handle_vblank(rdev, 3);
7930 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7931 DRM_DEBUG("IH: D4 vblank\n");
7932 }
7933 break;
7934 case 1: /* D4 vline */
7935 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7936 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7937 DRM_DEBUG("IH: D4 vline\n");
7938 }
7939 break;
7940 default:
7941 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7942 break;
7943 }
7944 break;
7945 case 5: /* D5 vblank/vline */
7946 switch (src_data) {
7947 case 0: /* D5 vblank */
7948 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7949 if (rdev->irq.crtc_vblank_int[4]) {
7950 drm_handle_vblank(rdev->ddev, 4);
7951 rdev->pm.vblank_sync = true;
7952 wake_up(&rdev->irq.vblank_queue);
7953 }
7954 if (atomic_read(&rdev->irq.pflip[4]))
7955 radeon_crtc_handle_vblank(rdev, 4);
7956 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7957 DRM_DEBUG("IH: D5 vblank\n");
7958 }
7959 break;
7960 case 1: /* D5 vline */
7961 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7962 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7963 DRM_DEBUG("IH: D5 vline\n");
7964 }
7965 break;
7966 default:
7967 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7968 break;
7969 }
7970 break;
7971 case 6: /* D6 vblank/vline */
7972 switch (src_data) {
7973 case 0: /* D6 vblank */
7974 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7975 if (rdev->irq.crtc_vblank_int[5]) {
7976 drm_handle_vblank(rdev->ddev, 5);
7977 rdev->pm.vblank_sync = true;
7978 wake_up(&rdev->irq.vblank_queue);
7979 }
7980 if (atomic_read(&rdev->irq.pflip[5]))
7981 radeon_crtc_handle_vblank(rdev, 5);
7982 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7983 DRM_DEBUG("IH: D6 vblank\n");
7984 }
7985 break;
7986 case 1: /* D6 vline */
7987 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7988 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7989 DRM_DEBUG("IH: D6 vline\n");
7990 }
7991 break;
7992 default:
7993 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7994 break;
7995 }
7996 break;
7997 case 8: /* D1 page flip */
7998 case 10: /* D2 page flip */
7999 case 12: /* D3 page flip */
8000 case 14: /* D4 page flip */
8001 case 16: /* D5 page flip */
8002 case 18: /* D6 page flip */
8003 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
8004 if (radeon_use_pflipirq > 0)
8005 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
8006 break;
8007 case 42: /* HPD hotplug */
8008 switch (src_data) {
8009 case 0:
8010 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8011 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8012 queue_hotplug = true;
8013 DRM_DEBUG("IH: HPD1\n");
8014 }
8015 break;
8016 case 1:
8017 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8018 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8019 queue_hotplug = true;
8020 DRM_DEBUG("IH: HPD2\n");
8021 }
8022 break;
8023 case 2:
8024 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8025 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8026 queue_hotplug = true;
8027 DRM_DEBUG("IH: HPD3\n");
8028 }
8029 break;
8030 case 3:
8031 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8032 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8033 queue_hotplug = true;
8034 DRM_DEBUG("IH: HPD4\n");
8035 }
8036 break;
8037 case 4:
8038 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8039 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8040 queue_hotplug = true;
8041 DRM_DEBUG("IH: HPD5\n");
8042 }
8043 break;
8044 case 5:
8045 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8046 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8047 queue_hotplug = true;
8048 DRM_DEBUG("IH: HPD6\n");
8049 }
8050 break;
8051 default:
8052 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8053 break;
8054 }
8055 break;
8056 case 124: /* UVD */
8057 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8058 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8059 break;
8060 case 146:
8061 case 147:
8062 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8063 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8064 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8065 /* reset addr and status */
8066 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8067 if (addr == 0x0 && status == 0x0)
8068 break;
8069 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8070 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
8071 addr);
8072 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8073 status);
8074 cik_vm_decode_fault(rdev, status, addr, mc_client);
8075 break;
8076 case 167: /* VCE */
8077 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8078 switch (src_data) {
8079 case 0:
8080 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8081 break;
8082 case 1:
8083 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8084 break;
8085 default:
8086 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8087 break;
8088 }
8089 break;
8090 case 176: /* GFX RB CP_INT */
8091 case 177: /* GFX IB CP_INT */
8092 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8093 break;
8094 case 181: /* CP EOP event */
8095 DRM_DEBUG("IH: CP EOP\n");
8096 /* XXX check the bitfield order! */
8097 me_id = (ring_id & 0x60) >> 5;
8098 pipe_id = (ring_id & 0x18) >> 3;
8099 queue_id = (ring_id & 0x7) >> 0;
8100 switch (me_id) {
8101 case 0:
8102 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8103 break;
8104 case 1:
8105 case 2:
8106 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8107 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8108 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8109 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8110 break;
8111 }
8112 break;
8113 case 184: /* CP Privileged reg access */
8114 DRM_ERROR("Illegal register access in command stream\n");
8115 /* XXX check the bitfield order! */
8116 me_id = (ring_id & 0x60) >> 5;
8117 pipe_id = (ring_id & 0x18) >> 3;
8118 queue_id = (ring_id & 0x7) >> 0;
8119 switch (me_id) {
8120 case 0:
8121 /* This results in a full GPU reset, but all we need to do is soft
8122 * reset the CP for gfx
8123 */
8124 queue_reset = true;
8125 break;
8126 case 1:
8127 /* XXX compute */
8128 queue_reset = true;
8129 break;
8130 case 2:
8131 /* XXX compute */
8132 queue_reset = true;
8133 break;
8134 }
8135 break;
8136 case 185: /* CP Privileged inst */
8137 DRM_ERROR("Illegal instruction in command stream\n");
8138 /* XXX check the bitfield order! */
8139 me_id = (ring_id & 0x60) >> 5;
8140 pipe_id = (ring_id & 0x18) >> 3;
8141 queue_id = (ring_id & 0x7) >> 0;
8142 switch (me_id) {
8143 case 0:
8144 /* This results in a full GPU reset, but all we need to do is soft
8145 * reset the CP for gfx
8146 */
8147 queue_reset = true;
8148 break;
8149 case 1:
8150 /* XXX compute */
8151 queue_reset = true;
8152 break;
8153 case 2:
8154 /* XXX compute */
8155 queue_reset = true;
8156 break;
8157 }
8158 break;
8159 case 224: /* SDMA trap event */
8160 /* XXX check the bitfield order! */
8161 me_id = (ring_id & 0x3) >> 0;
8162 queue_id = (ring_id & 0xc) >> 2;
8163 DRM_DEBUG("IH: SDMA trap\n");
8164 switch (me_id) {
8165 case 0:
8166 switch (queue_id) {
8167 case 0:
8168 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8169 break;
8170 case 1:
8171 /* XXX compute */
8172 break;
8173 case 2:
8174 /* XXX compute */
8175 break;
8176 }
8177 break;
8178 case 1:
8179 switch (queue_id) {
8180 case 0:
8181 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8182 break;
8183 case 1:
8184 /* XXX compute */
8185 break;
8186 case 2:
8187 /* XXX compute */
8188 break;
8189 }
8190 break;
8191 }
8192 break;
8193 case 230: /* thermal low to high */
8194 DRM_DEBUG("IH: thermal low to high\n");
8195 rdev->pm.dpm.thermal.high_to_low = false;
8196 queue_thermal = true;
8197 break;
8198 case 231: /* thermal high to low */
8199 DRM_DEBUG("IH: thermal high to low\n");
8200 rdev->pm.dpm.thermal.high_to_low = true;
8201 queue_thermal = true;
8202 break;
8203 case 233: /* GUI IDLE */
8204 DRM_DEBUG("IH: GUI idle\n");
8205 break;
8206 case 241: /* SDMA Privileged inst */
8207 case 247: /* SDMA Privileged inst */
8208 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8209 /* XXX check the bitfield order! */
8210 me_id = (ring_id & 0x3) >> 0;
8211 queue_id = (ring_id & 0xc) >> 2;
8212 switch (me_id) {
8213 case 0:
8214 switch (queue_id) {
8215 case 0:
8216 queue_reset = true;
8217 break;
8218 case 1:
8219 /* XXX compute */
8220 queue_reset = true;
8221 break;
8222 case 2:
8223 /* XXX compute */
8224 queue_reset = true;
8225 break;
8226 }
8227 break;
8228 case 1:
8229 switch (queue_id) {
8230 case 0:
8231 queue_reset = true;
8232 break;
8233 case 1:
8234 /* XXX compute */
8235 queue_reset = true;
8236 break;
8237 case 2:
8238 /* XXX compute */
8239 queue_reset = true;
8240 break;
8241 }
8242 break;
8243 }
8244 break;
8245 default:
8246 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8247 break;
8248 }
8249
8250 /* wptr/rptr are in bytes! */
8251 rptr += 16;
8252 rptr &= rdev->ih.ptr_mask;
8253 WREG32(IH_RB_RPTR, rptr);
8254 }
8255 if (queue_hotplug)
8256 schedule_work(&rdev->hotplug_work);
8257 if (queue_reset) {
8258 rdev->needs_reset = true;
8259 wake_up_all(&rdev->fence_queue);
8260 }
8261 if (queue_thermal)
8262 schedule_work(&rdev->pm.dpm.thermal.work);
8263 rdev->ih.rptr = rptr;
8264 atomic_set(&rdev->ih.lock, 0);
8265
8266 /* make sure wptr hasn't changed while processing */
8267 wptr = cik_get_ih_wptr(rdev);
8268 if (wptr != rptr)
8269 goto restart_ih;
8270
8271 return IRQ_HANDLED;
8272 }
8273
8274 /*
8275 * startup/shutdown callbacks
8276 */
8277 /**
8278 * cik_startup - program the asic to a functional state
8279 *
8280 * @rdev: radeon_device pointer
8281 *
8282 * Programs the asic to a functional state (CIK).
8283 * Called by cik_init() and cik_resume().
8284 * Returns 0 for success, error for failure.
8285 */
8286 static int cik_startup(struct radeon_device *rdev)
8287 {
8288 struct radeon_ring *ring;
8289 u32 nop;
8290 int r;
8291
8292 /* enable pcie gen2/3 link */
8293 cik_pcie_gen3_enable(rdev);
8294 /* enable aspm */
8295 cik_program_aspm(rdev);
8296
8297 /* scratch needs to be initialized before MC */
8298 r = r600_vram_scratch_init(rdev);
8299 if (r)
8300 return r;
8301
8302 cik_mc_program(rdev);
8303
8304 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8305 r = ci_mc_load_microcode(rdev);
8306 if (r) {
8307 DRM_ERROR("Failed to load MC firmware!\n");
8308 return r;
8309 }
8310 }
8311
8312 r = cik_pcie_gart_enable(rdev);
8313 if (r)
8314 return r;
8315 cik_gpu_init(rdev);
8316
8317 /* allocate rlc buffers */
8318 if (rdev->flags & RADEON_IS_IGP) {
8319 if (rdev->family == CHIP_KAVERI) {
8320 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8321 rdev->rlc.reg_list_size =
8322 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8323 } else {
8324 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8325 rdev->rlc.reg_list_size =
8326 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8327 }
8328 }
8329 rdev->rlc.cs_data = ci_cs_data;
8330 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8331 r = sumo_rlc_init(rdev);
8332 if (r) {
8333 DRM_ERROR("Failed to init rlc BOs!\n");
8334 return r;
8335 }
8336
8337 /* allocate wb buffer */
8338 r = radeon_wb_init(rdev);
8339 if (r)
8340 return r;
8341
8342 /* allocate mec buffers */
8343 r = cik_mec_init(rdev);
8344 if (r) {
8345 DRM_ERROR("Failed to init MEC BOs!\n");
8346 return r;
8347 }
8348
8349 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8350 if (r) {
8351 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8352 return r;
8353 }
8354
8355 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8356 if (r) {
8357 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8358 return r;
8359 }
8360
8361 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8362 if (r) {
8363 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8364 return r;
8365 }
8366
8367 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8368 if (r) {
8369 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8370 return r;
8371 }
8372
8373 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8374 if (r) {
8375 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8376 return r;
8377 }
8378
8379 r = radeon_uvd_resume(rdev);
8380 if (!r) {
8381 r = uvd_v4_2_resume(rdev);
8382 if (!r) {
8383 r = radeon_fence_driver_start_ring(rdev,
8384 R600_RING_TYPE_UVD_INDEX);
8385 if (r)
8386 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8387 }
8388 }
8389 if (r)
8390 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8391
8392 r = radeon_vce_resume(rdev);
8393 if (!r) {
8394 r = vce_v2_0_resume(rdev);
8395 if (!r)
8396 r = radeon_fence_driver_start_ring(rdev,
8397 TN_RING_TYPE_VCE1_INDEX);
8398 if (!r)
8399 r = radeon_fence_driver_start_ring(rdev,
8400 TN_RING_TYPE_VCE2_INDEX);
8401 }
8402 if (r) {
8403 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8404 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8405 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8406 }
8407
8408 /* Enable IRQ */
8409 if (!rdev->irq.installed) {
8410 r = radeon_irq_kms_init(rdev);
8411 if (r)
8412 return r;
8413 }
8414
8415 r = cik_irq_init(rdev);
8416 if (r) {
8417 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8418 radeon_irq_kms_fini(rdev);
8419 return r;
8420 }
8421 cik_irq_set(rdev);
8422
8423 if (rdev->family == CHIP_HAWAII) {
8424 if (rdev->new_fw)
8425 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8426 else
8427 nop = RADEON_CP_PACKET2;
8428 } else {
8429 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8430 }
8431
8432 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8433 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8434 nop);
8435 if (r)
8436 return r;
8437
8438 /* set up the compute queues */
8439 /* type-2 packets are deprecated on MEC, use type-3 instead */
8440 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8441 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8442 nop);
8443 if (r)
8444 return r;
8445 ring->me = 1; /* first MEC */
8446 ring->pipe = 0; /* first pipe */
8447 ring->queue = 0; /* first queue */
8448 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8449
8450 /* type-2 packets are deprecated on MEC, use type-3 instead */
8451 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8452 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8453 nop);
8454 if (r)
8455 return r;
8456 /* dGPU only have 1 MEC */
8457 ring->me = 1; /* first MEC */
8458 ring->pipe = 0; /* first pipe */
8459 ring->queue = 1; /* second queue */
8460 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8461
8462 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8463 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8464 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8465 if (r)
8466 return r;
8467
8468 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8469 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8470 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8471 if (r)
8472 return r;
8473
8474 r = cik_cp_resume(rdev);
8475 if (r)
8476 return r;
8477
8478 r = cik_sdma_resume(rdev);
8479 if (r)
8480 return r;
8481
8482 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8483 if (ring->ring_size) {
8484 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8485 RADEON_CP_PACKET2);
8486 if (!r)
8487 r = uvd_v1_0_init(rdev);
8488 if (r)
8489 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8490 }
8491
8492 r = -ENOENT;
8493
8494 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8495 if (ring->ring_size)
8496 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8497 VCE_CMD_NO_OP);
8498
8499 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8500 if (ring->ring_size)
8501 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8502 VCE_CMD_NO_OP);
8503
8504 if (!r)
8505 r = vce_v1_0_init(rdev);
8506 else if (r != -ENOENT)
8507 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8508
8509 r = radeon_ib_pool_init(rdev);
8510 if (r) {
8511 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8512 return r;
8513 }
8514
8515 r = radeon_vm_manager_init(rdev);
8516 if (r) {
8517 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8518 return r;
8519 }
8520
8521 r = radeon_audio_init(rdev);
8522 if (r)
8523 return r;
8524
8525 r = radeon_kfd_resume(rdev);
8526 if (r)
8527 return r;
8528
8529 return 0;
8530 }
8531
8532 /**
8533 * cik_resume - resume the asic to a functional state
8534 *
8535 * @rdev: radeon_device pointer
8536 *
8537 * Programs the asic to a functional state (CIK).
8538 * Called at resume.
8539 * Returns 0 for success, error for failure.
8540 */
8541 int cik_resume(struct radeon_device *rdev)
8542 {
8543 int r;
8544
8545 /* post card */
8546 atom_asic_init(rdev->mode_info.atom_context);
8547
8548 /* init golden registers */
8549 cik_init_golden_registers(rdev);
8550
8551 if (rdev->pm.pm_method == PM_METHOD_DPM)
8552 radeon_pm_resume(rdev);
8553
8554 rdev->accel_working = true;
8555 r = cik_startup(rdev);
8556 if (r) {
8557 DRM_ERROR("cik startup failed on resume\n");
8558 rdev->accel_working = false;
8559 return r;
8560 }
8561
8562 return r;
8563
8564 }
8565
8566 /**
8567 * cik_suspend - suspend the asic
8568 *
8569 * @rdev: radeon_device pointer
8570 *
8571 * Bring the chip into a state suitable for suspend (CIK).
8572 * Called at suspend.
8573 * Returns 0 for success.
8574 */
8575 int cik_suspend(struct radeon_device *rdev)
8576 {
8577 radeon_kfd_suspend(rdev);
8578 radeon_pm_suspend(rdev);
8579 radeon_audio_fini(rdev);
8580 radeon_vm_manager_fini(rdev);
8581 cik_cp_enable(rdev, false);
8582 cik_sdma_enable(rdev, false);
8583 uvd_v1_0_fini(rdev);
8584 radeon_uvd_suspend(rdev);
8585 radeon_vce_suspend(rdev);
8586 cik_fini_pg(rdev);
8587 cik_fini_cg(rdev);
8588 cik_irq_suspend(rdev);
8589 radeon_wb_disable(rdev);
8590 cik_pcie_gart_disable(rdev);
8591 return 0;
8592 }
8593
8594 /* Plan is to move initialization in that function and use
8595 * helper function so that radeon_device_init pretty much
8596 * do nothing more than calling asic specific function. This
8597 * should also allow to remove a bunch of callback function
8598 * like vram_info.
8599 */
8600 /**
8601 * cik_init - asic specific driver and hw init
8602 *
8603 * @rdev: radeon_device pointer
8604 *
8605 * Setup asic specific driver variables and program the hw
8606 * to a functional state (CIK).
8607 * Called at driver startup.
8608 * Returns 0 for success, errors for failure.
8609 */
8610 int cik_init(struct radeon_device *rdev)
8611 {
8612 struct radeon_ring *ring;
8613 int r;
8614
8615 /* Read BIOS */
8616 if (!radeon_get_bios(rdev)) {
8617 if (ASIC_IS_AVIVO(rdev))
8618 return -EINVAL;
8619 }
8620 /* Must be an ATOMBIOS */
8621 if (!rdev->is_atom_bios) {
8622 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8623 return -EINVAL;
8624 }
8625 r = radeon_atombios_init(rdev);
8626 if (r)
8627 return r;
8628
8629 /* Post card if necessary */
8630 if (!radeon_card_posted(rdev)) {
8631 if (!rdev->bios) {
8632 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8633 return -EINVAL;
8634 }
8635 DRM_INFO("GPU not posted. posting now...\n");
8636 atom_asic_init(rdev->mode_info.atom_context);
8637 }
8638 /* init golden registers */
8639 cik_init_golden_registers(rdev);
8640 /* Initialize scratch registers */
8641 cik_scratch_init(rdev);
8642 /* Initialize surface registers */
8643 radeon_surface_init(rdev);
8644 /* Initialize clocks */
8645 radeon_get_clock_info(rdev->ddev);
8646
8647 /* Fence driver */
8648 r = radeon_fence_driver_init(rdev);
8649 if (r)
8650 return r;
8651
8652 /* initialize memory controller */
8653 r = cik_mc_init(rdev);
8654 if (r)
8655 return r;
8656 /* Memory manager */
8657 r = radeon_bo_init(rdev);
8658 if (r)
8659 return r;
8660
8661 if (rdev->flags & RADEON_IS_IGP) {
8662 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8663 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8664 r = cik_init_microcode(rdev);
8665 if (r) {
8666 DRM_ERROR("Failed to load firmware!\n");
8667 return r;
8668 }
8669 }
8670 } else {
8671 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8672 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8673 !rdev->mc_fw) {
8674 r = cik_init_microcode(rdev);
8675 if (r) {
8676 DRM_ERROR("Failed to load firmware!\n");
8677 return r;
8678 }
8679 }
8680 }
8681
8682 /* Initialize power management */
8683 radeon_pm_init(rdev);
8684
8685 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8686 ring->ring_obj = NULL;
8687 r600_ring_init(rdev, ring, 1024 * 1024);
8688
8689 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8690 ring->ring_obj = NULL;
8691 r600_ring_init(rdev, ring, 1024 * 1024);
8692 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8693 if (r)
8694 return r;
8695
8696 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8697 ring->ring_obj = NULL;
8698 r600_ring_init(rdev, ring, 1024 * 1024);
8699 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8700 if (r)
8701 return r;
8702
8703 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8704 ring->ring_obj = NULL;
8705 r600_ring_init(rdev, ring, 256 * 1024);
8706
8707 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8708 ring->ring_obj = NULL;
8709 r600_ring_init(rdev, ring, 256 * 1024);
8710
8711 r = radeon_uvd_init(rdev);
8712 if (!r) {
8713 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8714 ring->ring_obj = NULL;
8715 r600_ring_init(rdev, ring, 4096);
8716 }
8717
8718 r = radeon_vce_init(rdev);
8719 if (!r) {
8720 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8721 ring->ring_obj = NULL;
8722 r600_ring_init(rdev, ring, 4096);
8723
8724 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8725 ring->ring_obj = NULL;
8726 r600_ring_init(rdev, ring, 4096);
8727 }
8728
8729 rdev->ih.ring_obj = NULL;
8730 r600_ih_ring_init(rdev, 64 * 1024);
8731
8732 r = r600_pcie_gart_init(rdev);
8733 if (r)
8734 return r;
8735
8736 rdev->accel_working = true;
8737 r = cik_startup(rdev);
8738 if (r) {
8739 dev_err(rdev->dev, "disabling GPU acceleration\n");
8740 cik_cp_fini(rdev);
8741 cik_sdma_fini(rdev);
8742 cik_irq_fini(rdev);
8743 sumo_rlc_fini(rdev);
8744 cik_mec_fini(rdev);
8745 radeon_wb_fini(rdev);
8746 radeon_ib_pool_fini(rdev);
8747 radeon_vm_manager_fini(rdev);
8748 radeon_irq_kms_fini(rdev);
8749 cik_pcie_gart_fini(rdev);
8750 rdev->accel_working = false;
8751 }
8752
8753 /* Don't start up if the MC ucode is missing.
8754 * The default clocks and voltages before the MC ucode
8755 * is loaded are not suffient for advanced operations.
8756 */
8757 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8758 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8759 return -EINVAL;
8760 }
8761
8762 return 0;
8763 }
8764
8765 /**
8766 * cik_fini - asic specific driver and hw fini
8767 *
8768 * @rdev: radeon_device pointer
8769 *
8770 * Tear down the asic specific driver variables and program the hw
8771 * to an idle state (CIK).
8772 * Called at driver unload.
8773 */
8774 void cik_fini(struct radeon_device *rdev)
8775 {
8776 radeon_pm_fini(rdev);
8777 cik_cp_fini(rdev);
8778 cik_sdma_fini(rdev);
8779 cik_fini_pg(rdev);
8780 cik_fini_cg(rdev);
8781 cik_irq_fini(rdev);
8782 sumo_rlc_fini(rdev);
8783 cik_mec_fini(rdev);
8784 radeon_wb_fini(rdev);
8785 radeon_vm_manager_fini(rdev);
8786 radeon_ib_pool_fini(rdev);
8787 radeon_irq_kms_fini(rdev);
8788 uvd_v1_0_fini(rdev);
8789 radeon_uvd_fini(rdev);
8790 radeon_vce_fini(rdev);
8791 cik_pcie_gart_fini(rdev);
8792 r600_vram_scratch_fini(rdev);
8793 radeon_gem_fini(rdev);
8794 radeon_fence_driver_fini(rdev);
8795 radeon_bo_fini(rdev);
8796 radeon_atombios_fini(rdev);
8797 kfree(rdev->bios);
8798 rdev->bios = NULL;
8799 }
8800
8801 void dce8_program_fmt(struct drm_encoder *encoder)
8802 {
8803 struct drm_device *dev = encoder->dev;
8804 struct radeon_device *rdev = dev->dev_private;
8805 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8806 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8807 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8808 int bpc = 0;
8809 u32 tmp = 0;
8810 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8811
8812 if (connector) {
8813 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8814 bpc = radeon_get_monitor_bpc(connector);
8815 dither = radeon_connector->dither;
8816 }
8817
8818 /* LVDS/eDP FMT is set up by atom */
8819 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8820 return;
8821
8822 /* not needed for analog */
8823 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8824 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8825 return;
8826
8827 if (bpc == 0)
8828 return;
8829
8830 switch (bpc) {
8831 case 6:
8832 if (dither == RADEON_FMT_DITHER_ENABLE)
8833 /* XXX sort out optimal dither settings */
8834 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8835 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8836 else
8837 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8838 break;
8839 case 8:
8840 if (dither == RADEON_FMT_DITHER_ENABLE)
8841 /* XXX sort out optimal dither settings */
8842 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8843 FMT_RGB_RANDOM_ENABLE |
8844 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8845 else
8846 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8847 break;
8848 case 10:
8849 if (dither == RADEON_FMT_DITHER_ENABLE)
8850 /* XXX sort out optimal dither settings */
8851 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8852 FMT_RGB_RANDOM_ENABLE |
8853 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8854 else
8855 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8856 break;
8857 default:
8858 /* not needed */
8859 break;
8860 }
8861
8862 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8863 }
8864
8865 /* display watermark setup */
8866 /**
8867 * dce8_line_buffer_adjust - Set up the line buffer
8868 *
8869 * @rdev: radeon_device pointer
8870 * @radeon_crtc: the selected display controller
8871 * @mode: the current display mode on the selected display
8872 * controller
8873 *
8874 * Setup up the line buffer allocation for
8875 * the selected display controller (CIK).
8876 * Returns the line buffer size in pixels.
8877 */
8878 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8879 struct radeon_crtc *radeon_crtc,
8880 struct drm_display_mode *mode)
8881 {
8882 u32 tmp, buffer_alloc, i;
8883 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8884 /*
8885 * Line Buffer Setup
8886 * There are 6 line buffers, one for each display controllers.
8887 * There are 3 partitions per LB. Select the number of partitions
8888 * to enable based on the display width. For display widths larger
8889 * than 4096, you need use to use 2 display controllers and combine
8890 * them using the stereo blender.
8891 */
8892 if (radeon_crtc->base.enabled && mode) {
8893 if (mode->crtc_hdisplay < 1920) {
8894 tmp = 1;
8895 buffer_alloc = 2;
8896 } else if (mode->crtc_hdisplay < 2560) {
8897 tmp = 2;
8898 buffer_alloc = 2;
8899 } else if (mode->crtc_hdisplay < 4096) {
8900 tmp = 0;
8901 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8902 } else {
8903 DRM_DEBUG_KMS("Mode too big for LB!\n");
8904 tmp = 0;
8905 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8906 }
8907 } else {
8908 tmp = 1;
8909 buffer_alloc = 0;
8910 }
8911
8912 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8913 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8914
8915 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8916 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8917 for (i = 0; i < rdev->usec_timeout; i++) {
8918 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8919 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8920 break;
8921 udelay(1);
8922 }
8923
8924 if (radeon_crtc->base.enabled && mode) {
8925 switch (tmp) {
8926 case 0:
8927 default:
8928 return 4096 * 2;
8929 case 1:
8930 return 1920 * 2;
8931 case 2:
8932 return 2560 * 2;
8933 }
8934 }
8935
8936 /* controller not enabled, so no lb used */
8937 return 0;
8938 }
8939
8940 /**
8941 * cik_get_number_of_dram_channels - get the number of dram channels
8942 *
8943 * @rdev: radeon_device pointer
8944 *
8945 * Look up the number of video ram channels (CIK).
8946 * Used for display watermark bandwidth calculations
8947 * Returns the number of dram channels
8948 */
8949 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8950 {
8951 u32 tmp = RREG32(MC_SHARED_CHMAP);
8952
8953 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8954 case 0:
8955 default:
8956 return 1;
8957 case 1:
8958 return 2;
8959 case 2:
8960 return 4;
8961 case 3:
8962 return 8;
8963 case 4:
8964 return 3;
8965 case 5:
8966 return 6;
8967 case 6:
8968 return 10;
8969 case 7:
8970 return 12;
8971 case 8:
8972 return 16;
8973 }
8974 }
8975
8976 struct dce8_wm_params {
8977 u32 dram_channels; /* number of dram channels */
8978 u32 yclk; /* bandwidth per dram data pin in kHz */
8979 u32 sclk; /* engine clock in kHz */
8980 u32 disp_clk; /* display clock in kHz */
8981 u32 src_width; /* viewport width */
8982 u32 active_time; /* active display time in ns */
8983 u32 blank_time; /* blank time in ns */
8984 bool interlaced; /* mode is interlaced */
8985 fixed20_12 vsc; /* vertical scale ratio */
8986 u32 num_heads; /* number of active crtcs */
8987 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8988 u32 lb_size; /* line buffer allocated to pipe */
8989 u32 vtaps; /* vertical scaler taps */
8990 };
8991
8992 /**
8993 * dce8_dram_bandwidth - get the dram bandwidth
8994 *
8995 * @wm: watermark calculation data
8996 *
8997 * Calculate the raw dram bandwidth (CIK).
8998 * Used for display watermark bandwidth calculations
8999 * Returns the dram bandwidth in MBytes/s
9000 */
9001 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
9002 {
9003 /* Calculate raw DRAM Bandwidth */
9004 fixed20_12 dram_efficiency; /* 0.7 */
9005 fixed20_12 yclk, dram_channels, bandwidth;
9006 fixed20_12 a;
9007
9008 a.full = dfixed_const(1000);
9009 yclk.full = dfixed_const(wm->yclk);
9010 yclk.full = dfixed_div(yclk, a);
9011 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9012 a.full = dfixed_const(10);
9013 dram_efficiency.full = dfixed_const(7);
9014 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9015 bandwidth.full = dfixed_mul(dram_channels, yclk);
9016 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9017
9018 return dfixed_trunc(bandwidth);
9019 }
9020
9021 /**
9022 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9023 *
9024 * @wm: watermark calculation data
9025 *
9026 * Calculate the dram bandwidth used for display (CIK).
9027 * Used for display watermark bandwidth calculations
9028 * Returns the dram bandwidth for display in MBytes/s
9029 */
9030 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9031 {
9032 /* Calculate DRAM Bandwidth and the part allocated to display. */
9033 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9034 fixed20_12 yclk, dram_channels, bandwidth;
9035 fixed20_12 a;
9036
9037 a.full = dfixed_const(1000);
9038 yclk.full = dfixed_const(wm->yclk);
9039 yclk.full = dfixed_div(yclk, a);
9040 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9041 a.full = dfixed_const(10);
9042 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9043 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9044 bandwidth.full = dfixed_mul(dram_channels, yclk);
9045 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9046
9047 return dfixed_trunc(bandwidth);
9048 }
9049
9050 /**
9051 * dce8_data_return_bandwidth - get the data return bandwidth
9052 *
9053 * @wm: watermark calculation data
9054 *
9055 * Calculate the data return bandwidth used for display (CIK).
9056 * Used for display watermark bandwidth calculations
9057 * Returns the data return bandwidth in MBytes/s
9058 */
9059 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9060 {
9061 /* Calculate the display Data return Bandwidth */
9062 fixed20_12 return_efficiency; /* 0.8 */
9063 fixed20_12 sclk, bandwidth;
9064 fixed20_12 a;
9065
9066 a.full = dfixed_const(1000);
9067 sclk.full = dfixed_const(wm->sclk);
9068 sclk.full = dfixed_div(sclk, a);
9069 a.full = dfixed_const(10);
9070 return_efficiency.full = dfixed_const(8);
9071 return_efficiency.full = dfixed_div(return_efficiency, a);
9072 a.full = dfixed_const(32);
9073 bandwidth.full = dfixed_mul(a, sclk);
9074 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9075
9076 return dfixed_trunc(bandwidth);
9077 }
9078
9079 /**
9080 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9081 *
9082 * @wm: watermark calculation data
9083 *
9084 * Calculate the dmif bandwidth used for display (CIK).
9085 * Used for display watermark bandwidth calculations
9086 * Returns the dmif bandwidth in MBytes/s
9087 */
9088 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9089 {
9090 /* Calculate the DMIF Request Bandwidth */
9091 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9092 fixed20_12 disp_clk, bandwidth;
9093 fixed20_12 a, b;
9094
9095 a.full = dfixed_const(1000);
9096 disp_clk.full = dfixed_const(wm->disp_clk);
9097 disp_clk.full = dfixed_div(disp_clk, a);
9098 a.full = dfixed_const(32);
9099 b.full = dfixed_mul(a, disp_clk);
9100
9101 a.full = dfixed_const(10);
9102 disp_clk_request_efficiency.full = dfixed_const(8);
9103 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9104
9105 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9106
9107 return dfixed_trunc(bandwidth);
9108 }
9109
9110 /**
9111 * dce8_available_bandwidth - get the min available bandwidth
9112 *
9113 * @wm: watermark calculation data
9114 *
9115 * Calculate the min available bandwidth used for display (CIK).
9116 * Used for display watermark bandwidth calculations
9117 * Returns the min available bandwidth in MBytes/s
9118 */
9119 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9120 {
9121 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9122 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9123 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9124 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9125
9126 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9127 }
9128
9129 /**
9130 * dce8_average_bandwidth - get the average available bandwidth
9131 *
9132 * @wm: watermark calculation data
9133 *
9134 * Calculate the average available bandwidth used for display (CIK).
9135 * Used for display watermark bandwidth calculations
9136 * Returns the average available bandwidth in MBytes/s
9137 */
9138 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9139 {
9140 /* Calculate the display mode Average Bandwidth
9141 * DisplayMode should contain the source and destination dimensions,
9142 * timing, etc.
9143 */
9144 fixed20_12 bpp;
9145 fixed20_12 line_time;
9146 fixed20_12 src_width;
9147 fixed20_12 bandwidth;
9148 fixed20_12 a;
9149
9150 a.full = dfixed_const(1000);
9151 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9152 line_time.full = dfixed_div(line_time, a);
9153 bpp.full = dfixed_const(wm->bytes_per_pixel);
9154 src_width.full = dfixed_const(wm->src_width);
9155 bandwidth.full = dfixed_mul(src_width, bpp);
9156 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9157 bandwidth.full = dfixed_div(bandwidth, line_time);
9158
9159 return dfixed_trunc(bandwidth);
9160 }
9161
9162 /**
9163 * dce8_latency_watermark - get the latency watermark
9164 *
9165 * @wm: watermark calculation data
9166 *
9167 * Calculate the latency watermark (CIK).
9168 * Used for display watermark bandwidth calculations
9169 * Returns the latency watermark in ns
9170 */
9171 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9172 {
9173 /* First calculate the latency in ns */
9174 u32 mc_latency = 2000; /* 2000 ns. */
9175 u32 available_bandwidth = dce8_available_bandwidth(wm);
9176 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9177 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9178 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9179 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9180 (wm->num_heads * cursor_line_pair_return_time);
9181 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9182 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9183 u32 tmp, dmif_size = 12288;
9184 fixed20_12 a, b, c;
9185
9186 if (wm->num_heads == 0)
9187 return 0;
9188
9189 a.full = dfixed_const(2);
9190 b.full = dfixed_const(1);
9191 if ((wm->vsc.full > a.full) ||
9192 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9193 (wm->vtaps >= 5) ||
9194 ((wm->vsc.full >= a.full) && wm->interlaced))
9195 max_src_lines_per_dst_line = 4;
9196 else
9197 max_src_lines_per_dst_line = 2;
9198
9199 a.full = dfixed_const(available_bandwidth);
9200 b.full = dfixed_const(wm->num_heads);
9201 a.full = dfixed_div(a, b);
9202
9203 b.full = dfixed_const(mc_latency + 512);
9204 c.full = dfixed_const(wm->disp_clk);
9205 b.full = dfixed_div(b, c);
9206
9207 c.full = dfixed_const(dmif_size);
9208 b.full = dfixed_div(c, b);
9209
9210 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9211
9212 b.full = dfixed_const(1000);
9213 c.full = dfixed_const(wm->disp_clk);
9214 b.full = dfixed_div(c, b);
9215 c.full = dfixed_const(wm->bytes_per_pixel);
9216 b.full = dfixed_mul(b, c);
9217
9218 lb_fill_bw = min(tmp, dfixed_trunc(b));
9219
9220 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9221 b.full = dfixed_const(1000);
9222 c.full = dfixed_const(lb_fill_bw);
9223 b.full = dfixed_div(c, b);
9224 a.full = dfixed_div(a, b);
9225 line_fill_time = dfixed_trunc(a);
9226
9227 if (line_fill_time < wm->active_time)
9228 return latency;
9229 else
9230 return latency + (line_fill_time - wm->active_time);
9231
9232 }
9233
9234 /**
9235 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9236 * average and available dram bandwidth
9237 *
9238 * @wm: watermark calculation data
9239 *
9240 * Check if the display average bandwidth fits in the display
9241 * dram bandwidth (CIK).
9242 * Used for display watermark bandwidth calculations
9243 * Returns true if the display fits, false if not.
9244 */
9245 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9246 {
9247 if (dce8_average_bandwidth(wm) <=
9248 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9249 return true;
9250 else
9251 return false;
9252 }
9253
9254 /**
9255 * dce8_average_bandwidth_vs_available_bandwidth - check
9256 * average and available bandwidth
9257 *
9258 * @wm: watermark calculation data
9259 *
9260 * Check if the display average bandwidth fits in the display
9261 * available bandwidth (CIK).
9262 * Used for display watermark bandwidth calculations
9263 * Returns true if the display fits, false if not.
9264 */
9265 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9266 {
9267 if (dce8_average_bandwidth(wm) <=
9268 (dce8_available_bandwidth(wm) / wm->num_heads))
9269 return true;
9270 else
9271 return false;
9272 }
9273
9274 /**
9275 * dce8_check_latency_hiding - check latency hiding
9276 *
9277 * @wm: watermark calculation data
9278 *
9279 * Check latency hiding (CIK).
9280 * Used for display watermark bandwidth calculations
9281 * Returns true if the display fits, false if not.
9282 */
9283 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9284 {
9285 u32 lb_partitions = wm->lb_size / wm->src_width;
9286 u32 line_time = wm->active_time + wm->blank_time;
9287 u32 latency_tolerant_lines;
9288 u32 latency_hiding;
9289 fixed20_12 a;
9290
9291 a.full = dfixed_const(1);
9292 if (wm->vsc.full > a.full)
9293 latency_tolerant_lines = 1;
9294 else {
9295 if (lb_partitions <= (wm->vtaps + 1))
9296 latency_tolerant_lines = 1;
9297 else
9298 latency_tolerant_lines = 2;
9299 }
9300
9301 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9302
9303 if (dce8_latency_watermark(wm) <= latency_hiding)
9304 return true;
9305 else
9306 return false;
9307 }
9308
9309 /**
9310 * dce8_program_watermarks - program display watermarks
9311 *
9312 * @rdev: radeon_device pointer
9313 * @radeon_crtc: the selected display controller
9314 * @lb_size: line buffer size
9315 * @num_heads: number of display controllers in use
9316 *
9317 * Calculate and program the display watermarks for the
9318 * selected display controller (CIK).
9319 */
9320 static void dce8_program_watermarks(struct radeon_device *rdev,
9321 struct radeon_crtc *radeon_crtc,
9322 u32 lb_size, u32 num_heads)
9323 {
9324 struct drm_display_mode *mode = &radeon_crtc->base.mode;
9325 struct dce8_wm_params wm_low, wm_high;
9326 u32 pixel_period;
9327 u32 line_time = 0;
9328 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9329 u32 tmp, wm_mask;
9330
9331 if (radeon_crtc->base.enabled && num_heads && mode) {
9332 pixel_period = 1000000 / (u32)mode->clock;
9333 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9334
9335 /* watermark for high clocks */
9336 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9337 rdev->pm.dpm_enabled) {
9338 wm_high.yclk =
9339 radeon_dpm_get_mclk(rdev, false) * 10;
9340 wm_high.sclk =
9341 radeon_dpm_get_sclk(rdev, false) * 10;
9342 } else {
9343 wm_high.yclk = rdev->pm.current_mclk * 10;
9344 wm_high.sclk = rdev->pm.current_sclk * 10;
9345 }
9346
9347 wm_high.disp_clk = mode->clock;
9348 wm_high.src_width = mode->crtc_hdisplay;
9349 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9350 wm_high.blank_time = line_time - wm_high.active_time;
9351 wm_high.interlaced = false;
9352 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9353 wm_high.interlaced = true;
9354 wm_high.vsc = radeon_crtc->vsc;
9355 wm_high.vtaps = 1;
9356 if (radeon_crtc->rmx_type != RMX_OFF)
9357 wm_high.vtaps = 2;
9358 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9359 wm_high.lb_size = lb_size;
9360 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9361 wm_high.num_heads = num_heads;
9362
9363 /* set for high clocks */
9364 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9365
9366 /* possibly force display priority to high */
9367 /* should really do this at mode validation time... */
9368 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9369 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9370 !dce8_check_latency_hiding(&wm_high) ||
9371 (rdev->disp_priority == 2)) {
9372 DRM_DEBUG_KMS("force priority to high\n");
9373 }
9374
9375 /* watermark for low clocks */
9376 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9377 rdev->pm.dpm_enabled) {
9378 wm_low.yclk =
9379 radeon_dpm_get_mclk(rdev, true) * 10;
9380 wm_low.sclk =
9381 radeon_dpm_get_sclk(rdev, true) * 10;
9382 } else {
9383 wm_low.yclk = rdev->pm.current_mclk * 10;
9384 wm_low.sclk = rdev->pm.current_sclk * 10;
9385 }
9386
9387 wm_low.disp_clk = mode->clock;
9388 wm_low.src_width = mode->crtc_hdisplay;
9389 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9390 wm_low.blank_time = line_time - wm_low.active_time;
9391 wm_low.interlaced = false;
9392 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9393 wm_low.interlaced = true;
9394 wm_low.vsc = radeon_crtc->vsc;
9395 wm_low.vtaps = 1;
9396 if (radeon_crtc->rmx_type != RMX_OFF)
9397 wm_low.vtaps = 2;
9398 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9399 wm_low.lb_size = lb_size;
9400 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9401 wm_low.num_heads = num_heads;
9402
9403 /* set for low clocks */
9404 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9405
9406 /* possibly force display priority to high */
9407 /* should really do this at mode validation time... */
9408 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9409 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9410 !dce8_check_latency_hiding(&wm_low) ||
9411 (rdev->disp_priority == 2)) {
9412 DRM_DEBUG_KMS("force priority to high\n");
9413 }
9414 }
9415
9416 /* select wm A */
9417 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9418 tmp = wm_mask;
9419 tmp &= ~LATENCY_WATERMARK_MASK(3);
9420 tmp |= LATENCY_WATERMARK_MASK(1);
9421 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9422 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9423 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9424 LATENCY_HIGH_WATERMARK(line_time)));
9425 /* select wm B */
9426 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9427 tmp &= ~LATENCY_WATERMARK_MASK(3);
9428 tmp |= LATENCY_WATERMARK_MASK(2);
9429 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9430 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9431 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9432 LATENCY_HIGH_WATERMARK(line_time)));
9433 /* restore original selection */
9434 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9435
9436 /* save values for DPM */
9437 radeon_crtc->line_time = line_time;
9438 radeon_crtc->wm_high = latency_watermark_a;
9439 radeon_crtc->wm_low = latency_watermark_b;
9440 }
9441
9442 /**
9443 * dce8_bandwidth_update - program display watermarks
9444 *
9445 * @rdev: radeon_device pointer
9446 *
9447 * Calculate and program the display watermarks and line
9448 * buffer allocation (CIK).
9449 */
9450 void dce8_bandwidth_update(struct radeon_device *rdev)
9451 {
9452 struct drm_display_mode *mode = NULL;
9453 u32 num_heads = 0, lb_size;
9454 int i;
9455
9456 if (!rdev->mode_info.mode_config_initialized)
9457 return;
9458
9459 radeon_update_display_priority(rdev);
9460
9461 for (i = 0; i < rdev->num_crtc; i++) {
9462 if (rdev->mode_info.crtcs[i]->base.enabled)
9463 num_heads++;
9464 }
9465 for (i = 0; i < rdev->num_crtc; i++) {
9466 mode = &rdev->mode_info.crtcs[i]->base.mode;
9467 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9468 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9469 }
9470 }
9471
9472 /**
9473 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9474 *
9475 * @rdev: radeon_device pointer
9476 *
9477 * Fetches a GPU clock counter snapshot (SI).
9478 * Returns the 64 bit clock counter snapshot.
9479 */
9480 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9481 {
9482 uint64_t clock;
9483
9484 mutex_lock(&rdev->gpu_clock_mutex);
9485 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9486 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9487 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9488 mutex_unlock(&rdev->gpu_clock_mutex);
9489 return clock;
9490 }
9491
9492 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9493 u32 cntl_reg, u32 status_reg)
9494 {
9495 int r, i;
9496 struct atom_clock_dividers dividers;
9497 uint32_t tmp;
9498
9499 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9500 clock, false, &dividers);
9501 if (r)
9502 return r;
9503
9504 tmp = RREG32_SMC(cntl_reg);
9505 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9506 tmp |= dividers.post_divider;
9507 WREG32_SMC(cntl_reg, tmp);
9508
9509 for (i = 0; i < 100; i++) {
9510 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9511 break;
9512 mdelay(10);
9513 }
9514 if (i == 100)
9515 return -ETIMEDOUT;
9516
9517 return 0;
9518 }
9519
9520 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9521 {
9522 int r = 0;
9523
9524 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9525 if (r)
9526 return r;
9527
9528 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9529 return r;
9530 }
9531
9532 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9533 {
9534 int r, i;
9535 struct atom_clock_dividers dividers;
9536 u32 tmp;
9537
9538 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9539 ecclk, false, &dividers);
9540 if (r)
9541 return r;
9542
9543 for (i = 0; i < 100; i++) {
9544 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9545 break;
9546 mdelay(10);
9547 }
9548 if (i == 100)
9549 return -ETIMEDOUT;
9550
9551 tmp = RREG32_SMC(CG_ECLK_CNTL);
9552 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9553 tmp |= dividers.post_divider;
9554 WREG32_SMC(CG_ECLK_CNTL, tmp);
9555
9556 for (i = 0; i < 100; i++) {
9557 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9558 break;
9559 mdelay(10);
9560 }
9561 if (i == 100)
9562 return -ETIMEDOUT;
9563
9564 return 0;
9565 }
9566
9567 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9568 {
9569 struct pci_dev *root = rdev->pdev->bus->self;
9570 int bridge_pos, gpu_pos;
9571 u32 speed_cntl, mask, current_data_rate;
9572 int ret, i;
9573 u16 tmp16;
9574
9575 if (pci_is_root_bus(rdev->pdev->bus))
9576 return;
9577
9578 if (radeon_pcie_gen2 == 0)
9579 return;
9580
9581 if (rdev->flags & RADEON_IS_IGP)
9582 return;
9583
9584 if (!(rdev->flags & RADEON_IS_PCIE))
9585 return;
9586
9587 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9588 if (ret != 0)
9589 return;
9590
9591 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9592 return;
9593
9594 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9595 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9596 LC_CURRENT_DATA_RATE_SHIFT;
9597 if (mask & DRM_PCIE_SPEED_80) {
9598 if (current_data_rate == 2) {
9599 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9600 return;
9601 }
9602 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9603 } else if (mask & DRM_PCIE_SPEED_50) {
9604 if (current_data_rate == 1) {
9605 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9606 return;
9607 }
9608 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9609 }
9610
9611 bridge_pos = pci_pcie_cap(root);
9612 if (!bridge_pos)
9613 return;
9614
9615 gpu_pos = pci_pcie_cap(rdev->pdev);
9616 if (!gpu_pos)
9617 return;
9618
9619 if (mask & DRM_PCIE_SPEED_80) {
9620 /* re-try equalization if gen3 is not already enabled */
9621 if (current_data_rate != 2) {
9622 u16 bridge_cfg, gpu_cfg;
9623 u16 bridge_cfg2, gpu_cfg2;
9624 u32 max_lw, current_lw, tmp;
9625
9626 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9627 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9628
9629 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9630 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9631
9632 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9633 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9634
9635 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9636 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9637 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9638
9639 if (current_lw < max_lw) {
9640 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9641 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9642 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9643 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9644 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9645 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9646 }
9647 }
9648
9649 for (i = 0; i < 10; i++) {
9650 /* check status */
9651 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9652 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9653 break;
9654
9655 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9656 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9657
9658 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9659 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9660
9661 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9662 tmp |= LC_SET_QUIESCE;
9663 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9664
9665 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9666 tmp |= LC_REDO_EQ;
9667 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9668
9669 mdelay(100);
9670
9671 /* linkctl */
9672 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9673 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9674 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9675 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9676
9677 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9678 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9679 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9680 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9681
9682 /* linkctl2 */
9683 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9684 tmp16 &= ~((1 << 4) | (7 << 9));
9685 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9686 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9687
9688 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9689 tmp16 &= ~((1 << 4) | (7 << 9));
9690 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9691 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9692
9693 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9694 tmp &= ~LC_SET_QUIESCE;
9695 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9696 }
9697 }
9698 }
9699
9700 /* set the link speed */
9701 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9702 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9703 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9704
9705 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9706 tmp16 &= ~0xf;
9707 if (mask & DRM_PCIE_SPEED_80)
9708 tmp16 |= 3; /* gen3 */
9709 else if (mask & DRM_PCIE_SPEED_50)
9710 tmp16 |= 2; /* gen2 */
9711 else
9712 tmp16 |= 1; /* gen1 */
9713 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9714
9715 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9716 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9717 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9718
9719 for (i = 0; i < rdev->usec_timeout; i++) {
9720 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9721 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9722 break;
9723 udelay(1);
9724 }
9725 }
9726
9727 static void cik_program_aspm(struct radeon_device *rdev)
9728 {
9729 u32 data, orig;
9730 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9731 bool disable_clkreq = false;
9732
9733 if (radeon_aspm == 0)
9734 return;
9735
9736 /* XXX double check IGPs */
9737 if (rdev->flags & RADEON_IS_IGP)
9738 return;
9739
9740 if (!(rdev->flags & RADEON_IS_PCIE))
9741 return;
9742
9743 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9744 data &= ~LC_XMIT_N_FTS_MASK;
9745 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9746 if (orig != data)
9747 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9748
9749 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9750 data |= LC_GO_TO_RECOVERY;
9751 if (orig != data)
9752 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9753
9754 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9755 data |= P_IGNORE_EDB_ERR;
9756 if (orig != data)
9757 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9758
9759 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9760 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9761 data |= LC_PMI_TO_L1_DIS;
9762 if (!disable_l0s)
9763 data |= LC_L0S_INACTIVITY(7);
9764
9765 if (!disable_l1) {
9766 data |= LC_L1_INACTIVITY(7);
9767 data &= ~LC_PMI_TO_L1_DIS;
9768 if (orig != data)
9769 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9770
9771 if (!disable_plloff_in_l1) {
9772 bool clk_req_support;
9773
9774 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9775 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9776 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9777 if (orig != data)
9778 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9779
9780 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9781 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9782 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9783 if (orig != data)
9784 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9785
9786 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9787 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9788 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9789 if (orig != data)
9790 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9791
9792 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9793 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9794 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9795 if (orig != data)
9796 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9797
9798 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9799 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9800 data |= LC_DYN_LANES_PWR_STATE(3);
9801 if (orig != data)
9802 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9803
9804 if (!disable_clkreq &&
9805 !pci_is_root_bus(rdev->pdev->bus)) {
9806 struct pci_dev *root = rdev->pdev->bus->self;
9807 u32 lnkcap;
9808
9809 clk_req_support = false;
9810 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9811 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9812 clk_req_support = true;
9813 } else {
9814 clk_req_support = false;
9815 }
9816
9817 if (clk_req_support) {
9818 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9819 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9820 if (orig != data)
9821 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9822
9823 orig = data = RREG32_SMC(THM_CLK_CNTL);
9824 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9825 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9826 if (orig != data)
9827 WREG32_SMC(THM_CLK_CNTL, data);
9828
9829 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9830 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9831 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9832 if (orig != data)
9833 WREG32_SMC(MISC_CLK_CTRL, data);
9834
9835 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9836 data &= ~BCLK_AS_XCLK;
9837 if (orig != data)
9838 WREG32_SMC(CG_CLKPIN_CNTL, data);
9839
9840 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9841 data &= ~FORCE_BIF_REFCLK_EN;
9842 if (orig != data)
9843 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9844
9845 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9846 data &= ~MPLL_CLKOUT_SEL_MASK;
9847 data |= MPLL_CLKOUT_SEL(4);
9848 if (orig != data)
9849 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9850 }
9851 }
9852 } else {
9853 if (orig != data)
9854 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9855 }
9856
9857 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9858 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9859 if (orig != data)
9860 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9861
9862 if (!disable_l0s) {
9863 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9864 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9865 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9866 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9867 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9868 data &= ~LC_L0S_INACTIVITY_MASK;
9869 if (orig != data)
9870 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9871 }
9872 }
9873 }
9874 }
Linux kernel - Deliverables
This page took 0.285694 seconds and 5 git commands to generate.