projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: drop CP page table updates & cleanup v2
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
02c81327
AD		 44MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
45MODULE_FIRMWARE("radeon/KAVERI_me.bin");
46MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
47MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
48MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 49MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
02c81327
AD		 50MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
51MODULE_FIRMWARE("radeon/KABINI_me.bin");
52MODULE_FIRMWARE("radeon/KABINI_ce.bin");
53MODULE_FIRMWARE("radeon/KABINI_mec.bin");
54MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 55MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
02c81327 56
a59781bb
AD		 57extern int r600_ih_ring_alloc(struct radeon_device *rdev);
58extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 59extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
60extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 61extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 62extern void sumo_rlc_fini(struct radeon_device *rdev);
63extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 64extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 65extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 66extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
2483b4ea
CK		 67extern int cik_sdma_resume(struct radeon_device *rdev);
68extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
69extern void cik_sdma_fini(struct radeon_device *rdev);
cc066715 70static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 71static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 72static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 73static void cik_init_pg(struct radeon_device *rdev);
74static void cik_init_cg(struct radeon_device *rdev);
4214faf6
AD		 75static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
76 bool enable);
6f2043ce 77
286d9cc6
AD		 78/* get temperature in millidegrees */
79int ci_get_temp(struct radeon_device *rdev)
80{
81 u32 temp;
82 int actual_temp = 0;
83
84 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
85 CTF_TEMP_SHIFT;
86
87 if (temp & 0x200)
88 actual_temp = 255;
89 else
90 actual_temp = temp & 0x1ff;
91
92 actual_temp = actual_temp * 1000;
93
94 return actual_temp;
95}
96
97/* get temperature in millidegrees */
98int kv_get_temp(struct radeon_device *rdev)
99{
100 u32 temp;
101 int actual_temp = 0;
102
103 temp = RREG32_SMC(0xC0300E0C);
104
105 if (temp)
106 actual_temp = (temp / 8) - 49;
107 else
108 actual_temp = 0;
109
110 actual_temp = actual_temp * 1000;
111
112 return actual_temp;
113}
6f2043ce 114
6e2c3c0a
AD		 115/*
116 * Indirect registers accessor
117 */
118u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
119{
0a5b7b0b 120 unsigned long flags;
6e2c3c0a
AD		 121 u32 r;
122
0a5b7b0b 123 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 124 WREG32(PCIE_INDEX, reg);
125 (void)RREG32(PCIE_INDEX);
126 r = RREG32(PCIE_DATA);
0a5b7b0b 127 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 128 return r;
129}
130
131void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
132{
0a5b7b0b
AD		 133 unsigned long flags;
134
135 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 136 WREG32(PCIE_INDEX, reg);
137 (void)RREG32(PCIE_INDEX);
138 WREG32(PCIE_DATA, v);
139 (void)RREG32(PCIE_DATA);
0a5b7b0b 140 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 141}
142
22c775ce
AD		 143static const u32 spectre_rlc_save_restore_register_list[] =
144{
145 (0x0e00 << 16) | (0xc12c >> 2),
146 0x00000000,
147 (0x0e00 << 16) | (0xc140 >> 2),
148 0x00000000,
149 (0x0e00 << 16) | (0xc150 >> 2),
150 0x00000000,
151 (0x0e00 << 16) | (0xc15c >> 2),
152 0x00000000,
153 (0x0e00 << 16) | (0xc168 >> 2),
154 0x00000000,
155 (0x0e00 << 16) | (0xc170 >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc178 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc204 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc2b4 >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc2b8 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc2bc >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc2c0 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0x8228 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0x829c >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0x869c >> 2),
174 0x00000000,
175 (0x0600 << 16) | (0x98f4 >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0x98f8 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x9900 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0xc260 >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x90e8 >> 2),
184 0x00000000,
185 (0x0e00 << 16) | (0x3c000 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x3c00c >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x8c1c >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0x9700 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0xcd20 >> 2),
194 0x00000000,
195 (0x4e00 << 16) | (0xcd20 >> 2),
196 0x00000000,
197 (0x5e00 << 16) | (0xcd20 >> 2),
198 0x00000000,
199 (0x6e00 << 16) | (0xcd20 >> 2),
200 0x00000000,
201 (0x7e00 << 16) | (0xcd20 >> 2),
202 0x00000000,
203 (0x8e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x9e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0xae00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0xbe00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x0e00 << 16) | (0x89bc >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0x8900 >> 2),
214 0x00000000,
215 0x3,
216 (0x0e00 << 16) | (0xc130 >> 2),
217 0x00000000,
218 (0x0e00 << 16) | (0xc134 >> 2),
219 0x00000000,
220 (0x0e00 << 16) | (0xc1fc >> 2),
221 0x00000000,
222 (0x0e00 << 16) | (0xc208 >> 2),
223 0x00000000,
224 (0x0e00 << 16) | (0xc264 >> 2),
225 0x00000000,
226 (0x0e00 << 16) | (0xc268 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc26c >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc270 >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc274 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc278 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc27c >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc280 >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc284 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc288 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc28c >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc290 >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc294 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc298 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc29c >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc2a0 >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc2a4 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc2a8 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc2ac >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc2b0 >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0x301d0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0x30238 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0x30250 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0x30254 >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0x30258 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x3025c >> 2),
275 0x00000000,
276 (0x4e00 << 16) | (0xc900 >> 2),
277 0x00000000,
278 (0x5e00 << 16) | (0xc900 >> 2),
279 0x00000000,
280 (0x6e00 << 16) | (0xc900 >> 2),
281 0x00000000,
282 (0x7e00 << 16) | (0xc900 >> 2),
283 0x00000000,
284 (0x8e00 << 16) | (0xc900 >> 2),
285 0x00000000,
286 (0x9e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0xae00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0xbe00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x4e00 << 16) | (0xc904 >> 2),
293 0x00000000,
294 (0x5e00 << 16) | (0xc904 >> 2),
295 0x00000000,
296 (0x6e00 << 16) | (0xc904 >> 2),
297 0x00000000,
298 (0x7e00 << 16) | (0xc904 >> 2),
299 0x00000000,
300 (0x8e00 << 16) | (0xc904 >> 2),
301 0x00000000,
302 (0x9e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0xae00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0xbe00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x4e00 << 16) | (0xc908 >> 2),
309 0x00000000,
310 (0x5e00 << 16) | (0xc908 >> 2),
311 0x00000000,
312 (0x6e00 << 16) | (0xc908 >> 2),
313 0x00000000,
314 (0x7e00 << 16) | (0xc908 >> 2),
315 0x00000000,
316 (0x8e00 << 16) | (0xc908 >> 2),
317 0x00000000,
318 (0x9e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0xae00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0xbe00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x4e00 << 16) | (0xc90c >> 2),
325 0x00000000,
326 (0x5e00 << 16) | (0xc90c >> 2),
327 0x00000000,
328 (0x6e00 << 16) | (0xc90c >> 2),
329 0x00000000,
330 (0x7e00 << 16) | (0xc90c >> 2),
331 0x00000000,
332 (0x8e00 << 16) | (0xc90c >> 2),
333 0x00000000,
334 (0x9e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0xae00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0xbe00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x4e00 << 16) | (0xc910 >> 2),
341 0x00000000,
342 (0x5e00 << 16) | (0xc910 >> 2),
343 0x00000000,
344 (0x6e00 << 16) | (0xc910 >> 2),
345 0x00000000,
346 (0x7e00 << 16) | (0xc910 >> 2),
347 0x00000000,
348 (0x8e00 << 16) | (0xc910 >> 2),
349 0x00000000,
350 (0x9e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0xae00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0xbe00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x0e00 << 16) | (0xc99c >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0x9834 >> 2),
359 0x00000000,
360 (0x0000 << 16) | (0x30f00 >> 2),
361 0x00000000,
362 (0x0001 << 16) | (0x30f00 >> 2),
363 0x00000000,
364 (0x0000 << 16) | (0x30f04 >> 2),
365 0x00000000,
366 (0x0001 << 16) | (0x30f04 >> 2),
367 0x00000000,
368 (0x0000 << 16) | (0x30f08 >> 2),
369 0x00000000,
370 (0x0001 << 16) | (0x30f08 >> 2),
371 0x00000000,
372 (0x0000 << 16) | (0x30f0c >> 2),
373 0x00000000,
374 (0x0001 << 16) | (0x30f0c >> 2),
375 0x00000000,
376 (0x0600 << 16) | (0x9b7c >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0x8a14 >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0x8a18 >> 2),
381 0x00000000,
382 (0x0600 << 16) | (0x30a00 >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0x8bf0 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0x8bcc >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8b24 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x30a04 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a10 >> 2),
393 0x00000000,
394 (0x0600 << 16) | (0x30a14 >> 2),
395 0x00000000,
396 (0x0600 << 16) | (0x30a18 >> 2),
397 0x00000000,
398 (0x0600 << 16) | (0x30a2c >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0xc700 >> 2),
401 0x00000000,
402 (0x0e00 << 16) | (0xc704 >> 2),
403 0x00000000,
404 (0x0e00 << 16) | (0xc708 >> 2),
405 0x00000000,
406 (0x0e00 << 16) | (0xc768 >> 2),
407 0x00000000,
408 (0x0400 << 16) | (0xc770 >> 2),
409 0x00000000,
410 (0x0400 << 16) | (0xc774 >> 2),
411 0x00000000,
412 (0x0400 << 16) | (0xc778 >> 2),
413 0x00000000,
414 (0x0400 << 16) | (0xc77c >> 2),
415 0x00000000,
416 (0x0400 << 16) | (0xc780 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc784 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc788 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc78c >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc798 >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc79c >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc7a0 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc7a4 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc7a8 >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc7ac >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc7b0 >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7b4 >> 2),
439 0x00000000,
440 (0x0e00 << 16) | (0x9100 >> 2),
441 0x00000000,
442 (0x0e00 << 16) | (0x3c010 >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x92a8 >> 2),
445 0x00000000,
446 (0x0e00 << 16) | (0x92ac >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0x92b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x92b8 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x92bc >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92c0 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92c4 >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92c8 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92cc >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92d0 >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x8c00 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x8c04 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x8c20 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x8c38 >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x8c3c >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0xae00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x9604 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0xac08 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0xac0c >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0xac10 >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xac14 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0xac58 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac68 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac6c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac70 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac74 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac78 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac7c >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac80 >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac84 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac88 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac8c >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x970c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x9714 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x9718 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x971c >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x31068 >> 2),
517 0x00000000,
518 (0x4e00 << 16) | (0x31068 >> 2),
519 0x00000000,
520 (0x5e00 << 16) | (0x31068 >> 2),
521 0x00000000,
522 (0x6e00 << 16) | (0x31068 >> 2),
523 0x00000000,
524 (0x7e00 << 16) | (0x31068 >> 2),
525 0x00000000,
526 (0x8e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x9e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0xae00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0xbe00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0xcd10 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0xcd14 >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0x88b0 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0x88b4 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0x88b8 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0x88bc >> 2),
545 0x00000000,
546 (0x0400 << 16) | (0x89c0 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88c4 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88c8 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88d0 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88d4 >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0x88d8 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x8980 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x30938 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x3093c >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x30940 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x89a0 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x30900 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30904 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x89b4 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x3c210 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x3c214 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x3c218 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x8904 >> 2),
581 0x00000000,
582 0x5,
583 (0x0e00 << 16) | (0x8c28 >> 2),
584 (0x0e00 << 16) | (0x8c2c >> 2),
585 (0x0e00 << 16) | (0x8c30 >> 2),
586 (0x0e00 << 16) | (0x8c34 >> 2),
587 (0x0e00 << 16) | (0x9600 >> 2),
588};
589
590static const u32 kalindi_rlc_save_restore_register_list[] =
591{
592 (0x0e00 << 16) | (0xc12c >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0xc140 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0xc150 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xc15c >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xc168 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xc170 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc204 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc2b4 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc2b8 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc2bc >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc2c0 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x8228 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x829c >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x869c >> 2),
619 0x00000000,
620 (0x0600 << 16) | (0x98f4 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0x98f8 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x9900 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xc260 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x90e8 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x3c000 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x3c00c >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x8c1c >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x9700 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0xcd20 >> 2),
639 0x00000000,
640 (0x4e00 << 16) | (0xcd20 >> 2),
641 0x00000000,
642 (0x5e00 << 16) | (0xcd20 >> 2),
643 0x00000000,
644 (0x6e00 << 16) | (0xcd20 >> 2),
645 0x00000000,
646 (0x7e00 << 16) | (0xcd20 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0x89bc >> 2),
649 0x00000000,
650 (0x0e00 << 16) | (0x8900 >> 2),
651 0x00000000,
652 0x3,
653 (0x0e00 << 16) | (0xc130 >> 2),
654 0x00000000,
655 (0x0e00 << 16) | (0xc134 >> 2),
656 0x00000000,
657 (0x0e00 << 16) | (0xc1fc >> 2),
658 0x00000000,
659 (0x0e00 << 16) | (0xc208 >> 2),
660 0x00000000,
661 (0x0e00 << 16) | (0xc264 >> 2),
662 0x00000000,
663 (0x0e00 << 16) | (0xc268 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc26c >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc270 >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc274 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc28c >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc290 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc294 >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc298 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc2a0 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc2a4 >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc2a8 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc2ac >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0x301d0 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0x30238 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0x30250 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0x30254 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0x30258 >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x3025c >> 2),
698 0x00000000,
699 (0x4e00 << 16) | (0xc900 >> 2),
700 0x00000000,
701 (0x5e00 << 16) | (0xc900 >> 2),
702 0x00000000,
703 (0x6e00 << 16) | (0xc900 >> 2),
704 0x00000000,
705 (0x7e00 << 16) | (0xc900 >> 2),
706 0x00000000,
707 (0x4e00 << 16) | (0xc904 >> 2),
708 0x00000000,
709 (0x5e00 << 16) | (0xc904 >> 2),
710 0x00000000,
711 (0x6e00 << 16) | (0xc904 >> 2),
712 0x00000000,
713 (0x7e00 << 16) | (0xc904 >> 2),
714 0x00000000,
715 (0x4e00 << 16) | (0xc908 >> 2),
716 0x00000000,
717 (0x5e00 << 16) | (0xc908 >> 2),
718 0x00000000,
719 (0x6e00 << 16) | (0xc908 >> 2),
720 0x00000000,
721 (0x7e00 << 16) | (0xc908 >> 2),
722 0x00000000,
723 (0x4e00 << 16) | (0xc90c >> 2),
724 0x00000000,
725 (0x5e00 << 16) | (0xc90c >> 2),
726 0x00000000,
727 (0x6e00 << 16) | (0xc90c >> 2),
728 0x00000000,
729 (0x7e00 << 16) | (0xc90c >> 2),
730 0x00000000,
731 (0x4e00 << 16) | (0xc910 >> 2),
732 0x00000000,
733 (0x5e00 << 16) | (0xc910 >> 2),
734 0x00000000,
735 (0x6e00 << 16) | (0xc910 >> 2),
736 0x00000000,
737 (0x7e00 << 16) | (0xc910 >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0xc99c >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0x9834 >> 2),
742 0x00000000,
743 (0x0000 << 16) | (0x30f00 >> 2),
744 0x00000000,
745 (0x0000 << 16) | (0x30f04 >> 2),
746 0x00000000,
747 (0x0000 << 16) | (0x30f08 >> 2),
748 0x00000000,
749 (0x0000 << 16) | (0x30f0c >> 2),
750 0x00000000,
751 (0x0600 << 16) | (0x9b7c >> 2),
752 0x00000000,
753 (0x0e00 << 16) | (0x8a14 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x8a18 >> 2),
756 0x00000000,
757 (0x0600 << 16) | (0x30a00 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x8bf0 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x8bcc >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8b24 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x30a04 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a10 >> 2),
768 0x00000000,
769 (0x0600 << 16) | (0x30a14 >> 2),
770 0x00000000,
771 (0x0600 << 16) | (0x30a18 >> 2),
772 0x00000000,
773 (0x0600 << 16) | (0x30a2c >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0xc700 >> 2),
776 0x00000000,
777 (0x0e00 << 16) | (0xc704 >> 2),
778 0x00000000,
779 (0x0e00 << 16) | (0xc708 >> 2),
780 0x00000000,
781 (0x0e00 << 16) | (0xc768 >> 2),
782 0x00000000,
783 (0x0400 << 16) | (0xc770 >> 2),
784 0x00000000,
785 (0x0400 << 16) | (0xc774 >> 2),
786 0x00000000,
787 (0x0400 << 16) | (0xc798 >> 2),
788 0x00000000,
789 (0x0400 << 16) | (0xc79c >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0x9100 >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0x3c010 >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0x8c00 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0x8c04 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0x8c20 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x8c38 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x8c3c >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xae00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x9604 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xac08 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xac0c >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0xac10 >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xac14 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0xac58 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac68 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac6c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac70 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac74 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac78 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac7c >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac80 >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac84 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac88 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac8c >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0x970c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0x9714 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0x9718 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0x971c >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0x31068 >> 2),
848 0x00000000,
849 (0x4e00 << 16) | (0x31068 >> 2),
850 0x00000000,
851 (0x5e00 << 16) | (0x31068 >> 2),
852 0x00000000,
853 (0x6e00 << 16) | (0x31068 >> 2),
854 0x00000000,
855 (0x7e00 << 16) | (0x31068 >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0xcd10 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0xcd14 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x88b0 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x88b4 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x88b8 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x88bc >> 2),
868 0x00000000,
869 (0x0400 << 16) | (0x89c0 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88c4 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88c8 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88d0 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88d4 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0x88d8 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x8980 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x30938 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x3093c >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x30940 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x89a0 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x30900 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30904 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x89b4 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x3e1fc >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x3c210 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x3c214 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x3c218 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x8904 >> 2),
906 0x00000000,
907 0x5,
908 (0x0e00 << 16) | (0x8c28 >> 2),
909 (0x0e00 << 16) | (0x8c2c >> 2),
910 (0x0e00 << 16) | (0x8c30 >> 2),
911 (0x0e00 << 16) | (0x8c34 >> 2),
912 (0x0e00 << 16) | (0x9600 >> 2),
913};
914
0aafd313
AD		 915static const u32 bonaire_golden_spm_registers[] =
916{
917 0x30800, 0xe0ffffff, 0xe0000000
918};
919
920static const u32 bonaire_golden_common_registers[] =
921{
922 0xc770, 0xffffffff, 0x00000800,
923 0xc774, 0xffffffff, 0x00000800,
924 0xc798, 0xffffffff, 0x00007fbf,
925 0xc79c, 0xffffffff, 0x00007faf
926};
927
928static const u32 bonaire_golden_registers[] =
929{
930 0x3354, 0x00000333, 0x00000333,
931 0x3350, 0x000c0fc0, 0x00040200,
932 0x9a10, 0x00010000, 0x00058208,
933 0x3c000, 0xffff1fff, 0x00140000,
934 0x3c200, 0xfdfc0fff, 0x00000100,
935 0x3c234, 0x40000000, 0x40000200,
936 0x9830, 0xffffffff, 0x00000000,
937 0x9834, 0xf00fffff, 0x00000400,
938 0x9838, 0x0002021c, 0x00020200,
939 0xc78, 0x00000080, 0x00000000,
940 0x5bb0, 0x000000f0, 0x00000070,
941 0x5bc0, 0xf0311fff, 0x80300000,
942 0x98f8, 0x73773777, 0x12010001,
943 0x350c, 0x00810000, 0x408af000,
944 0x7030, 0x31000111, 0x00000011,
945 0x2f48, 0x73773777, 0x12010001,
946 0x220c, 0x00007fb6, 0x0021a1b1,
947 0x2210, 0x00007fb6, 0x002021b1,
948 0x2180, 0x00007fb6, 0x00002191,
949 0x2218, 0x00007fb6, 0x002121b1,
950 0x221c, 0x00007fb6, 0x002021b1,
951 0x21dc, 0x00007fb6, 0x00002191,
952 0x21e0, 0x00007fb6, 0x00002191,
953 0x3628, 0x0000003f, 0x0000000a,
954 0x362c, 0x0000003f, 0x0000000a,
955 0x2ae4, 0x00073ffe, 0x000022a2,
956 0x240c, 0x000007ff, 0x00000000,
957 0x8a14, 0xf000003f, 0x00000007,
958 0x8bf0, 0x00002001, 0x00000001,
959 0x8b24, 0xffffffff, 0x00ffffff,
960 0x30a04, 0x0000ff0f, 0x00000000,
961 0x28a4c, 0x07ffffff, 0x06000000,
962 0x4d8, 0x00000fff, 0x00000100,
963 0x3e78, 0x00000001, 0x00000002,
964 0x9100, 0x03000000, 0x0362c688,
965 0x8c00, 0x000000ff, 0x00000001,
966 0xe40, 0x00001fff, 0x00001fff,
967 0x9060, 0x0000007f, 0x00000020,
968 0x9508, 0x00010000, 0x00010000,
969 0xac14, 0x000003ff, 0x000000f3,
970 0xac0c, 0xffffffff, 0x00001032
971};
972
973static const u32 bonaire_mgcg_cgcg_init[] =
974{
975 0xc420, 0xffffffff, 0xfffffffc,
976 0x30800, 0xffffffff, 0xe0000000,
977 0x3c2a0, 0xffffffff, 0x00000100,
978 0x3c208, 0xffffffff, 0x00000100,
979 0x3c2c0, 0xffffffff, 0xc0000100,
980 0x3c2c8, 0xffffffff, 0xc0000100,
981 0x3c2c4, 0xffffffff, 0xc0000100,
982 0x55e4, 0xffffffff, 0x00600100,
983 0x3c280, 0xffffffff, 0x00000100,
984 0x3c214, 0xffffffff, 0x06000100,
985 0x3c220, 0xffffffff, 0x00000100,
986 0x3c218, 0xffffffff, 0x06000100,
987 0x3c204, 0xffffffff, 0x00000100,
988 0x3c2e0, 0xffffffff, 0x00000100,
989 0x3c224, 0xffffffff, 0x00000100,
990 0x3c200, 0xffffffff, 0x00000100,
991 0x3c230, 0xffffffff, 0x00000100,
992 0x3c234, 0xffffffff, 0x00000100,
993 0x3c250, 0xffffffff, 0x00000100,
994 0x3c254, 0xffffffff, 0x00000100,
995 0x3c258, 0xffffffff, 0x00000100,
996 0x3c25c, 0xffffffff, 0x00000100,
997 0x3c260, 0xffffffff, 0x00000100,
998 0x3c27c, 0xffffffff, 0x00000100,
999 0x3c278, 0xffffffff, 0x00000100,
1000 0x3c210, 0xffffffff, 0x06000100,
1001 0x3c290, 0xffffffff, 0x00000100,
1002 0x3c274, 0xffffffff, 0x00000100,
1003 0x3c2b4, 0xffffffff, 0x00000100,
1004 0x3c2b0, 0xffffffff, 0x00000100,
1005 0x3c270, 0xffffffff, 0x00000100,
1006 0x30800, 0xffffffff, 0xe0000000,
1007 0x3c020, 0xffffffff, 0x00010000,
1008 0x3c024, 0xffffffff, 0x00030002,
1009 0x3c028, 0xffffffff, 0x00040007,
1010 0x3c02c, 0xffffffff, 0x00060005,
1011 0x3c030, 0xffffffff, 0x00090008,
1012 0x3c034, 0xffffffff, 0x00010000,
1013 0x3c038, 0xffffffff, 0x00030002,
1014 0x3c03c, 0xffffffff, 0x00040007,
1015 0x3c040, 0xffffffff, 0x00060005,
1016 0x3c044, 0xffffffff, 0x00090008,
1017 0x3c048, 0xffffffff, 0x00010000,
1018 0x3c04c, 0xffffffff, 0x00030002,
1019 0x3c050, 0xffffffff, 0x00040007,
1020 0x3c054, 0xffffffff, 0x00060005,
1021 0x3c058, 0xffffffff, 0x00090008,
1022 0x3c05c, 0xffffffff, 0x00010000,
1023 0x3c060, 0xffffffff, 0x00030002,
1024 0x3c064, 0xffffffff, 0x00040007,
1025 0x3c068, 0xffffffff, 0x00060005,
1026 0x3c06c, 0xffffffff, 0x00090008,
1027 0x3c070, 0xffffffff, 0x00010000,
1028 0x3c074, 0xffffffff, 0x00030002,
1029 0x3c078, 0xffffffff, 0x00040007,
1030 0x3c07c, 0xffffffff, 0x00060005,
1031 0x3c080, 0xffffffff, 0x00090008,
1032 0x3c084, 0xffffffff, 0x00010000,
1033 0x3c088, 0xffffffff, 0x00030002,
1034 0x3c08c, 0xffffffff, 0x00040007,
1035 0x3c090, 0xffffffff, 0x00060005,
1036 0x3c094, 0xffffffff, 0x00090008,
1037 0x3c098, 0xffffffff, 0x00010000,
1038 0x3c09c, 0xffffffff, 0x00030002,
1039 0x3c0a0, 0xffffffff, 0x00040007,
1040 0x3c0a4, 0xffffffff, 0x00060005,
1041 0x3c0a8, 0xffffffff, 0x00090008,
1042 0x3c000, 0xffffffff, 0x96e00200,
1043 0x8708, 0xffffffff, 0x00900100,
1044 0xc424, 0xffffffff, 0x0020003f,
1045 0x38, 0xffffffff, 0x0140001c,
1046 0x3c, 0x000f0000, 0x000f0000,
1047 0x220, 0xffffffff, 0xC060000C,
1048 0x224, 0xc0000fff, 0x00000100,
1049 0xf90, 0xffffffff, 0x00000100,
1050 0xf98, 0x00000101, 0x00000000,
1051 0x20a8, 0xffffffff, 0x00000104,
1052 0x55e4, 0xff000fff, 0x00000100,
1053 0x30cc, 0xc0000fff, 0x00000104,
1054 0xc1e4, 0x00000001, 0x00000001,
1055 0xd00c, 0xff000ff0, 0x00000100,
1056 0xd80c, 0xff000ff0, 0x00000100
1057};
1058
1059static const u32 spectre_golden_spm_registers[] =
1060{
1061 0x30800, 0xe0ffffff, 0xe0000000
1062};
1063
1064static const u32 spectre_golden_common_registers[] =
1065{
1066 0xc770, 0xffffffff, 0x00000800,
1067 0xc774, 0xffffffff, 0x00000800,
1068 0xc798, 0xffffffff, 0x00007fbf,
1069 0xc79c, 0xffffffff, 0x00007faf
1070};
1071
1072static const u32 spectre_golden_registers[] =
1073{
1074 0x3c000, 0xffff1fff, 0x96940200,
1075 0x3c00c, 0xffff0001, 0xff000000,
1076 0x3c200, 0xfffc0fff, 0x00000100,
1077 0x6ed8, 0x00010101, 0x00010000,
1078 0x9834, 0xf00fffff, 0x00000400,
1079 0x9838, 0xfffffffc, 0x00020200,
1080 0x5bb0, 0x000000f0, 0x00000070,
1081 0x5bc0, 0xf0311fff, 0x80300000,
1082 0x98f8, 0x73773777, 0x12010001,
1083 0x9b7c, 0x00ff0000, 0x00fc0000,
1084 0x2f48, 0x73773777, 0x12010001,
1085 0x8a14, 0xf000003f, 0x00000007,
1086 0x8b24, 0xffffffff, 0x00ffffff,
1087 0x28350, 0x3f3f3fff, 0x00000082,
1088 0x28355, 0x0000003f, 0x00000000,
1089 0x3e78, 0x00000001, 0x00000002,
1090 0x913c, 0xffff03df, 0x00000004,
1091 0xc768, 0x00000008, 0x00000008,
1092 0x8c00, 0x000008ff, 0x00000800,
1093 0x9508, 0x00010000, 0x00010000,
1094 0xac0c, 0xffffffff, 0x54763210,
1095 0x214f8, 0x01ff01ff, 0x00000002,
1096 0x21498, 0x007ff800, 0x00200000,
1097 0x2015c, 0xffffffff, 0x00000f40,
1098 0x30934, 0xffffffff, 0x00000001
1099};
1100
1101static const u32 spectre_mgcg_cgcg_init[] =
1102{
1103 0xc420, 0xffffffff, 0xfffffffc,
1104 0x30800, 0xffffffff, 0xe0000000,
1105 0x3c2a0, 0xffffffff, 0x00000100,
1106 0x3c208, 0xffffffff, 0x00000100,
1107 0x3c2c0, 0xffffffff, 0x00000100,
1108 0x3c2c8, 0xffffffff, 0x00000100,
1109 0x3c2c4, 0xffffffff, 0x00000100,
1110 0x55e4, 0xffffffff, 0x00600100,
1111 0x3c280, 0xffffffff, 0x00000100,
1112 0x3c214, 0xffffffff, 0x06000100,
1113 0x3c220, 0xffffffff, 0x00000100,
1114 0x3c218, 0xffffffff, 0x06000100,
1115 0x3c204, 0xffffffff, 0x00000100,
1116 0x3c2e0, 0xffffffff, 0x00000100,
1117 0x3c224, 0xffffffff, 0x00000100,
1118 0x3c200, 0xffffffff, 0x00000100,
1119 0x3c230, 0xffffffff, 0x00000100,
1120 0x3c234, 0xffffffff, 0x00000100,
1121 0x3c250, 0xffffffff, 0x00000100,
1122 0x3c254, 0xffffffff, 0x00000100,
1123 0x3c258, 0xffffffff, 0x00000100,
1124 0x3c25c, 0xffffffff, 0x00000100,
1125 0x3c260, 0xffffffff, 0x00000100,
1126 0x3c27c, 0xffffffff, 0x00000100,
1127 0x3c278, 0xffffffff, 0x00000100,
1128 0x3c210, 0xffffffff, 0x06000100,
1129 0x3c290, 0xffffffff, 0x00000100,
1130 0x3c274, 0xffffffff, 0x00000100,
1131 0x3c2b4, 0xffffffff, 0x00000100,
1132 0x3c2b0, 0xffffffff, 0x00000100,
1133 0x3c270, 0xffffffff, 0x00000100,
1134 0x30800, 0xffffffff, 0xe0000000,
1135 0x3c020, 0xffffffff, 0x00010000,
1136 0x3c024, 0xffffffff, 0x00030002,
1137 0x3c028, 0xffffffff, 0x00040007,
1138 0x3c02c, 0xffffffff, 0x00060005,
1139 0x3c030, 0xffffffff, 0x00090008,
1140 0x3c034, 0xffffffff, 0x00010000,
1141 0x3c038, 0xffffffff, 0x00030002,
1142 0x3c03c, 0xffffffff, 0x00040007,
1143 0x3c040, 0xffffffff, 0x00060005,
1144 0x3c044, 0xffffffff, 0x00090008,
1145 0x3c048, 0xffffffff, 0x00010000,
1146 0x3c04c, 0xffffffff, 0x00030002,
1147 0x3c050, 0xffffffff, 0x00040007,
1148 0x3c054, 0xffffffff, 0x00060005,
1149 0x3c058, 0xffffffff, 0x00090008,
1150 0x3c05c, 0xffffffff, 0x00010000,
1151 0x3c060, 0xffffffff, 0x00030002,
1152 0x3c064, 0xffffffff, 0x00040007,
1153 0x3c068, 0xffffffff, 0x00060005,
1154 0x3c06c, 0xffffffff, 0x00090008,
1155 0x3c070, 0xffffffff, 0x00010000,
1156 0x3c074, 0xffffffff, 0x00030002,
1157 0x3c078, 0xffffffff, 0x00040007,
1158 0x3c07c, 0xffffffff, 0x00060005,
1159 0x3c080, 0xffffffff, 0x00090008,
1160 0x3c084, 0xffffffff, 0x00010000,
1161 0x3c088, 0xffffffff, 0x00030002,
1162 0x3c08c, 0xffffffff, 0x00040007,
1163 0x3c090, 0xffffffff, 0x00060005,
1164 0x3c094, 0xffffffff, 0x00090008,
1165 0x3c098, 0xffffffff, 0x00010000,
1166 0x3c09c, 0xffffffff, 0x00030002,
1167 0x3c0a0, 0xffffffff, 0x00040007,
1168 0x3c0a4, 0xffffffff, 0x00060005,
1169 0x3c0a8, 0xffffffff, 0x00090008,
1170 0x3c0ac, 0xffffffff, 0x00010000,
1171 0x3c0b0, 0xffffffff, 0x00030002,
1172 0x3c0b4, 0xffffffff, 0x00040007,
1173 0x3c0b8, 0xffffffff, 0x00060005,
1174 0x3c0bc, 0xffffffff, 0x00090008,
1175 0x3c000, 0xffffffff, 0x96e00200,
1176 0x8708, 0xffffffff, 0x00900100,
1177 0xc424, 0xffffffff, 0x0020003f,
1178 0x38, 0xffffffff, 0x0140001c,
1179 0x3c, 0x000f0000, 0x000f0000,
1180 0x220, 0xffffffff, 0xC060000C,
1181 0x224, 0xc0000fff, 0x00000100,
1182 0xf90, 0xffffffff, 0x00000100,
1183 0xf98, 0x00000101, 0x00000000,
1184 0x20a8, 0xffffffff, 0x00000104,
1185 0x55e4, 0xff000fff, 0x00000100,
1186 0x30cc, 0xc0000fff, 0x00000104,
1187 0xc1e4, 0x00000001, 0x00000001,
1188 0xd00c, 0xff000ff0, 0x00000100,
1189 0xd80c, 0xff000ff0, 0x00000100
1190};
1191
1192static const u32 kalindi_golden_spm_registers[] =
1193{
1194 0x30800, 0xe0ffffff, 0xe0000000
1195};
1196
1197static const u32 kalindi_golden_common_registers[] =
1198{
1199 0xc770, 0xffffffff, 0x00000800,
1200 0xc774, 0xffffffff, 0x00000800,
1201 0xc798, 0xffffffff, 0x00007fbf,
1202 0xc79c, 0xffffffff, 0x00007faf
1203};
1204
1205static const u32 kalindi_golden_registers[] =
1206{
1207 0x3c000, 0xffffdfff, 0x6e944040,
1208 0x55e4, 0xff607fff, 0xfc000100,
1209 0x3c220, 0xff000fff, 0x00000100,
1210 0x3c224, 0xff000fff, 0x00000100,
1211 0x3c200, 0xfffc0fff, 0x00000100,
1212 0x6ed8, 0x00010101, 0x00010000,
1213 0x9830, 0xffffffff, 0x00000000,
1214 0x9834, 0xf00fffff, 0x00000400,
1215 0x5bb0, 0x000000f0, 0x00000070,
1216 0x5bc0, 0xf0311fff, 0x80300000,
1217 0x98f8, 0x73773777, 0x12010001,
1218 0x98fc, 0xffffffff, 0x00000010,
1219 0x9b7c, 0x00ff0000, 0x00fc0000,
1220 0x8030, 0x00001f0f, 0x0000100a,
1221 0x2f48, 0x73773777, 0x12010001,
1222 0x2408, 0x000fffff, 0x000c007f,
1223 0x8a14, 0xf000003f, 0x00000007,
1224 0x8b24, 0x3fff3fff, 0x00ffcfff,
1225 0x30a04, 0x0000ff0f, 0x00000000,
1226 0x28a4c, 0x07ffffff, 0x06000000,
1227 0x4d8, 0x00000fff, 0x00000100,
1228 0x3e78, 0x00000001, 0x00000002,
1229 0xc768, 0x00000008, 0x00000008,
1230 0x8c00, 0x000000ff, 0x00000003,
1231 0x214f8, 0x01ff01ff, 0x00000002,
1232 0x21498, 0x007ff800, 0x00200000,
1233 0x2015c, 0xffffffff, 0x00000f40,
1234 0x88c4, 0x001f3ae3, 0x00000082,
1235 0x88d4, 0x0000001f, 0x00000010,
1236 0x30934, 0xffffffff, 0x00000000
1237};
1238
1239static const u32 kalindi_mgcg_cgcg_init[] =
1240{
1241 0xc420, 0xffffffff, 0xfffffffc,
1242 0x30800, 0xffffffff, 0xe0000000,
1243 0x3c2a0, 0xffffffff, 0x00000100,
1244 0x3c208, 0xffffffff, 0x00000100,
1245 0x3c2c0, 0xffffffff, 0x00000100,
1246 0x3c2c8, 0xffffffff, 0x00000100,
1247 0x3c2c4, 0xffffffff, 0x00000100,
1248 0x55e4, 0xffffffff, 0x00600100,
1249 0x3c280, 0xffffffff, 0x00000100,
1250 0x3c214, 0xffffffff, 0x06000100,
1251 0x3c220, 0xffffffff, 0x00000100,
1252 0x3c218, 0xffffffff, 0x06000100,
1253 0x3c204, 0xffffffff, 0x00000100,
1254 0x3c2e0, 0xffffffff, 0x00000100,
1255 0x3c224, 0xffffffff, 0x00000100,
1256 0x3c200, 0xffffffff, 0x00000100,
1257 0x3c230, 0xffffffff, 0x00000100,
1258 0x3c234, 0xffffffff, 0x00000100,
1259 0x3c250, 0xffffffff, 0x00000100,
1260 0x3c254, 0xffffffff, 0x00000100,
1261 0x3c258, 0xffffffff, 0x00000100,
1262 0x3c25c, 0xffffffff, 0x00000100,
1263 0x3c260, 0xffffffff, 0x00000100,
1264 0x3c27c, 0xffffffff, 0x00000100,
1265 0x3c278, 0xffffffff, 0x00000100,
1266 0x3c210, 0xffffffff, 0x06000100,
1267 0x3c290, 0xffffffff, 0x00000100,
1268 0x3c274, 0xffffffff, 0x00000100,
1269 0x3c2b4, 0xffffffff, 0x00000100,
1270 0x3c2b0, 0xffffffff, 0x00000100,
1271 0x3c270, 0xffffffff, 0x00000100,
1272 0x30800, 0xffffffff, 0xe0000000,
1273 0x3c020, 0xffffffff, 0x00010000,
1274 0x3c024, 0xffffffff, 0x00030002,
1275 0x3c028, 0xffffffff, 0x00040007,
1276 0x3c02c, 0xffffffff, 0x00060005,
1277 0x3c030, 0xffffffff, 0x00090008,
1278 0x3c034, 0xffffffff, 0x00010000,
1279 0x3c038, 0xffffffff, 0x00030002,
1280 0x3c03c, 0xffffffff, 0x00040007,
1281 0x3c040, 0xffffffff, 0x00060005,
1282 0x3c044, 0xffffffff, 0x00090008,
1283 0x3c000, 0xffffffff, 0x96e00200,
1284 0x8708, 0xffffffff, 0x00900100,
1285 0xc424, 0xffffffff, 0x0020003f,
1286 0x38, 0xffffffff, 0x0140001c,
1287 0x3c, 0x000f0000, 0x000f0000,
1288 0x220, 0xffffffff, 0xC060000C,
1289 0x224, 0xc0000fff, 0x00000100,
1290 0x20a8, 0xffffffff, 0x00000104,
1291 0x55e4, 0xff000fff, 0x00000100,
1292 0x30cc, 0xc0000fff, 0x00000104,
1293 0xc1e4, 0x00000001, 0x00000001,
1294 0xd00c, 0xff000ff0, 0x00000100,
1295 0xd80c, 0xff000ff0, 0x00000100
1296};
1297
1298static void cik_init_golden_registers(struct radeon_device *rdev)
1299{
1300 switch (rdev->family) {
1301 case CHIP_BONAIRE:
1302 radeon_program_register_sequence(rdev,
1303 bonaire_mgcg_cgcg_init,
1304 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1305 radeon_program_register_sequence(rdev,
1306 bonaire_golden_registers,
1307 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1308 radeon_program_register_sequence(rdev,
1309 bonaire_golden_common_registers,
1310 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1311 radeon_program_register_sequence(rdev,
1312 bonaire_golden_spm_registers,
1313 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1314 break;
1315 case CHIP_KABINI:
1316 radeon_program_register_sequence(rdev,
1317 kalindi_mgcg_cgcg_init,
1318 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1319 radeon_program_register_sequence(rdev,
1320 kalindi_golden_registers,
1321 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1322 radeon_program_register_sequence(rdev,
1323 kalindi_golden_common_registers,
1324 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1325 radeon_program_register_sequence(rdev,
1326 kalindi_golden_spm_registers,
1327 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1328 break;
1329 case CHIP_KAVERI:
1330 radeon_program_register_sequence(rdev,
1331 spectre_mgcg_cgcg_init,
1332 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1333 radeon_program_register_sequence(rdev,
1334 spectre_golden_registers,
1335 (const u32)ARRAY_SIZE(spectre_golden_registers));
1336 radeon_program_register_sequence(rdev,
1337 spectre_golden_common_registers,
1338 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1339 radeon_program_register_sequence(rdev,
1340 spectre_golden_spm_registers,
1341 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1342 break;
1343 default:
1344 break;
1345 }
1346}
1347
2c67912c
AD		 1348/**
1349 * cik_get_xclk - get the xclk
1350 *
1351 * @rdev: radeon_device pointer
1352 *
1353 * Returns the reference clock used by the gfx engine
1354 * (CIK).
1355 */
1356u32 cik_get_xclk(struct radeon_device *rdev)
1357{
1358 u32 reference_clock = rdev->clock.spll.reference_freq;
1359
1360 if (rdev->flags & RADEON_IS_IGP) {
1361 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1362 return reference_clock / 2;
1363 } else {
1364 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1365 return reference_clock / 4;
1366 }
1367 return reference_clock;
1368}
1369
75efdee1
AD		 1370/**
1371 * cik_mm_rdoorbell - read a doorbell dword
1372 *
1373 * @rdev: radeon_device pointer
1374 * @offset: byte offset into the aperture
1375 *
1376 * Returns the value in the doorbell aperture at the
1377 * requested offset (CIK).
1378 */
1379u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1380{
1381 if (offset < rdev->doorbell.size) {
1382 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1383 } else {
1384 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1385 return 0;
1386 }
1387}
1388
1389/**
1390 * cik_mm_wdoorbell - write a doorbell dword
1391 *
1392 * @rdev: radeon_device pointer
1393 * @offset: byte offset into the aperture
1394 * @v: value to write
1395 *
1396 * Writes @v to the doorbell aperture at the
1397 * requested offset (CIK).
1398 */
1399void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1400{
1401 if (offset < rdev->doorbell.size) {
1402 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1403 } else {
1404 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1405 }
1406}
1407
bc8273fe
AD		 1408#define BONAIRE_IO_MC_REGS_SIZE 36
1409
1410static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1411{
1412 {0x00000070, 0x04400000},
1413 {0x00000071, 0x80c01803},
1414 {0x00000072, 0x00004004},
1415 {0x00000073, 0x00000100},
1416 {0x00000074, 0x00ff0000},
1417 {0x00000075, 0x34000000},
1418 {0x00000076, 0x08000014},
1419 {0x00000077, 0x00cc08ec},
1420 {0x00000078, 0x00000400},
1421 {0x00000079, 0x00000000},
1422 {0x0000007a, 0x04090000},
1423 {0x0000007c, 0x00000000},
1424 {0x0000007e, 0x4408a8e8},
1425 {0x0000007f, 0x00000304},
1426 {0x00000080, 0x00000000},
1427 {0x00000082, 0x00000001},
1428 {0x00000083, 0x00000002},
1429 {0x00000084, 0xf3e4f400},
1430 {0x00000085, 0x052024e3},
1431 {0x00000087, 0x00000000},
1432 {0x00000088, 0x01000000},
1433 {0x0000008a, 0x1c0a0000},
1434 {0x0000008b, 0xff010000},
1435 {0x0000008d, 0xffffefff},
1436 {0x0000008e, 0xfff3efff},
1437 {0x0000008f, 0xfff3efbf},
1438 {0x00000092, 0xf7ffffff},
1439 {0x00000093, 0xffffff7f},
1440 {0x00000095, 0x00101101},
1441 {0x00000096, 0x00000fff},
1442 {0x00000097, 0x00116fff},
1443 {0x00000098, 0x60010000},
1444 {0x00000099, 0x10010000},
1445 {0x0000009a, 0x00006000},
1446 {0x0000009b, 0x00001000},
1447 {0x0000009f, 0x00b48000}
1448};
1449
b556b12e
AD		 1450/**
1451 * cik_srbm_select - select specific register instances
1452 *
1453 * @rdev: radeon_device pointer
1454 * @me: selected ME (micro engine)
1455 * @pipe: pipe
1456 * @queue: queue
1457 * @vmid: VMID
1458 *
1459 * Switches the currently active registers instances. Some
1460 * registers are instanced per VMID, others are instanced per
1461 * me/pipe/queue combination.
1462 */
1463static void cik_srbm_select(struct radeon_device *rdev,
1464 u32 me, u32 pipe, u32 queue, u32 vmid)
1465{
1466 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1467 MEID(me & 0x3) |
1468 VMID(vmid & 0xf) |
1469 QUEUEID(queue & 0x7));
1470 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1471}
1472
bc8273fe
AD		 1473/* ucode loading */
1474/**
1475 * ci_mc_load_microcode - load MC ucode into the hw
1476 *
1477 * @rdev: radeon_device pointer
1478 *
1479 * Load the GDDR MC ucode into the hw (CIK).
1480 * Returns 0 on success, error on failure.
1481 */
1482static int ci_mc_load_microcode(struct radeon_device *rdev)
1483{
1484 const __be32 *fw_data;
1485 u32 running, blackout = 0;
1486 u32 *io_mc_regs;
1487 int i, ucode_size, regs_size;
1488
1489 if (!rdev->mc_fw)
1490 return -EINVAL;
1491
1492 switch (rdev->family) {
1493 case CHIP_BONAIRE:
1494 default:
1495 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1496 ucode_size = CIK_MC_UCODE_SIZE;
1497 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1498 break;
1499 }
1500
1501 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1502
1503 if (running == 0) {
1504 if (running) {
1505 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1506 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1507 }
1508
1509 /* reset the engine and set to writable */
1510 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1511 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1512
1513 /* load mc io regs */
1514 for (i = 0; i < regs_size; i++) {
1515 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1516 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1517 }
1518 /* load the MC ucode */
1519 fw_data = (const __be32 *)rdev->mc_fw->data;
1520 for (i = 0; i < ucode_size; i++)
1521 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1522
1523 /* put the engine back into the active state */
1524 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1525 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1526 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1527
1528 /* wait for training to complete */
1529 for (i = 0; i < rdev->usec_timeout; i++) {
1530 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1531 break;
1532 udelay(1);
1533 }
1534 for (i = 0; i < rdev->usec_timeout; i++) {
1535 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1536 break;
1537 udelay(1);
1538 }
1539
1540 if (running)
1541 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1542 }
1543
1544 return 0;
1545}
1546
02c81327
AD		 1547/**
1548 * cik_init_microcode - load ucode images from disk
1549 *
1550 * @rdev: radeon_device pointer
1551 *
1552 * Use the firmware interface to load the ucode images into
1553 * the driver (not loaded into hw).
1554 * Returns 0 on success, error on failure.
1555 */
1556static int cik_init_microcode(struct radeon_device *rdev)
1557{
02c81327
AD		 1558 const char *chip_name;
1559 size_t pfp_req_size, me_req_size, ce_req_size,
21a93e13 1560 mec_req_size, rlc_req_size, mc_req_size,
cc8dbbb4 1561 sdma_req_size, smc_req_size;
02c81327
AD		 1562 char fw_name[30];
1563 int err;
1564
1565 DRM_DEBUG("\n");
1566
02c81327
AD		 1567 switch (rdev->family) {
1568 case CHIP_BONAIRE:
1569 chip_name = "BONAIRE";
1570 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1571 me_req_size = CIK_ME_UCODE_SIZE * 4;
1572 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1573 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1574 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1575 mc_req_size = CIK_MC_UCODE_SIZE * 4;
21a93e13 1576 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1577 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
02c81327
AD		 1578 break;
1579 case CHIP_KAVERI:
1580 chip_name = "KAVERI";
1581 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1582 me_req_size = CIK_ME_UCODE_SIZE * 4;
1583 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1584 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1585 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1586 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1587 break;
1588 case CHIP_KABINI:
1589 chip_name = "KABINI";
1590 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1591 me_req_size = CIK_ME_UCODE_SIZE * 4;
1592 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1593 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1594 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1595 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1596 break;
1597 default: BUG();
1598 }
1599
1600 DRM_INFO("Loading %s Microcode\n", chip_name);
1601
1602 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1603 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
02c81327
AD		 1604 if (err)
1605 goto out;
1606 if (rdev->pfp_fw->size != pfp_req_size) {
1607 printk(KERN_ERR
1608 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1609 rdev->pfp_fw->size, fw_name);
1610 err = -EINVAL;
1611 goto out;
1612 }
1613
1614 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1615 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
02c81327
AD		 1616 if (err)
1617 goto out;
1618 if (rdev->me_fw->size != me_req_size) {
1619 printk(KERN_ERR
1620 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1621 rdev->me_fw->size, fw_name);
1622 err = -EINVAL;
1623 }
1624
1625 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1626 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
02c81327
AD		 1627 if (err)
1628 goto out;
1629 if (rdev->ce_fw->size != ce_req_size) {
1630 printk(KERN_ERR
1631 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1632 rdev->ce_fw->size, fw_name);
1633 err = -EINVAL;
1634 }
1635
1636 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
0a168933 1637 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
02c81327
AD		 1638 if (err)
1639 goto out;
1640 if (rdev->mec_fw->size != mec_req_size) {
1641 printk(KERN_ERR
1642 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1643 rdev->mec_fw->size, fw_name);
1644 err = -EINVAL;
1645 }
1646
1647 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
0a168933 1648 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
02c81327
AD		 1649 if (err)
1650 goto out;
1651 if (rdev->rlc_fw->size != rlc_req_size) {
1652 printk(KERN_ERR
1653 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1654 rdev->rlc_fw->size, fw_name);
1655 err = -EINVAL;
1656 }
1657
21a93e13 1658 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
0a168933 1659 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
21a93e13
AD		 1660 if (err)
1661 goto out;
1662 if (rdev->sdma_fw->size != sdma_req_size) {
1663 printk(KERN_ERR
1664 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1665 rdev->sdma_fw->size, fw_name);
1666 err = -EINVAL;
1667 }
1668
cc8dbbb4 1669 /* No SMC, MC ucode on APUs */
02c81327
AD		 1670 if (!(rdev->flags & RADEON_IS_IGP)) {
1671 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1672 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
02c81327
AD		 1673 if (err)
1674 goto out;
1675 if (rdev->mc_fw->size != mc_req_size) {
1676 printk(KERN_ERR
1677 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1678 rdev->mc_fw->size, fw_name);
1679 err = -EINVAL;
1680 }
cc8dbbb4
AD		 1681
1682 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1683 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1684 if (err) {
1685 printk(KERN_ERR
1686 "smc: error loading firmware \"%s\"\n",
1687 fw_name);
1688 release_firmware(rdev->smc_fw);
1689 rdev->smc_fw = NULL;
1690 } else if (rdev->smc_fw->size != smc_req_size) {
1691 printk(KERN_ERR
1692 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1693 rdev->smc_fw->size, fw_name);
1694 err = -EINVAL;
1695 }
02c81327
AD		 1696 }
1697
1698out:
02c81327
AD		 1699 if (err) {
1700 if (err != -EINVAL)
1701 printk(KERN_ERR
1702 "cik_cp: Failed to load firmware \"%s\"\n",
1703 fw_name);
1704 release_firmware(rdev->pfp_fw);
1705 rdev->pfp_fw = NULL;
1706 release_firmware(rdev->me_fw);
1707 rdev->me_fw = NULL;
1708 release_firmware(rdev->ce_fw);
1709 rdev->ce_fw = NULL;
1710 release_firmware(rdev->rlc_fw);
1711 rdev->rlc_fw = NULL;
1712 release_firmware(rdev->mc_fw);
1713 rdev->mc_fw = NULL;
cc8dbbb4
AD		 1714 release_firmware(rdev->smc_fw);
1715 rdev->smc_fw = NULL;
02c81327
AD		 1716 }
1717 return err;
1718}
1719
8cc1a532
AD		 1720/*
1721 * Core functions
1722 */
1723/**
1724 * cik_tiling_mode_table_init - init the hw tiling table
1725 *
1726 * @rdev: radeon_device pointer
1727 *
1728 * Starting with SI, the tiling setup is done globally in a
1729 * set of 32 tiling modes. Rather than selecting each set of
1730 * parameters per surface as on older asics, we just select
1731 * which index in the tiling table we want to use, and the
1732 * surface uses those parameters (CIK).
1733 */
1734static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1735{
1736 const u32 num_tile_mode_states = 32;
1737 const u32 num_secondary_tile_mode_states = 16;
1738 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1739 u32 num_pipe_configs;
1740 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1741 rdev->config.cik.max_shader_engines;
1742
1743 switch (rdev->config.cik.mem_row_size_in_kb) {
1744 case 1:
1745 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1746 break;
1747 case 2:
1748 default:
1749 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1750 break;
1751 case 4:
1752 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1753 break;
1754 }
1755
1756 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1757 if (num_pipe_configs > 8)
1758 num_pipe_configs = 8; /* ??? */
1759
1760 if (num_pipe_configs == 8) {
1761 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1762 switch (reg_offset) {
1763 case 0:
1764 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1765 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1766 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1767 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1768 break;
1769 case 1:
1770 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1771 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1772 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1773 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1774 break;
1775 case 2:
1776 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1777 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1778 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1779 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1780 break;
1781 case 3:
1782 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1783 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1784 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1785 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1786 break;
1787 case 4:
1788 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1789 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1790 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1791 TILE_SPLIT(split_equal_to_row_size));
1792 break;
1793 case 5:
1794 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1796 break;
1797 case 6:
1798 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1799 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1800 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1801 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1802 break;
1803 case 7:
1804 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1805 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1806 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1807 TILE_SPLIT(split_equal_to_row_size));
1808 break;
1809 case 8:
1810 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1811 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
1812 break;
1813 case 9:
1814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1815 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1816 break;
1817 case 10:
1818 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1819 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1820 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1821 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1822 break;
1823 case 11:
1824 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1825 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1826 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1827 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1828 break;
1829 case 12:
1830 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1831 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1832 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1833 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1834 break;
1835 case 13:
1836 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1837 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1838 break;
1839 case 14:
1840 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1841 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1842 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1843 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1844 break;
1845 case 16:
1846 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1847 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1848 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1849 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1850 break;
1851 case 17:
1852 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1853 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1854 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1855 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1856 break;
1857 case 27:
1858 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1859 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1860 break;
1861 case 28:
1862 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1863 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1864 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1865 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1866 break;
1867 case 29:
1868 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1869 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1870 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1871 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1872 break;
1873 case 30:
1874 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1875 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1876 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1877 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1878 break;
1879 default:
1880 gb_tile_moden = 0;
1881 break;
1882 }
39aee490 1883 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 1884 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1885 }
1886 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1887 switch (reg_offset) {
1888 case 0:
1889 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1890 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1891 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1892 NUM_BANKS(ADDR_SURF_16_BANK));
1893 break;
1894 case 1:
1895 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1896 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1897 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1898 NUM_BANKS(ADDR_SURF_16_BANK));
1899 break;
1900 case 2:
1901 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1902 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1903 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1904 NUM_BANKS(ADDR_SURF_16_BANK));
1905 break;
1906 case 3:
1907 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1908 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1909 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1910 NUM_BANKS(ADDR_SURF_16_BANK));
1911 break;
1912 case 4:
1913 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1914 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1915 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1916 NUM_BANKS(ADDR_SURF_8_BANK));
1917 break;
1918 case 5:
1919 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1920 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1921 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1922 NUM_BANKS(ADDR_SURF_4_BANK));
1923 break;
1924 case 6:
1925 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1926 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1927 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1928 NUM_BANKS(ADDR_SURF_2_BANK));
1929 break;
1930 case 8:
1931 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1932 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1933 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1934 NUM_BANKS(ADDR_SURF_16_BANK));
1935 break;
1936 case 9:
1937 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1938 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1939 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1940 NUM_BANKS(ADDR_SURF_16_BANK));
1941 break;
1942 case 10:
1943 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1944 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1945 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1946 NUM_BANKS(ADDR_SURF_16_BANK));
1947 break;
1948 case 11:
1949 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1950 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1951 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1952 NUM_BANKS(ADDR_SURF_16_BANK));
1953 break;
1954 case 12:
1955 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1956 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1957 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1958 NUM_BANKS(ADDR_SURF_8_BANK));
1959 break;
1960 case 13:
1961 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1962 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1963 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1964 NUM_BANKS(ADDR_SURF_4_BANK));
1965 break;
1966 case 14:
1967 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1968 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1969 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1970 NUM_BANKS(ADDR_SURF_2_BANK));
1971 break;
1972 default:
1973 gb_tile_moden = 0;
1974 break;
1975 }
1976 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1977 }
1978 } else if (num_pipe_configs == 4) {
1979 if (num_rbs == 4) {
1980 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1981 switch (reg_offset) {
1982 case 0:
1983 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1984 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1985 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1986 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1987 break;
1988 case 1:
1989 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1990 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1991 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1992 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1993 break;
1994 case 2:
1995 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1996 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1997 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1998 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1999 break;
2000 case 3:
2001 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2002 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2003 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2004 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2005 break;
2006 case 4:
2007 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2008 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2009 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2010 TILE_SPLIT(split_equal_to_row_size));
2011 break;
2012 case 5:
2013 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2014 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2015 break;
2016 case 6:
2017 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2018 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2019 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2020 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2021 break;
2022 case 7:
2023 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2024 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2025 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2026 TILE_SPLIT(split_equal_to_row_size));
2027 break;
2028 case 8:
2029 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2030 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2031 break;
2032 case 9:
2033 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2034 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2035 break;
2036 case 10:
2037 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2038 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2039 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2040 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2041 break;
2042 case 11:
2043 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2044 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2045 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2046 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2047 break;
2048 case 12:
2049 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2050 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2051 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2052 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2053 break;
2054 case 13:
2055 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2056 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2057 break;
2058 case 14:
2059 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2060 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2061 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2062 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2063 break;
2064 case 16:
2065 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2066 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2067 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2068 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2069 break;
2070 case 17:
2071 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2072 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2073 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2074 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2075 break;
2076 case 27:
2077 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2078 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2079 break;
2080 case 28:
2081 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2082 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2083 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2084 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2085 break;
2086 case 29:
2087 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2088 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2089 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2090 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2091 break;
2092 case 30:
2093 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2094 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2095 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2096 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2097 break;
2098 default:
2099 gb_tile_moden = 0;
2100 break;
2101 }
39aee490 2102 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2103 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2104 }
2105 } else if (num_rbs < 4) {
2106 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2107 switch (reg_offset) {
2108 case 0:
2109 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2111 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2113 break;
2114 case 1:
2115 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2116 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2117 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2118 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2119 break;
2120 case 2:
2121 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2122 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2123 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2124 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2125 break;
2126 case 3:
2127 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2128 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2129 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2130 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2131 break;
2132 case 4:
2133 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2134 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2135 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2136 TILE_SPLIT(split_equal_to_row_size));
2137 break;
2138 case 5:
2139 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2140 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2141 break;
2142 case 6:
2143 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2144 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2145 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2146 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2147 break;
2148 case 7:
2149 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2150 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2151 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2152 TILE_SPLIT(split_equal_to_row_size));
2153 break;
2154 case 8:
2155 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2156 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2157 break;
2158 case 9:
2159 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2160 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2161 break;
2162 case 10:
2163 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2164 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2165 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2166 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2167 break;
2168 case 11:
2169 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2170 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2171 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2172 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2173 break;
2174 case 12:
2175 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2176 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2177 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2178 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2179 break;
2180 case 13:
2181 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2182 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2183 break;
2184 case 14:
2185 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2186 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2187 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2188 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2189 break;
2190 case 16:
2191 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2192 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2193 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2194 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2195 break;
2196 case 17:
2197 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2198 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2199 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2200 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2201 break;
2202 case 27:
2203 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2204 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2205 break;
2206 case 28:
2207 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2208 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2209 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2210 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2211 break;
2212 case 29:
2213 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2214 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2215 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2216 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2217 break;
2218 case 30:
2219 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2220 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2221 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2222 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2223 break;
2224 default:
2225 gb_tile_moden = 0;
2226 break;
2227 }
39aee490 2228 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2229 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2230 }
2231 }
2232 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2233 switch (reg_offset) {
2234 case 0:
2235 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2236 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2237 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2238 NUM_BANKS(ADDR_SURF_16_BANK));
2239 break;
2240 case 1:
2241 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2242 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2243 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2244 NUM_BANKS(ADDR_SURF_16_BANK));
2245 break;
2246 case 2:
2247 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2248 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2249 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2250 NUM_BANKS(ADDR_SURF_16_BANK));
2251 break;
2252 case 3:
2253 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2254 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2255 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2256 NUM_BANKS(ADDR_SURF_16_BANK));
2257 break;
2258 case 4:
2259 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2260 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2261 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2262 NUM_BANKS(ADDR_SURF_16_BANK));
2263 break;
2264 case 5:
2265 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2266 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2267 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2268 NUM_BANKS(ADDR_SURF_8_BANK));
2269 break;
2270 case 6:
2271 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2272 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2273 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2274 NUM_BANKS(ADDR_SURF_4_BANK));
2275 break;
2276 case 8:
2277 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2278 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2279 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2280 NUM_BANKS(ADDR_SURF_16_BANK));
2281 break;
2282 case 9:
2283 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2284 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2285 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2286 NUM_BANKS(ADDR_SURF_16_BANK));
2287 break;
2288 case 10:
2289 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2290 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2291 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2292 NUM_BANKS(ADDR_SURF_16_BANK));
2293 break;
2294 case 11:
2295 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2296 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2297 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2298 NUM_BANKS(ADDR_SURF_16_BANK));
2299 break;
2300 case 12:
2301 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2302 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2303 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2304 NUM_BANKS(ADDR_SURF_16_BANK));
2305 break;
2306 case 13:
2307 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2308 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2309 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2310 NUM_BANKS(ADDR_SURF_8_BANK));
2311 break;
2312 case 14:
2313 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2316 NUM_BANKS(ADDR_SURF_4_BANK));
2317 break;
2318 default:
2319 gb_tile_moden = 0;
2320 break;
2321 }
2322 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2323 }
2324 } else if (num_pipe_configs == 2) {
2325 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2326 switch (reg_offset) {
2327 case 0:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P2) |
2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2332 break;
2333 case 1:
2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2335 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2336 PIPE_CONFIG(ADDR_SURF_P2) |
2337 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2338 break;
2339 case 2:
2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2342 PIPE_CONFIG(ADDR_SURF_P2) |
2343 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2344 break;
2345 case 3:
2346 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2347 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2348 PIPE_CONFIG(ADDR_SURF_P2) |
2349 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2350 break;
2351 case 4:
2352 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2353 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2354 PIPE_CONFIG(ADDR_SURF_P2) |
2355 TILE_SPLIT(split_equal_to_row_size));
2356 break;
2357 case 5:
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2359 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2360 break;
2361 case 6:
2362 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2363 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2364 PIPE_CONFIG(ADDR_SURF_P2) |
2365 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2366 break;
2367 case 7:
2368 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2369 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2370 PIPE_CONFIG(ADDR_SURF_P2) |
2371 TILE_SPLIT(split_equal_to_row_size));
2372 break;
2373 case 8:
2374 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2375 break;
2376 case 9:
2377 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2378 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2379 break;
2380 case 10:
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P2) |
2384 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2385 break;
2386 case 11:
2387 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2389 PIPE_CONFIG(ADDR_SURF_P2) |
2390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2391 break;
2392 case 12:
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P2) |
2396 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397 break;
2398 case 13:
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2400 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2401 break;
2402 case 14:
2403 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2404 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2405 PIPE_CONFIG(ADDR_SURF_P2) |
2406 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2407 break;
2408 case 16:
2409 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2410 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P2) |
2412 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2413 break;
2414 case 17:
2415 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2416 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2417 PIPE_CONFIG(ADDR_SURF_P2) |
2418 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2419 break;
2420 case 27:
2421 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2422 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2423 break;
2424 case 28:
2425 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2426 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2427 PIPE_CONFIG(ADDR_SURF_P2) |
2428 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2429 break;
2430 case 29:
2431 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2432 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2433 PIPE_CONFIG(ADDR_SURF_P2) |
2434 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2435 break;
2436 case 30:
2437 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P2) |
2440 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2441 break;
2442 default:
2443 gb_tile_moden = 0;
2444 break;
2445 }
39aee490 2446 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2447 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2448 }
2449 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2450 switch (reg_offset) {
2451 case 0:
2452 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2453 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2454 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2455 NUM_BANKS(ADDR_SURF_16_BANK));
2456 break;
2457 case 1:
2458 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2459 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2460 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2461 NUM_BANKS(ADDR_SURF_16_BANK));
2462 break;
2463 case 2:
2464 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2467 NUM_BANKS(ADDR_SURF_16_BANK));
2468 break;
2469 case 3:
2470 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2471 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2472 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2473 NUM_BANKS(ADDR_SURF_16_BANK));
2474 break;
2475 case 4:
2476 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2479 NUM_BANKS(ADDR_SURF_16_BANK));
2480 break;
2481 case 5:
2482 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2483 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2484 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2485 NUM_BANKS(ADDR_SURF_16_BANK));
2486 break;
2487 case 6:
2488 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2491 NUM_BANKS(ADDR_SURF_8_BANK));
2492 break;
2493 case 8:
2494 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2495 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2496 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2497 NUM_BANKS(ADDR_SURF_16_BANK));
2498 break;
2499 case 9:
2500 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2503 NUM_BANKS(ADDR_SURF_16_BANK));
2504 break;
2505 case 10:
2506 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2507 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2508 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2509 NUM_BANKS(ADDR_SURF_16_BANK));
2510 break;
2511 case 11:
2512 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2513 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2514 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2515 NUM_BANKS(ADDR_SURF_16_BANK));
2516 break;
2517 case 12:
2518 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2519 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2520 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2521 NUM_BANKS(ADDR_SURF_16_BANK));
2522 break;
2523 case 13:
2524 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2525 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2526 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2527 NUM_BANKS(ADDR_SURF_16_BANK));
2528 break;
2529 case 14:
2530 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2531 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2532 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2533 NUM_BANKS(ADDR_SURF_8_BANK));
2534 break;
2535 default:
2536 gb_tile_moden = 0;
2537 break;
2538 }
2539 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2540 }
2541 } else
2542 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2543}
2544
2545/**
2546 * cik_select_se_sh - select which SE, SH to address
2547 *
2548 * @rdev: radeon_device pointer
2549 * @se_num: shader engine to address
2550 * @sh_num: sh block to address
2551 *
2552 * Select which SE, SH combinations to address. Certain
2553 * registers are instanced per SE or SH. 0xffffffff means
2554 * broadcast to all SEs or SHs (CIK).
2555 */
2556static void cik_select_se_sh(struct radeon_device *rdev,
2557 u32 se_num, u32 sh_num)
2558{
2559 u32 data = INSTANCE_BROADCAST_WRITES;
2560
2561 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 2562 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 2563 else if (se_num == 0xffffffff)
2564 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2565 else if (sh_num == 0xffffffff)
2566 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2567 else
2568 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2569 WREG32(GRBM_GFX_INDEX, data);
2570}
2571
2572/**
2573 * cik_create_bitmask - create a bitmask
2574 *
2575 * @bit_width: length of the mask
2576 *
2577 * create a variable length bit mask (CIK).
2578 * Returns the bitmask.
2579 */
2580static u32 cik_create_bitmask(u32 bit_width)
2581{
2582 u32 i, mask = 0;
2583
2584 for (i = 0; i < bit_width; i++) {
2585 mask <<= 1;
2586 mask |= 1;
2587 }
2588 return mask;
2589}
2590
2591/**
2592 * cik_select_se_sh - select which SE, SH to address
2593 *
2594 * @rdev: radeon_device pointer
2595 * @max_rb_num: max RBs (render backends) for the asic
2596 * @se_num: number of SEs (shader engines) for the asic
2597 * @sh_per_se: number of SH blocks per SE for the asic
2598 *
2599 * Calculates the bitmask of disabled RBs (CIK).
2600 * Returns the disabled RB bitmask.
2601 */
2602static u32 cik_get_rb_disabled(struct radeon_device *rdev,
2603 u32 max_rb_num, u32 se_num,
2604 u32 sh_per_se)
2605{
2606 u32 data, mask;
2607
2608 data = RREG32(CC_RB_BACKEND_DISABLE);
2609 if (data & 1)
2610 data &= BACKEND_DISABLE_MASK;
2611 else
2612 data = 0;
2613 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2614
2615 data >>= BACKEND_DISABLE_SHIFT;
2616
2617 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
2618
2619 return data & mask;
2620}
2621
2622/**
2623 * cik_setup_rb - setup the RBs on the asic
2624 *
2625 * @rdev: radeon_device pointer
2626 * @se_num: number of SEs (shader engines) for the asic
2627 * @sh_per_se: number of SH blocks per SE for the asic
2628 * @max_rb_num: max RBs (render backends) for the asic
2629 *
2630 * Configures per-SE/SH RB registers (CIK).
2631 */
2632static void cik_setup_rb(struct radeon_device *rdev,
2633 u32 se_num, u32 sh_per_se,
2634 u32 max_rb_num)
2635{
2636 int i, j;
2637 u32 data, mask;
2638 u32 disabled_rbs = 0;
2639 u32 enabled_rbs = 0;
2640
2641 for (i = 0; i < se_num; i++) {
2642 for (j = 0; j < sh_per_se; j++) {
2643 cik_select_se_sh(rdev, i, j);
2644 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2645 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
2646 }
2647 }
2648 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2649
2650 mask = 1;
2651 for (i = 0; i < max_rb_num; i++) {
2652 if (!(disabled_rbs & mask))
2653 enabled_rbs |= mask;
2654 mask <<= 1;
2655 }
2656
2657 for (i = 0; i < se_num; i++) {
2658 cik_select_se_sh(rdev, i, 0xffffffff);
2659 data = 0;
2660 for (j = 0; j < sh_per_se; j++) {
2661 switch (enabled_rbs & 3) {
2662 case 1:
2663 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2664 break;
2665 case 2:
2666 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2667 break;
2668 case 3:
2669 default:
2670 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2671 break;
2672 }
2673 enabled_rbs >>= 2;
2674 }
2675 WREG32(PA_SC_RASTER_CONFIG, data);
2676 }
2677 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2678}
2679
2680/**
2681 * cik_gpu_init - setup the 3D engine
2682 *
2683 * @rdev: radeon_device pointer
2684 *
2685 * Configures the 3D engine and tiling configuration
2686 * registers so that the 3D engine is usable.
2687 */
2688static void cik_gpu_init(struct radeon_device *rdev)
2689{
2690 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
2691 u32 mc_shared_chmap, mc_arb_ramcfg;
2692 u32 hdp_host_path_cntl;
2693 u32 tmp;
2694 int i, j;
2695
2696 switch (rdev->family) {
2697 case CHIP_BONAIRE:
2698 rdev->config.cik.max_shader_engines = 2;
2699 rdev->config.cik.max_tile_pipes = 4;
2700 rdev->config.cik.max_cu_per_sh = 7;
2701 rdev->config.cik.max_sh_per_se = 1;
2702 rdev->config.cik.max_backends_per_se = 2;
2703 rdev->config.cik.max_texture_channel_caches = 4;
2704 rdev->config.cik.max_gprs = 256;
2705 rdev->config.cik.max_gs_threads = 32;
2706 rdev->config.cik.max_hw_contexts = 8;
2707
2708 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2709 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2710 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2711 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2712 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2713 break;
2714 case CHIP_KAVERI:
b2e4c70a
AD		 2715 rdev->config.cik.max_shader_engines = 1;
2716 rdev->config.cik.max_tile_pipes = 4;
2717 if ((rdev->pdev->device == 0x1304) ||
2718 (rdev->pdev->device == 0x1305) ||
2719 (rdev->pdev->device == 0x130C) ||
2720 (rdev->pdev->device == 0x130F) ||
2721 (rdev->pdev->device == 0x1310) ||
2722 (rdev->pdev->device == 0x1311) ||
2723 (rdev->pdev->device == 0x131C)) {
2724 rdev->config.cik.max_cu_per_sh = 8;
2725 rdev->config.cik.max_backends_per_se = 2;
2726 } else if ((rdev->pdev->device == 0x1309) ||
2727 (rdev->pdev->device == 0x130A) ||
2728 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 2729 (rdev->pdev->device == 0x1313) ||
2730 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 2731 rdev->config.cik.max_cu_per_sh = 6;
2732 rdev->config.cik.max_backends_per_se = 2;
2733 } else if ((rdev->pdev->device == 0x1306) ||
2734 (rdev->pdev->device == 0x1307) ||
2735 (rdev->pdev->device == 0x130B) ||
2736 (rdev->pdev->device == 0x130E) ||
2737 (rdev->pdev->device == 0x1315) ||
2738 (rdev->pdev->device == 0x131B)) {
2739 rdev->config.cik.max_cu_per_sh = 4;
2740 rdev->config.cik.max_backends_per_se = 1;
2741 } else {
2742 rdev->config.cik.max_cu_per_sh = 3;
2743 rdev->config.cik.max_backends_per_se = 1;
2744 }
2745 rdev->config.cik.max_sh_per_se = 1;
2746 rdev->config.cik.max_texture_channel_caches = 4;
2747 rdev->config.cik.max_gprs = 256;
2748 rdev->config.cik.max_gs_threads = 16;
2749 rdev->config.cik.max_hw_contexts = 8;
2750
2751 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2752 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2753 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2754 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2755 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 2756 break;
2757 case CHIP_KABINI:
2758 default:
2759 rdev->config.cik.max_shader_engines = 1;
2760 rdev->config.cik.max_tile_pipes = 2;
2761 rdev->config.cik.max_cu_per_sh = 2;
2762 rdev->config.cik.max_sh_per_se = 1;
2763 rdev->config.cik.max_backends_per_se = 1;
2764 rdev->config.cik.max_texture_channel_caches = 2;
2765 rdev->config.cik.max_gprs = 256;
2766 rdev->config.cik.max_gs_threads = 16;
2767 rdev->config.cik.max_hw_contexts = 8;
2768
2769 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2770 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2771 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2772 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2773 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2774 break;
2775 }
2776
2777 /* Initialize HDP */
2778 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2779 WREG32((0x2c14 + j), 0x00000000);
2780 WREG32((0x2c18 + j), 0x00000000);
2781 WREG32((0x2c1c + j), 0x00000000);
2782 WREG32((0x2c20 + j), 0x00000000);
2783 WREG32((0x2c24 + j), 0x00000000);
2784 }
2785
2786 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2787
2788 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2789
2790 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2791 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2792
2793 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
2794 rdev->config.cik.mem_max_burst_length_bytes = 256;
2795 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2796 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2797 if (rdev->config.cik.mem_row_size_in_kb > 4)
2798 rdev->config.cik.mem_row_size_in_kb = 4;
2799 /* XXX use MC settings? */
2800 rdev->config.cik.shader_engine_tile_size = 32;
2801 rdev->config.cik.num_gpus = 1;
2802 rdev->config.cik.multi_gpu_tile_size = 64;
2803
2804 /* fix up row size */
2805 gb_addr_config &= ~ROW_SIZE_MASK;
2806 switch (rdev->config.cik.mem_row_size_in_kb) {
2807 case 1:
2808 default:
2809 gb_addr_config |= ROW_SIZE(0);
2810 break;
2811 case 2:
2812 gb_addr_config |= ROW_SIZE(1);
2813 break;
2814 case 4:
2815 gb_addr_config |= ROW_SIZE(2);
2816 break;
2817 }
2818
2819 /* setup tiling info dword. gb_addr_config is not adequate since it does
2820 * not have bank info, so create a custom tiling dword.
2821 * bits 3:0 num_pipes
2822 * bits 7:4 num_banks
2823 * bits 11:8 group_size
2824 * bits 15:12 row_size
2825 */
2826 rdev->config.cik.tile_config = 0;
2827 switch (rdev->config.cik.num_tile_pipes) {
2828 case 1:
2829 rdev->config.cik.tile_config |= (0 << 0);
2830 break;
2831 case 2:
2832 rdev->config.cik.tile_config |= (1 << 0);
2833 break;
2834 case 4:
2835 rdev->config.cik.tile_config |= (2 << 0);
2836 break;
2837 case 8:
2838 default:
2839 /* XXX what about 12? */
2840 rdev->config.cik.tile_config |= (3 << 0);
2841 break;
2842 }
a537314e
MD		 2843 rdev->config.cik.tile_config |=
2844 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 2845 rdev->config.cik.tile_config |=
2846 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2847 rdev->config.cik.tile_config |=
2848 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2849
2850 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2851 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2852 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 2853 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
2854 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 2855 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2856 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2857 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 2858
2859 cik_tiling_mode_table_init(rdev);
2860
2861 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
2862 rdev->config.cik.max_sh_per_se,
2863 rdev->config.cik.max_backends_per_se);
2864
2865 /* set HW defaults for 3D engine */
2866 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2867
2868 WREG32(SX_DEBUG_1, 0x20);
2869
2870 WREG32(TA_CNTL_AUX, 0x00010000);
2871
2872 tmp = RREG32(SPI_CONFIG_CNTL);
2873 tmp |= 0x03000000;
2874 WREG32(SPI_CONFIG_CNTL, tmp);
2875
2876 WREG32(SQ_CONFIG, 1);
2877
2878 WREG32(DB_DEBUG, 0);
2879
2880 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
2881 tmp |= 0x00000400;
2882 WREG32(DB_DEBUG2, tmp);
2883
2884 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
2885 tmp |= 0x00020200;
2886 WREG32(DB_DEBUG3, tmp);
2887
2888 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
2889 tmp |= 0x00018208;
2890 WREG32(CB_HW_CONTROL, tmp);
2891
2892 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2893
2894 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
2895 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
2896 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
2897 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
2898
2899 WREG32(VGT_NUM_INSTANCES, 1);
2900
2901 WREG32(CP_PERFMON_CNTL, 0);
2902
2903 WREG32(SQ_CONFIG, 0);
2904
2905 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2906 FORCE_EOV_MAX_REZ_CNT(255)));
2907
2908 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2909 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2910
2911 WREG32(VGT_GS_VERTEX_REUSE, 16);
2912 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2913
2914 tmp = RREG32(HDP_MISC_CNTL);
2915 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2916 WREG32(HDP_MISC_CNTL, tmp);
2917
2918 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2919 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2920
2921 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2922 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
2923
2924 udelay(50);
2925}
2926
2cae3bc3
AD		 2927/*
2928 * GPU scratch registers helpers function.
2929 */
2930/**
2931 * cik_scratch_init - setup driver info for CP scratch regs
2932 *
2933 * @rdev: radeon_device pointer
2934 *
2935 * Set up the number and offset of the CP scratch registers.
2936 * NOTE: use of CP scratch registers is a legacy inferface and
2937 * is not used by default on newer asics (r6xx+). On newer asics,
2938 * memory buffers are used for fences rather than scratch regs.
2939 */
2940static void cik_scratch_init(struct radeon_device *rdev)
2941{
2942 int i;
2943
2944 rdev->scratch.num_reg = 7;
2945 rdev->scratch.reg_base = SCRATCH_REG0;
2946 for (i = 0; i < rdev->scratch.num_reg; i++) {
2947 rdev->scratch.free[i] = true;
2948 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2949 }
2950}
2951
fbc832c7
AD		 2952/**
2953 * cik_ring_test - basic gfx ring test
2954 *
2955 * @rdev: radeon_device pointer
2956 * @ring: radeon_ring structure holding ring information
2957 *
2958 * Allocate a scratch register and write to it using the gfx ring (CIK).
2959 * Provides a basic gfx ring test to verify that the ring is working.
2960 * Used by cik_cp_gfx_resume();
2961 * Returns 0 on success, error on failure.
2962 */
2963int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2964{
2965 uint32_t scratch;
2966 uint32_t tmp = 0;
2967 unsigned i;
2968 int r;
2969
2970 r = radeon_scratch_get(rdev, &scratch);
2971 if (r) {
2972 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2973 return r;
2974 }
2975 WREG32(scratch, 0xCAFEDEAD);
2976 r = radeon_ring_lock(rdev, ring, 3);
2977 if (r) {
2978 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2979 radeon_scratch_free(rdev, scratch);
2980 return r;
2981 }
2982 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2983 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
2984 radeon_ring_write(ring, 0xDEADBEEF);
2985 radeon_ring_unlock_commit(rdev, ring);
963e81f9 2986
fbc832c7
AD		 2987 for (i = 0; i < rdev->usec_timeout; i++) {
2988 tmp = RREG32(scratch);
2989 if (tmp == 0xDEADBEEF)
2990 break;
2991 DRM_UDELAY(1);
2992 }
2993 if (i < rdev->usec_timeout) {
2994 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2995 } else {
2996 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2997 ring->idx, scratch, tmp);
2998 r = -EINVAL;
2999 }
3000 radeon_scratch_free(rdev, scratch);
3001 return r;
3002}
3003
2cae3bc3 3004/**
b07fdd38 3005 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3006 *
3007 * @rdev: radeon_device pointer
3008 * @fence: radeon fence object
3009 *
3010 * Emits a fence sequnce number on the gfx ring and flushes
3011 * GPU caches.
3012 */
b07fdd38
AD		 3013void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3014 struct radeon_fence *fence)
2cae3bc3
AD		 3015{
3016 struct radeon_ring *ring = &rdev->ring[fence->ring];
3017 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3018
3019 /* EVENT_WRITE_EOP - flush caches, send int */
3020 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3021 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3022 EOP_TC_ACTION_EN |
3023 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3024 EVENT_INDEX(5)));
3025 radeon_ring_write(ring, addr & 0xfffffffc);
3026 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3027 radeon_ring_write(ring, fence->seq);
3028 radeon_ring_write(ring, 0);
3029 /* HDP flush */
3030 /* We should be using the new WAIT_REG_MEM special op packet here
3031 * but it causes the CP to hang
3032 */
3033 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3034 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3035 WRITE_DATA_DST_SEL(0)));
3036 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3037 radeon_ring_write(ring, 0);
3038 radeon_ring_write(ring, 0);
3039}
3040
b07fdd38
AD		 3041/**
3042 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3043 *
3044 * @rdev: radeon_device pointer
3045 * @fence: radeon fence object
3046 *
3047 * Emits a fence sequnce number on the compute ring and flushes
3048 * GPU caches.
3049 */
3050void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3051 struct radeon_fence *fence)
3052{
3053 struct radeon_ring *ring = &rdev->ring[fence->ring];
3054 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3055
3056 /* RELEASE_MEM - flush caches, send int */
3057 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3058 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3059 EOP_TC_ACTION_EN |
3060 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3061 EVENT_INDEX(5)));
3062 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3063 radeon_ring_write(ring, addr & 0xfffffffc);
3064 radeon_ring_write(ring, upper_32_bits(addr));
3065 radeon_ring_write(ring, fence->seq);
3066 radeon_ring_write(ring, 0);
3067 /* HDP flush */
3068 /* We should be using the new WAIT_REG_MEM special op packet here
3069 * but it causes the CP to hang
3070 */
3071 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3072 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3073 WRITE_DATA_DST_SEL(0)));
3074 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3075 radeon_ring_write(ring, 0);
3076 radeon_ring_write(ring, 0);
3077}
3078
2cae3bc3
AD		 3079void cik_semaphore_ring_emit(struct radeon_device *rdev,
3080 struct radeon_ring *ring,
3081 struct radeon_semaphore *semaphore,
3082 bool emit_wait)
3083{
3084 uint64_t addr = semaphore->gpu_addr;
3085 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3086
3087 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3088 radeon_ring_write(ring, addr & 0xffffffff);
3089 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3090}
3091
c9dbd705
AD		 3092/**
3093 * cik_copy_cpdma - copy pages using the CP DMA engine
3094 *
3095 * @rdev: radeon_device pointer
3096 * @src_offset: src GPU address
3097 * @dst_offset: dst GPU address
3098 * @num_gpu_pages: number of GPU pages to xfer
3099 * @fence: radeon fence object
3100 *
3101 * Copy GPU paging using the CP DMA engine (CIK+).
3102 * Used by the radeon ttm implementation to move pages if
3103 * registered as the asic copy callback.
3104 */
3105int cik_copy_cpdma(struct radeon_device *rdev,
3106 uint64_t src_offset, uint64_t dst_offset,
3107 unsigned num_gpu_pages,
3108 struct radeon_fence **fence)
3109{
3110 struct radeon_semaphore *sem = NULL;
3111 int ring_index = rdev->asic->copy.blit_ring_index;
3112 struct radeon_ring *ring = &rdev->ring[ring_index];
3113 u32 size_in_bytes, cur_size_in_bytes, control;
3114 int i, num_loops;
3115 int r = 0;
3116
3117 r = radeon_semaphore_create(rdev, &sem);
3118 if (r) {
3119 DRM_ERROR("radeon: moving bo (%d).\n", r);
3120 return r;
3121 }
3122
3123 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3124 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3125 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3126 if (r) {
3127 DRM_ERROR("radeon: moving bo (%d).\n", r);
3128 radeon_semaphore_free(rdev, &sem, NULL);
3129 return r;
3130 }
3131
3132 if (radeon_fence_need_sync(*fence, ring->idx)) {
3133 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3134 ring->idx);
3135 radeon_fence_note_sync(*fence, ring->idx);
3136 } else {
3137 radeon_semaphore_free(rdev, &sem, NULL);
3138 }
3139
3140 for (i = 0; i < num_loops; i++) {
3141 cur_size_in_bytes = size_in_bytes;
3142 if (cur_size_in_bytes > 0x1fffff)
3143 cur_size_in_bytes = 0x1fffff;
3144 size_in_bytes -= cur_size_in_bytes;
3145 control = 0;
3146 if (size_in_bytes == 0)
3147 control |= PACKET3_DMA_DATA_CP_SYNC;
3148 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3149 radeon_ring_write(ring, control);
3150 radeon_ring_write(ring, lower_32_bits(src_offset));
3151 radeon_ring_write(ring, upper_32_bits(src_offset));
3152 radeon_ring_write(ring, lower_32_bits(dst_offset));
3153 radeon_ring_write(ring, upper_32_bits(dst_offset));
3154 radeon_ring_write(ring, cur_size_in_bytes);
3155 src_offset += cur_size_in_bytes;
3156 dst_offset += cur_size_in_bytes;
3157 }
3158
3159 r = radeon_fence_emit(rdev, fence, ring->idx);
3160 if (r) {
3161 radeon_ring_unlock_undo(rdev, ring);
3162 return r;
3163 }
3164
3165 radeon_ring_unlock_commit(rdev, ring);
3166 radeon_semaphore_free(rdev, &sem, *fence);
3167
3168 return r;
3169}
3170
2cae3bc3
AD		 3171/*
3172 * IB stuff
3173 */
3174/**
3175 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3176 *
3177 * @rdev: radeon_device pointer
3178 * @ib: radeon indirect buffer object
3179 *
3180 * Emits an DE (drawing engine) or CE (constant engine) IB
3181 * on the gfx ring. IBs are usually generated by userspace
3182 * acceleration drivers and submitted to the kernel for
3183 * sheduling on the ring. This function schedules the IB
3184 * on the gfx ring for execution by the GPU.
3185 */
3186void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3187{
3188 struct radeon_ring *ring = &rdev->ring[ib->ring];
3189 u32 header, control = INDIRECT_BUFFER_VALID;
3190
3191 if (ib->is_const_ib) {
3192 /* set switch buffer packet before const IB */
3193 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3194 radeon_ring_write(ring, 0);
3195
3196 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3197 } else {
3198 u32 next_rptr;
3199 if (ring->rptr_save_reg) {
3200 next_rptr = ring->wptr + 3 + 4;
3201 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3202 radeon_ring_write(ring, ((ring->rptr_save_reg -
3203 PACKET3_SET_UCONFIG_REG_START) >> 2));
3204 radeon_ring_write(ring, next_rptr);
3205 } else if (rdev->wb.enabled) {
3206 next_rptr = ring->wptr + 5 + 4;
3207 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3208 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3209 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3210 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3211 radeon_ring_write(ring, next_rptr);
3212 }
3213
3214 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3215 }
3216
3217 control |= ib->length_dw |
3218 (ib->vm ? (ib->vm->id << 24) : 0);
3219
3220 radeon_ring_write(ring, header);
3221 radeon_ring_write(ring,
3222#ifdef __BIG_ENDIAN
3223 (2 << 0) |
3224#endif
3225 (ib->gpu_addr & 0xFFFFFFFC));
3226 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3227 radeon_ring_write(ring, control);
3228}
3229
fbc832c7
AD		 3230/**
3231 * cik_ib_test - basic gfx ring IB test
3232 *
3233 * @rdev: radeon_device pointer
3234 * @ring: radeon_ring structure holding ring information
3235 *
3236 * Allocate an IB and execute it on the gfx ring (CIK).
3237 * Provides a basic gfx ring test to verify that IBs are working.
3238 * Returns 0 on success, error on failure.
3239 */
3240int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3241{
3242 struct radeon_ib ib;
3243 uint32_t scratch;
3244 uint32_t tmp = 0;
3245 unsigned i;
3246 int r;
3247
3248 r = radeon_scratch_get(rdev, &scratch);
3249 if (r) {
3250 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3251 return r;
3252 }
3253 WREG32(scratch, 0xCAFEDEAD);
3254 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3255 if (r) {
3256 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3257 return r;
3258 }
3259 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3260 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3261 ib.ptr[2] = 0xDEADBEEF;
3262 ib.length_dw = 3;
3263 r = radeon_ib_schedule(rdev, &ib, NULL);
3264 if (r) {
3265 radeon_scratch_free(rdev, scratch);
3266 radeon_ib_free(rdev, &ib);
3267 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3268 return r;
3269 }
3270 r = radeon_fence_wait(ib.fence, false);
3271 if (r) {
3272 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3273 return r;
3274 }
3275 for (i = 0; i < rdev->usec_timeout; i++) {
3276 tmp = RREG32(scratch);
3277 if (tmp == 0xDEADBEEF)
3278 break;
3279 DRM_UDELAY(1);
3280 }
3281 if (i < rdev->usec_timeout) {
3282 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3283 } else {
3284 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3285 scratch, tmp);
3286 r = -EINVAL;
3287 }
3288 radeon_scratch_free(rdev, scratch);
3289 radeon_ib_free(rdev, &ib);
3290 return r;
3291}
3292
841cf442
AD		 3293/*
3294 * CP.
3295 * On CIK, gfx and compute now have independant command processors.
3296 *
3297 * GFX
3298 * Gfx consists of a single ring and can process both gfx jobs and
3299 * compute jobs. The gfx CP consists of three microengines (ME):
3300 * PFP - Pre-Fetch Parser
3301 * ME - Micro Engine
3302 * CE - Constant Engine
3303 * The PFP and ME make up what is considered the Drawing Engine (DE).
3304 * The CE is an asynchronous engine used for updating buffer desciptors
3305 * used by the DE so that they can be loaded into cache in parallel
3306 * while the DE is processing state update packets.
3307 *
3308 * Compute
3309 * The compute CP consists of two microengines (ME):
3310 * MEC1 - Compute MicroEngine 1
3311 * MEC2 - Compute MicroEngine 2
3312 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3313 * The queues are exposed to userspace and are programmed directly
3314 * by the compute runtime.
3315 */
3316/**
3317 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3318 *
3319 * @rdev: radeon_device pointer
3320 * @enable: enable or disable the MEs
3321 *
3322 * Halts or unhalts the gfx MEs.
3323 */
3324static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3325{
3326 if (enable)
3327 WREG32(CP_ME_CNTL, 0);
3328 else {
3329 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3330 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3331 }
3332 udelay(50);
3333}
3334
3335/**
3336 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3337 *
3338 * @rdev: radeon_device pointer
3339 *
3340 * Loads the gfx PFP, ME, and CE ucode.
3341 * Returns 0 for success, -EINVAL if the ucode is not available.
3342 */
3343static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3344{
3345 const __be32 *fw_data;
3346 int i;
3347
3348 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3349 return -EINVAL;
3350
3351 cik_cp_gfx_enable(rdev, false);
3352
3353 /* PFP */
3354 fw_data = (const __be32 *)rdev->pfp_fw->data;
3355 WREG32(CP_PFP_UCODE_ADDR, 0);
3356 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3357 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3358 WREG32(CP_PFP_UCODE_ADDR, 0);
3359
3360 /* CE */
3361 fw_data = (const __be32 *)rdev->ce_fw->data;
3362 WREG32(CP_CE_UCODE_ADDR, 0);
3363 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3364 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3365 WREG32(CP_CE_UCODE_ADDR, 0);
3366
3367 /* ME */
3368 fw_data = (const __be32 *)rdev->me_fw->data;
3369 WREG32(CP_ME_RAM_WADDR, 0);
3370 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3371 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3372 WREG32(CP_ME_RAM_WADDR, 0);
3373
3374 WREG32(CP_PFP_UCODE_ADDR, 0);
3375 WREG32(CP_CE_UCODE_ADDR, 0);
3376 WREG32(CP_ME_RAM_WADDR, 0);
3377 WREG32(CP_ME_RAM_RADDR, 0);
3378 return 0;
3379}
3380
3381/**
3382 * cik_cp_gfx_start - start the gfx ring
3383 *
3384 * @rdev: radeon_device pointer
3385 *
3386 * Enables the ring and loads the clear state context and other
3387 * packets required to init the ring.
3388 * Returns 0 for success, error for failure.
3389 */
3390static int cik_cp_gfx_start(struct radeon_device *rdev)
3391{
3392 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3393 int r, i;
3394
3395 /* init the CP */
3396 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3397 WREG32(CP_ENDIAN_SWAP, 0);
3398 WREG32(CP_DEVICE_ID, 1);
3399
3400 cik_cp_gfx_enable(rdev, true);
3401
3402 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3403 if (r) {
3404 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3405 return r;
3406 }
3407
3408 /* init the CE partitions. CE only used for gfx on CIK */
3409 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3410 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3411 radeon_ring_write(ring, 0xc000);
3412 radeon_ring_write(ring, 0xc000);
3413
3414 /* setup clear context state */
3415 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3416 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3417
3418 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3419 radeon_ring_write(ring, 0x80000000);
3420 radeon_ring_write(ring, 0x80000000);
3421
3422 for (i = 0; i < cik_default_size; i++)
3423 radeon_ring_write(ring, cik_default_state[i]);
3424
3425 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3426 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3427
3428 /* set clear context state */
3429 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3430 radeon_ring_write(ring, 0);
3431
3432 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3433 radeon_ring_write(ring, 0x00000316);
3434 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3435 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3436
3437 radeon_ring_unlock_commit(rdev, ring);
3438
3439 return 0;
3440}
3441
3442/**
3443 * cik_cp_gfx_fini - stop the gfx ring
3444 *
3445 * @rdev: radeon_device pointer
3446 *
3447 * Stop the gfx ring and tear down the driver ring
3448 * info.
3449 */
3450static void cik_cp_gfx_fini(struct radeon_device *rdev)
3451{
3452 cik_cp_gfx_enable(rdev, false);
3453 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3454}
3455
3456/**
3457 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3458 *
3459 * @rdev: radeon_device pointer
3460 *
3461 * Program the location and size of the gfx ring buffer
3462 * and test it to make sure it's working.
3463 * Returns 0 for success, error for failure.
3464 */
3465static int cik_cp_gfx_resume(struct radeon_device *rdev)
3466{
3467 struct radeon_ring *ring;
3468 u32 tmp;
3469 u32 rb_bufsz;
3470 u64 rb_addr;
3471 int r;
3472
3473 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3474 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3475
3476 /* Set the write pointer delay */
3477 WREG32(CP_RB_WPTR_DELAY, 0);
3478
3479 /* set the RB to use vmid 0 */
3480 WREG32(CP_RB_VMID, 0);
3481
3482 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3483
3484 /* ring 0 - compute and gfx */
3485 /* Set ring buffer size */
3486 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3487 rb_bufsz = order_base_2(ring->ring_size / 8);
3488 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 3489#ifdef __BIG_ENDIAN
3490 tmp |= BUF_SWAP_32BIT;
3491#endif
3492 WREG32(CP_RB0_CNTL, tmp);
3493
3494 /* Initialize the ring buffer's read and write pointers */
3495 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3496 ring->wptr = 0;
3497 WREG32(CP_RB0_WPTR, ring->wptr);
3498
3499 /* set the wb address wether it's enabled or not */
3500 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3501 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3502
3503 /* scratch register shadowing is no longer supported */
3504 WREG32(SCRATCH_UMSK, 0);
3505
3506 if (!rdev->wb.enabled)
3507 tmp |= RB_NO_UPDATE;
3508
3509 mdelay(1);
3510 WREG32(CP_RB0_CNTL, tmp);
3511
3512 rb_addr = ring->gpu_addr >> 8;
3513 WREG32(CP_RB0_BASE, rb_addr);
3514 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3515
3516 ring->rptr = RREG32(CP_RB0_RPTR);
3517
3518 /* start the ring */
3519 cik_cp_gfx_start(rdev);
3520 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3521 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3522 if (r) {
3523 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3524 return r;
3525 }
3526 return 0;
3527}
3528
963e81f9
AD		 3529u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
3530 struct radeon_ring *ring)
3531{
3532 u32 rptr;
3533
3534
3535
3536 if (rdev->wb.enabled) {
3537 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
3538 } else {
f61d5b46 3539 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3540 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3541 rptr = RREG32(CP_HQD_PQ_RPTR);
3542 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3543 mutex_unlock(&rdev->srbm_mutex);
963e81f9 3544 }
963e81f9
AD		 3545
3546 return rptr;
3547}
3548
3549u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
3550 struct radeon_ring *ring)
3551{
3552 u32 wptr;
3553
3554 if (rdev->wb.enabled) {
3555 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
3556 } else {
f61d5b46 3557 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3558 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3559 wptr = RREG32(CP_HQD_PQ_WPTR);
3560 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3561 mutex_unlock(&rdev->srbm_mutex);
963e81f9 3562 }
963e81f9
AD		 3563
3564 return wptr;
3565}
3566
3567void cik_compute_ring_set_wptr(struct radeon_device *rdev,
3568 struct radeon_ring *ring)
3569{
2e1e6dad
CK		 3570 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
3571 WDOORBELL32(ring->doorbell_offset, ring->wptr);
963e81f9
AD		 3572}
3573
841cf442
AD		 3574/**
3575 * cik_cp_compute_enable - enable/disable the compute CP MEs
3576 *
3577 * @rdev: radeon_device pointer
3578 * @enable: enable or disable the MEs
3579 *
3580 * Halts or unhalts the compute MEs.
3581 */
3582static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
3583{
3584 if (enable)
3585 WREG32(CP_MEC_CNTL, 0);
3586 else
3587 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
3588 udelay(50);
3589}
3590
3591/**
3592 * cik_cp_compute_load_microcode - load the compute CP ME ucode
3593 *
3594 * @rdev: radeon_device pointer
3595 *
3596 * Loads the compute MEC1&2 ucode.
3597 * Returns 0 for success, -EINVAL if the ucode is not available.
3598 */
3599static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
3600{
3601 const __be32 *fw_data;
3602 int i;
3603
3604 if (!rdev->mec_fw)
3605 return -EINVAL;
3606
3607 cik_cp_compute_enable(rdev, false);
3608
3609 /* MEC1 */
3610 fw_data = (const __be32 *)rdev->mec_fw->data;
3611 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3612 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3613 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
3614 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3615
3616 if (rdev->family == CHIP_KAVERI) {
3617 /* MEC2 */
3618 fw_data = (const __be32 *)rdev->mec_fw->data;
3619 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3620 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3621 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
3622 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3623 }
3624
3625 return 0;
3626}
3627
3628/**
3629 * cik_cp_compute_start - start the compute queues
3630 *
3631 * @rdev: radeon_device pointer
3632 *
3633 * Enable the compute queues.
3634 * Returns 0 for success, error for failure.
3635 */
3636static int cik_cp_compute_start(struct radeon_device *rdev)
3637{
963e81f9
AD		 3638 cik_cp_compute_enable(rdev, true);
3639
841cf442
AD		 3640 return 0;
3641}
3642
3643/**
3644 * cik_cp_compute_fini - stop the compute queues
3645 *
3646 * @rdev: radeon_device pointer
3647 *
3648 * Stop the compute queues and tear down the driver queue
3649 * info.
3650 */
3651static void cik_cp_compute_fini(struct radeon_device *rdev)
3652{
963e81f9
AD		 3653 int i, idx, r;
3654
841cf442 3655 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 3656
3657 for (i = 0; i < 2; i++) {
3658 if (i == 0)
3659 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3660 else
3661 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3662
3663 if (rdev->ring[idx].mqd_obj) {
3664 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3665 if (unlikely(r != 0))
3666 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
3667
3668 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
3669 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3670
3671 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
3672 rdev->ring[idx].mqd_obj = NULL;
3673 }
3674 }
841cf442
AD		 3675}
3676
963e81f9
AD		 3677static void cik_mec_fini(struct radeon_device *rdev)
3678{
3679 int r;
3680
3681 if (rdev->mec.hpd_eop_obj) {
3682 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3683 if (unlikely(r != 0))
3684 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
3685 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
3686 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3687
3688 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
3689 rdev->mec.hpd_eop_obj = NULL;
3690 }
3691}
3692
3693#define MEC_HPD_SIZE 2048
3694
3695static int cik_mec_init(struct radeon_device *rdev)
3696{
3697 int r;
3698 u32 *hpd;
3699
3700 /*
3701 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
3702 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
3703 */
3704 if (rdev->family == CHIP_KAVERI)
3705 rdev->mec.num_mec = 2;
3706 else
3707 rdev->mec.num_mec = 1;
3708 rdev->mec.num_pipe = 4;
3709 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
3710
3711 if (rdev->mec.hpd_eop_obj == NULL) {
3712 r = radeon_bo_create(rdev,
3713 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
3714 PAGE_SIZE, true,
3715 RADEON_GEM_DOMAIN_GTT, NULL,
3716 &rdev->mec.hpd_eop_obj);
3717 if (r) {
3718 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
3719 return r;
3720 }
3721 }
3722
3723 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3724 if (unlikely(r != 0)) {
3725 cik_mec_fini(rdev);
3726 return r;
3727 }
3728 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
3729 &rdev->mec.hpd_eop_gpu_addr);
3730 if (r) {
3731 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
3732 cik_mec_fini(rdev);
3733 return r;
3734 }
3735 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
3736 if (r) {
3737 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
3738 cik_mec_fini(rdev);
3739 return r;
3740 }
3741
3742 /* clear memory. Not sure if this is required or not */
3743 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
3744
3745 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
3746 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3747
3748 return 0;
3749}
3750
3751struct hqd_registers
3752{
3753 u32 cp_mqd_base_addr;
3754 u32 cp_mqd_base_addr_hi;
3755 u32 cp_hqd_active;
3756 u32 cp_hqd_vmid;
3757 u32 cp_hqd_persistent_state;
3758 u32 cp_hqd_pipe_priority;
3759 u32 cp_hqd_queue_priority;
3760 u32 cp_hqd_quantum;
3761 u32 cp_hqd_pq_base;
3762 u32 cp_hqd_pq_base_hi;
3763 u32 cp_hqd_pq_rptr;
3764 u32 cp_hqd_pq_rptr_report_addr;
3765 u32 cp_hqd_pq_rptr_report_addr_hi;
3766 u32 cp_hqd_pq_wptr_poll_addr;
3767 u32 cp_hqd_pq_wptr_poll_addr_hi;
3768 u32 cp_hqd_pq_doorbell_control;
3769 u32 cp_hqd_pq_wptr;
3770 u32 cp_hqd_pq_control;
3771 u32 cp_hqd_ib_base_addr;
3772 u32 cp_hqd_ib_base_addr_hi;
3773 u32 cp_hqd_ib_rptr;
3774 u32 cp_hqd_ib_control;
3775 u32 cp_hqd_iq_timer;
3776 u32 cp_hqd_iq_rptr;
3777 u32 cp_hqd_dequeue_request;
3778 u32 cp_hqd_dma_offload;
3779 u32 cp_hqd_sema_cmd;
3780 u32 cp_hqd_msg_type;
3781 u32 cp_hqd_atomic0_preop_lo;
3782 u32 cp_hqd_atomic0_preop_hi;
3783 u32 cp_hqd_atomic1_preop_lo;
3784 u32 cp_hqd_atomic1_preop_hi;
3785 u32 cp_hqd_hq_scheduler0;
3786 u32 cp_hqd_hq_scheduler1;
3787 u32 cp_mqd_control;
3788};
3789
3790struct bonaire_mqd
3791{
3792 u32 header;
3793 u32 dispatch_initiator;
3794 u32 dimensions[3];
3795 u32 start_idx[3];
3796 u32 num_threads[3];
3797 u32 pipeline_stat_enable;
3798 u32 perf_counter_enable;
3799 u32 pgm[2];
3800 u32 tba[2];
3801 u32 tma[2];
3802 u32 pgm_rsrc[2];
3803 u32 vmid;
3804 u32 resource_limits;
3805 u32 static_thread_mgmt01[2];
3806 u32 tmp_ring_size;
3807 u32 static_thread_mgmt23[2];
3808 u32 restart[3];
3809 u32 thread_trace_enable;
3810 u32 reserved1;
3811 u32 user_data[16];
3812 u32 vgtcs_invoke_count[2];
3813 struct hqd_registers queue_state;
3814 u32 dequeue_cntr;
3815 u32 interrupt_queue[64];
3816};
3817
841cf442
AD		 3818/**
3819 * cik_cp_compute_resume - setup the compute queue registers
3820 *
3821 * @rdev: radeon_device pointer
3822 *
3823 * Program the compute queues and test them to make sure they
3824 * are working.
3825 * Returns 0 for success, error for failure.
3826 */
3827static int cik_cp_compute_resume(struct radeon_device *rdev)
3828{
963e81f9
AD		 3829 int r, i, idx;
3830 u32 tmp;
3831 bool use_doorbell = true;
3832 u64 hqd_gpu_addr;
3833 u64 mqd_gpu_addr;
3834 u64 eop_gpu_addr;
3835 u64 wb_gpu_addr;
3836 u32 *buf;
3837 struct bonaire_mqd *mqd;
841cf442 3838
841cf442
AD		 3839 r = cik_cp_compute_start(rdev);
3840 if (r)
3841 return r;
963e81f9
AD		 3842
3843 /* fix up chicken bits */
3844 tmp = RREG32(CP_CPF_DEBUG);
3845 tmp |= (1 << 23);
3846 WREG32(CP_CPF_DEBUG, tmp);
3847
3848 /* init the pipes */
f61d5b46 3849 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3850 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
3851 int me = (i < 4) ? 1 : 2;
3852 int pipe = (i < 4) ? i : (i - 4);
3853
3854 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
3855
3856 cik_srbm_select(rdev, me, pipe, 0, 0);
3857
3858 /* write the EOP addr */
3859 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
3860 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
3861
3862 /* set the VMID assigned */
3863 WREG32(CP_HPD_EOP_VMID, 0);
3864
3865 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3866 tmp = RREG32(CP_HPD_EOP_CONTROL);
3867 tmp &= ~EOP_SIZE_MASK;
b72a8925 3868 tmp |= order_base_2(MEC_HPD_SIZE / 8);
963e81f9
AD		 3869 WREG32(CP_HPD_EOP_CONTROL, tmp);
3870 }
3871 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 3872 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 3873
3874 /* init the queues. Just two for now. */
3875 for (i = 0; i < 2; i++) {
3876 if (i == 0)
3877 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3878 else
3879 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3880
3881 if (rdev->ring[idx].mqd_obj == NULL) {
3882 r = radeon_bo_create(rdev,
3883 sizeof(struct bonaire_mqd),
3884 PAGE_SIZE, true,
3885 RADEON_GEM_DOMAIN_GTT, NULL,
3886 &rdev->ring[idx].mqd_obj);
3887 if (r) {
3888 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
3889 return r;
3890 }
3891 }
3892
3893 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3894 if (unlikely(r != 0)) {
3895 cik_cp_compute_fini(rdev);
3896 return r;
3897 }
3898 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
3899 &mqd_gpu_addr);
3900 if (r) {
3901 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
3902 cik_cp_compute_fini(rdev);
3903 return r;
3904 }
3905 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
3906 if (r) {
3907 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
3908 cik_cp_compute_fini(rdev);
3909 return r;
3910 }
3911
3912 /* doorbell offset */
3913 rdev->ring[idx].doorbell_offset =
3914 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
3915
3916 /* init the mqd struct */
3917 memset(buf, 0, sizeof(struct bonaire_mqd));
3918
3919 mqd = (struct bonaire_mqd *)buf;
3920 mqd->header = 0xC0310800;
3921 mqd->static_thread_mgmt01[0] = 0xffffffff;
3922 mqd->static_thread_mgmt01[1] = 0xffffffff;
3923 mqd->static_thread_mgmt23[0] = 0xffffffff;
3924 mqd->static_thread_mgmt23[1] = 0xffffffff;
3925
f61d5b46 3926 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 3927 cik_srbm_select(rdev, rdev->ring[idx].me,
3928 rdev->ring[idx].pipe,
3929 rdev->ring[idx].queue, 0);
3930
3931 /* disable wptr polling */
3932 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
3933 tmp &= ~WPTR_POLL_EN;
3934 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
3935
3936 /* enable doorbell? */
3937 mqd->queue_state.cp_hqd_pq_doorbell_control =
3938 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3939 if (use_doorbell)
3940 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3941 else
3942 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
3943 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3944 mqd->queue_state.cp_hqd_pq_doorbell_control);
3945
3946 /* disable the queue if it's active */
3947 mqd->queue_state.cp_hqd_dequeue_request = 0;
3948 mqd->queue_state.cp_hqd_pq_rptr = 0;
3949 mqd->queue_state.cp_hqd_pq_wptr= 0;
3950 if (RREG32(CP_HQD_ACTIVE) & 1) {
3951 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
3952 for (i = 0; i < rdev->usec_timeout; i++) {
3953 if (!(RREG32(CP_HQD_ACTIVE) & 1))
3954 break;
3955 udelay(1);
3956 }
3957 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
3958 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
3959 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3960 }
3961
3962 /* set the pointer to the MQD */
3963 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
3964 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3965 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
3966 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
3967 /* set MQD vmid to 0 */
3968 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
3969 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
3970 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
3971
3972 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3973 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
3974 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
3975 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3976 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
3977 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
3978
3979 /* set up the HQD, this is similar to CP_RB0_CNTL */
3980 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
3981 mqd->queue_state.cp_hqd_pq_control &=
3982 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
3983
3984 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 3985 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 3986 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 3987 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 3988#ifdef __BIG_ENDIAN
3989 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
3990#endif
3991 mqd->queue_state.cp_hqd_pq_control &=
3992 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
3993 mqd->queue_state.cp_hqd_pq_control |=
3994 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
3995 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
3996
3997 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
3998 if (i == 0)
3999 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4000 else
4001 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4002 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4003 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4004 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4005 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4006 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4007
4008 /* set the wb address wether it's enabled or not */
4009 if (i == 0)
4010 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4011 else
4012 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4013 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4014 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4015 upper_32_bits(wb_gpu_addr) & 0xffff;
4016 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4017 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4018 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4019 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4020
4021 /* enable the doorbell if requested */
4022 if (use_doorbell) {
4023 mqd->queue_state.cp_hqd_pq_doorbell_control =
4024 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4025 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4026 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4027 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
4028 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4029 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4030 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4031
4032 } else {
4033 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4034 }
4035 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4036 mqd->queue_state.cp_hqd_pq_doorbell_control);
4037
4038 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4039 rdev->ring[idx].wptr = 0;
4040 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4041 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4042 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4043 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4044
4045 /* set the vmid for the queue */
4046 mqd->queue_state.cp_hqd_vmid = 0;
4047 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4048
4049 /* activate the queue */
4050 mqd->queue_state.cp_hqd_active = 1;
4051 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4052
4053 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4054 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4055
4056 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4057 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4058
4059 rdev->ring[idx].ready = true;
4060 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4061 if (r)
4062 rdev->ring[idx].ready = false;
4063 }
4064
841cf442
AD		 4065 return 0;
4066}
4067
841cf442
AD		 4068static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4069{
4070 cik_cp_gfx_enable(rdev, enable);
4071 cik_cp_compute_enable(rdev, enable);
4072}
4073
841cf442
AD		 4074static int cik_cp_load_microcode(struct radeon_device *rdev)
4075{
4076 int r;
4077
4078 r = cik_cp_gfx_load_microcode(rdev);
4079 if (r)
4080 return r;
4081 r = cik_cp_compute_load_microcode(rdev);
4082 if (r)
4083 return r;
4084
4085 return 0;
4086}
4087
841cf442
AD		 4088static void cik_cp_fini(struct radeon_device *rdev)
4089{
4090 cik_cp_gfx_fini(rdev);
4091 cik_cp_compute_fini(rdev);
4092}
4093
841cf442
AD		 4094static int cik_cp_resume(struct radeon_device *rdev)
4095{
4096 int r;
4097
4214faf6
AD		 4098 cik_enable_gui_idle_interrupt(rdev, false);
4099
841cf442
AD		 4100 r = cik_cp_load_microcode(rdev);
4101 if (r)
4102 return r;
4103
4104 r = cik_cp_gfx_resume(rdev);
4105 if (r)
4106 return r;
4107 r = cik_cp_compute_resume(rdev);
4108 if (r)
4109 return r;
4110
4214faf6
AD		 4111 cik_enable_gui_idle_interrupt(rdev, true);
4112
841cf442
AD		 4113 return 0;
4114}
4115
cc066715 4116static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4117{
6f2043ce
AD		 4118 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4119 RREG32(GRBM_STATUS));
4120 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4121 RREG32(GRBM_STATUS2));
4122 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4123 RREG32(GRBM_STATUS_SE0));
4124 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4125 RREG32(GRBM_STATUS_SE1));
4126 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4127 RREG32(GRBM_STATUS_SE2));
4128 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4129 RREG32(GRBM_STATUS_SE3));
4130 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4131 RREG32(SRBM_STATUS));
4132 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4133 RREG32(SRBM_STATUS2));
cc066715
AD		 4134 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4135 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4136 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4137 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4138 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4139 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4140 RREG32(CP_STALLED_STAT1));
4141 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4142 RREG32(CP_STALLED_STAT2));
4143 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4144 RREG32(CP_STALLED_STAT3));
4145 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4146 RREG32(CP_CPF_BUSY_STAT));
4147 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4148 RREG32(CP_CPF_STALLED_STAT1));
4149 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4150 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4151 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4152 RREG32(CP_CPC_STALLED_STAT1));
4153 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4154}
6f2043ce 4155
21a93e13 4156/**
cc066715 4157 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4158 *
4159 * @rdev: radeon_device pointer
21a93e13 4160 *
cc066715
AD		 4161 * Check which blocks are busy and return the relevant reset
4162 * mask to be used by cik_gpu_soft_reset().
4163 * Returns a mask of the blocks to be reset.
21a93e13 4164 */
2483b4ea 4165u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4166{
cc066715
AD		 4167 u32 reset_mask = 0;
4168 u32 tmp;
21a93e13 4169
cc066715
AD		 4170 /* GRBM_STATUS */
4171 tmp = RREG32(GRBM_STATUS);
4172 if (tmp & (PA_BUSY | SC_BUSY |
4173 BCI_BUSY | SX_BUSY |
4174 TA_BUSY | VGT_BUSY |
4175 DB_BUSY | CB_BUSY |
4176 GDS_BUSY | SPI_BUSY |
4177 IA_BUSY | IA_BUSY_NO_DMA))
4178 reset_mask |= RADEON_RESET_GFX;
21a93e13 4179
cc066715
AD		 4180 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4181 reset_mask |= RADEON_RESET_CP;
21a93e13 4182
cc066715
AD		 4183 /* GRBM_STATUS2 */
4184 tmp = RREG32(GRBM_STATUS2);
4185 if (tmp & RLC_BUSY)
4186 reset_mask |= RADEON_RESET_RLC;
21a93e13 4187
cc066715
AD		 4188 /* SDMA0_STATUS_REG */
4189 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4190 if (!(tmp & SDMA_IDLE))
4191 reset_mask |= RADEON_RESET_DMA;
21a93e13 4192
cc066715
AD		 4193 /* SDMA1_STATUS_REG */
4194 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4195 if (!(tmp & SDMA_IDLE))
4196 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4197
cc066715
AD		 4198 /* SRBM_STATUS2 */
4199 tmp = RREG32(SRBM_STATUS2);
4200 if (tmp & SDMA_BUSY)
4201 reset_mask |= RADEON_RESET_DMA;
21a93e13 4202
cc066715
AD		 4203 if (tmp & SDMA1_BUSY)
4204 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4205
cc066715
AD		 4206 /* SRBM_STATUS */
4207 tmp = RREG32(SRBM_STATUS);
21a93e13 4208
cc066715
AD		 4209 if (tmp & IH_BUSY)
4210 reset_mask |= RADEON_RESET_IH;
21a93e13 4211
cc066715
AD		 4212 if (tmp & SEM_BUSY)
4213 reset_mask |= RADEON_RESET_SEM;
21a93e13 4214
cc066715
AD		 4215 if (tmp & GRBM_RQ_PENDING)
4216 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4217
cc066715
AD		 4218 if (tmp & VMC_BUSY)
4219 reset_mask |= RADEON_RESET_VMC;
21a93e13 4220
cc066715
AD		 4221 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4222 MCC_BUSY | MCD_BUSY))
4223 reset_mask |= RADEON_RESET_MC;
21a93e13 4224
cc066715
AD		 4225 if (evergreen_is_display_hung(rdev))
4226 reset_mask |= RADEON_RESET_DISPLAY;
4227
4228 /* Skip MC reset as it's mostly likely not hung, just busy */
4229 if (reset_mask & RADEON_RESET_MC) {
4230 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4231 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4232 }
cc066715
AD		 4233
4234 return reset_mask;
21a93e13
AD		 4235}
4236
4237/**
cc066715 4238 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4239 *
4240 * @rdev: radeon_device pointer
cc066715 4241 * @reset_mask: mask of which blocks to reset
21a93e13 4242 *
cc066715 4243 * Soft reset the blocks specified in @reset_mask.
21a93e13 4244 */
cc066715 4245static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4246{
6f2043ce 4247 struct evergreen_mc_save save;
cc066715
AD		 4248 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4249 u32 tmp;
21a93e13 4250
cc066715
AD		 4251 if (reset_mask == 0)
4252 return;
21a93e13 4253
cc066715 4254 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 4255
cc066715
AD		 4256 cik_print_gpu_status_regs(rdev);
4257 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4258 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4259 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4260 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 4261
cc066715
AD		 4262 /* stop the rlc */
4263 cik_rlc_stop(rdev);
21a93e13 4264
cc066715
AD		 4265 /* Disable GFX parsing/prefetching */
4266 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 4267
cc066715
AD		 4268 /* Disable MEC parsing/prefetching */
4269 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 4270
cc066715
AD		 4271 if (reset_mask & RADEON_RESET_DMA) {
4272 /* sdma0 */
4273 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4274 tmp |= SDMA_HALT;
4275 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4276 }
4277 if (reset_mask & RADEON_RESET_DMA1) {
4278 /* sdma1 */
4279 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4280 tmp |= SDMA_HALT;
4281 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4282 }
21a93e13 4283
6f2043ce 4284 evergreen_mc_stop(rdev, &save);
cc066715 4285 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 4286 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4287 }
21a93e13 4288
cc066715
AD		 4289 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4290 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 4291
cc066715
AD		 4292 if (reset_mask & RADEON_RESET_CP) {
4293 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 4294
cc066715
AD		 4295 srbm_soft_reset |= SOFT_RESET_GRBM;
4296 }
21a93e13 4297
cc066715
AD		 4298 if (reset_mask & RADEON_RESET_DMA)
4299 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 4300
cc066715
AD		 4301 if (reset_mask & RADEON_RESET_DMA1)
4302 srbm_soft_reset |= SOFT_RESET_SDMA1;
4303
4304 if (reset_mask & RADEON_RESET_DISPLAY)
4305 srbm_soft_reset |= SOFT_RESET_DC;
4306
4307 if (reset_mask & RADEON_RESET_RLC)
4308 grbm_soft_reset |= SOFT_RESET_RLC;
4309
4310 if (reset_mask & RADEON_RESET_SEM)
4311 srbm_soft_reset |= SOFT_RESET_SEM;
4312
4313 if (reset_mask & RADEON_RESET_IH)
4314 srbm_soft_reset |= SOFT_RESET_IH;
4315
4316 if (reset_mask & RADEON_RESET_GRBM)
4317 srbm_soft_reset |= SOFT_RESET_GRBM;
4318
4319 if (reset_mask & RADEON_RESET_VMC)
4320 srbm_soft_reset |= SOFT_RESET_VMC;
4321
4322 if (!(rdev->flags & RADEON_IS_IGP)) {
4323 if (reset_mask & RADEON_RESET_MC)
4324 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 4325 }
4326
cc066715
AD		 4327 if (grbm_soft_reset) {
4328 tmp = RREG32(GRBM_SOFT_RESET);
4329 tmp |= grbm_soft_reset;
4330 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4331 WREG32(GRBM_SOFT_RESET, tmp);
4332 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 4333
cc066715 4334 udelay(50);
21a93e13 4335
cc066715
AD		 4336 tmp &= ~grbm_soft_reset;
4337 WREG32(GRBM_SOFT_RESET, tmp);
4338 tmp = RREG32(GRBM_SOFT_RESET);
4339 }
21a93e13 4340
cc066715
AD		 4341 if (srbm_soft_reset) {
4342 tmp = RREG32(SRBM_SOFT_RESET);
4343 tmp |= srbm_soft_reset;
4344 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4345 WREG32(SRBM_SOFT_RESET, tmp);
4346 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 4347
cc066715 4348 udelay(50);
21a93e13 4349
cc066715
AD		 4350 tmp &= ~srbm_soft_reset;
4351 WREG32(SRBM_SOFT_RESET, tmp);
4352 tmp = RREG32(SRBM_SOFT_RESET);
4353 }
21a93e13 4354
6f2043ce
AD		 4355 /* Wait a little for things to settle down */
4356 udelay(50);
21a93e13 4357
6f2043ce 4358 evergreen_mc_resume(rdev, &save);
cc066715
AD		 4359 udelay(50);
4360
4361 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 4362}
4363
4364/**
cc066715 4365 * cik_asic_reset - soft reset GPU
21a93e13
AD		 4366 *
4367 * @rdev: radeon_device pointer
4368 *
cc066715
AD		 4369 * Look up which blocks are hung and attempt
4370 * to reset them.
6f2043ce 4371 * Returns 0 for success.
21a93e13 4372 */
6f2043ce 4373int cik_asic_reset(struct radeon_device *rdev)
21a93e13 4374{
cc066715 4375 u32 reset_mask;
21a93e13 4376
cc066715 4377 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 4378
cc066715
AD		 4379 if (reset_mask)
4380 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 4381
cc066715 4382 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 4383
cc066715
AD		 4384 reset_mask = cik_gpu_check_soft_reset(rdev);
4385
4386 if (!reset_mask)
4387 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 4388
4389 return 0;
4390}
4391
4392/**
cc066715 4393 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 4394 *
4395 * @rdev: radeon_device pointer
cc066715 4396 * @ring: radeon_ring structure holding ring information
21a93e13 4397 *
cc066715
AD		 4398 * Check if the 3D engine is locked up (CIK).
4399 * Returns true if the engine is locked, false if not.
21a93e13 4400 */
cc066715 4401bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 4402{
cc066715 4403 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 4404
cc066715
AD		 4405 if (!(reset_mask & (RADEON_RESET_GFX |
4406 RADEON_RESET_COMPUTE |
4407 RADEON_RESET_CP))) {
4408 radeon_ring_lockup_update(ring);
4409 return false;
21a93e13 4410 }
cc066715
AD		 4411 /* force CP activities */
4412 radeon_ring_force_activity(rdev, ring);
4413 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 4414}
4415
1c49165d 4416/* MC */
21a93e13 4417/**
1c49165d 4418 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 4419 *
4420 * @rdev: radeon_device pointer
21a93e13 4421 *
1c49165d
AD		 4422 * Set the location of vram, gart, and AGP in the GPU's
4423 * physical address space (CIK).
21a93e13 4424 */
1c49165d 4425static void cik_mc_program(struct radeon_device *rdev)
21a93e13 4426{
1c49165d 4427 struct evergreen_mc_save save;
21a93e13 4428 u32 tmp;
1c49165d 4429 int i, j;
21a93e13 4430
1c49165d
AD		 4431 /* Initialize HDP */
4432 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4433 WREG32((0x2c14 + j), 0x00000000);
4434 WREG32((0x2c18 + j), 0x00000000);
4435 WREG32((0x2c1c + j), 0x00000000);
4436 WREG32((0x2c20 + j), 0x00000000);
4437 WREG32((0x2c24 + j), 0x00000000);
21a93e13 4438 }
1c49165d 4439 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 4440
1c49165d
AD		 4441 evergreen_mc_stop(rdev, &save);
4442 if (radeon_mc_wait_for_idle(rdev)) {
4443 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 4444 }
1c49165d
AD		 4445 /* Lockout access through VGA aperture*/
4446 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4447 /* Update configuration */
4448 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4449 rdev->mc.vram_start >> 12);
4450 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4451 rdev->mc.vram_end >> 12);
4452 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4453 rdev->vram_scratch.gpu_addr >> 12);
4454 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4455 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4456 WREG32(MC_VM_FB_LOCATION, tmp);
4457 /* XXX double check these! */
4458 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4459 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4460 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4461 WREG32(MC_VM_AGP_BASE, 0);
4462 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4463 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4464 if (radeon_mc_wait_for_idle(rdev)) {
4465 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 4466 }
1c49165d
AD		 4467 evergreen_mc_resume(rdev, &save);
4468 /* we need to own VRAM, so turn off the VGA renderer here
4469 * to stop it overwriting our objects */
4470 rv515_vga_render_disable(rdev);
21a93e13
AD		 4471}
4472
4473/**
1c49165d 4474 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 4475 *
4476 * @rdev: radeon_device pointer
21a93e13 4477 *
1c49165d
AD		 4478 * Look up the amount of vram, vram width, and decide how to place
4479 * vram and gart within the GPU's physical address space (CIK).
4480 * Returns 0 for success.
21a93e13 4481 */
1c49165d 4482static int cik_mc_init(struct radeon_device *rdev)
21a93e13 4483{
1c49165d
AD		 4484 u32 tmp;
4485 int chansize, numchan;
21a93e13 4486
1c49165d
AD		 4487 /* Get VRAM informations */
4488 rdev->mc.vram_is_ddr = true;
4489 tmp = RREG32(MC_ARB_RAMCFG);
4490 if (tmp & CHANSIZE_MASK) {
4491 chansize = 64;
21a93e13 4492 } else {
1c49165d 4493 chansize = 32;
21a93e13 4494 }
1c49165d
AD		 4495 tmp = RREG32(MC_SHARED_CHMAP);
4496 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4497 case 0:
4498 default:
4499 numchan = 1;
4500 break;
4501 case 1:
4502 numchan = 2;
4503 break;
4504 case 2:
4505 numchan = 4;
4506 break;
4507 case 3:
4508 numchan = 8;
4509 break;
4510 case 4:
4511 numchan = 3;
4512 break;
4513 case 5:
4514 numchan = 6;
4515 break;
4516 case 6:
4517 numchan = 10;
4518 break;
4519 case 7:
4520 numchan = 12;
4521 break;
4522 case 8:
4523 numchan = 16;
4524 break;
4525 }
4526 rdev->mc.vram_width = numchan * chansize;
4527 /* Could aper size report 0 ? */
4528 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4529 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4530 /* size in MB on si */
13c5bfda
AD		 4531 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
4532 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 4533 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4534 si_vram_gtt_location(rdev, &rdev->mc);
4535 radeon_update_bandwidth_info(rdev);
4536
4537 return 0;
4538}
4539
4540/*
4541 * GART
4542 * VMID 0 is the physical GPU addresses as used by the kernel.
4543 * VMIDs 1-15 are used for userspace clients and are handled
4544 * by the radeon vm/hsa code.
4545 */
4546/**
4547 * cik_pcie_gart_tlb_flush - gart tlb flush callback
4548 *
4549 * @rdev: radeon_device pointer
4550 *
4551 * Flush the TLB for the VMID 0 page table (CIK).
4552 */
4553void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4554{
4555 /* flush hdp cache */
4556 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4557
4558 /* bits 0-15 are the VM contexts0-15 */
4559 WREG32(VM_INVALIDATE_REQUEST, 0x1);
4560}
4561
4562/**
4563 * cik_pcie_gart_enable - gart enable
4564 *
4565 * @rdev: radeon_device pointer
4566 *
4567 * This sets up the TLBs, programs the page tables for VMID0,
4568 * sets up the hw for VMIDs 1-15 which are allocated on
4569 * demand, and sets up the global locations for the LDS, GDS,
4570 * and GPUVM for FSA64 clients (CIK).
4571 * Returns 0 for success, errors for failure.
4572 */
4573static int cik_pcie_gart_enable(struct radeon_device *rdev)
4574{
4575 int r, i;
4576
4577 if (rdev->gart.robj == NULL) {
4578 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4579 return -EINVAL;
4580 }
4581 r = radeon_gart_table_vram_pin(rdev);
4582 if (r)
4583 return r;
4584 radeon_gart_restore(rdev);
4585 /* Setup TLB control */
4586 WREG32(MC_VM_MX_L1_TLB_CNTL,
4587 (0xA << 7) |
4588 ENABLE_L1_TLB |
4589 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4590 ENABLE_ADVANCED_DRIVER_MODEL |
4591 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4592 /* Setup L2 cache */
4593 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4594 ENABLE_L2_FRAGMENT_PROCESSING |
4595 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4596 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4597 EFFECTIVE_L2_QUEUE_SIZE(7) |
4598 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4599 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4600 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4601 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4602 /* setup context0 */
4603 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4604 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4605 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4606 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4607 (u32)(rdev->dummy_page.addr >> 12));
4608 WREG32(VM_CONTEXT0_CNTL2, 0);
4609 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4610 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4611
4612 WREG32(0x15D4, 0);
4613 WREG32(0x15D8, 0);
4614 WREG32(0x15DC, 0);
4615
4616 /* empty context1-15 */
4617 /* FIXME start with 4G, once using 2 level pt switch to full
4618 * vm size space
4619 */
4620 /* set vm size, must be a multiple of 4 */
4621 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4622 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4623 for (i = 1; i < 16; i++) {
4624 if (i < 8)
4625 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4626 rdev->gart.table_addr >> 12);
4627 else
4628 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4629 rdev->gart.table_addr >> 12);
4630 }
4631
4632 /* enable context1-15 */
4633 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4634 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 4635 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 4636 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
a00024b0
AD		 4637 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4638 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4639 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4640 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4641 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4642 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4643 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4644 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4645 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4646 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4647 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4648 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d
AD		 4649
4650 /* TC cache setup ??? */
4651 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
4652 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
4653 WREG32(TC_CFG_L1_STORE_POLICY, 0);
4654
4655 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
4656 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
4657 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
4658 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
4659 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
4660
4661 WREG32(TC_CFG_L1_VOLATILE, 0);
4662 WREG32(TC_CFG_L2_VOLATILE, 0);
4663
4664 if (rdev->family == CHIP_KAVERI) {
4665 u32 tmp = RREG32(CHUB_CONTROL);
4666 tmp &= ~BYPASS_VM;
4667 WREG32(CHUB_CONTROL, tmp);
4668 }
4669
4670 /* XXX SH_MEM regs */
4671 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 4672 mutex_lock(&rdev->srbm_mutex);
1c49165d 4673 for (i = 0; i < 16; i++) {
b556b12e 4674 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 4675 /* CP and shaders */
1c49165d
AD		 4676 WREG32(SH_MEM_CONFIG, 0);
4677 WREG32(SH_MEM_APE1_BASE, 1);
4678 WREG32(SH_MEM_APE1_LIMIT, 0);
4679 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 4680 /* SDMA GFX */
4681 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
4682 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
4683 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
4684 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
4685 /* XXX SDMA RLC - todo */
1c49165d 4686 }
b556b12e 4687 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4688 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 4689
4690 cik_pcie_gart_tlb_flush(rdev);
4691 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4692 (unsigned)(rdev->mc.gtt_size >> 20),
4693 (unsigned long long)rdev->gart.table_addr);
4694 rdev->gart.ready = true;
4695 return 0;
4696}
4697
4698/**
4699 * cik_pcie_gart_disable - gart disable
4700 *
4701 * @rdev: radeon_device pointer
4702 *
4703 * This disables all VM page table (CIK).
4704 */
4705static void cik_pcie_gart_disable(struct radeon_device *rdev)
4706{
4707 /* Disable all tables */
4708 WREG32(VM_CONTEXT0_CNTL, 0);
4709 WREG32(VM_CONTEXT1_CNTL, 0);
4710 /* Setup TLB control */
4711 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4712 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4713 /* Setup L2 cache */
4714 WREG32(VM_L2_CNTL,
4715 ENABLE_L2_FRAGMENT_PROCESSING |
4716 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4717 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4718 EFFECTIVE_L2_QUEUE_SIZE(7) |
4719 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4720 WREG32(VM_L2_CNTL2, 0);
4721 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4722 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4723 radeon_gart_table_vram_unpin(rdev);
4724}
4725
4726/**
4727 * cik_pcie_gart_fini - vm fini callback
4728 *
4729 * @rdev: radeon_device pointer
4730 *
4731 * Tears down the driver GART/VM setup (CIK).
4732 */
4733static void cik_pcie_gart_fini(struct radeon_device *rdev)
4734{
4735 cik_pcie_gart_disable(rdev);
4736 radeon_gart_table_vram_free(rdev);
4737 radeon_gart_fini(rdev);
4738}
4739
4740/* vm parser */
4741/**
4742 * cik_ib_parse - vm ib_parse callback
4743 *
4744 * @rdev: radeon_device pointer
4745 * @ib: indirect buffer pointer
4746 *
4747 * CIK uses hw IB checking so this is a nop (CIK).
4748 */
4749int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4750{
4751 return 0;
4752}
4753
4754/*
4755 * vm
4756 * VMID 0 is the physical GPU addresses as used by the kernel.
4757 * VMIDs 1-15 are used for userspace clients and are handled
4758 * by the radeon vm/hsa code.
4759 */
4760/**
4761 * cik_vm_init - cik vm init callback
4762 *
4763 * @rdev: radeon_device pointer
4764 *
4765 * Inits cik specific vm parameters (number of VMs, base of vram for
4766 * VMIDs 1-15) (CIK).
4767 * Returns 0 for success.
4768 */
4769int cik_vm_init(struct radeon_device *rdev)
4770{
4771 /* number of VMs */
4772 rdev->vm_manager.nvm = 16;
4773 /* base offset of vram pages */
4774 if (rdev->flags & RADEON_IS_IGP) {
4775 u64 tmp = RREG32(MC_VM_FB_OFFSET);
4776 tmp <<= 22;
4777 rdev->vm_manager.vram_base_offset = tmp;
4778 } else
4779 rdev->vm_manager.vram_base_offset = 0;
4780
4781 return 0;
4782}
4783
4784/**
4785 * cik_vm_fini - cik vm fini callback
4786 *
4787 * @rdev: radeon_device pointer
4788 *
4789 * Tear down any asic specific VM setup (CIK).
4790 */
4791void cik_vm_fini(struct radeon_device *rdev)
4792{
4793}
4794
3ec7d11b
AD		 4795/**
4796 * cik_vm_decode_fault - print human readable fault info
4797 *
4798 * @rdev: radeon_device pointer
4799 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4800 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4801 *
4802 * Print human readable fault information (CIK).
4803 */
4804static void cik_vm_decode_fault(struct radeon_device *rdev,
4805 u32 status, u32 addr, u32 mc_client)
4806{
4807 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4808 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4809 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 4810 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
4811 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 4812
328a50c7 4813 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 4814 protections, vmid, addr,
4815 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 4816 block, mc_client, mc_id);
3ec7d11b
AD		 4817}
4818
f96ab484
AD		 4819/**
4820 * cik_vm_flush - cik vm flush using the CP
4821 *
4822 * @rdev: radeon_device pointer
4823 *
4824 * Update the page table base and flush the VM TLB
4825 * using the CP (CIK).
4826 */
4827void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4828{
4829 struct radeon_ring *ring = &rdev->ring[ridx];
4830
4831 if (vm == NULL)
4832 return;
4833
4834 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4835 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4836 WRITE_DATA_DST_SEL(0)));
4837 if (vm->id < 8) {
4838 radeon_ring_write(ring,
4839 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4840 } else {
4841 radeon_ring_write(ring,
4842 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4843 }
4844 radeon_ring_write(ring, 0);
4845 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4846
4847 /* update SH_MEM_* regs */
4848 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4849 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4850 WRITE_DATA_DST_SEL(0)));
4851 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4852 radeon_ring_write(ring, 0);
4853 radeon_ring_write(ring, VMID(vm->id));
4854
4855 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4856 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4857 WRITE_DATA_DST_SEL(0)));
4858 radeon_ring_write(ring, SH_MEM_BASES >> 2);
4859 radeon_ring_write(ring, 0);
4860
4861 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
4862 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
4863 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
4864 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 4865
f96ab484
AD		 4866 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4867 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4868 WRITE_DATA_DST_SEL(0)));
4869 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4870 radeon_ring_write(ring, 0);
4871 radeon_ring_write(ring, VMID(0));
6f2043ce 4872
f96ab484
AD		 4873 /* HDP flush */
4874 /* We should be using the WAIT_REG_MEM packet here like in
4875 * cik_fence_ring_emit(), but it causes the CP to hang in this
4876 * context...
4877 */
4878 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4879 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4880 WRITE_DATA_DST_SEL(0)));
4881 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4882 radeon_ring_write(ring, 0);
4883 radeon_ring_write(ring, 0);
4884
4885 /* bits 0-15 are the VM contexts0-15 */
4886 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4887 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4888 WRITE_DATA_DST_SEL(0)));
4889 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4890 radeon_ring_write(ring, 0);
4891 radeon_ring_write(ring, 1 << vm->id);
4892
b07fdd38
AD		 4893 /* compute doesn't have PFP */
4894 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
4895 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4896 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4897 radeon_ring_write(ring, 0x0);
4898 }
cc066715 4899}
6f2043ce 4900
f6796cae
AD		 4901/*
4902 * RLC
4903 * The RLC is a multi-purpose microengine that handles a
4904 * variety of functions, the most important of which is
4905 * the interrupt controller.
4906 */
866d83de
AD		 4907static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
4908 bool enable)
f6796cae 4909{
866d83de 4910 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 4911
866d83de
AD		 4912 if (enable)
4913 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4914 else
4915 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 4916 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 4917}
f6796cae 4918
866d83de 4919static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 4920{
cc066715 4921 u32 tmp;
6f2043ce 4922
866d83de
AD		 4923 tmp = RREG32(RLC_LB_CNTL);
4924 if (enable)
4925 tmp |= LOAD_BALANCE_ENABLE;
4926 else
4927 tmp &= ~LOAD_BALANCE_ENABLE;
4928 WREG32(RLC_LB_CNTL, tmp);
4929}
cc066715 4930
866d83de
AD		 4931static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
4932{
4933 u32 i, j, k;
4934 u32 mask;
cc066715 4935
f6796cae
AD		 4936 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
4937 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
4938 cik_select_se_sh(rdev, i, j);
4939 for (k = 0; k < rdev->usec_timeout; k++) {
4940 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
4941 break;
4942 udelay(1);
4943 }
4944 }
4945 }
4946 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 4947
f6796cae
AD		 4948 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
4949 for (k = 0; k < rdev->usec_timeout; k++) {
4950 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
4951 break;
4952 udelay(1);
4953 }
4954}
cc066715 4955
22c775ce
AD		 4956static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
4957{
4958 u32 tmp;
cc066715 4959
22c775ce
AD		 4960 tmp = RREG32(RLC_CNTL);
4961 if (tmp != rlc)
4962 WREG32(RLC_CNTL, rlc);
4963}
cc066715 4964
22c775ce
AD		 4965static u32 cik_halt_rlc(struct radeon_device *rdev)
4966{
4967 u32 data, orig;
cc066715 4968
22c775ce 4969 orig = data = RREG32(RLC_CNTL);
cc066715 4970
22c775ce
AD		 4971 if (data & RLC_ENABLE) {
4972 u32 i;
cc066715 4973
22c775ce
AD		 4974 data &= ~RLC_ENABLE;
4975 WREG32(RLC_CNTL, data);
cc066715 4976
22c775ce
AD		 4977 for (i = 0; i < rdev->usec_timeout; i++) {
4978 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
4979 break;
4980 udelay(1);
4981 }
cc066715 4982
22c775ce
AD		 4983 cik_wait_for_rlc_serdes(rdev);
4984 }
cc066715 4985
22c775ce
AD		 4986 return orig;
4987}
cc066715 4988
a412fce0
AD		 4989void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
4990{
4991 u32 tmp, i, mask;
4992
4993 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
4994 WREG32(RLC_GPR_REG2, tmp);
4995
4996 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
4997 for (i = 0; i < rdev->usec_timeout; i++) {
4998 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
4999 break;
5000 udelay(1);
5001 }
5002
5003 for (i = 0; i < rdev->usec_timeout; i++) {
5004 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5005 break;
5006 udelay(1);
5007 }
5008}
5009
5010void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5011{
5012 u32 tmp;
5013
5014 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5015 WREG32(RLC_GPR_REG2, tmp);
5016}
5017
866d83de
AD		 5018/**
5019 * cik_rlc_stop - stop the RLC ME
5020 *
5021 * @rdev: radeon_device pointer
5022 *
5023 * Halt the RLC ME (MicroEngine) (CIK).
5024 */
5025static void cik_rlc_stop(struct radeon_device *rdev)
5026{
22c775ce 5027 WREG32(RLC_CNTL, 0);
866d83de
AD		 5028
5029 cik_enable_gui_idle_interrupt(rdev, false);
5030
866d83de
AD		 5031 cik_wait_for_rlc_serdes(rdev);
5032}
5033
f6796cae
AD		 5034/**
5035 * cik_rlc_start - start the RLC ME
5036 *
5037 * @rdev: radeon_device pointer
5038 *
5039 * Unhalt the RLC ME (MicroEngine) (CIK).
5040 */
5041static void cik_rlc_start(struct radeon_device *rdev)
5042{
f6796cae 5043 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5044
866d83de 5045 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5046
f6796cae 5047 udelay(50);
6f2043ce
AD		 5048}
5049
5050/**
f6796cae 5051 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 5052 *
5053 * @rdev: radeon_device pointer
5054 *
f6796cae
AD		 5055 * Initialize the RLC registers, load the ucode,
5056 * and start the RLC (CIK).
5057 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 5058 */
f6796cae 5059static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 5060{
22c775ce 5061 u32 i, size, tmp;
f6796cae 5062 const __be32 *fw_data;
cc066715 5063
f6796cae
AD		 5064 if (!rdev->rlc_fw)
5065 return -EINVAL;
cc066715 5066
f6796cae
AD		 5067 switch (rdev->family) {
5068 case CHIP_BONAIRE:
5069 default:
5070 size = BONAIRE_RLC_UCODE_SIZE;
5071 break;
5072 case CHIP_KAVERI:
5073 size = KV_RLC_UCODE_SIZE;
5074 break;
5075 case CHIP_KABINI:
5076 size = KB_RLC_UCODE_SIZE;
5077 break;
5078 }
cc066715 5079
cc066715
AD		 5080 cik_rlc_stop(rdev);
5081
22c775ce
AD		 5082 /* disable CG */
5083 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5084 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 5085
866d83de 5086 si_rlc_reset(rdev);
6f2043ce 5087
22c775ce 5088 cik_init_pg(rdev);
6f2043ce 5089
22c775ce 5090 cik_init_cg(rdev);
cc066715 5091
f6796cae
AD		 5092 WREG32(RLC_LB_CNTR_INIT, 0);
5093 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 5094
f6796cae
AD		 5095 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5096 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5097 WREG32(RLC_LB_PARAMS, 0x00600408);
5098 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 5099
f6796cae
AD		 5100 WREG32(RLC_MC_CNTL, 0);
5101 WREG32(RLC_UCODE_CNTL, 0);
cc066715 5102
f6796cae
AD		 5103 fw_data = (const __be32 *)rdev->rlc_fw->data;
5104 WREG32(RLC_GPM_UCODE_ADDR, 0);
5105 for (i = 0; i < size; i++)
5106 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5107 WREG32(RLC_GPM_UCODE_ADDR, 0);
cc066715 5108
866d83de
AD		 5109 /* XXX - find out what chips support lbpw */
5110 cik_enable_lbpw(rdev, false);
cc066715 5111
22c775ce
AD		 5112 if (rdev->family == CHIP_BONAIRE)
5113 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 5114
f6796cae 5115 cik_rlc_start(rdev);
cc066715 5116
f6796cae
AD		 5117 return 0;
5118}
cc066715 5119
22c775ce
AD		 5120static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5121{
5122 u32 data, orig, tmp, tmp2;
cc066715 5123
22c775ce 5124 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 5125
473359bc 5126 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 5127 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5128
22c775ce 5129 tmp = cik_halt_rlc(rdev);
cc066715 5130
22c775ce
AD		 5131 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5132 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5133 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5134 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5135 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 5136
22c775ce 5137 cik_update_rlc(rdev, tmp);
cc066715 5138
22c775ce
AD		 5139 data |= CGCG_EN | CGLS_EN;
5140 } else {
ddc76ff6 5141 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 5142
22c775ce
AD		 5143 RREG32(CB_CGTT_SCLK_CTRL);
5144 RREG32(CB_CGTT_SCLK_CTRL);
5145 RREG32(CB_CGTT_SCLK_CTRL);
5146 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 5147
22c775ce 5148 data &= ~(CGCG_EN | CGLS_EN);
cc066715 5149 }
6f2043ce 5150
22c775ce
AD		 5151 if (orig != data)
5152 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 5153
6f2043ce
AD		 5154}
5155
22c775ce 5156static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 5157{
22c775ce
AD		 5158 u32 data, orig, tmp = 0;
5159
473359bc
AD		 5160 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5161 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5162 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5163 orig = data = RREG32(CP_MEM_SLP_CNTL);
5164 data |= CP_MEM_LS_EN;
5165 if (orig != data)
5166 WREG32(CP_MEM_SLP_CNTL, data);
5167 }
5168 }
cc066715 5169
22c775ce
AD		 5170 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5171 data &= 0xfffffffd;
5172 if (orig != data)
5173 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5174
5175 tmp = cik_halt_rlc(rdev);
5176
5177 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5178 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5179 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5180 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5181 WREG32(RLC_SERDES_WR_CTRL, data);
5182
5183 cik_update_rlc(rdev, tmp);
5184
473359bc
AD		 5185 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5186 orig = data = RREG32(CGTS_SM_CTRL_REG);
5187 data &= ~SM_MODE_MASK;
5188 data |= SM_MODE(0x2);
5189 data |= SM_MODE_ENABLE;
5190 data &= ~CGTS_OVERRIDE;
5191 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5192 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5193 data &= ~CGTS_LS_OVERRIDE;
5194 data &= ~ON_MONITOR_ADD_MASK;
5195 data |= ON_MONITOR_ADD_EN;
5196 data |= ON_MONITOR_ADD(0x96);
5197 if (orig != data)
5198 WREG32(CGTS_SM_CTRL_REG, data);
5199 }
22c775ce
AD		 5200 } else {
5201 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5202 data |= 0x00000002;
5203 if (orig != data)
5204 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5205
5206 data = RREG32(RLC_MEM_SLP_CNTL);
5207 if (data & RLC_MEM_LS_EN) {
5208 data &= ~RLC_MEM_LS_EN;
5209 WREG32(RLC_MEM_SLP_CNTL, data);
5210 }
6f2043ce 5211
22c775ce
AD		 5212 data = RREG32(CP_MEM_SLP_CNTL);
5213 if (data & CP_MEM_LS_EN) {
5214 data &= ~CP_MEM_LS_EN;
5215 WREG32(CP_MEM_SLP_CNTL, data);
5216 }
cc066715 5217
22c775ce
AD		 5218 orig = data = RREG32(CGTS_SM_CTRL_REG);
5219 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5220 if (orig != data)
5221 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 5222
22c775ce 5223 tmp = cik_halt_rlc(rdev);
cc066715 5224
22c775ce
AD		 5225 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5226 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5227 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5228 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5229 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 5230
22c775ce 5231 cik_update_rlc(rdev, tmp);
cc066715 5232 }
6f2043ce 5233}
1c49165d 5234
22c775ce 5235static const u32 mc_cg_registers[] =
21a93e13 5236{
22c775ce
AD		 5237 MC_HUB_MISC_HUB_CG,
5238 MC_HUB_MISC_SIP_CG,
5239 MC_HUB_MISC_VM_CG,
5240 MC_XPB_CLK_GAT,
5241 ATC_MISC_CG,
5242 MC_CITF_MISC_WR_CG,
5243 MC_CITF_MISC_RD_CG,
5244 MC_CITF_MISC_VM_CG,
5245 VM_L2_CG,
5246};
21a93e13 5247
22c775ce
AD		 5248static void cik_enable_mc_ls(struct radeon_device *rdev,
5249 bool enable)
1c49165d 5250{
22c775ce
AD		 5251 int i;
5252 u32 orig, data;
1c49165d 5253
22c775ce
AD		 5254 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5255 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5256 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 5257 data |= MC_LS_ENABLE;
5258 else
5259 data &= ~MC_LS_ENABLE;
5260 if (data != orig)
5261 WREG32(mc_cg_registers[i], data);
1c49165d 5262 }
22c775ce 5263}
1c49165d 5264
22c775ce
AD		 5265static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5266 bool enable)
5267{
5268 int i;
5269 u32 orig, data;
5270
5271 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5272 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5273 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 5274 data |= MC_CG_ENABLE;
5275 else
5276 data &= ~MC_CG_ENABLE;
5277 if (data != orig)
5278 WREG32(mc_cg_registers[i], data);
1c49165d 5279 }
1c49165d
AD		 5280}
5281
22c775ce
AD		 5282static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5283 bool enable)
1c49165d 5284{
22c775ce 5285 u32 orig, data;
1c49165d 5286
473359bc 5287 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 5288 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5289 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 5290 } else {
22c775ce
AD		 5291 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5292 data |= 0xff000000;
5293 if (data != orig)
5294 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5295
22c775ce
AD		 5296 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5297 data |= 0xff000000;
5298 if (data != orig)
5299 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5300 }
1c49165d
AD		 5301}
5302
22c775ce
AD		 5303static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5304 bool enable)
1c49165d 5305{
22c775ce
AD		 5306 u32 orig, data;
5307
473359bc 5308 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 5309 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5310 data |= 0x100;
5311 if (orig != data)
5312 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5313
5314 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5315 data |= 0x100;
5316 if (orig != data)
5317 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5318 } else {
5319 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5320 data &= ~0x100;
5321 if (orig != data)
5322 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5323
22c775ce
AD		 5324 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5325 data &= ~0x100;
5326 if (orig != data)
5327 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5328 }
1c49165d
AD		 5329}
5330
22c775ce
AD		 5331static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5332 bool enable)
1c49165d 5333{
22c775ce 5334 u32 orig, data;
1c49165d 5335
473359bc 5336 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 5337 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5338 data = 0xfff;
5339 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 5340
22c775ce
AD		 5341 orig = data = RREG32(UVD_CGC_CTRL);
5342 data |= DCM;
5343 if (orig != data)
5344 WREG32(UVD_CGC_CTRL, data);
5345 } else {
5346 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5347 data &= ~0xfff;
5348 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 5349
22c775ce
AD		 5350 orig = data = RREG32(UVD_CGC_CTRL);
5351 data &= ~DCM;
5352 if (orig != data)
5353 WREG32(UVD_CGC_CTRL, data);
1c49165d 5354 }
22c775ce 5355}
1c49165d 5356
473359bc
AD		 5357static void cik_enable_bif_mgls(struct radeon_device *rdev,
5358 bool enable)
5359{
5360 u32 orig, data;
1c49165d 5361
473359bc 5362 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 5363
473359bc
AD		 5364 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5365 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5366 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5367 else
5368 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5369 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 5370
473359bc
AD		 5371 if (orig != data)
5372 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5373}
1c49165d 5374
22c775ce
AD		 5375static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5376 bool enable)
5377{
5378 u32 orig, data;
1c49165d 5379
22c775ce 5380 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 5381
473359bc 5382 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 5383 data &= ~CLOCK_GATING_DIS;
5384 else
5385 data |= CLOCK_GATING_DIS;
5386
5387 if (orig != data)
5388 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 5389}
5390
22c775ce
AD		 5391static void cik_enable_hdp_ls(struct radeon_device *rdev,
5392 bool enable)
1c49165d 5393{
22c775ce
AD		 5394 u32 orig, data;
5395
5396 orig = data = RREG32(HDP_MEM_POWER_LS);
5397
473359bc 5398 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 5399 data |= HDP_LS_ENABLE;
5400 else
5401 data &= ~HDP_LS_ENABLE;
5402
5403 if (orig != data)
5404 WREG32(HDP_MEM_POWER_LS, data);
5405}
5406
5407void cik_update_cg(struct radeon_device *rdev,
5408 u32 block, bool enable)
5409{
4214faf6 5410
22c775ce 5411 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 5412 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 5413 /* order matters! */
5414 if (enable) {
5415 cik_enable_mgcg(rdev, true);
5416 cik_enable_cgcg(rdev, true);
5417 } else {
5418 cik_enable_cgcg(rdev, false);
5419 cik_enable_mgcg(rdev, false);
5420 }
4214faf6 5421 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 5422 }
5423
5424 if (block & RADEON_CG_BLOCK_MC) {
5425 if (!(rdev->flags & RADEON_IS_IGP)) {
5426 cik_enable_mc_mgcg(rdev, enable);
5427 cik_enable_mc_ls(rdev, enable);
5428 }
5429 }
5430
5431 if (block & RADEON_CG_BLOCK_SDMA) {
5432 cik_enable_sdma_mgcg(rdev, enable);
5433 cik_enable_sdma_mgls(rdev, enable);
5434 }
5435
473359bc
AD		 5436 if (block & RADEON_CG_BLOCK_BIF) {
5437 cik_enable_bif_mgls(rdev, enable);
5438 }
5439
22c775ce
AD		 5440 if (block & RADEON_CG_BLOCK_UVD) {
5441 if (rdev->has_uvd)
5442 cik_enable_uvd_mgcg(rdev, enable);
5443 }
5444
5445 if (block & RADEON_CG_BLOCK_HDP) {
5446 cik_enable_hdp_mgcg(rdev, enable);
5447 cik_enable_hdp_ls(rdev, enable);
5448 }
1c49165d
AD		 5449}
5450
22c775ce 5451static void cik_init_cg(struct radeon_device *rdev)
1c49165d 5452{
22c775ce 5453
ddc76ff6 5454 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 5455
5456 if (rdev->has_uvd)
5457 si_init_uvd_internal_cg(rdev);
5458
5459 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5460 RADEON_CG_BLOCK_SDMA |
473359bc 5461 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 5462 RADEON_CG_BLOCK_UVD |
5463 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 5464}
5465
473359bc 5466static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 5467{
473359bc
AD		 5468 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5469 RADEON_CG_BLOCK_SDMA |
5470 RADEON_CG_BLOCK_BIF |
5471 RADEON_CG_BLOCK_UVD |
5472 RADEON_CG_BLOCK_HDP), false);
5473
5474 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 5475}
5476
22c775ce
AD		 5477static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5478 bool enable)
1c49165d 5479{
22c775ce 5480 u32 data, orig;
1c49165d 5481
22c775ce 5482 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5483 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5484 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5485 else
5486 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5487 if (orig != data)
5488 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 5489}
5490
22c775ce
AD		 5491static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5492 bool enable)
1c49165d 5493{
22c775ce
AD		 5494 u32 data, orig;
5495
5496 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5497 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5498 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5499 else
5500 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5501 if (orig != data)
5502 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 5503}
5504
22c775ce 5505static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 5506{
22c775ce 5507 u32 data, orig;
3ec7d11b 5508
22c775ce 5509 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5510 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 5511 data &= ~DISABLE_CP_PG;
5512 else
5513 data |= DISABLE_CP_PG;
5514 if (orig != data)
5515 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 5516}
5517
22c775ce 5518static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 5519{
22c775ce 5520 u32 data, orig;
f96ab484 5521
22c775ce 5522 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5523 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 5524 data &= ~DISABLE_GDS_PG;
5525 else
5526 data |= DISABLE_GDS_PG;
5527 if (orig != data)
5528 WREG32(RLC_PG_CNTL, data);
5529}
5530
5531#define CP_ME_TABLE_SIZE 96
5532#define CP_ME_TABLE_OFFSET 2048
5533#define CP_MEC_TABLE_OFFSET 4096
5534
5535void cik_init_cp_pg_table(struct radeon_device *rdev)
5536{
5537 const __be32 *fw_data;
5538 volatile u32 *dst_ptr;
5539 int me, i, max_me = 4;
5540 u32 bo_offset = 0;
5541 u32 table_offset;
5542
5543 if (rdev->family == CHIP_KAVERI)
5544 max_me = 5;
5545
5546 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 5547 return;
5548
22c775ce
AD		 5549 /* write the cp table buffer */
5550 dst_ptr = rdev->rlc.cp_table_ptr;
5551 for (me = 0; me < max_me; me++) {
5552 if (me == 0) {
5553 fw_data = (const __be32 *)rdev->ce_fw->data;
5554 table_offset = CP_ME_TABLE_OFFSET;
5555 } else if (me == 1) {
5556 fw_data = (const __be32 *)rdev->pfp_fw->data;
5557 table_offset = CP_ME_TABLE_OFFSET;
5558 } else if (me == 2) {
5559 fw_data = (const __be32 *)rdev->me_fw->data;
5560 table_offset = CP_ME_TABLE_OFFSET;
5561 } else {
5562 fw_data = (const __be32 *)rdev->mec_fw->data;
5563 table_offset = CP_MEC_TABLE_OFFSET;
5564 }
5565
5566 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6ba81e53 5567 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
22c775ce
AD		 5568 }
5569 bo_offset += CP_ME_TABLE_SIZE;
f96ab484 5570 }
22c775ce 5571}
f96ab484 5572
22c775ce
AD		 5573static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
5574 bool enable)
5575{
5576 u32 data, orig;
5577
2b19d17f 5578 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 5579 orig = data = RREG32(RLC_PG_CNTL);
5580 data |= GFX_PG_ENABLE;
5581 if (orig != data)
5582 WREG32(RLC_PG_CNTL, data);
5583
5584 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5585 data |= AUTO_PG_EN;
5586 if (orig != data)
5587 WREG32(RLC_AUTO_PG_CTRL, data);
5588 } else {
5589 orig = data = RREG32(RLC_PG_CNTL);
5590 data &= ~GFX_PG_ENABLE;
5591 if (orig != data)
5592 WREG32(RLC_PG_CNTL, data);
f96ab484 5593
22c775ce
AD		 5594 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5595 data &= ~AUTO_PG_EN;
5596 if (orig != data)
5597 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 5598
22c775ce
AD		 5599 data = RREG32(DB_RENDER_CONTROL);
5600 }
5601}
f96ab484 5602
22c775ce
AD		 5603static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5604{
5605 u32 mask = 0, tmp, tmp1;
5606 int i;
f96ab484 5607
22c775ce
AD		 5608 cik_select_se_sh(rdev, se, sh);
5609 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5610 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5611 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 5612
22c775ce 5613 tmp &= 0xffff0000;
f96ab484 5614
22c775ce
AD		 5615 tmp |= tmp1;
5616 tmp >>= 16;
5617
5618 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
5619 mask <<= 1;
5620 mask |= 1;
b07fdd38 5621 }
22c775ce
AD		 5622
5623 return (~tmp) & mask;
f96ab484
AD		 5624}
5625
22c775ce 5626static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 5627{
22c775ce
AD		 5628 u32 i, j, k, active_cu_number = 0;
5629 u32 mask, counter, cu_bitmap;
5630 u32 tmp = 0;
d0e092d9 5631
22c775ce
AD		 5632 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5633 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5634 mask = 1;
5635 cu_bitmap = 0;
5636 counter = 0;
5637 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
5638 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
5639 if (counter < 2)
5640 cu_bitmap |= mask;
5641 counter ++;
d0e092d9 5642 }
22c775ce 5643 mask <<= 1;
d0e092d9 5644 }
d0e092d9 5645
22c775ce
AD		 5646 active_cu_number += counter;
5647 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 5648 }
d0e092d9 5649 }
22c775ce
AD		 5650
5651 WREG32(RLC_PG_AO_CU_MASK, tmp);
5652
5653 tmp = RREG32(RLC_MAX_PG_CU);
5654 tmp &= ~MAX_PU_CU_MASK;
5655 tmp |= MAX_PU_CU(active_cu_number);
5656 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 5657}
5658
22c775ce
AD		 5659static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
5660 bool enable)
605de6b9 5661{
22c775ce 5662 u32 data, orig;
605de6b9 5663
22c775ce 5664 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5665 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 5666 data |= STATIC_PER_CU_PG_ENABLE;
5667 else
5668 data &= ~STATIC_PER_CU_PG_ENABLE;
5669 if (orig != data)
5670 WREG32(RLC_PG_CNTL, data);
5671}
5672
5673static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
5674 bool enable)
5675{
5676 u32 data, orig;
605de6b9 5677
22c775ce 5678 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5679 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 5680 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 5681 else
22c775ce
AD		 5682 data &= ~DYN_PER_CU_PG_ENABLE;
5683 if (orig != data)
5684 WREG32(RLC_PG_CNTL, data);
5685}
605de6b9 5686
22c775ce
AD		 5687#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
5688#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
5689
5690static void cik_init_gfx_cgpg(struct radeon_device *rdev)
5691{
5692 u32 data, orig;
5693 u32 i;
5694
5695 if (rdev->rlc.cs_data) {
5696 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5697 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 5698 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 5699 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 5700 } else {
22c775ce
AD		 5701 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5702 for (i = 0; i < 3; i++)
5703 WREG32(RLC_GPM_SCRATCH_DATA, 0);
5704 }
5705 if (rdev->rlc.reg_list) {
5706 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
5707 for (i = 0; i < rdev->rlc.reg_list_size; i++)
5708 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 5709 }
605de6b9 5710
22c775ce
AD		 5711 orig = data = RREG32(RLC_PG_CNTL);
5712 data |= GFX_PG_SRC;
5713 if (orig != data)
5714 WREG32(RLC_PG_CNTL, data);
605de6b9 5715
22c775ce
AD		 5716 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5717 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 5718
22c775ce
AD		 5719 data = RREG32(CP_RB_WPTR_POLL_CNTL);
5720 data &= ~IDLE_POLL_COUNT_MASK;
5721 data |= IDLE_POLL_COUNT(0x60);
5722 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 5723
22c775ce
AD		 5724 data = 0x10101010;
5725 WREG32(RLC_PG_DELAY, data);
605de6b9 5726
22c775ce
AD		 5727 data = RREG32(RLC_PG_DELAY_2);
5728 data &= ~0xff;
5729 data |= 0x3;
5730 WREG32(RLC_PG_DELAY_2, data);
605de6b9 5731
22c775ce
AD		 5732 data = RREG32(RLC_AUTO_PG_CTRL);
5733 data &= ~GRBM_REG_SGIT_MASK;
5734 data |= GRBM_REG_SGIT(0x700);
5735 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 5736
605de6b9
AD		 5737}
5738
22c775ce 5739static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 5740{
473359bc
AD		 5741 cik_enable_gfx_cgpg(rdev, enable);
5742 cik_enable_gfx_static_mgpg(rdev, enable);
5743 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 5744}
f6796cae 5745
a0f38609
AD		 5746u32 cik_get_csb_size(struct radeon_device *rdev)
5747{
5748 u32 count = 0;
5749 const struct cs_section_def *sect = NULL;
5750 const struct cs_extent_def *ext = NULL;
f6796cae 5751
a0f38609
AD		 5752 if (rdev->rlc.cs_data == NULL)
5753 return 0;
f6796cae 5754
a0f38609
AD		 5755 /* begin clear state */
5756 count += 2;
5757 /* context control state */
5758 count += 3;
5759
5760 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5761 for (ext = sect->section; ext->extent != NULL; ++ext) {
5762 if (sect->id == SECT_CONTEXT)
5763 count += 2 + ext->reg_count;
5764 else
5765 return 0;
f6796cae
AD		 5766 }
5767 }
a0f38609
AD		 5768 /* pa_sc_raster_config/pa_sc_raster_config1 */
5769 count += 4;
5770 /* end clear state */
5771 count += 2;
5772 /* clear state */
5773 count += 2;
f6796cae 5774
a0f38609 5775 return count;
f6796cae
AD		 5776}
5777
a0f38609 5778void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 5779{
a0f38609
AD		 5780 u32 count = 0, i;
5781 const struct cs_section_def *sect = NULL;
5782 const struct cs_extent_def *ext = NULL;
f6796cae 5783
a0f38609
AD		 5784 if (rdev->rlc.cs_data == NULL)
5785 return;
5786 if (buffer == NULL)
5787 return;
f6796cae 5788
6ba81e53
AD		 5789 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5790 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 5791
6ba81e53
AD		 5792 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5793 buffer[count++] = cpu_to_le32(0x80000000);
5794 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 5795
5796 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5797 for (ext = sect->section; ext->extent != NULL; ++ext) {
5798 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 5799 buffer[count++] =
5800 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5801 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 5802 for (i = 0; i < ext->reg_count; i++)
6ba81e53 5803 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 5804 } else {
5805 return;
5806 }
5807 }
5808 }
f6796cae 5809
6ba81e53
AD		 5810 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
5811 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 5812 switch (rdev->family) {
5813 case CHIP_BONAIRE:
6ba81e53
AD		 5814 buffer[count++] = cpu_to_le32(0x16000012);
5815 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 5816 break;
5817 case CHIP_KAVERI:
6ba81e53
AD		 5818 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
5819 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 5820 break;
5821 case CHIP_KABINI:
6ba81e53
AD		 5822 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
5823 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609
AD		 5824 break;
5825 default:
6ba81e53
AD		 5826 buffer[count++] = cpu_to_le32(0x00000000);
5827 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 5828 break;
5829 }
5830
6ba81e53
AD		 5831 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5832 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 5833
6ba81e53
AD		 5834 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5835 buffer[count++] = cpu_to_le32(0);
a0f38609 5836}
f6796cae 5837
473359bc 5838static void cik_init_pg(struct radeon_device *rdev)
22c775ce 5839{
473359bc 5840 if (rdev->pg_flags) {
22c775ce
AD		 5841 cik_enable_sck_slowdown_on_pu(rdev, true);
5842 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 5843 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 5844 cik_init_gfx_cgpg(rdev);
5845 cik_enable_cp_pg(rdev, true);
5846 cik_enable_gds_pg(rdev, true);
5847 }
22c775ce
AD		 5848 cik_init_ao_cu_mask(rdev);
5849 cik_update_gfx_pg(rdev, true);
5850 }
5851}
f6796cae 5852
473359bc
AD		 5853static void cik_fini_pg(struct radeon_device *rdev)
5854{
5855 if (rdev->pg_flags) {
5856 cik_update_gfx_pg(rdev, false);
2b19d17f 5857 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 5858 cik_enable_cp_pg(rdev, false);
5859 cik_enable_gds_pg(rdev, false);
5860 }
5861 }
f6796cae 5862}
a59781bb
AD		 5863
5864/*
5865 * Interrupts
5866 * Starting with r6xx, interrupts are handled via a ring buffer.
5867 * Ring buffers are areas of GPU accessible memory that the GPU
5868 * writes interrupt vectors into and the host reads vectors out of.
5869 * There is a rptr (read pointer) that determines where the
5870 * host is currently reading, and a wptr (write pointer)
5871 * which determines where the GPU has written. When the
5872 * pointers are equal, the ring is idle. When the GPU
5873 * writes vectors to the ring buffer, it increments the
5874 * wptr. When there is an interrupt, the host then starts
5875 * fetching commands and processing them until the pointers are
5876 * equal again at which point it updates the rptr.
5877 */
5878
5879/**
5880 * cik_enable_interrupts - Enable the interrupt ring buffer
5881 *
5882 * @rdev: radeon_device pointer
5883 *
5884 * Enable the interrupt ring buffer (CIK).
5885 */
5886static void cik_enable_interrupts(struct radeon_device *rdev)
5887{
5888 u32 ih_cntl = RREG32(IH_CNTL);
5889 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5890
5891 ih_cntl |= ENABLE_INTR;
5892 ih_rb_cntl |= IH_RB_ENABLE;
5893 WREG32(IH_CNTL, ih_cntl);
5894 WREG32(IH_RB_CNTL, ih_rb_cntl);
5895 rdev->ih.enabled = true;
5896}
5897
5898/**
5899 * cik_disable_interrupts - Disable the interrupt ring buffer
5900 *
5901 * @rdev: radeon_device pointer
5902 *
5903 * Disable the interrupt ring buffer (CIK).
5904 */
5905static void cik_disable_interrupts(struct radeon_device *rdev)
5906{
5907 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5908 u32 ih_cntl = RREG32(IH_CNTL);
5909
5910 ih_rb_cntl &= ~IH_RB_ENABLE;
5911 ih_cntl &= ~ENABLE_INTR;
5912 WREG32(IH_RB_CNTL, ih_rb_cntl);
5913 WREG32(IH_CNTL, ih_cntl);
5914 /* set rptr, wptr to 0 */
5915 WREG32(IH_RB_RPTR, 0);
5916 WREG32(IH_RB_WPTR, 0);
5917 rdev->ih.enabled = false;
5918 rdev->ih.rptr = 0;
5919}
5920
5921/**
5922 * cik_disable_interrupt_state - Disable all interrupt sources
5923 *
5924 * @rdev: radeon_device pointer
5925 *
5926 * Clear all interrupt enable bits used by the driver (CIK).
5927 */
5928static void cik_disable_interrupt_state(struct radeon_device *rdev)
5929{
5930 u32 tmp;
5931
5932 /* gfx ring */
4214faf6
AD		 5933 tmp = RREG32(CP_INT_CNTL_RING0) &
5934 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5935 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 5936 /* sdma */
5937 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5938 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5939 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5940 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 5941 /* compute queues */
5942 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
5943 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
5944 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
5945 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
5946 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
5947 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
5948 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
5949 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
5950 /* grbm */
5951 WREG32(GRBM_INT_CNTL, 0);
5952 /* vline/vblank, etc. */
5953 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5954 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5955 if (rdev->num_crtc >= 4) {
5956 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5957 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5958 }
5959 if (rdev->num_crtc >= 6) {
5960 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5961 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5962 }
5963
5964 /* dac hotplug */
5965 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5966
5967 /* digital hotplug */
5968 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5969 WREG32(DC_HPD1_INT_CONTROL, tmp);
5970 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5971 WREG32(DC_HPD2_INT_CONTROL, tmp);
5972 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5973 WREG32(DC_HPD3_INT_CONTROL, tmp);
5974 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5975 WREG32(DC_HPD4_INT_CONTROL, tmp);
5976 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5977 WREG32(DC_HPD5_INT_CONTROL, tmp);
5978 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5979 WREG32(DC_HPD6_INT_CONTROL, tmp);
5980
5981}
5982
5983/**
5984 * cik_irq_init - init and enable the interrupt ring
5985 *
5986 * @rdev: radeon_device pointer
5987 *
5988 * Allocate a ring buffer for the interrupt controller,
5989 * enable the RLC, disable interrupts, enable the IH
5990 * ring buffer and enable it (CIK).
5991 * Called at device load and reume.
5992 * Returns 0 for success, errors for failure.
5993 */
5994static int cik_irq_init(struct radeon_device *rdev)
5995{
5996 int ret = 0;
5997 int rb_bufsz;
5998 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5999
6000 /* allocate ring */
6001 ret = r600_ih_ring_alloc(rdev);
6002 if (ret)
6003 return ret;
6004
6005 /* disable irqs */
6006 cik_disable_interrupts(rdev);
6007
6008 /* init rlc */
6009 ret = cik_rlc_resume(rdev);
6010 if (ret) {
6011 r600_ih_ring_fini(rdev);
6012 return ret;
6013 }
6014
6015 /* setup interrupt control */
6016 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6017 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6018 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6019 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6020 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6021 */
6022 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6023 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6024 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6025 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6026
6027 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 6028 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 6029
6030 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6031 IH_WPTR_OVERFLOW_CLEAR |
6032 (rb_bufsz << 1));
6033
6034 if (rdev->wb.enabled)
6035 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6036
6037 /* set the writeback address whether it's enabled or not */
6038 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6039 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6040
6041 WREG32(IH_RB_CNTL, ih_rb_cntl);
6042
6043 /* set rptr, wptr to 0 */
6044 WREG32(IH_RB_RPTR, 0);
6045 WREG32(IH_RB_WPTR, 0);
6046
6047 /* Default settings for IH_CNTL (disabled at first) */
6048 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6049 /* RPTR_REARM only works if msi's are enabled */
6050 if (rdev->msi_enabled)
6051 ih_cntl |= RPTR_REARM;
6052 WREG32(IH_CNTL, ih_cntl);
6053
6054 /* force the active interrupt state to all disabled */
6055 cik_disable_interrupt_state(rdev);
6056
6057 pci_set_master(rdev->pdev);
6058
6059 /* enable irqs */
6060 cik_enable_interrupts(rdev);
6061
6062 return ret;
6063}
6064
6065/**
6066 * cik_irq_set - enable/disable interrupt sources
6067 *
6068 * @rdev: radeon_device pointer
6069 *
6070 * Enable interrupt sources on the GPU (vblanks, hpd,
6071 * etc.) (CIK).
6072 * Returns 0 for success, errors for failure.
6073 */
6074int cik_irq_set(struct radeon_device *rdev)
6075{
4214faf6 6076 u32 cp_int_cntl;
2b0781a6
AD		 6077 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6078 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 6079 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6080 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6081 u32 grbm_int_cntl = 0;
21a93e13 6082 u32 dma_cntl, dma_cntl1;
41a524ab 6083 u32 thermal_int;
a59781bb
AD		 6084
6085 if (!rdev->irq.installed) {
6086 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6087 return -EINVAL;
6088 }
6089 /* don't enable anything if the ih is disabled */
6090 if (!rdev->ih.enabled) {
6091 cik_disable_interrupts(rdev);
6092 /* force the active interrupt state to all disabled */
6093 cik_disable_interrupt_state(rdev);
6094 return 0;
6095 }
6096
4214faf6
AD		 6097 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6098 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6099 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6100
a59781bb
AD		 6101 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6102 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6103 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6104 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6105 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6106 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6107
21a93e13
AD		 6108 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6109 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6110
2b0781a6
AD		 6111 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6112 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6113 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6114 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6115 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6116 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6117 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6118 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6119
cc8dbbb4
AD		 6120 if (rdev->flags & RADEON_IS_IGP)
6121 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6122 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6123 else
6124 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6125 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 6126
a59781bb
AD		 6127 /* enable CP interrupts on all rings */
6128 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6129 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6130 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6131 }
2b0781a6
AD		 6132 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6133 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6134 DRM_DEBUG("si_irq_set: sw int cp1\n");
6135 if (ring->me == 1) {
6136 switch (ring->pipe) {
6137 case 0:
6138 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6139 break;
6140 case 1:
6141 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6142 break;
6143 case 2:
6144 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6145 break;
6146 case 3:
6147 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6148 break;
6149 default:
6150 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6151 break;
6152 }
6153 } else if (ring->me == 2) {
6154 switch (ring->pipe) {
6155 case 0:
6156 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6157 break;
6158 case 1:
6159 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6160 break;
6161 case 2:
6162 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6163 break;
6164 case 3:
6165 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6166 break;
6167 default:
6168 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6169 break;
6170 }
6171 } else {
6172 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6173 }
6174 }
6175 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6176 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6177 DRM_DEBUG("si_irq_set: sw int cp2\n");
6178 if (ring->me == 1) {
6179 switch (ring->pipe) {
6180 case 0:
6181 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6182 break;
6183 case 1:
6184 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6185 break;
6186 case 2:
6187 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6188 break;
6189 case 3:
6190 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6191 break;
6192 default:
6193 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6194 break;
6195 }
6196 } else if (ring->me == 2) {
6197 switch (ring->pipe) {
6198 case 0:
6199 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6200 break;
6201 case 1:
6202 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6203 break;
6204 case 2:
6205 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6206 break;
6207 case 3:
6208 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6209 break;
6210 default:
6211 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6212 break;
6213 }
6214 } else {
6215 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6216 }
6217 }
a59781bb 6218
21a93e13
AD		 6219 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6220 DRM_DEBUG("cik_irq_set: sw int dma\n");
6221 dma_cntl |= TRAP_ENABLE;
6222 }
6223
6224 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6225 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6226 dma_cntl1 |= TRAP_ENABLE;
6227 }
6228
a59781bb
AD		 6229 if (rdev->irq.crtc_vblank_int[0] ||
6230 atomic_read(&rdev->irq.pflip[0])) {
6231 DRM_DEBUG("cik_irq_set: vblank 0\n");
6232 crtc1 |= VBLANK_INTERRUPT_MASK;
6233 }
6234 if (rdev->irq.crtc_vblank_int[1] ||
6235 atomic_read(&rdev->irq.pflip[1])) {
6236 DRM_DEBUG("cik_irq_set: vblank 1\n");
6237 crtc2 |= VBLANK_INTERRUPT_MASK;
6238 }
6239 if (rdev->irq.crtc_vblank_int[2] ||
6240 atomic_read(&rdev->irq.pflip[2])) {
6241 DRM_DEBUG("cik_irq_set: vblank 2\n");
6242 crtc3 |= VBLANK_INTERRUPT_MASK;
6243 }
6244 if (rdev->irq.crtc_vblank_int[3] ||
6245 atomic_read(&rdev->irq.pflip[3])) {
6246 DRM_DEBUG("cik_irq_set: vblank 3\n");
6247 crtc4 |= VBLANK_INTERRUPT_MASK;
6248 }
6249 if (rdev->irq.crtc_vblank_int[4] ||
6250 atomic_read(&rdev->irq.pflip[4])) {
6251 DRM_DEBUG("cik_irq_set: vblank 4\n");
6252 crtc5 |= VBLANK_INTERRUPT_MASK;
6253 }
6254 if (rdev->irq.crtc_vblank_int[5] ||
6255 atomic_read(&rdev->irq.pflip[5])) {
6256 DRM_DEBUG("cik_irq_set: vblank 5\n");
6257 crtc6 |= VBLANK_INTERRUPT_MASK;
6258 }
6259 if (rdev->irq.hpd[0]) {
6260 DRM_DEBUG("cik_irq_set: hpd 1\n");
6261 hpd1 |= DC_HPDx_INT_EN;
6262 }
6263 if (rdev->irq.hpd[1]) {
6264 DRM_DEBUG("cik_irq_set: hpd 2\n");
6265 hpd2 |= DC_HPDx_INT_EN;
6266 }
6267 if (rdev->irq.hpd[2]) {
6268 DRM_DEBUG("cik_irq_set: hpd 3\n");
6269 hpd3 |= DC_HPDx_INT_EN;
6270 }
6271 if (rdev->irq.hpd[3]) {
6272 DRM_DEBUG("cik_irq_set: hpd 4\n");
6273 hpd4 |= DC_HPDx_INT_EN;
6274 }
6275 if (rdev->irq.hpd[4]) {
6276 DRM_DEBUG("cik_irq_set: hpd 5\n");
6277 hpd5 |= DC_HPDx_INT_EN;
6278 }
6279 if (rdev->irq.hpd[5]) {
6280 DRM_DEBUG("cik_irq_set: hpd 6\n");
6281 hpd6 |= DC_HPDx_INT_EN;
6282 }
6283
41a524ab
AD		 6284 if (rdev->irq.dpm_thermal) {
6285 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 6286 if (rdev->flags & RADEON_IS_IGP)
6287 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6288 else
6289 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 6290 }
6291
a59781bb
AD		 6292 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6293
21a93e13
AD		 6294 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6295 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6296
2b0781a6
AD		 6297 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6298 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6299 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6300 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6301 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6302 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6303 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6304 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6305
a59781bb
AD		 6306 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6307
6308 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6309 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6310 if (rdev->num_crtc >= 4) {
6311 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6312 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6313 }
6314 if (rdev->num_crtc >= 6) {
6315 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6316 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6317 }
6318
6319 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6320 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6321 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6322 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6323 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6324 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6325
cc8dbbb4
AD		 6326 if (rdev->flags & RADEON_IS_IGP)
6327 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6328 else
6329 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 6330
a59781bb
AD		 6331 return 0;
6332}
6333
6334/**
6335 * cik_irq_ack - ack interrupt sources
6336 *
6337 * @rdev: radeon_device pointer
6338 *
6339 * Ack interrupt sources on the GPU (vblanks, hpd,
6340 * etc.) (CIK). Certain interrupts sources are sw
6341 * generated and do not require an explicit ack.
6342 */
6343static inline void cik_irq_ack(struct radeon_device *rdev)
6344{
6345 u32 tmp;
6346
6347 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6348 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6349 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6350 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6351 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6352 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6353 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6354
6355 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6356 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6357 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6358 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6359 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6360 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6361 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6362 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6363
6364 if (rdev->num_crtc >= 4) {
6365 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6366 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6367 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6368 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6369 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6370 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6371 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6372 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6373 }
6374
6375 if (rdev->num_crtc >= 6) {
6376 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6377 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6378 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6379 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6380 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6381 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6382 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6383 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6384 }
6385
6386 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6387 tmp = RREG32(DC_HPD1_INT_CONTROL);
6388 tmp |= DC_HPDx_INT_ACK;
6389 WREG32(DC_HPD1_INT_CONTROL, tmp);
6390 }
6391 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6392 tmp = RREG32(DC_HPD2_INT_CONTROL);
6393 tmp |= DC_HPDx_INT_ACK;
6394 WREG32(DC_HPD2_INT_CONTROL, tmp);
6395 }
6396 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6397 tmp = RREG32(DC_HPD3_INT_CONTROL);
6398 tmp |= DC_HPDx_INT_ACK;
6399 WREG32(DC_HPD3_INT_CONTROL, tmp);
6400 }
6401 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6402 tmp = RREG32(DC_HPD4_INT_CONTROL);
6403 tmp |= DC_HPDx_INT_ACK;
6404 WREG32(DC_HPD4_INT_CONTROL, tmp);
6405 }
6406 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6407 tmp = RREG32(DC_HPD5_INT_CONTROL);
6408 tmp |= DC_HPDx_INT_ACK;
6409 WREG32(DC_HPD5_INT_CONTROL, tmp);
6410 }
6411 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6412 tmp = RREG32(DC_HPD5_INT_CONTROL);
6413 tmp |= DC_HPDx_INT_ACK;
6414 WREG32(DC_HPD6_INT_CONTROL, tmp);
6415 }
6416}
6417
6418/**
6419 * cik_irq_disable - disable interrupts
6420 *
6421 * @rdev: radeon_device pointer
6422 *
6423 * Disable interrupts on the hw (CIK).
6424 */
6425static void cik_irq_disable(struct radeon_device *rdev)
6426{
6427 cik_disable_interrupts(rdev);
6428 /* Wait and acknowledge irq */
6429 mdelay(1);
6430 cik_irq_ack(rdev);
6431 cik_disable_interrupt_state(rdev);
6432}
6433
6434/**
6435 * cik_irq_disable - disable interrupts for suspend
6436 *
6437 * @rdev: radeon_device pointer
6438 *
6439 * Disable interrupts and stop the RLC (CIK).
6440 * Used for suspend.
6441 */
6442static void cik_irq_suspend(struct radeon_device *rdev)
6443{
6444 cik_irq_disable(rdev);
6445 cik_rlc_stop(rdev);
6446}
6447
6448/**
6449 * cik_irq_fini - tear down interrupt support
6450 *
6451 * @rdev: radeon_device pointer
6452 *
6453 * Disable interrupts on the hw and free the IH ring
6454 * buffer (CIK).
6455 * Used for driver unload.
6456 */
6457static void cik_irq_fini(struct radeon_device *rdev)
6458{
6459 cik_irq_suspend(rdev);
6460 r600_ih_ring_fini(rdev);
6461}
6462
6463/**
6464 * cik_get_ih_wptr - get the IH ring buffer wptr
6465 *
6466 * @rdev: radeon_device pointer
6467 *
6468 * Get the IH ring buffer wptr from either the register
6469 * or the writeback memory buffer (CIK). Also check for
6470 * ring buffer overflow and deal with it.
6471 * Used by cik_irq_process().
6472 * Returns the value of the wptr.
6473 */
6474static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6475{
6476 u32 wptr, tmp;
6477
6478 if (rdev->wb.enabled)
6479 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6480 else
6481 wptr = RREG32(IH_RB_WPTR);
6482
6483 if (wptr & RB_OVERFLOW) {
6484 /* When a ring buffer overflow happen start parsing interrupt
6485 * from the last not overwritten vector (wptr + 16). Hopefully
6486 * this should allow us to catchup.
6487 */
6488 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6489 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6490 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6491 tmp = RREG32(IH_RB_CNTL);
6492 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6493 WREG32(IH_RB_CNTL, tmp);
6494 }
6495 return (wptr & rdev->ih.ptr_mask);
6496}
6497
6498/* CIK IV Ring
6499 * Each IV ring entry is 128 bits:
6500 * [7:0] - interrupt source id
6501 * [31:8] - reserved
6502 * [59:32] - interrupt source data
6503 * [63:60] - reserved
21a93e13
AD		 6504 * [71:64] - RINGID
6505 * CP:
6506 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 6507 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
6508 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
6509 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
6510 * PIPE_ID - ME0 0=3D
6511 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 6512 * SDMA:
6513 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
6514 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
6515 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 6516 * [79:72] - VMID
6517 * [95:80] - PASID
6518 * [127:96] - reserved
6519 */
6520/**
6521 * cik_irq_process - interrupt handler
6522 *
6523 * @rdev: radeon_device pointer
6524 *
6525 * Interrupt hander (CIK). Walk the IH ring,
6526 * ack interrupts and schedule work to handle
6527 * interrupt events.
6528 * Returns irq process return code.
6529 */
6530int cik_irq_process(struct radeon_device *rdev)
6531{
2b0781a6
AD		 6532 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6533 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 6534 u32 wptr;
6535 u32 rptr;
6536 u32 src_id, src_data, ring_id;
6537 u8 me_id, pipe_id, queue_id;
6538 u32 ring_index;
6539 bool queue_hotplug = false;
6540 bool queue_reset = false;
3ec7d11b 6541 u32 addr, status, mc_client;
41a524ab 6542 bool queue_thermal = false;
a59781bb
AD		 6543
6544 if (!rdev->ih.enabled || rdev->shutdown)
6545 return IRQ_NONE;
6546
6547 wptr = cik_get_ih_wptr(rdev);
6548
6549restart_ih:
6550 /* is somebody else already processing irqs? */
6551 if (atomic_xchg(&rdev->ih.lock, 1))
6552 return IRQ_NONE;
6553
6554 rptr = rdev->ih.rptr;
6555 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6556
6557 /* Order reading of wptr vs. reading of IH ring data */
6558 rmb();
6559
6560 /* display interrupts */
6561 cik_irq_ack(rdev);
6562
6563 while (rptr != wptr) {
6564 /* wptr/rptr are in bytes! */
6565 ring_index = rptr / 4;
6566 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6567 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6568 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 6569
6570 switch (src_id) {
6571 case 1: /* D1 vblank/vline */
6572 switch (src_data) {
6573 case 0: /* D1 vblank */
6574 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
6575 if (rdev->irq.crtc_vblank_int[0]) {
6576 drm_handle_vblank(rdev->ddev, 0);
6577 rdev->pm.vblank_sync = true;
6578 wake_up(&rdev->irq.vblank_queue);
6579 }
6580 if (atomic_read(&rdev->irq.pflip[0]))
6581 radeon_crtc_handle_flip(rdev, 0);
6582 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6583 DRM_DEBUG("IH: D1 vblank\n");
6584 }
6585 break;
6586 case 1: /* D1 vline */
6587 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
6588 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6589 DRM_DEBUG("IH: D1 vline\n");
6590 }
6591 break;
6592 default:
6593 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6594 break;
6595 }
6596 break;
6597 case 2: /* D2 vblank/vline */
6598 switch (src_data) {
6599 case 0: /* D2 vblank */
6600 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6601 if (rdev->irq.crtc_vblank_int[1]) {
6602 drm_handle_vblank(rdev->ddev, 1);
6603 rdev->pm.vblank_sync = true;
6604 wake_up(&rdev->irq.vblank_queue);
6605 }
6606 if (atomic_read(&rdev->irq.pflip[1]))
6607 radeon_crtc_handle_flip(rdev, 1);
6608 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6609 DRM_DEBUG("IH: D2 vblank\n");
6610 }
6611 break;
6612 case 1: /* D2 vline */
6613 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6614 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6615 DRM_DEBUG("IH: D2 vline\n");
6616 }
6617 break;
6618 default:
6619 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6620 break;
6621 }
6622 break;
6623 case 3: /* D3 vblank/vline */
6624 switch (src_data) {
6625 case 0: /* D3 vblank */
6626 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6627 if (rdev->irq.crtc_vblank_int[2]) {
6628 drm_handle_vblank(rdev->ddev, 2);
6629 rdev->pm.vblank_sync = true;
6630 wake_up(&rdev->irq.vblank_queue);
6631 }
6632 if (atomic_read(&rdev->irq.pflip[2]))
6633 radeon_crtc_handle_flip(rdev, 2);
6634 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6635 DRM_DEBUG("IH: D3 vblank\n");
6636 }
6637 break;
6638 case 1: /* D3 vline */
6639 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6640 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6641 DRM_DEBUG("IH: D3 vline\n");
6642 }
6643 break;
6644 default:
6645 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6646 break;
6647 }
6648 break;
6649 case 4: /* D4 vblank/vline */
6650 switch (src_data) {
6651 case 0: /* D4 vblank */
6652 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6653 if (rdev->irq.crtc_vblank_int[3]) {
6654 drm_handle_vblank(rdev->ddev, 3);
6655 rdev->pm.vblank_sync = true;
6656 wake_up(&rdev->irq.vblank_queue);
6657 }
6658 if (atomic_read(&rdev->irq.pflip[3]))
6659 radeon_crtc_handle_flip(rdev, 3);
6660 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6661 DRM_DEBUG("IH: D4 vblank\n");
6662 }
6663 break;
6664 case 1: /* D4 vline */
6665 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6666 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6667 DRM_DEBUG("IH: D4 vline\n");
6668 }
6669 break;
6670 default:
6671 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6672 break;
6673 }
6674 break;
6675 case 5: /* D5 vblank/vline */
6676 switch (src_data) {
6677 case 0: /* D5 vblank */
6678 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6679 if (rdev->irq.crtc_vblank_int[4]) {
6680 drm_handle_vblank(rdev->ddev, 4);
6681 rdev->pm.vblank_sync = true;
6682 wake_up(&rdev->irq.vblank_queue);
6683 }
6684 if (atomic_read(&rdev->irq.pflip[4]))
6685 radeon_crtc_handle_flip(rdev, 4);
6686 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6687 DRM_DEBUG("IH: D5 vblank\n");
6688 }
6689 break;
6690 case 1: /* D5 vline */
6691 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6692 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6693 DRM_DEBUG("IH: D5 vline\n");
6694 }
6695 break;
6696 default:
6697 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6698 break;
6699 }
6700 break;
6701 case 6: /* D6 vblank/vline */
6702 switch (src_data) {
6703 case 0: /* D6 vblank */
6704 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6705 if (rdev->irq.crtc_vblank_int[5]) {
6706 drm_handle_vblank(rdev->ddev, 5);
6707 rdev->pm.vblank_sync = true;
6708 wake_up(&rdev->irq.vblank_queue);
6709 }
6710 if (atomic_read(&rdev->irq.pflip[5]))
6711 radeon_crtc_handle_flip(rdev, 5);
6712 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6713 DRM_DEBUG("IH: D6 vblank\n");
6714 }
6715 break;
6716 case 1: /* D6 vline */
6717 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6718 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6719 DRM_DEBUG("IH: D6 vline\n");
6720 }
6721 break;
6722 default:
6723 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6724 break;
6725 }
6726 break;
6727 case 42: /* HPD hotplug */
6728 switch (src_data) {
6729 case 0:
6730 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6731 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
6732 queue_hotplug = true;
6733 DRM_DEBUG("IH: HPD1\n");
6734 }
6735 break;
6736 case 1:
6737 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6738 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6739 queue_hotplug = true;
6740 DRM_DEBUG("IH: HPD2\n");
6741 }
6742 break;
6743 case 2:
6744 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6745 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6746 queue_hotplug = true;
6747 DRM_DEBUG("IH: HPD3\n");
6748 }
6749 break;
6750 case 3:
6751 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6752 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6753 queue_hotplug = true;
6754 DRM_DEBUG("IH: HPD4\n");
6755 }
6756 break;
6757 case 4:
6758 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6759 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6760 queue_hotplug = true;
6761 DRM_DEBUG("IH: HPD5\n");
6762 }
6763 break;
6764 case 5:
6765 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6766 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6767 queue_hotplug = true;
6768 DRM_DEBUG("IH: HPD6\n");
6769 }
6770 break;
6771 default:
6772 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6773 break;
6774 }
6775 break;
6a3808b8
CK		 6776 case 124: /* UVD */
6777 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6778 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6779 break;
9d97c99b
AD		 6780 case 146:
6781 case 147:
3ec7d11b
AD		 6782 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6783 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6784 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9d97c99b
AD		 6785 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6786 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 6787 addr);
9d97c99b 6788 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 6789 status);
6790 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b
AD		 6791 /* reset addr and status */
6792 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6793 break;
a59781bb
AD		 6794 case 176: /* GFX RB CP_INT */
6795 case 177: /* GFX IB CP_INT */
6796 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6797 break;
6798 case 181: /* CP EOP event */
6799 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 6800 /* XXX check the bitfield order! */
6801 me_id = (ring_id & 0x60) >> 5;
6802 pipe_id = (ring_id & 0x18) >> 3;
6803 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 6804 switch (me_id) {
6805 case 0:
6806 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6807 break;
6808 case 1:
a59781bb 6809 case 2:
2b0781a6
AD		 6810 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
6811 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6812 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
6813 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 6814 break;
6815 }
6816 break;
6817 case 184: /* CP Privileged reg access */
6818 DRM_ERROR("Illegal register access in command stream\n");
6819 /* XXX check the bitfield order! */
6820 me_id = (ring_id & 0x60) >> 5;
6821 pipe_id = (ring_id & 0x18) >> 3;
6822 queue_id = (ring_id & 0x7) >> 0;
6823 switch (me_id) {
6824 case 0:
6825 /* This results in a full GPU reset, but all we need to do is soft
6826 * reset the CP for gfx
6827 */
6828 queue_reset = true;
6829 break;
6830 case 1:
6831 /* XXX compute */
2b0781a6 6832 queue_reset = true;
a59781bb
AD		 6833 break;
6834 case 2:
6835 /* XXX compute */
2b0781a6 6836 queue_reset = true;
a59781bb
AD		 6837 break;
6838 }
6839 break;
6840 case 185: /* CP Privileged inst */
6841 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 6842 /* XXX check the bitfield order! */
6843 me_id = (ring_id & 0x60) >> 5;
6844 pipe_id = (ring_id & 0x18) >> 3;
6845 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 6846 switch (me_id) {
6847 case 0:
6848 /* This results in a full GPU reset, but all we need to do is soft
6849 * reset the CP for gfx
6850 */
6851 queue_reset = true;
6852 break;
6853 case 1:
6854 /* XXX compute */
2b0781a6 6855 queue_reset = true;
a59781bb
AD		 6856 break;
6857 case 2:
6858 /* XXX compute */
2b0781a6 6859 queue_reset = true;
a59781bb
AD		 6860 break;
6861 }
6862 break;
21a93e13
AD		 6863 case 224: /* SDMA trap event */
6864 /* XXX check the bitfield order! */
6865 me_id = (ring_id & 0x3) >> 0;
6866 queue_id = (ring_id & 0xc) >> 2;
6867 DRM_DEBUG("IH: SDMA trap\n");
6868 switch (me_id) {
6869 case 0:
6870 switch (queue_id) {
6871 case 0:
6872 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6873 break;
6874 case 1:
6875 /* XXX compute */
6876 break;
6877 case 2:
6878 /* XXX compute */
6879 break;
6880 }
6881 break;
6882 case 1:
6883 switch (queue_id) {
6884 case 0:
6885 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6886 break;
6887 case 1:
6888 /* XXX compute */
6889 break;
6890 case 2:
6891 /* XXX compute */
6892 break;
6893 }
6894 break;
6895 }
6896 break;
41a524ab
AD		 6897 case 230: /* thermal low to high */
6898 DRM_DEBUG("IH: thermal low to high\n");
6899 rdev->pm.dpm.thermal.high_to_low = false;
6900 queue_thermal = true;
6901 break;
6902 case 231: /* thermal high to low */
6903 DRM_DEBUG("IH: thermal high to low\n");
6904 rdev->pm.dpm.thermal.high_to_low = true;
6905 queue_thermal = true;
6906 break;
6907 case 233: /* GUI IDLE */
6908 DRM_DEBUG("IH: GUI idle\n");
6909 break;
21a93e13
AD		 6910 case 241: /* SDMA Privileged inst */
6911 case 247: /* SDMA Privileged inst */
6912 DRM_ERROR("Illegal instruction in SDMA command stream\n");
6913 /* XXX check the bitfield order! */
6914 me_id = (ring_id & 0x3) >> 0;
6915 queue_id = (ring_id & 0xc) >> 2;
6916 switch (me_id) {
6917 case 0:
6918 switch (queue_id) {
6919 case 0:
6920 queue_reset = true;
6921 break;
6922 case 1:
6923 /* XXX compute */
6924 queue_reset = true;
6925 break;
6926 case 2:
6927 /* XXX compute */
6928 queue_reset = true;
6929 break;
6930 }
6931 break;
6932 case 1:
6933 switch (queue_id) {
6934 case 0:
6935 queue_reset = true;
6936 break;
6937 case 1:
6938 /* XXX compute */
6939 queue_reset = true;
6940 break;
6941 case 2:
6942 /* XXX compute */
6943 queue_reset = true;
6944 break;
6945 }
6946 break;
6947 }
6948 break;
a59781bb
AD		 6949 default:
6950 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6951 break;
6952 }
6953
6954 /* wptr/rptr are in bytes! */
6955 rptr += 16;
6956 rptr &= rdev->ih.ptr_mask;
6957 }
6958 if (queue_hotplug)
6959 schedule_work(&rdev->hotplug_work);
6960 if (queue_reset)
6961 schedule_work(&rdev->reset_work);
41a524ab
AD		 6962 if (queue_thermal)
6963 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb
AD		 6964 rdev->ih.rptr = rptr;
6965 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6966 atomic_set(&rdev->ih.lock, 0);
6967
6968 /* make sure wptr hasn't changed while processing */
6969 wptr = cik_get_ih_wptr(rdev);
6970 if (wptr != rptr)
6971 goto restart_ih;
6972
6973 return IRQ_HANDLED;
6974}
7bf94a2c
AD		 6975
6976/*
6977 * startup/shutdown callbacks
6978 */
6979/**
6980 * cik_startup - program the asic to a functional state
6981 *
6982 * @rdev: radeon_device pointer
6983 *
6984 * Programs the asic to a functional state (CIK).
6985 * Called by cik_init() and cik_resume().
6986 * Returns 0 for success, error for failure.
6987 */
6988static int cik_startup(struct radeon_device *rdev)
6989{
6990 struct radeon_ring *ring;
6991 int r;
6992
8a7cd276
AD		 6993 /* enable pcie gen2/3 link */
6994 cik_pcie_gen3_enable(rdev);
7235711a
AD		 6995 /* enable aspm */
6996 cik_program_aspm(rdev);
8a7cd276 6997
e5903d39
AD		 6998 /* scratch needs to be initialized before MC */
6999 r = r600_vram_scratch_init(rdev);
7000 if (r)
7001 return r;
7002
6fab3feb
AD		 7003 cik_mc_program(rdev);
7004
7bf94a2c
AD		 7005 if (rdev->flags & RADEON_IS_IGP) {
7006 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7007 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7008 r = cik_init_microcode(rdev);
7009 if (r) {
7010 DRM_ERROR("Failed to load firmware!\n");
7011 return r;
7012 }
7013 }
7014 } else {
7015 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7016 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7017 !rdev->mc_fw) {
7018 r = cik_init_microcode(rdev);
7019 if (r) {
7020 DRM_ERROR("Failed to load firmware!\n");
7021 return r;
7022 }
7023 }
7024
7025 r = ci_mc_load_microcode(rdev);
7026 if (r) {
7027 DRM_ERROR("Failed to load MC firmware!\n");
7028 return r;
7029 }
7030 }
7031
7bf94a2c
AD		 7032 r = cik_pcie_gart_enable(rdev);
7033 if (r)
7034 return r;
7035 cik_gpu_init(rdev);
7036
7037 /* allocate rlc buffers */
22c775ce
AD		 7038 if (rdev->flags & RADEON_IS_IGP) {
7039 if (rdev->family == CHIP_KAVERI) {
7040 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7041 rdev->rlc.reg_list_size =
7042 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7043 } else {
7044 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7045 rdev->rlc.reg_list_size =
7046 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7047 }
7048 }
7049 rdev->rlc.cs_data = ci_cs_data;
7050 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 7051 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 7052 if (r) {
7053 DRM_ERROR("Failed to init rlc BOs!\n");
7054 return r;
7055 }
7056
7057 /* allocate wb buffer */
7058 r = radeon_wb_init(rdev);
7059 if (r)
7060 return r;
7061
963e81f9
AD		 7062 /* allocate mec buffers */
7063 r = cik_mec_init(rdev);
7064 if (r) {
7065 DRM_ERROR("Failed to init MEC BOs!\n");
7066 return r;
7067 }
7068
7bf94a2c
AD		 7069 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7070 if (r) {
7071 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7072 return r;
7073 }
7074
963e81f9
AD		 7075 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7076 if (r) {
7077 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7078 return r;
7079 }
7080
7081 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7082 if (r) {
7083 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7084 return r;
7085 }
7086
7bf94a2c
AD		 7087 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7088 if (r) {
7089 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7090 return r;
7091 }
7092
7093 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7094 if (r) {
7095 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7096 return r;
7097 }
7098
2ce529da 7099 r = radeon_uvd_resume(rdev);
87167bb1 7100 if (!r) {
2ce529da
AD		 7101 r = uvd_v4_2_resume(rdev);
7102 if (!r) {
7103 r = radeon_fence_driver_start_ring(rdev,
7104 R600_RING_TYPE_UVD_INDEX);
7105 if (r)
7106 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7107 }
87167bb1
CK		 7108 }
7109 if (r)
7110 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7111
7bf94a2c
AD		 7112 /* Enable IRQ */
7113 if (!rdev->irq.installed) {
7114 r = radeon_irq_kms_init(rdev);
7115 if (r)
7116 return r;
7117 }
7118
7119 r = cik_irq_init(rdev);
7120 if (r) {
7121 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7122 radeon_irq_kms_fini(rdev);
7123 return r;
7124 }
7125 cik_irq_set(rdev);
7126
7127 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7128 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7129 CP_RB0_RPTR, CP_RB0_WPTR,
1dac28eb 7130 PACKET3(PACKET3_NOP, 0x3FFF));
7bf94a2c
AD		 7131 if (r)
7132 return r;
7133
963e81f9 7134 /* set up the compute queues */
2615b53a 7135 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7136 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7137 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7138 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7139 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7140 if (r)
7141 return r;
7142 ring->me = 1; /* first MEC */
7143 ring->pipe = 0; /* first pipe */
7144 ring->queue = 0; /* first queue */
7145 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7146
2615b53a 7147 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7148 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7149 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7150 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7151 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7152 if (r)
7153 return r;
7154 /* dGPU only have 1 MEC */
7155 ring->me = 1; /* first MEC */
7156 ring->pipe = 0; /* first pipe */
7157 ring->queue = 1; /* second queue */
7158 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7159
7bf94a2c
AD		 7160 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7161 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7162 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7163 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
2e1e6dad 7164 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7165 if (r)
7166 return r;
7167
7168 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7169 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7170 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7171 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
2e1e6dad 7172 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7173 if (r)
7174 return r;
7175
7176 r = cik_cp_resume(rdev);
7177 if (r)
7178 return r;
7179
7180 r = cik_sdma_resume(rdev);
7181 if (r)
7182 return r;
7183
87167bb1
CK		 7184 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7185 if (ring->ring_size) {
02c9f7fa 7186 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
87167bb1 7187 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 7188 RADEON_CP_PACKET2);
87167bb1 7189 if (!r)
e409b128 7190 r = uvd_v1_0_init(rdev);
87167bb1
CK		 7191 if (r)
7192 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7193 }
7194
7bf94a2c
AD		 7195 r = radeon_ib_pool_init(rdev);
7196 if (r) {
7197 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7198 return r;
7199 }
7200
7201 r = radeon_vm_manager_init(rdev);
7202 if (r) {
7203 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7204 return r;
7205 }
7206
b530602f
AD		 7207 r = dce6_audio_init(rdev);
7208 if (r)
7209 return r;
7210
7bf94a2c
AD		 7211 return 0;
7212}
7213
7214/**
7215 * cik_resume - resume the asic to a functional state
7216 *
7217 * @rdev: radeon_device pointer
7218 *
7219 * Programs the asic to a functional state (CIK).
7220 * Called at resume.
7221 * Returns 0 for success, error for failure.
7222 */
7223int cik_resume(struct radeon_device *rdev)
7224{
7225 int r;
7226
7227 /* post card */
7228 atom_asic_init(rdev->mode_info.atom_context);
7229
0aafd313
AD		 7230 /* init golden registers */
7231 cik_init_golden_registers(rdev);
7232
7bf94a2c
AD		 7233 rdev->accel_working = true;
7234 r = cik_startup(rdev);
7235 if (r) {
7236 DRM_ERROR("cik startup failed on resume\n");
7237 rdev->accel_working = false;
7238 return r;
7239 }
7240
7241 return r;
7242
7243}
7244
7245/**
7246 * cik_suspend - suspend the asic
7247 *
7248 * @rdev: radeon_device pointer
7249 *
7250 * Bring the chip into a state suitable for suspend (CIK).
7251 * Called at suspend.
7252 * Returns 0 for success.
7253 */
7254int cik_suspend(struct radeon_device *rdev)
7255{
b530602f 7256 dce6_audio_fini(rdev);
7bf94a2c
AD		 7257 radeon_vm_manager_fini(rdev);
7258 cik_cp_enable(rdev, false);
7259 cik_sdma_enable(rdev, false);
e409b128 7260 uvd_v1_0_fini(rdev);
87167bb1 7261 radeon_uvd_suspend(rdev);
473359bc
AD		 7262 cik_fini_pg(rdev);
7263 cik_fini_cg(rdev);
7bf94a2c
AD		 7264 cik_irq_suspend(rdev);
7265 radeon_wb_disable(rdev);
7266 cik_pcie_gart_disable(rdev);
7267 return 0;
7268}
7269
7270/* Plan is to move initialization in that function and use
7271 * helper function so that radeon_device_init pretty much
7272 * do nothing more than calling asic specific function. This
7273 * should also allow to remove a bunch of callback function
7274 * like vram_info.
7275 */
7276/**
7277 * cik_init - asic specific driver and hw init
7278 *
7279 * @rdev: radeon_device pointer
7280 *
7281 * Setup asic specific driver variables and program the hw
7282 * to a functional state (CIK).
7283 * Called at driver startup.
7284 * Returns 0 for success, errors for failure.
7285 */
7286int cik_init(struct radeon_device *rdev)
7287{
7288 struct radeon_ring *ring;
7289 int r;
7290
7291 /* Read BIOS */
7292 if (!radeon_get_bios(rdev)) {
7293 if (ASIC_IS_AVIVO(rdev))
7294 return -EINVAL;
7295 }
7296 /* Must be an ATOMBIOS */
7297 if (!rdev->is_atom_bios) {
7298 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7299 return -EINVAL;
7300 }
7301 r = radeon_atombios_init(rdev);
7302 if (r)
7303 return r;
7304
7305 /* Post card if necessary */
7306 if (!radeon_card_posted(rdev)) {
7307 if (!rdev->bios) {
7308 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7309 return -EINVAL;
7310 }
7311 DRM_INFO("GPU not posted. posting now...\n");
7312 atom_asic_init(rdev->mode_info.atom_context);
7313 }
0aafd313
AD		 7314 /* init golden registers */
7315 cik_init_golden_registers(rdev);
7bf94a2c
AD		 7316 /* Initialize scratch registers */
7317 cik_scratch_init(rdev);
7318 /* Initialize surface registers */
7319 radeon_surface_init(rdev);
7320 /* Initialize clocks */
7321 radeon_get_clock_info(rdev->ddev);
7322
7323 /* Fence driver */
7324 r = radeon_fence_driver_init(rdev);
7325 if (r)
7326 return r;
7327
7328 /* initialize memory controller */
7329 r = cik_mc_init(rdev);
7330 if (r)
7331 return r;
7332 /* Memory manager */
7333 r = radeon_bo_init(rdev);
7334 if (r)
7335 return r;
7336
7337 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7338 ring->ring_obj = NULL;
7339 r600_ring_init(rdev, ring, 1024 * 1024);
7340
963e81f9
AD		 7341 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7342 ring->ring_obj = NULL;
7343 r600_ring_init(rdev, ring, 1024 * 1024);
7344 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7345 if (r)
7346 return r;
7347
7348 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7349 ring->ring_obj = NULL;
7350 r600_ring_init(rdev, ring, 1024 * 1024);
7351 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7352 if (r)
7353 return r;
7354
7bf94a2c
AD		 7355 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7356 ring->ring_obj = NULL;
7357 r600_ring_init(rdev, ring, 256 * 1024);
7358
7359 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7360 ring->ring_obj = NULL;
7361 r600_ring_init(rdev, ring, 256 * 1024);
7362
87167bb1
CK		 7363 r = radeon_uvd_init(rdev);
7364 if (!r) {
7365 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7366 ring->ring_obj = NULL;
7367 r600_ring_init(rdev, ring, 4096);
7368 }
7369
7bf94a2c
AD		 7370 rdev->ih.ring_obj = NULL;
7371 r600_ih_ring_init(rdev, 64 * 1024);
7372
7373 r = r600_pcie_gart_init(rdev);
7374 if (r)
7375 return r;
7376
7377 rdev->accel_working = true;
7378 r = cik_startup(rdev);
7379 if (r) {
7380 dev_err(rdev->dev, "disabling GPU acceleration\n");
7381 cik_cp_fini(rdev);
7382 cik_sdma_fini(rdev);
7383 cik_irq_fini(rdev);
1fd11777 7384 sumo_rlc_fini(rdev);
963e81f9 7385 cik_mec_fini(rdev);
7bf94a2c
AD		 7386 radeon_wb_fini(rdev);
7387 radeon_ib_pool_fini(rdev);
7388 radeon_vm_manager_fini(rdev);
7389 radeon_irq_kms_fini(rdev);
7390 cik_pcie_gart_fini(rdev);
7391 rdev->accel_working = false;
7392 }
7393
7394 /* Don't start up if the MC ucode is missing.
7395 * The default clocks and voltages before the MC ucode
7396 * is loaded are not suffient for advanced operations.
7397 */
7398 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7399 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7400 return -EINVAL;
7401 }
7402
7403 return 0;
7404}
7405
7406/**
7407 * cik_fini - asic specific driver and hw fini
7408 *
7409 * @rdev: radeon_device pointer
7410 *
7411 * Tear down the asic specific driver variables and program the hw
7412 * to an idle state (CIK).
7413 * Called at driver unload.
7414 */
7415void cik_fini(struct radeon_device *rdev)
7416{
7417 cik_cp_fini(rdev);
7418 cik_sdma_fini(rdev);
473359bc
AD		 7419 cik_fini_pg(rdev);
7420 cik_fini_cg(rdev);
7bf94a2c 7421 cik_irq_fini(rdev);
1fd11777 7422 sumo_rlc_fini(rdev);
963e81f9 7423 cik_mec_fini(rdev);
7bf94a2c
AD		 7424 radeon_wb_fini(rdev);
7425 radeon_vm_manager_fini(rdev);
7426 radeon_ib_pool_fini(rdev);
7427 radeon_irq_kms_fini(rdev);
e409b128 7428 uvd_v1_0_fini(rdev);
87167bb1 7429 radeon_uvd_fini(rdev);
7bf94a2c
AD		 7430 cik_pcie_gart_fini(rdev);
7431 r600_vram_scratch_fini(rdev);
7432 radeon_gem_fini(rdev);
7433 radeon_fence_driver_fini(rdev);
7434 radeon_bo_fini(rdev);
7435 radeon_atombios_fini(rdev);
7436 kfree(rdev->bios);
7437 rdev->bios = NULL;
7438}
cd84a27d 7439
134b480f
AD		 7440void dce8_program_fmt(struct drm_encoder *encoder)
7441{
7442 struct drm_device *dev = encoder->dev;
7443 struct radeon_device *rdev = dev->dev_private;
7444 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7445 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7446 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7447 int bpc = 0;
7448 u32 tmp = 0;
6214bb74 7449 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 7450
6214bb74
AD		 7451 if (connector) {
7452 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 7453 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 7454 dither = radeon_connector->dither;
7455 }
134b480f
AD		 7456
7457 /* LVDS/eDP FMT is set up by atom */
7458 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7459 return;
7460
7461 /* not needed for analog */
7462 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7463 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7464 return;
7465
7466 if (bpc == 0)
7467 return;
7468
7469 switch (bpc) {
7470 case 6:
6214bb74 7471 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7472 /* XXX sort out optimal dither settings */
7473 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7474 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7475 else
7476 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7477 break;
7478 case 8:
6214bb74 7479 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7480 /* XXX sort out optimal dither settings */
7481 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7482 FMT_RGB_RANDOM_ENABLE |
7483 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7484 else
7485 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7486 break;
7487 case 10:
6214bb74 7488 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7489 /* XXX sort out optimal dither settings */
7490 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7491 FMT_RGB_RANDOM_ENABLE |
7492 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7493 else
7494 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7495 break;
7496 default:
7497 /* not needed */
7498 break;
7499 }
7500
7501 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
7502}
7503
cd84a27d
AD		 7504/* display watermark setup */
7505/**
7506 * dce8_line_buffer_adjust - Set up the line buffer
7507 *
7508 * @rdev: radeon_device pointer
7509 * @radeon_crtc: the selected display controller
7510 * @mode: the current display mode on the selected display
7511 * controller
7512 *
7513 * Setup up the line buffer allocation for
7514 * the selected display controller (CIK).
7515 * Returns the line buffer size in pixels.
7516 */
7517static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
7518 struct radeon_crtc *radeon_crtc,
7519 struct drm_display_mode *mode)
7520{
bc01a8c7
AD		 7521 u32 tmp, buffer_alloc, i;
7522 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 7523 /*
7524 * Line Buffer Setup
7525 * There are 6 line buffers, one for each display controllers.
7526 * There are 3 partitions per LB. Select the number of partitions
7527 * to enable based on the display width. For display widths larger
7528 * than 4096, you need use to use 2 display controllers and combine
7529 * them using the stereo blender.
7530 */
7531 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 7532 if (mode->crtc_hdisplay < 1920) {
cd84a27d 7533 tmp = 1;
bc01a8c7
AD		 7534 buffer_alloc = 2;
7535 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 7536 tmp = 2;
bc01a8c7
AD		 7537 buffer_alloc = 2;
7538 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 7539 tmp = 0;
bc01a8c7
AD		 7540 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7541 } else {
cd84a27d
AD		 7542 DRM_DEBUG_KMS("Mode too big for LB!\n");
7543 tmp = 0;
bc01a8c7 7544 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 7545 }
bc01a8c7 7546 } else {
cd84a27d 7547 tmp = 1;
bc01a8c7
AD		 7548 buffer_alloc = 0;
7549 }
cd84a27d
AD		 7550
7551 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
7552 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
7553
bc01a8c7
AD		 7554 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
7555 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
7556 for (i = 0; i < rdev->usec_timeout; i++) {
7557 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
7558 DMIF_BUFFERS_ALLOCATED_COMPLETED)
7559 break;
7560 udelay(1);
7561 }
7562
cd84a27d
AD		 7563 if (radeon_crtc->base.enabled && mode) {
7564 switch (tmp) {
7565 case 0:
7566 default:
7567 return 4096 * 2;
7568 case 1:
7569 return 1920 * 2;
7570 case 2:
7571 return 2560 * 2;
7572 }
7573 }
7574
7575 /* controller not enabled, so no lb used */
7576 return 0;
7577}
7578
7579/**
7580 * cik_get_number_of_dram_channels - get the number of dram channels
7581 *
7582 * @rdev: radeon_device pointer
7583 *
7584 * Look up the number of video ram channels (CIK).
7585 * Used for display watermark bandwidth calculations
7586 * Returns the number of dram channels
7587 */
7588static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
7589{
7590 u32 tmp = RREG32(MC_SHARED_CHMAP);
7591
7592 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
7593 case 0:
7594 default:
7595 return 1;
7596 case 1:
7597 return 2;
7598 case 2:
7599 return 4;
7600 case 3:
7601 return 8;
7602 case 4:
7603 return 3;
7604 case 5:
7605 return 6;
7606 case 6:
7607 return 10;
7608 case 7:
7609 return 12;
7610 case 8:
7611 return 16;
7612 }
7613}
7614
7615struct dce8_wm_params {
7616 u32 dram_channels; /* number of dram channels */
7617 u32 yclk; /* bandwidth per dram data pin in kHz */
7618 u32 sclk; /* engine clock in kHz */
7619 u32 disp_clk; /* display clock in kHz */
7620 u32 src_width; /* viewport width */
7621 u32 active_time; /* active display time in ns */
7622 u32 blank_time; /* blank time in ns */
7623 bool interlaced; /* mode is interlaced */
7624 fixed20_12 vsc; /* vertical scale ratio */
7625 u32 num_heads; /* number of active crtcs */
7626 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
7627 u32 lb_size; /* line buffer allocated to pipe */
7628 u32 vtaps; /* vertical scaler taps */
7629};
7630
7631/**
7632 * dce8_dram_bandwidth - get the dram bandwidth
7633 *
7634 * @wm: watermark calculation data
7635 *
7636 * Calculate the raw dram bandwidth (CIK).
7637 * Used for display watermark bandwidth calculations
7638 * Returns the dram bandwidth in MBytes/s
7639 */
7640static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
7641{
7642 /* Calculate raw DRAM Bandwidth */
7643 fixed20_12 dram_efficiency; /* 0.7 */
7644 fixed20_12 yclk, dram_channels, bandwidth;
7645 fixed20_12 a;
7646
7647 a.full = dfixed_const(1000);
7648 yclk.full = dfixed_const(wm->yclk);
7649 yclk.full = dfixed_div(yclk, a);
7650 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7651 a.full = dfixed_const(10);
7652 dram_efficiency.full = dfixed_const(7);
7653 dram_efficiency.full = dfixed_div(dram_efficiency, a);
7654 bandwidth.full = dfixed_mul(dram_channels, yclk);
7655 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
7656
7657 return dfixed_trunc(bandwidth);
7658}
7659
7660/**
7661 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
7662 *
7663 * @wm: watermark calculation data
7664 *
7665 * Calculate the dram bandwidth used for display (CIK).
7666 * Used for display watermark bandwidth calculations
7667 * Returns the dram bandwidth for display in MBytes/s
7668 */
7669static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7670{
7671 /* Calculate DRAM Bandwidth and the part allocated to display. */
7672 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
7673 fixed20_12 yclk, dram_channels, bandwidth;
7674 fixed20_12 a;
7675
7676 a.full = dfixed_const(1000);
7677 yclk.full = dfixed_const(wm->yclk);
7678 yclk.full = dfixed_div(yclk, a);
7679 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7680 a.full = dfixed_const(10);
7681 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
7682 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
7683 bandwidth.full = dfixed_mul(dram_channels, yclk);
7684 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
7685
7686 return dfixed_trunc(bandwidth);
7687}
7688
7689/**
7690 * dce8_data_return_bandwidth - get the data return bandwidth
7691 *
7692 * @wm: watermark calculation data
7693 *
7694 * Calculate the data return bandwidth used for display (CIK).
7695 * Used for display watermark bandwidth calculations
7696 * Returns the data return bandwidth in MBytes/s
7697 */
7698static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
7699{
7700 /* Calculate the display Data return Bandwidth */
7701 fixed20_12 return_efficiency; /* 0.8 */
7702 fixed20_12 sclk, bandwidth;
7703 fixed20_12 a;
7704
7705 a.full = dfixed_const(1000);
7706 sclk.full = dfixed_const(wm->sclk);
7707 sclk.full = dfixed_div(sclk, a);
7708 a.full = dfixed_const(10);
7709 return_efficiency.full = dfixed_const(8);
7710 return_efficiency.full = dfixed_div(return_efficiency, a);
7711 a.full = dfixed_const(32);
7712 bandwidth.full = dfixed_mul(a, sclk);
7713 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
7714
7715 return dfixed_trunc(bandwidth);
7716}
7717
7718/**
7719 * dce8_dmif_request_bandwidth - get the dmif bandwidth
7720 *
7721 * @wm: watermark calculation data
7722 *
7723 * Calculate the dmif bandwidth used for display (CIK).
7724 * Used for display watermark bandwidth calculations
7725 * Returns the dmif bandwidth in MBytes/s
7726 */
7727static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
7728{
7729 /* Calculate the DMIF Request Bandwidth */
7730 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
7731 fixed20_12 disp_clk, bandwidth;
7732 fixed20_12 a, b;
7733
7734 a.full = dfixed_const(1000);
7735 disp_clk.full = dfixed_const(wm->disp_clk);
7736 disp_clk.full = dfixed_div(disp_clk, a);
7737 a.full = dfixed_const(32);
7738 b.full = dfixed_mul(a, disp_clk);
7739
7740 a.full = dfixed_const(10);
7741 disp_clk_request_efficiency.full = dfixed_const(8);
7742 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
7743
7744 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
7745
7746 return dfixed_trunc(bandwidth);
7747}
7748
7749/**
7750 * dce8_available_bandwidth - get the min available bandwidth
7751 *
7752 * @wm: watermark calculation data
7753 *
7754 * Calculate the min available bandwidth used for display (CIK).
7755 * Used for display watermark bandwidth calculations
7756 * Returns the min available bandwidth in MBytes/s
7757 */
7758static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
7759{
7760 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
7761 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
7762 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
7763 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
7764
7765 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
7766}
7767
7768/**
7769 * dce8_average_bandwidth - get the average available bandwidth
7770 *
7771 * @wm: watermark calculation data
7772 *
7773 * Calculate the average available bandwidth used for display (CIK).
7774 * Used for display watermark bandwidth calculations
7775 * Returns the average available bandwidth in MBytes/s
7776 */
7777static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
7778{
7779 /* Calculate the display mode Average Bandwidth
7780 * DisplayMode should contain the source and destination dimensions,
7781 * timing, etc.
7782 */
7783 fixed20_12 bpp;
7784 fixed20_12 line_time;
7785 fixed20_12 src_width;
7786 fixed20_12 bandwidth;
7787 fixed20_12 a;
7788
7789 a.full = dfixed_const(1000);
7790 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
7791 line_time.full = dfixed_div(line_time, a);
7792 bpp.full = dfixed_const(wm->bytes_per_pixel);
7793 src_width.full = dfixed_const(wm->src_width);
7794 bandwidth.full = dfixed_mul(src_width, bpp);
7795 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
7796 bandwidth.full = dfixed_div(bandwidth, line_time);
7797
7798 return dfixed_trunc(bandwidth);
7799}
7800
7801/**
7802 * dce8_latency_watermark - get the latency watermark
7803 *
7804 * @wm: watermark calculation data
7805 *
7806 * Calculate the latency watermark (CIK).
7807 * Used for display watermark bandwidth calculations
7808 * Returns the latency watermark in ns
7809 */
7810static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
7811{
7812 /* First calculate the latency in ns */
7813 u32 mc_latency = 2000; /* 2000 ns. */
7814 u32 available_bandwidth = dce8_available_bandwidth(wm);
7815 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
7816 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
7817 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
7818 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
7819 (wm->num_heads * cursor_line_pair_return_time);
7820 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
7821 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
7822 u32 tmp, dmif_size = 12288;
7823 fixed20_12 a, b, c;
7824
7825 if (wm->num_heads == 0)
7826 return 0;
7827
7828 a.full = dfixed_const(2);
7829 b.full = dfixed_const(1);
7830 if ((wm->vsc.full > a.full) ||
7831 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
7832 (wm->vtaps >= 5) ||
7833 ((wm->vsc.full >= a.full) && wm->interlaced))
7834 max_src_lines_per_dst_line = 4;
7835 else
7836 max_src_lines_per_dst_line = 2;
7837
7838 a.full = dfixed_const(available_bandwidth);
7839 b.full = dfixed_const(wm->num_heads);
7840 a.full = dfixed_div(a, b);
7841
7842 b.full = dfixed_const(mc_latency + 512);
7843 c.full = dfixed_const(wm->disp_clk);
7844 b.full = dfixed_div(b, c);
7845
7846 c.full = dfixed_const(dmif_size);
7847 b.full = dfixed_div(c, b);
7848
7849 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
7850
7851 b.full = dfixed_const(1000);
7852 c.full = dfixed_const(wm->disp_clk);
7853 b.full = dfixed_div(c, b);
7854 c.full = dfixed_const(wm->bytes_per_pixel);
7855 b.full = dfixed_mul(b, c);
7856
7857 lb_fill_bw = min(tmp, dfixed_trunc(b));
7858
7859 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
7860 b.full = dfixed_const(1000);
7861 c.full = dfixed_const(lb_fill_bw);
7862 b.full = dfixed_div(c, b);
7863 a.full = dfixed_div(a, b);
7864 line_fill_time = dfixed_trunc(a);
7865
7866 if (line_fill_time < wm->active_time)
7867 return latency;
7868 else
7869 return latency + (line_fill_time - wm->active_time);
7870
7871}
7872
7873/**
7874 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
7875 * average and available dram bandwidth
7876 *
7877 * @wm: watermark calculation data
7878 *
7879 * Check if the display average bandwidth fits in the display
7880 * dram bandwidth (CIK).
7881 * Used for display watermark bandwidth calculations
7882 * Returns true if the display fits, false if not.
7883 */
7884static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7885{
7886 if (dce8_average_bandwidth(wm) <=
7887 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
7888 return true;
7889 else
7890 return false;
7891}
7892
7893/**
7894 * dce8_average_bandwidth_vs_available_bandwidth - check
7895 * average and available bandwidth
7896 *
7897 * @wm: watermark calculation data
7898 *
7899 * Check if the display average bandwidth fits in the display
7900 * available bandwidth (CIK).
7901 * Used for display watermark bandwidth calculations
7902 * Returns true if the display fits, false if not.
7903 */
7904static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
7905{
7906 if (dce8_average_bandwidth(wm) <=
7907 (dce8_available_bandwidth(wm) / wm->num_heads))
7908 return true;
7909 else
7910 return false;
7911}
7912
7913/**
7914 * dce8_check_latency_hiding - check latency hiding
7915 *
7916 * @wm: watermark calculation data
7917 *
7918 * Check latency hiding (CIK).
7919 * Used for display watermark bandwidth calculations
7920 * Returns true if the display fits, false if not.
7921 */
7922static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
7923{
7924 u32 lb_partitions = wm->lb_size / wm->src_width;
7925 u32 line_time = wm->active_time + wm->blank_time;
7926 u32 latency_tolerant_lines;
7927 u32 latency_hiding;
7928 fixed20_12 a;
7929
7930 a.full = dfixed_const(1);
7931 if (wm->vsc.full > a.full)
7932 latency_tolerant_lines = 1;
7933 else {
7934 if (lb_partitions <= (wm->vtaps + 1))
7935 latency_tolerant_lines = 1;
7936 else
7937 latency_tolerant_lines = 2;
7938 }
7939
7940 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
7941
7942 if (dce8_latency_watermark(wm) <= latency_hiding)
7943 return true;
7944 else
7945 return false;
7946}
7947
7948/**
7949 * dce8_program_watermarks - program display watermarks
7950 *
7951 * @rdev: radeon_device pointer
7952 * @radeon_crtc: the selected display controller
7953 * @lb_size: line buffer size
7954 * @num_heads: number of display controllers in use
7955 *
7956 * Calculate and program the display watermarks for the
7957 * selected display controller (CIK).
7958 */
7959static void dce8_program_watermarks(struct radeon_device *rdev,
7960 struct radeon_crtc *radeon_crtc,
7961 u32 lb_size, u32 num_heads)
7962{
7963 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 7964 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 7965 u32 pixel_period;
7966 u32 line_time = 0;
7967 u32 latency_watermark_a = 0, latency_watermark_b = 0;
7968 u32 tmp, wm_mask;
7969
7970 if (radeon_crtc->base.enabled && num_heads && mode) {
7971 pixel_period = 1000000 / (u32)mode->clock;
7972 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
7973
58ea2dea
AD		 7974 /* watermark for high clocks */
7975 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7976 rdev->pm.dpm_enabled) {
7977 wm_high.yclk =
7978 radeon_dpm_get_mclk(rdev, false) * 10;
7979 wm_high.sclk =
7980 radeon_dpm_get_sclk(rdev, false) * 10;
7981 } else {
7982 wm_high.yclk = rdev->pm.current_mclk * 10;
7983 wm_high.sclk = rdev->pm.current_sclk * 10;
7984 }
7985
7986 wm_high.disp_clk = mode->clock;
7987 wm_high.src_width = mode->crtc_hdisplay;
7988 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
7989 wm_high.blank_time = line_time - wm_high.active_time;
7990 wm_high.interlaced = false;
cd84a27d 7991 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 7992 wm_high.interlaced = true;
7993 wm_high.vsc = radeon_crtc->vsc;
7994 wm_high.vtaps = 1;
cd84a27d 7995 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 7996 wm_high.vtaps = 2;
7997 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
7998 wm_high.lb_size = lb_size;
7999 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8000 wm_high.num_heads = num_heads;
cd84a27d
AD		 8001
8002 /* set for high clocks */
58ea2dea
AD		 8003 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8004
8005 /* possibly force display priority to high */
8006 /* should really do this at mode validation time... */
8007 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8008 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8009 !dce8_check_latency_hiding(&wm_high) ||
8010 (rdev->disp_priority == 2)) {
8011 DRM_DEBUG_KMS("force priority to high\n");
8012 }
8013
8014 /* watermark for low clocks */
8015 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8016 rdev->pm.dpm_enabled) {
8017 wm_low.yclk =
8018 radeon_dpm_get_mclk(rdev, true) * 10;
8019 wm_low.sclk =
8020 radeon_dpm_get_sclk(rdev, true) * 10;
8021 } else {
8022 wm_low.yclk = rdev->pm.current_mclk * 10;
8023 wm_low.sclk = rdev->pm.current_sclk * 10;
8024 }
8025
8026 wm_low.disp_clk = mode->clock;
8027 wm_low.src_width = mode->crtc_hdisplay;
8028 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8029 wm_low.blank_time = line_time - wm_low.active_time;
8030 wm_low.interlaced = false;
8031 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8032 wm_low.interlaced = true;
8033 wm_low.vsc = radeon_crtc->vsc;
8034 wm_low.vtaps = 1;
8035 if (radeon_crtc->rmx_type != RMX_OFF)
8036 wm_low.vtaps = 2;
8037 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8038 wm_low.lb_size = lb_size;
8039 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8040 wm_low.num_heads = num_heads;
8041
cd84a27d 8042 /* set for low clocks */
58ea2dea 8043 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 8044
8045 /* possibly force display priority to high */
8046 /* should really do this at mode validation time... */
58ea2dea
AD		 8047 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8048 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8049 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 8050 (rdev->disp_priority == 2)) {
8051 DRM_DEBUG_KMS("force priority to high\n");
8052 }
8053 }
8054
8055 /* select wm A */
8056 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8057 tmp = wm_mask;
8058 tmp &= ~LATENCY_WATERMARK_MASK(3);
8059 tmp |= LATENCY_WATERMARK_MASK(1);
8060 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8061 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8062 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8063 LATENCY_HIGH_WATERMARK(line_time)));
8064 /* select wm B */
8065 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8066 tmp &= ~LATENCY_WATERMARK_MASK(3);
8067 tmp |= LATENCY_WATERMARK_MASK(2);
8068 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8069 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8070 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8071 LATENCY_HIGH_WATERMARK(line_time)));
8072 /* restore original selection */
8073 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 8074
8075 /* save values for DPM */
8076 radeon_crtc->line_time = line_time;
8077 radeon_crtc->wm_high = latency_watermark_a;
8078 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 8079}
8080
8081/**
8082 * dce8_bandwidth_update - program display watermarks
8083 *
8084 * @rdev: radeon_device pointer
8085 *
8086 * Calculate and program the display watermarks and line
8087 * buffer allocation (CIK).
8088 */
8089void dce8_bandwidth_update(struct radeon_device *rdev)
8090{
8091 struct drm_display_mode *mode = NULL;
8092 u32 num_heads = 0, lb_size;
8093 int i;
8094
8095 radeon_update_display_priority(rdev);
8096
8097 for (i = 0; i < rdev->num_crtc; i++) {
8098 if (rdev->mode_info.crtcs[i]->base.enabled)
8099 num_heads++;
8100 }
8101 for (i = 0; i < rdev->num_crtc; i++) {
8102 mode = &rdev->mode_info.crtcs[i]->base.mode;
8103 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8104 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8105 }
8106}
44fa346f
AD		 8107
8108/**
8109 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8110 *
8111 * @rdev: radeon_device pointer
8112 *
8113 * Fetches a GPU clock counter snapshot (SI).
8114 * Returns the 64 bit clock counter snapshot.
8115 */
8116uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8117{
8118 uint64_t clock;
8119
8120 mutex_lock(&rdev->gpu_clock_mutex);
8121 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8122 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8123 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8124 mutex_unlock(&rdev->gpu_clock_mutex);
8125 return clock;
8126}
8127
87167bb1
CK		 8128static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8129 u32 cntl_reg, u32 status_reg)
8130{
8131 int r, i;
8132 struct atom_clock_dividers dividers;
8133 uint32_t tmp;
8134
8135 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8136 clock, false, &dividers);
8137 if (r)
8138 return r;
8139
8140 tmp = RREG32_SMC(cntl_reg);
8141 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8142 tmp |= dividers.post_divider;
8143 WREG32_SMC(cntl_reg, tmp);
8144
8145 for (i = 0; i < 100; i++) {
8146 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8147 break;
8148 mdelay(10);
8149 }
8150 if (i == 100)
8151 return -ETIMEDOUT;
8152
8153 return 0;
8154}
8155
8156int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8157{
8158 int r = 0;
8159
8160 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8161 if (r)
8162 return r;
8163
8164 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8165 return r;
8166}
8167
8a7cd276 8168static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 8169{
8a7cd276
AD		 8170 struct pci_dev *root = rdev->pdev->bus->self;
8171 int bridge_pos, gpu_pos;
8172 u32 speed_cntl, mask, current_data_rate;
8173 int ret, i;
8174 u16 tmp16;
87167bb1 8175
8a7cd276
AD		 8176 if (radeon_pcie_gen2 == 0)
8177 return;
87167bb1 8178
8a7cd276
AD		 8179 if (rdev->flags & RADEON_IS_IGP)
8180 return;
87167bb1 8181
8a7cd276
AD		 8182 if (!(rdev->flags & RADEON_IS_PCIE))
8183 return;
87167bb1 8184
8a7cd276
AD		 8185 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8186 if (ret != 0)
8187 return;
87167bb1 8188
8a7cd276
AD		 8189 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8190 return;
87167bb1 8191
8a7cd276
AD		 8192 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8193 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8194 LC_CURRENT_DATA_RATE_SHIFT;
8195 if (mask & DRM_PCIE_SPEED_80) {
8196 if (current_data_rate == 2) {
8197 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8198 return;
8199 }
8200 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8201 } else if (mask & DRM_PCIE_SPEED_50) {
8202 if (current_data_rate == 1) {
8203 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8204 return;
8205 }
8206 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8207 }
87167bb1 8208
8a7cd276
AD		 8209 bridge_pos = pci_pcie_cap(root);
8210 if (!bridge_pos)
8211 return;
8212
8213 gpu_pos = pci_pcie_cap(rdev->pdev);
8214 if (!gpu_pos)
8215 return;
8216
8217 if (mask & DRM_PCIE_SPEED_80) {
8218 /* re-try equalization if gen3 is not already enabled */
8219 if (current_data_rate != 2) {
8220 u16 bridge_cfg, gpu_cfg;
8221 u16 bridge_cfg2, gpu_cfg2;
8222 u32 max_lw, current_lw, tmp;
8223
8224 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8225 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8226
8227 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8228 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8229
8230 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8231 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8232
8233 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8234 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8235 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8236
8237 if (current_lw < max_lw) {
8238 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8239 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8240 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8241 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8242 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8243 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8244 }
8245 }
8246
8247 for (i = 0; i < 10; i++) {
8248 /* check status */
8249 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8250 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8251 break;
8252
8253 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8254 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8255
8256 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8257 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8258
8259 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8260 tmp |= LC_SET_QUIESCE;
8261 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8262
8263 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8264 tmp |= LC_REDO_EQ;
8265 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8266
8267 mdelay(100);
8268
8269 /* linkctl */
8270 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8271 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8272 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8273 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8274
8275 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8276 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8277 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8278 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8279
8280 /* linkctl2 */
8281 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8282 tmp16 &= ~((1 << 4) | (7 << 9));
8283 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8284 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8285
8286 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8287 tmp16 &= ~((1 << 4) | (7 << 9));
8288 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8289 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8290
8291 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8292 tmp &= ~LC_SET_QUIESCE;
8293 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8294 }
8295 }
8296 }
8297
8298 /* set the link speed */
8299 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8300 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8301 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8302
8303 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8304 tmp16 &= ~0xf;
8305 if (mask & DRM_PCIE_SPEED_80)
8306 tmp16 |= 3; /* gen3 */
8307 else if (mask & DRM_PCIE_SPEED_50)
8308 tmp16 |= 2; /* gen2 */
8309 else
8310 tmp16 |= 1; /* gen1 */
8311 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8312
8313 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8314 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8315 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8316
8317 for (i = 0; i < rdev->usec_timeout; i++) {
8318 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8319 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8320 break;
8321 udelay(1);
8322 }
8323}
7235711a
AD		 8324
8325static void cik_program_aspm(struct radeon_device *rdev)
8326{
8327 u32 data, orig;
8328 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8329 bool disable_clkreq = false;
8330
8331 if (radeon_aspm == 0)
8332 return;
8333
8334 /* XXX double check IGPs */
8335 if (rdev->flags & RADEON_IS_IGP)
8336 return;
8337
8338 if (!(rdev->flags & RADEON_IS_PCIE))
8339 return;
8340
8341 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8342 data &= ~LC_XMIT_N_FTS_MASK;
8343 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8344 if (orig != data)
8345 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8346
8347 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8348 data |= LC_GO_TO_RECOVERY;
8349 if (orig != data)
8350 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8351
8352 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8353 data |= P_IGNORE_EDB_ERR;
8354 if (orig != data)
8355 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8356
8357 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8358 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8359 data |= LC_PMI_TO_L1_DIS;
8360 if (!disable_l0s)
8361 data |= LC_L0S_INACTIVITY(7);
8362
8363 if (!disable_l1) {
8364 data |= LC_L1_INACTIVITY(7);
8365 data &= ~LC_PMI_TO_L1_DIS;
8366 if (orig != data)
8367 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8368
8369 if (!disable_plloff_in_l1) {
8370 bool clk_req_support;
8371
8372 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8373 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8374 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8375 if (orig != data)
8376 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8377
8378 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8379 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8380 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8381 if (orig != data)
8382 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8383
8384 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8385 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8386 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8387 if (orig != data)
8388 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8389
8390 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8391 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8392 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8393 if (orig != data)
8394 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8395
8396 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8397 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8398 data |= LC_DYN_LANES_PWR_STATE(3);
8399 if (orig != data)
8400 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8401
8402 if (!disable_clkreq) {
8403 struct pci_dev *root = rdev->pdev->bus->self;
8404 u32 lnkcap;
8405
8406 clk_req_support = false;
8407 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8408 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8409 clk_req_support = true;
8410 } else {
8411 clk_req_support = false;
8412 }
8413
8414 if (clk_req_support) {
8415 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8416 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8417 if (orig != data)
8418 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8419
8420 orig = data = RREG32_SMC(THM_CLK_CNTL);
8421 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8422 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8423 if (orig != data)
8424 WREG32_SMC(THM_CLK_CNTL, data);
8425
8426 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8427 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8428 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8429 if (orig != data)
8430 WREG32_SMC(MISC_CLK_CTRL, data);
8431
8432 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8433 data &= ~BCLK_AS_XCLK;
8434 if (orig != data)
8435 WREG32_SMC(CG_CLKPIN_CNTL, data);
8436
8437 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8438 data &= ~FORCE_BIF_REFCLK_EN;
8439 if (orig != data)
8440 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8441
8442 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8443 data &= ~MPLL_CLKOUT_SEL_MASK;
8444 data |= MPLL_CLKOUT_SEL(4);
8445 if (orig != data)
8446 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8447 }
8448 }
8449 } else {
8450 if (orig != data)
8451 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8452 }
8453
8454 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8455 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8456 if (orig != data)
8457 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8458
8459 if (!disable_l0s) {
8460 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8461 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8462 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8463 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8464 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8465 data &= ~LC_L0S_INACTIVITY_MASK;
8466 if (orig != data)
8467 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8468 }
8469 }
8470 }
87167bb1 8471}
Linux kernel - Deliverables
This page took 0.446579 seconds and 5 git commands to generate.