projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: Allow write-combined CPU mappings of BOs in GTT (v2)
[deliverable/linux.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 41MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 42MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 43MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 44MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 45
46MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
47MODULE_FIRMWARE("radeon/bonaire_me.bin");
48MODULE_FIRMWARE("radeon/bonaire_ce.bin");
49MODULE_FIRMWARE("radeon/bonaire_mec.bin");
50MODULE_FIRMWARE("radeon/bonaire_mc.bin");
51MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
52MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
53MODULE_FIRMWARE("radeon/bonaire_smc.bin");
54
d4775655
AD		 55MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
56MODULE_FIRMWARE("radeon/HAWAII_me.bin");
57MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
58MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
59MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 60MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 61MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
62MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
63MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 64
65MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
66MODULE_FIRMWARE("radeon/hawaii_me.bin");
67MODULE_FIRMWARE("radeon/hawaii_ce.bin");
68MODULE_FIRMWARE("radeon/hawaii_mec.bin");
69MODULE_FIRMWARE("radeon/hawaii_mc.bin");
70MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
71MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
72MODULE_FIRMWARE("radeon/hawaii_smc.bin");
73
02c81327
AD		 74MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
75MODULE_FIRMWARE("radeon/KAVERI_me.bin");
76MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
77MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
78MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 79MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 80
81MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
82MODULE_FIRMWARE("radeon/kaveri_me.bin");
83MODULE_FIRMWARE("radeon/kaveri_ce.bin");
84MODULE_FIRMWARE("radeon/kaveri_mec.bin");
85MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
86MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
87MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
88
02c81327
AD		 89MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
90MODULE_FIRMWARE("radeon/KABINI_me.bin");
91MODULE_FIRMWARE("radeon/KABINI_ce.bin");
92MODULE_FIRMWARE("radeon/KABINI_mec.bin");
93MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 94MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 95
96MODULE_FIRMWARE("radeon/kabini_pfp.bin");
97MODULE_FIRMWARE("radeon/kabini_me.bin");
98MODULE_FIRMWARE("radeon/kabini_ce.bin");
99MODULE_FIRMWARE("radeon/kabini_mec.bin");
100MODULE_FIRMWARE("radeon/kabini_rlc.bin");
101MODULE_FIRMWARE("radeon/kabini_sdma.bin");
102
f73a9e83
SL		 103MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
104MODULE_FIRMWARE("radeon/MULLINS_me.bin");
105MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
106MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
107MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
108MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 109
f2c6b0f4
AD		 110MODULE_FIRMWARE("radeon/mullins_pfp.bin");
111MODULE_FIRMWARE("radeon/mullins_me.bin");
112MODULE_FIRMWARE("radeon/mullins_ce.bin");
113MODULE_FIRMWARE("radeon/mullins_mec.bin");
114MODULE_FIRMWARE("radeon/mullins_rlc.bin");
115MODULE_FIRMWARE("radeon/mullins_sdma.bin");
116
a59781bb
AD		 117extern int r600_ih_ring_alloc(struct radeon_device *rdev);
118extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 119extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
120extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 121extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 122extern void sumo_rlc_fini(struct radeon_device *rdev);
123extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 124extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 125extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 126extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 127static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 128extern int cik_sdma_resume(struct radeon_device *rdev);
129extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
130extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 131extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 132static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 133static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 134static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 135static void cik_init_pg(struct radeon_device *rdev);
136static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 137static void cik_fini_pg(struct radeon_device *rdev);
138static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 139static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
140 bool enable);
6f2043ce 141
286d9cc6
AD		 142/* get temperature in millidegrees */
143int ci_get_temp(struct radeon_device *rdev)
144{
145 u32 temp;
146 int actual_temp = 0;
147
148 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
149 CTF_TEMP_SHIFT;
150
151 if (temp & 0x200)
152 actual_temp = 255;
153 else
154 actual_temp = temp & 0x1ff;
155
156 actual_temp = actual_temp * 1000;
157
158 return actual_temp;
159}
160
161/* get temperature in millidegrees */
162int kv_get_temp(struct radeon_device *rdev)
163{
164 u32 temp;
165 int actual_temp = 0;
166
167 temp = RREG32_SMC(0xC0300E0C);
168
169 if (temp)
170 actual_temp = (temp / 8) - 49;
171 else
172 actual_temp = 0;
173
174 actual_temp = actual_temp * 1000;
175
176 return actual_temp;
177}
6f2043ce 178
6e2c3c0a
AD		 179/*
180 * Indirect registers accessor
181 */
182u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
183{
0a5b7b0b 184 unsigned long flags;
6e2c3c0a
AD		 185 u32 r;
186
0a5b7b0b 187 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 188 WREG32(PCIE_INDEX, reg);
189 (void)RREG32(PCIE_INDEX);
190 r = RREG32(PCIE_DATA);
0a5b7b0b 191 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 192 return r;
193}
194
195void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
196{
0a5b7b0b
AD		 197 unsigned long flags;
198
199 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 200 WREG32(PCIE_INDEX, reg);
201 (void)RREG32(PCIE_INDEX);
202 WREG32(PCIE_DATA, v);
203 (void)RREG32(PCIE_DATA);
0a5b7b0b 204 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 205}
206
22c775ce
AD		 207static const u32 spectre_rlc_save_restore_register_list[] =
208{
209 (0x0e00 << 16) | (0xc12c >> 2),
210 0x00000000,
211 (0x0e00 << 16) | (0xc140 >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0xc150 >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0xc15c >> 2),
216 0x00000000,
217 (0x0e00 << 16) | (0xc168 >> 2),
218 0x00000000,
219 (0x0e00 << 16) | (0xc170 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0xc178 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0xc204 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0xc2b4 >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc2b8 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc2bc >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc2c0 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0x8228 >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0x829c >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0x869c >> 2),
238 0x00000000,
239 (0x0600 << 16) | (0x98f4 >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0x98f8 >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0x9900 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc260 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0x90e8 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0x3c000 >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0x3c00c >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0x8c1c >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0x9700 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xcd20 >> 2),
258 0x00000000,
259 (0x4e00 << 16) | (0xcd20 >> 2),
260 0x00000000,
261 (0x5e00 << 16) | (0xcd20 >> 2),
262 0x00000000,
263 (0x6e00 << 16) | (0xcd20 >> 2),
264 0x00000000,
265 (0x7e00 << 16) | (0xcd20 >> 2),
266 0x00000000,
267 (0x8e00 << 16) | (0xcd20 >> 2),
268 0x00000000,
269 (0x9e00 << 16) | (0xcd20 >> 2),
270 0x00000000,
271 (0xae00 << 16) | (0xcd20 >> 2),
272 0x00000000,
273 (0xbe00 << 16) | (0xcd20 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0x89bc >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x8900 >> 2),
278 0x00000000,
279 0x3,
280 (0x0e00 << 16) | (0xc130 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0xc134 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0xc1fc >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0xc208 >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0xc264 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0xc268 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xc26c >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0xc270 >> 2),
295 0x00000000,
296 (0x0e00 << 16) | (0xc274 >> 2),
297 0x00000000,
298 (0x0e00 << 16) | (0xc278 >> 2),
299 0x00000000,
300 (0x0e00 << 16) | (0xc27c >> 2),
301 0x00000000,
302 (0x0e00 << 16) | (0xc280 >> 2),
303 0x00000000,
304 (0x0e00 << 16) | (0xc284 >> 2),
305 0x00000000,
306 (0x0e00 << 16) | (0xc288 >> 2),
307 0x00000000,
308 (0x0e00 << 16) | (0xc28c >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0xc290 >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0xc294 >> 2),
313 0x00000000,
314 (0x0e00 << 16) | (0xc298 >> 2),
315 0x00000000,
316 (0x0e00 << 16) | (0xc29c >> 2),
317 0x00000000,
318 (0x0e00 << 16) | (0xc2a0 >> 2),
319 0x00000000,
320 (0x0e00 << 16) | (0xc2a4 >> 2),
321 0x00000000,
322 (0x0e00 << 16) | (0xc2a8 >> 2),
323 0x00000000,
324 (0x0e00 << 16) | (0xc2ac >> 2),
325 0x00000000,
326 (0x0e00 << 16) | (0xc2b0 >> 2),
327 0x00000000,
328 (0x0e00 << 16) | (0x301d0 >> 2),
329 0x00000000,
330 (0x0e00 << 16) | (0x30238 >> 2),
331 0x00000000,
332 (0x0e00 << 16) | (0x30250 >> 2),
333 0x00000000,
334 (0x0e00 << 16) | (0x30254 >> 2),
335 0x00000000,
336 (0x0e00 << 16) | (0x30258 >> 2),
337 0x00000000,
338 (0x0e00 << 16) | (0x3025c >> 2),
339 0x00000000,
340 (0x4e00 << 16) | (0xc900 >> 2),
341 0x00000000,
342 (0x5e00 << 16) | (0xc900 >> 2),
343 0x00000000,
344 (0x6e00 << 16) | (0xc900 >> 2),
345 0x00000000,
346 (0x7e00 << 16) | (0xc900 >> 2),
347 0x00000000,
348 (0x8e00 << 16) | (0xc900 >> 2),
349 0x00000000,
350 (0x9e00 << 16) | (0xc900 >> 2),
351 0x00000000,
352 (0xae00 << 16) | (0xc900 >> 2),
353 0x00000000,
354 (0xbe00 << 16) | (0xc900 >> 2),
355 0x00000000,
356 (0x4e00 << 16) | (0xc904 >> 2),
357 0x00000000,
358 (0x5e00 << 16) | (0xc904 >> 2),
359 0x00000000,
360 (0x6e00 << 16) | (0xc904 >> 2),
361 0x00000000,
362 (0x7e00 << 16) | (0xc904 >> 2),
363 0x00000000,
364 (0x8e00 << 16) | (0xc904 >> 2),
365 0x00000000,
366 (0x9e00 << 16) | (0xc904 >> 2),
367 0x00000000,
368 (0xae00 << 16) | (0xc904 >> 2),
369 0x00000000,
370 (0xbe00 << 16) | (0xc904 >> 2),
371 0x00000000,
372 (0x4e00 << 16) | (0xc908 >> 2),
373 0x00000000,
374 (0x5e00 << 16) | (0xc908 >> 2),
375 0x00000000,
376 (0x6e00 << 16) | (0xc908 >> 2),
377 0x00000000,
378 (0x7e00 << 16) | (0xc908 >> 2),
379 0x00000000,
380 (0x8e00 << 16) | (0xc908 >> 2),
381 0x00000000,
382 (0x9e00 << 16) | (0xc908 >> 2),
383 0x00000000,
384 (0xae00 << 16) | (0xc908 >> 2),
385 0x00000000,
386 (0xbe00 << 16) | (0xc908 >> 2),
387 0x00000000,
388 (0x4e00 << 16) | (0xc90c >> 2),
389 0x00000000,
390 (0x5e00 << 16) | (0xc90c >> 2),
391 0x00000000,
392 (0x6e00 << 16) | (0xc90c >> 2),
393 0x00000000,
394 (0x7e00 << 16) | (0xc90c >> 2),
395 0x00000000,
396 (0x8e00 << 16) | (0xc90c >> 2),
397 0x00000000,
398 (0x9e00 << 16) | (0xc90c >> 2),
399 0x00000000,
400 (0xae00 << 16) | (0xc90c >> 2),
401 0x00000000,
402 (0xbe00 << 16) | (0xc90c >> 2),
403 0x00000000,
404 (0x4e00 << 16) | (0xc910 >> 2),
405 0x00000000,
406 (0x5e00 << 16) | (0xc910 >> 2),
407 0x00000000,
408 (0x6e00 << 16) | (0xc910 >> 2),
409 0x00000000,
410 (0x7e00 << 16) | (0xc910 >> 2),
411 0x00000000,
412 (0x8e00 << 16) | (0xc910 >> 2),
413 0x00000000,
414 (0x9e00 << 16) | (0xc910 >> 2),
415 0x00000000,
416 (0xae00 << 16) | (0xc910 >> 2),
417 0x00000000,
418 (0xbe00 << 16) | (0xc910 >> 2),
419 0x00000000,
420 (0x0e00 << 16) | (0xc99c >> 2),
421 0x00000000,
422 (0x0e00 << 16) | (0x9834 >> 2),
423 0x00000000,
424 (0x0000 << 16) | (0x30f00 >> 2),
425 0x00000000,
426 (0x0001 << 16) | (0x30f00 >> 2),
427 0x00000000,
428 (0x0000 << 16) | (0x30f04 >> 2),
429 0x00000000,
430 (0x0001 << 16) | (0x30f04 >> 2),
431 0x00000000,
432 (0x0000 << 16) | (0x30f08 >> 2),
433 0x00000000,
434 (0x0001 << 16) | (0x30f08 >> 2),
435 0x00000000,
436 (0x0000 << 16) | (0x30f0c >> 2),
437 0x00000000,
438 (0x0001 << 16) | (0x30f0c >> 2),
439 0x00000000,
440 (0x0600 << 16) | (0x9b7c >> 2),
441 0x00000000,
442 (0x0e00 << 16) | (0x8a14 >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x8a18 >> 2),
445 0x00000000,
446 (0x0600 << 16) | (0x30a00 >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0x8bf0 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x8bcc >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x8b24 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x30a04 >> 2),
455 0x00000000,
456 (0x0600 << 16) | (0x30a10 >> 2),
457 0x00000000,
458 (0x0600 << 16) | (0x30a14 >> 2),
459 0x00000000,
460 (0x0600 << 16) | (0x30a18 >> 2),
461 0x00000000,
462 (0x0600 << 16) | (0x30a2c >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0xc700 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0xc704 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0xc708 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0xc768 >> 2),
471 0x00000000,
472 (0x0400 << 16) | (0xc770 >> 2),
473 0x00000000,
474 (0x0400 << 16) | (0xc774 >> 2),
475 0x00000000,
476 (0x0400 << 16) | (0xc778 >> 2),
477 0x00000000,
478 (0x0400 << 16) | (0xc77c >> 2),
479 0x00000000,
480 (0x0400 << 16) | (0xc780 >> 2),
481 0x00000000,
482 (0x0400 << 16) | (0xc784 >> 2),
483 0x00000000,
484 (0x0400 << 16) | (0xc788 >> 2),
485 0x00000000,
486 (0x0400 << 16) | (0xc78c >> 2),
487 0x00000000,
488 (0x0400 << 16) | (0xc798 >> 2),
489 0x00000000,
490 (0x0400 << 16) | (0xc79c >> 2),
491 0x00000000,
492 (0x0400 << 16) | (0xc7a0 >> 2),
493 0x00000000,
494 (0x0400 << 16) | (0xc7a4 >> 2),
495 0x00000000,
496 (0x0400 << 16) | (0xc7a8 >> 2),
497 0x00000000,
498 (0x0400 << 16) | (0xc7ac >> 2),
499 0x00000000,
500 (0x0400 << 16) | (0xc7b0 >> 2),
501 0x00000000,
502 (0x0400 << 16) | (0xc7b4 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x9100 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x3c010 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x92a8 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x92ac >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x92b4 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x92b8 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x92bc >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x92c0 >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x92c4 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x92c8 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x92cc >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x92d0 >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0x8c00 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0x8c04 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0x8c20 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0x8c38 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0x8c3c >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0xae00 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0x9604 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0xac08 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xac0c >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xac10 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0xac14 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0xac58 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0xac68 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0xac6c >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0xac70 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0xac74 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0xac78 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0xac7c >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0xac80 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0xac84 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0xac88 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0xac8c >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x970c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x9714 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x9718 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x971c >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x31068 >> 2),
581 0x00000000,
582 (0x4e00 << 16) | (0x31068 >> 2),
583 0x00000000,
584 (0x5e00 << 16) | (0x31068 >> 2),
585 0x00000000,
586 (0x6e00 << 16) | (0x31068 >> 2),
587 0x00000000,
588 (0x7e00 << 16) | (0x31068 >> 2),
589 0x00000000,
590 (0x8e00 << 16) | (0x31068 >> 2),
591 0x00000000,
592 (0x9e00 << 16) | (0x31068 >> 2),
593 0x00000000,
594 (0xae00 << 16) | (0x31068 >> 2),
595 0x00000000,
596 (0xbe00 << 16) | (0x31068 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xcd10 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xcd14 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0x88b0 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x88b4 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0x88b8 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0x88bc >> 2),
609 0x00000000,
610 (0x0400 << 16) | (0x89c0 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0x88c4 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x88c8 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x88d0 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x88d4 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x88d8 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0x8980 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x30938 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x3093c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x30940 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x89a0 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x30900 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x30904 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x89b4 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x3c210 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c214 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c218 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8904 >> 2),
645 0x00000000,
646 0x5,
647 (0x0e00 << 16) | (0x8c28 >> 2),
648 (0x0e00 << 16) | (0x8c2c >> 2),
649 (0x0e00 << 16) | (0x8c30 >> 2),
650 (0x0e00 << 16) | (0x8c34 >> 2),
651 (0x0e00 << 16) | (0x9600 >> 2),
652};
653
654static const u32 kalindi_rlc_save_restore_register_list[] =
655{
656 (0x0e00 << 16) | (0xc12c >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0xc140 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xc150 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xc15c >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xc168 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc170 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc204 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc2b4 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc2b8 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc2bc >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc2c0 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x8228 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x829c >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x869c >> 2),
683 0x00000000,
684 (0x0600 << 16) | (0x98f4 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x98f8 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x9900 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc260 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x90e8 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x3c000 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x3c00c >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x8c1c >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x9700 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0xcd20 >> 2),
703 0x00000000,
704 (0x4e00 << 16) | (0xcd20 >> 2),
705 0x00000000,
706 (0x5e00 << 16) | (0xcd20 >> 2),
707 0x00000000,
708 (0x6e00 << 16) | (0xcd20 >> 2),
709 0x00000000,
710 (0x7e00 << 16) | (0xcd20 >> 2),
711 0x00000000,
712 (0x0e00 << 16) | (0x89bc >> 2),
713 0x00000000,
714 (0x0e00 << 16) | (0x8900 >> 2),
715 0x00000000,
716 0x3,
717 (0x0e00 << 16) | (0xc130 >> 2),
718 0x00000000,
719 (0x0e00 << 16) | (0xc134 >> 2),
720 0x00000000,
721 (0x0e00 << 16) | (0xc1fc >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0xc208 >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc264 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0xc268 >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0xc26c >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0xc270 >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0xc274 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0xc28c >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0xc290 >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0xc294 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc298 >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0xc2a0 >> 2),
744 0x00000000,
745 (0x0e00 << 16) | (0xc2a4 >> 2),
746 0x00000000,
747 (0x0e00 << 16) | (0xc2a8 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc2ac >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x301d0 >> 2),
752 0x00000000,
753 (0x0e00 << 16) | (0x30238 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x30250 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x30254 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x30258 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x3025c >> 2),
762 0x00000000,
763 (0x4e00 << 16) | (0xc900 >> 2),
764 0x00000000,
765 (0x5e00 << 16) | (0xc900 >> 2),
766 0x00000000,
767 (0x6e00 << 16) | (0xc900 >> 2),
768 0x00000000,
769 (0x7e00 << 16) | (0xc900 >> 2),
770 0x00000000,
771 (0x4e00 << 16) | (0xc904 >> 2),
772 0x00000000,
773 (0x5e00 << 16) | (0xc904 >> 2),
774 0x00000000,
775 (0x6e00 << 16) | (0xc904 >> 2),
776 0x00000000,
777 (0x7e00 << 16) | (0xc904 >> 2),
778 0x00000000,
779 (0x4e00 << 16) | (0xc908 >> 2),
780 0x00000000,
781 (0x5e00 << 16) | (0xc908 >> 2),
782 0x00000000,
783 (0x6e00 << 16) | (0xc908 >> 2),
784 0x00000000,
785 (0x7e00 << 16) | (0xc908 >> 2),
786 0x00000000,
787 (0x4e00 << 16) | (0xc90c >> 2),
788 0x00000000,
789 (0x5e00 << 16) | (0xc90c >> 2),
790 0x00000000,
791 (0x6e00 << 16) | (0xc90c >> 2),
792 0x00000000,
793 (0x7e00 << 16) | (0xc90c >> 2),
794 0x00000000,
795 (0x4e00 << 16) | (0xc910 >> 2),
796 0x00000000,
797 (0x5e00 << 16) | (0xc910 >> 2),
798 0x00000000,
799 (0x6e00 << 16) | (0xc910 >> 2),
800 0x00000000,
801 (0x7e00 << 16) | (0xc910 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc99c >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x9834 >> 2),
806 0x00000000,
807 (0x0000 << 16) | (0x30f00 >> 2),
808 0x00000000,
809 (0x0000 << 16) | (0x30f04 >> 2),
810 0x00000000,
811 (0x0000 << 16) | (0x30f08 >> 2),
812 0x00000000,
813 (0x0000 << 16) | (0x30f0c >> 2),
814 0x00000000,
815 (0x0600 << 16) | (0x9b7c >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x8a14 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x8a18 >> 2),
820 0x00000000,
821 (0x0600 << 16) | (0x30a00 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x8bf0 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x8bcc >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0x8b24 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0x30a04 >> 2),
830 0x00000000,
831 (0x0600 << 16) | (0x30a10 >> 2),
832 0x00000000,
833 (0x0600 << 16) | (0x30a14 >> 2),
834 0x00000000,
835 (0x0600 << 16) | (0x30a18 >> 2),
836 0x00000000,
837 (0x0600 << 16) | (0x30a2c >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xc700 >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xc704 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xc708 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xc768 >> 2),
846 0x00000000,
847 (0x0400 << 16) | (0xc770 >> 2),
848 0x00000000,
849 (0x0400 << 16) | (0xc774 >> 2),
850 0x00000000,
851 (0x0400 << 16) | (0xc798 >> 2),
852 0x00000000,
853 (0x0400 << 16) | (0xc79c >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x9100 >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x3c010 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0x8c00 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x8c04 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x8c20 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x8c38 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x8c3c >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xae00 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x9604 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0xac08 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0xac0c >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0xac10 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0xac14 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0xac58 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0xac68 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0xac6c >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0xac70 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0xac74 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0xac78 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0xac7c >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0xac80 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0xac84 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0xac88 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xac8c >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x970c >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x9714 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x9718 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x971c >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x31068 >> 2),
912 0x00000000,
913 (0x4e00 << 16) | (0x31068 >> 2),
914 0x00000000,
915 (0x5e00 << 16) | (0x31068 >> 2),
916 0x00000000,
917 (0x6e00 << 16) | (0x31068 >> 2),
918 0x00000000,
919 (0x7e00 << 16) | (0x31068 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0xcd10 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0xcd14 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x88b0 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x88b4 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x88b8 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x88bc >> 2),
932 0x00000000,
933 (0x0400 << 16) | (0x89c0 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0x88c4 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0x88c8 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0x88d0 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0x88d4 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0x88d8 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0x8980 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0x30938 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0x3093c >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0x30940 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0x89a0 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0x30900 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0x30904 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0x89b4 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0x3e1fc >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0x3c210 >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x3c214 >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x3c218 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x8904 >> 2),
970 0x00000000,
971 0x5,
972 (0x0e00 << 16) | (0x8c28 >> 2),
973 (0x0e00 << 16) | (0x8c2c >> 2),
974 (0x0e00 << 16) | (0x8c30 >> 2),
975 (0x0e00 << 16) | (0x8c34 >> 2),
976 (0x0e00 << 16) | (0x9600 >> 2),
977};
978
0aafd313
AD		 979static const u32 bonaire_golden_spm_registers[] =
980{
981 0x30800, 0xe0ffffff, 0xe0000000
982};
983
984static const u32 bonaire_golden_common_registers[] =
985{
986 0xc770, 0xffffffff, 0x00000800,
987 0xc774, 0xffffffff, 0x00000800,
988 0xc798, 0xffffffff, 0x00007fbf,
989 0xc79c, 0xffffffff, 0x00007faf
990};
991
992static const u32 bonaire_golden_registers[] =
993{
994 0x3354, 0x00000333, 0x00000333,
995 0x3350, 0x000c0fc0, 0x00040200,
996 0x9a10, 0x00010000, 0x00058208,
997 0x3c000, 0xffff1fff, 0x00140000,
998 0x3c200, 0xfdfc0fff, 0x00000100,
999 0x3c234, 0x40000000, 0x40000200,
1000 0x9830, 0xffffffff, 0x00000000,
1001 0x9834, 0xf00fffff, 0x00000400,
1002 0x9838, 0x0002021c, 0x00020200,
1003 0xc78, 0x00000080, 0x00000000,
1004 0x5bb0, 0x000000f0, 0x00000070,
1005 0x5bc0, 0xf0311fff, 0x80300000,
1006 0x98f8, 0x73773777, 0x12010001,
1007 0x350c, 0x00810000, 0x408af000,
1008 0x7030, 0x31000111, 0x00000011,
1009 0x2f48, 0x73773777, 0x12010001,
1010 0x220c, 0x00007fb6, 0x0021a1b1,
1011 0x2210, 0x00007fb6, 0x002021b1,
1012 0x2180, 0x00007fb6, 0x00002191,
1013 0x2218, 0x00007fb6, 0x002121b1,
1014 0x221c, 0x00007fb6, 0x002021b1,
1015 0x21dc, 0x00007fb6, 0x00002191,
1016 0x21e0, 0x00007fb6, 0x00002191,
1017 0x3628, 0x0000003f, 0x0000000a,
1018 0x362c, 0x0000003f, 0x0000000a,
1019 0x2ae4, 0x00073ffe, 0x000022a2,
1020 0x240c, 0x000007ff, 0x00000000,
1021 0x8a14, 0xf000003f, 0x00000007,
1022 0x8bf0, 0x00002001, 0x00000001,
1023 0x8b24, 0xffffffff, 0x00ffffff,
1024 0x30a04, 0x0000ff0f, 0x00000000,
1025 0x28a4c, 0x07ffffff, 0x06000000,
1026 0x4d8, 0x00000fff, 0x00000100,
1027 0x3e78, 0x00000001, 0x00000002,
1028 0x9100, 0x03000000, 0x0362c688,
1029 0x8c00, 0x000000ff, 0x00000001,
1030 0xe40, 0x00001fff, 0x00001fff,
1031 0x9060, 0x0000007f, 0x00000020,
1032 0x9508, 0x00010000, 0x00010000,
1033 0xac14, 0x000003ff, 0x000000f3,
1034 0xac0c, 0xffffffff, 0x00001032
1035};
1036
1037static const u32 bonaire_mgcg_cgcg_init[] =
1038{
1039 0xc420, 0xffffffff, 0xfffffffc,
1040 0x30800, 0xffffffff, 0xe0000000,
1041 0x3c2a0, 0xffffffff, 0x00000100,
1042 0x3c208, 0xffffffff, 0x00000100,
1043 0x3c2c0, 0xffffffff, 0xc0000100,
1044 0x3c2c8, 0xffffffff, 0xc0000100,
1045 0x3c2c4, 0xffffffff, 0xc0000100,
1046 0x55e4, 0xffffffff, 0x00600100,
1047 0x3c280, 0xffffffff, 0x00000100,
1048 0x3c214, 0xffffffff, 0x06000100,
1049 0x3c220, 0xffffffff, 0x00000100,
1050 0x3c218, 0xffffffff, 0x06000100,
1051 0x3c204, 0xffffffff, 0x00000100,
1052 0x3c2e0, 0xffffffff, 0x00000100,
1053 0x3c224, 0xffffffff, 0x00000100,
1054 0x3c200, 0xffffffff, 0x00000100,
1055 0x3c230, 0xffffffff, 0x00000100,
1056 0x3c234, 0xffffffff, 0x00000100,
1057 0x3c250, 0xffffffff, 0x00000100,
1058 0x3c254, 0xffffffff, 0x00000100,
1059 0x3c258, 0xffffffff, 0x00000100,
1060 0x3c25c, 0xffffffff, 0x00000100,
1061 0x3c260, 0xffffffff, 0x00000100,
1062 0x3c27c, 0xffffffff, 0x00000100,
1063 0x3c278, 0xffffffff, 0x00000100,
1064 0x3c210, 0xffffffff, 0x06000100,
1065 0x3c290, 0xffffffff, 0x00000100,
1066 0x3c274, 0xffffffff, 0x00000100,
1067 0x3c2b4, 0xffffffff, 0x00000100,
1068 0x3c2b0, 0xffffffff, 0x00000100,
1069 0x3c270, 0xffffffff, 0x00000100,
1070 0x30800, 0xffffffff, 0xe0000000,
1071 0x3c020, 0xffffffff, 0x00010000,
1072 0x3c024, 0xffffffff, 0x00030002,
1073 0x3c028, 0xffffffff, 0x00040007,
1074 0x3c02c, 0xffffffff, 0x00060005,
1075 0x3c030, 0xffffffff, 0x00090008,
1076 0x3c034, 0xffffffff, 0x00010000,
1077 0x3c038, 0xffffffff, 0x00030002,
1078 0x3c03c, 0xffffffff, 0x00040007,
1079 0x3c040, 0xffffffff, 0x00060005,
1080 0x3c044, 0xffffffff, 0x00090008,
1081 0x3c048, 0xffffffff, 0x00010000,
1082 0x3c04c, 0xffffffff, 0x00030002,
1083 0x3c050, 0xffffffff, 0x00040007,
1084 0x3c054, 0xffffffff, 0x00060005,
1085 0x3c058, 0xffffffff, 0x00090008,
1086 0x3c05c, 0xffffffff, 0x00010000,
1087 0x3c060, 0xffffffff, 0x00030002,
1088 0x3c064, 0xffffffff, 0x00040007,
1089 0x3c068, 0xffffffff, 0x00060005,
1090 0x3c06c, 0xffffffff, 0x00090008,
1091 0x3c070, 0xffffffff, 0x00010000,
1092 0x3c074, 0xffffffff, 0x00030002,
1093 0x3c078, 0xffffffff, 0x00040007,
1094 0x3c07c, 0xffffffff, 0x00060005,
1095 0x3c080, 0xffffffff, 0x00090008,
1096 0x3c084, 0xffffffff, 0x00010000,
1097 0x3c088, 0xffffffff, 0x00030002,
1098 0x3c08c, 0xffffffff, 0x00040007,
1099 0x3c090, 0xffffffff, 0x00060005,
1100 0x3c094, 0xffffffff, 0x00090008,
1101 0x3c098, 0xffffffff, 0x00010000,
1102 0x3c09c, 0xffffffff, 0x00030002,
1103 0x3c0a0, 0xffffffff, 0x00040007,
1104 0x3c0a4, 0xffffffff, 0x00060005,
1105 0x3c0a8, 0xffffffff, 0x00090008,
1106 0x3c000, 0xffffffff, 0x96e00200,
1107 0x8708, 0xffffffff, 0x00900100,
1108 0xc424, 0xffffffff, 0x0020003f,
1109 0x38, 0xffffffff, 0x0140001c,
1110 0x3c, 0x000f0000, 0x000f0000,
1111 0x220, 0xffffffff, 0xC060000C,
1112 0x224, 0xc0000fff, 0x00000100,
1113 0xf90, 0xffffffff, 0x00000100,
1114 0xf98, 0x00000101, 0x00000000,
1115 0x20a8, 0xffffffff, 0x00000104,
1116 0x55e4, 0xff000fff, 0x00000100,
1117 0x30cc, 0xc0000fff, 0x00000104,
1118 0xc1e4, 0x00000001, 0x00000001,
1119 0xd00c, 0xff000ff0, 0x00000100,
1120 0xd80c, 0xff000ff0, 0x00000100
1121};
1122
1123static const u32 spectre_golden_spm_registers[] =
1124{
1125 0x30800, 0xe0ffffff, 0xe0000000
1126};
1127
1128static const u32 spectre_golden_common_registers[] =
1129{
1130 0xc770, 0xffffffff, 0x00000800,
1131 0xc774, 0xffffffff, 0x00000800,
1132 0xc798, 0xffffffff, 0x00007fbf,
1133 0xc79c, 0xffffffff, 0x00007faf
1134};
1135
1136static const u32 spectre_golden_registers[] =
1137{
1138 0x3c000, 0xffff1fff, 0x96940200,
1139 0x3c00c, 0xffff0001, 0xff000000,
1140 0x3c200, 0xfffc0fff, 0x00000100,
1141 0x6ed8, 0x00010101, 0x00010000,
1142 0x9834, 0xf00fffff, 0x00000400,
1143 0x9838, 0xfffffffc, 0x00020200,
1144 0x5bb0, 0x000000f0, 0x00000070,
1145 0x5bc0, 0xf0311fff, 0x80300000,
1146 0x98f8, 0x73773777, 0x12010001,
1147 0x9b7c, 0x00ff0000, 0x00fc0000,
1148 0x2f48, 0x73773777, 0x12010001,
1149 0x8a14, 0xf000003f, 0x00000007,
1150 0x8b24, 0xffffffff, 0x00ffffff,
1151 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1152 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1153 0x3e78, 0x00000001, 0x00000002,
1154 0x913c, 0xffff03df, 0x00000004,
1155 0xc768, 0x00000008, 0x00000008,
1156 0x8c00, 0x000008ff, 0x00000800,
1157 0x9508, 0x00010000, 0x00010000,
1158 0xac0c, 0xffffffff, 0x54763210,
1159 0x214f8, 0x01ff01ff, 0x00000002,
1160 0x21498, 0x007ff800, 0x00200000,
1161 0x2015c, 0xffffffff, 0x00000f40,
1162 0x30934, 0xffffffff, 0x00000001
1163};
1164
1165static const u32 spectre_mgcg_cgcg_init[] =
1166{
1167 0xc420, 0xffffffff, 0xfffffffc,
1168 0x30800, 0xffffffff, 0xe0000000,
1169 0x3c2a0, 0xffffffff, 0x00000100,
1170 0x3c208, 0xffffffff, 0x00000100,
1171 0x3c2c0, 0xffffffff, 0x00000100,
1172 0x3c2c8, 0xffffffff, 0x00000100,
1173 0x3c2c4, 0xffffffff, 0x00000100,
1174 0x55e4, 0xffffffff, 0x00600100,
1175 0x3c280, 0xffffffff, 0x00000100,
1176 0x3c214, 0xffffffff, 0x06000100,
1177 0x3c220, 0xffffffff, 0x00000100,
1178 0x3c218, 0xffffffff, 0x06000100,
1179 0x3c204, 0xffffffff, 0x00000100,
1180 0x3c2e0, 0xffffffff, 0x00000100,
1181 0x3c224, 0xffffffff, 0x00000100,
1182 0x3c200, 0xffffffff, 0x00000100,
1183 0x3c230, 0xffffffff, 0x00000100,
1184 0x3c234, 0xffffffff, 0x00000100,
1185 0x3c250, 0xffffffff, 0x00000100,
1186 0x3c254, 0xffffffff, 0x00000100,
1187 0x3c258, 0xffffffff, 0x00000100,
1188 0x3c25c, 0xffffffff, 0x00000100,
1189 0x3c260, 0xffffffff, 0x00000100,
1190 0x3c27c, 0xffffffff, 0x00000100,
1191 0x3c278, 0xffffffff, 0x00000100,
1192 0x3c210, 0xffffffff, 0x06000100,
1193 0x3c290, 0xffffffff, 0x00000100,
1194 0x3c274, 0xffffffff, 0x00000100,
1195 0x3c2b4, 0xffffffff, 0x00000100,
1196 0x3c2b0, 0xffffffff, 0x00000100,
1197 0x3c270, 0xffffffff, 0x00000100,
1198 0x30800, 0xffffffff, 0xe0000000,
1199 0x3c020, 0xffffffff, 0x00010000,
1200 0x3c024, 0xffffffff, 0x00030002,
1201 0x3c028, 0xffffffff, 0x00040007,
1202 0x3c02c, 0xffffffff, 0x00060005,
1203 0x3c030, 0xffffffff, 0x00090008,
1204 0x3c034, 0xffffffff, 0x00010000,
1205 0x3c038, 0xffffffff, 0x00030002,
1206 0x3c03c, 0xffffffff, 0x00040007,
1207 0x3c040, 0xffffffff, 0x00060005,
1208 0x3c044, 0xffffffff, 0x00090008,
1209 0x3c048, 0xffffffff, 0x00010000,
1210 0x3c04c, 0xffffffff, 0x00030002,
1211 0x3c050, 0xffffffff, 0x00040007,
1212 0x3c054, 0xffffffff, 0x00060005,
1213 0x3c058, 0xffffffff, 0x00090008,
1214 0x3c05c, 0xffffffff, 0x00010000,
1215 0x3c060, 0xffffffff, 0x00030002,
1216 0x3c064, 0xffffffff, 0x00040007,
1217 0x3c068, 0xffffffff, 0x00060005,
1218 0x3c06c, 0xffffffff, 0x00090008,
1219 0x3c070, 0xffffffff, 0x00010000,
1220 0x3c074, 0xffffffff, 0x00030002,
1221 0x3c078, 0xffffffff, 0x00040007,
1222 0x3c07c, 0xffffffff, 0x00060005,
1223 0x3c080, 0xffffffff, 0x00090008,
1224 0x3c084, 0xffffffff, 0x00010000,
1225 0x3c088, 0xffffffff, 0x00030002,
1226 0x3c08c, 0xffffffff, 0x00040007,
1227 0x3c090, 0xffffffff, 0x00060005,
1228 0x3c094, 0xffffffff, 0x00090008,
1229 0x3c098, 0xffffffff, 0x00010000,
1230 0x3c09c, 0xffffffff, 0x00030002,
1231 0x3c0a0, 0xffffffff, 0x00040007,
1232 0x3c0a4, 0xffffffff, 0x00060005,
1233 0x3c0a8, 0xffffffff, 0x00090008,
1234 0x3c0ac, 0xffffffff, 0x00010000,
1235 0x3c0b0, 0xffffffff, 0x00030002,
1236 0x3c0b4, 0xffffffff, 0x00040007,
1237 0x3c0b8, 0xffffffff, 0x00060005,
1238 0x3c0bc, 0xffffffff, 0x00090008,
1239 0x3c000, 0xffffffff, 0x96e00200,
1240 0x8708, 0xffffffff, 0x00900100,
1241 0xc424, 0xffffffff, 0x0020003f,
1242 0x38, 0xffffffff, 0x0140001c,
1243 0x3c, 0x000f0000, 0x000f0000,
1244 0x220, 0xffffffff, 0xC060000C,
1245 0x224, 0xc0000fff, 0x00000100,
1246 0xf90, 0xffffffff, 0x00000100,
1247 0xf98, 0x00000101, 0x00000000,
1248 0x20a8, 0xffffffff, 0x00000104,
1249 0x55e4, 0xff000fff, 0x00000100,
1250 0x30cc, 0xc0000fff, 0x00000104,
1251 0xc1e4, 0x00000001, 0x00000001,
1252 0xd00c, 0xff000ff0, 0x00000100,
1253 0xd80c, 0xff000ff0, 0x00000100
1254};
1255
1256static const u32 kalindi_golden_spm_registers[] =
1257{
1258 0x30800, 0xe0ffffff, 0xe0000000
1259};
1260
1261static const u32 kalindi_golden_common_registers[] =
1262{
1263 0xc770, 0xffffffff, 0x00000800,
1264 0xc774, 0xffffffff, 0x00000800,
1265 0xc798, 0xffffffff, 0x00007fbf,
1266 0xc79c, 0xffffffff, 0x00007faf
1267};
1268
1269static const u32 kalindi_golden_registers[] =
1270{
1271 0x3c000, 0xffffdfff, 0x6e944040,
1272 0x55e4, 0xff607fff, 0xfc000100,
1273 0x3c220, 0xff000fff, 0x00000100,
1274 0x3c224, 0xff000fff, 0x00000100,
1275 0x3c200, 0xfffc0fff, 0x00000100,
1276 0x6ed8, 0x00010101, 0x00010000,
1277 0x9830, 0xffffffff, 0x00000000,
1278 0x9834, 0xf00fffff, 0x00000400,
1279 0x5bb0, 0x000000f0, 0x00000070,
1280 0x5bc0, 0xf0311fff, 0x80300000,
1281 0x98f8, 0x73773777, 0x12010001,
1282 0x98fc, 0xffffffff, 0x00000010,
1283 0x9b7c, 0x00ff0000, 0x00fc0000,
1284 0x8030, 0x00001f0f, 0x0000100a,
1285 0x2f48, 0x73773777, 0x12010001,
1286 0x2408, 0x000fffff, 0x000c007f,
1287 0x8a14, 0xf000003f, 0x00000007,
1288 0x8b24, 0x3fff3fff, 0x00ffcfff,
1289 0x30a04, 0x0000ff0f, 0x00000000,
1290 0x28a4c, 0x07ffffff, 0x06000000,
1291 0x4d8, 0x00000fff, 0x00000100,
1292 0x3e78, 0x00000001, 0x00000002,
1293 0xc768, 0x00000008, 0x00000008,
1294 0x8c00, 0x000000ff, 0x00000003,
1295 0x214f8, 0x01ff01ff, 0x00000002,
1296 0x21498, 0x007ff800, 0x00200000,
1297 0x2015c, 0xffffffff, 0x00000f40,
1298 0x88c4, 0x001f3ae3, 0x00000082,
1299 0x88d4, 0x0000001f, 0x00000010,
1300 0x30934, 0xffffffff, 0x00000000
1301};
1302
1303static const u32 kalindi_mgcg_cgcg_init[] =
1304{
1305 0xc420, 0xffffffff, 0xfffffffc,
1306 0x30800, 0xffffffff, 0xe0000000,
1307 0x3c2a0, 0xffffffff, 0x00000100,
1308 0x3c208, 0xffffffff, 0x00000100,
1309 0x3c2c0, 0xffffffff, 0x00000100,
1310 0x3c2c8, 0xffffffff, 0x00000100,
1311 0x3c2c4, 0xffffffff, 0x00000100,
1312 0x55e4, 0xffffffff, 0x00600100,
1313 0x3c280, 0xffffffff, 0x00000100,
1314 0x3c214, 0xffffffff, 0x06000100,
1315 0x3c220, 0xffffffff, 0x00000100,
1316 0x3c218, 0xffffffff, 0x06000100,
1317 0x3c204, 0xffffffff, 0x00000100,
1318 0x3c2e0, 0xffffffff, 0x00000100,
1319 0x3c224, 0xffffffff, 0x00000100,
1320 0x3c200, 0xffffffff, 0x00000100,
1321 0x3c230, 0xffffffff, 0x00000100,
1322 0x3c234, 0xffffffff, 0x00000100,
1323 0x3c250, 0xffffffff, 0x00000100,
1324 0x3c254, 0xffffffff, 0x00000100,
1325 0x3c258, 0xffffffff, 0x00000100,
1326 0x3c25c, 0xffffffff, 0x00000100,
1327 0x3c260, 0xffffffff, 0x00000100,
1328 0x3c27c, 0xffffffff, 0x00000100,
1329 0x3c278, 0xffffffff, 0x00000100,
1330 0x3c210, 0xffffffff, 0x06000100,
1331 0x3c290, 0xffffffff, 0x00000100,
1332 0x3c274, 0xffffffff, 0x00000100,
1333 0x3c2b4, 0xffffffff, 0x00000100,
1334 0x3c2b0, 0xffffffff, 0x00000100,
1335 0x3c270, 0xffffffff, 0x00000100,
1336 0x30800, 0xffffffff, 0xe0000000,
1337 0x3c020, 0xffffffff, 0x00010000,
1338 0x3c024, 0xffffffff, 0x00030002,
1339 0x3c028, 0xffffffff, 0x00040007,
1340 0x3c02c, 0xffffffff, 0x00060005,
1341 0x3c030, 0xffffffff, 0x00090008,
1342 0x3c034, 0xffffffff, 0x00010000,
1343 0x3c038, 0xffffffff, 0x00030002,
1344 0x3c03c, 0xffffffff, 0x00040007,
1345 0x3c040, 0xffffffff, 0x00060005,
1346 0x3c044, 0xffffffff, 0x00090008,
1347 0x3c000, 0xffffffff, 0x96e00200,
1348 0x8708, 0xffffffff, 0x00900100,
1349 0xc424, 0xffffffff, 0x0020003f,
1350 0x38, 0xffffffff, 0x0140001c,
1351 0x3c, 0x000f0000, 0x000f0000,
1352 0x220, 0xffffffff, 0xC060000C,
1353 0x224, 0xc0000fff, 0x00000100,
1354 0x20a8, 0xffffffff, 0x00000104,
1355 0x55e4, 0xff000fff, 0x00000100,
1356 0x30cc, 0xc0000fff, 0x00000104,
1357 0xc1e4, 0x00000001, 0x00000001,
1358 0xd00c, 0xff000ff0, 0x00000100,
1359 0xd80c, 0xff000ff0, 0x00000100
1360};
1361
8efff337
AD		 1362static const u32 hawaii_golden_spm_registers[] =
1363{
1364 0x30800, 0xe0ffffff, 0xe0000000
1365};
1366
1367static const u32 hawaii_golden_common_registers[] =
1368{
1369 0x30800, 0xffffffff, 0xe0000000,
1370 0x28350, 0xffffffff, 0x3a00161a,
1371 0x28354, 0xffffffff, 0x0000002e,
1372 0x9a10, 0xffffffff, 0x00018208,
1373 0x98f8, 0xffffffff, 0x12011003
1374};
1375
1376static const u32 hawaii_golden_registers[] =
1377{
1378 0x3354, 0x00000333, 0x00000333,
1379 0x9a10, 0x00010000, 0x00058208,
1380 0x9830, 0xffffffff, 0x00000000,
1381 0x9834, 0xf00fffff, 0x00000400,
1382 0x9838, 0x0002021c, 0x00020200,
1383 0xc78, 0x00000080, 0x00000000,
1384 0x5bb0, 0x000000f0, 0x00000070,
1385 0x5bc0, 0xf0311fff, 0x80300000,
1386 0x350c, 0x00810000, 0x408af000,
1387 0x7030, 0x31000111, 0x00000011,
1388 0x2f48, 0x73773777, 0x12010001,
1389 0x2120, 0x0000007f, 0x0000001b,
1390 0x21dc, 0x00007fb6, 0x00002191,
1391 0x3628, 0x0000003f, 0x0000000a,
1392 0x362c, 0x0000003f, 0x0000000a,
1393 0x2ae4, 0x00073ffe, 0x000022a2,
1394 0x240c, 0x000007ff, 0x00000000,
1395 0x8bf0, 0x00002001, 0x00000001,
1396 0x8b24, 0xffffffff, 0x00ffffff,
1397 0x30a04, 0x0000ff0f, 0x00000000,
1398 0x28a4c, 0x07ffffff, 0x06000000,
1399 0x3e78, 0x00000001, 0x00000002,
1400 0xc768, 0x00000008, 0x00000008,
1401 0xc770, 0x00000f00, 0x00000800,
1402 0xc774, 0x00000f00, 0x00000800,
1403 0xc798, 0x00ffffff, 0x00ff7fbf,
1404 0xc79c, 0x00ffffff, 0x00ff7faf,
1405 0x8c00, 0x000000ff, 0x00000800,
1406 0xe40, 0x00001fff, 0x00001fff,
1407 0x9060, 0x0000007f, 0x00000020,
1408 0x9508, 0x00010000, 0x00010000,
1409 0xae00, 0x00100000, 0x000ff07c,
1410 0xac14, 0x000003ff, 0x0000000f,
1411 0xac10, 0xffffffff, 0x7564fdec,
1412 0xac0c, 0xffffffff, 0x3120b9a8,
1413 0xac08, 0x20000000, 0x0f9c0000
1414};
1415
1416static const u32 hawaii_mgcg_cgcg_init[] =
1417{
1418 0xc420, 0xffffffff, 0xfffffffd,
1419 0x30800, 0xffffffff, 0xe0000000,
1420 0x3c2a0, 0xffffffff, 0x00000100,
1421 0x3c208, 0xffffffff, 0x00000100,
1422 0x3c2c0, 0xffffffff, 0x00000100,
1423 0x3c2c8, 0xffffffff, 0x00000100,
1424 0x3c2c4, 0xffffffff, 0x00000100,
1425 0x55e4, 0xffffffff, 0x00200100,
1426 0x3c280, 0xffffffff, 0x00000100,
1427 0x3c214, 0xffffffff, 0x06000100,
1428 0x3c220, 0xffffffff, 0x00000100,
1429 0x3c218, 0xffffffff, 0x06000100,
1430 0x3c204, 0xffffffff, 0x00000100,
1431 0x3c2e0, 0xffffffff, 0x00000100,
1432 0x3c224, 0xffffffff, 0x00000100,
1433 0x3c200, 0xffffffff, 0x00000100,
1434 0x3c230, 0xffffffff, 0x00000100,
1435 0x3c234, 0xffffffff, 0x00000100,
1436 0x3c250, 0xffffffff, 0x00000100,
1437 0x3c254, 0xffffffff, 0x00000100,
1438 0x3c258, 0xffffffff, 0x00000100,
1439 0x3c25c, 0xffffffff, 0x00000100,
1440 0x3c260, 0xffffffff, 0x00000100,
1441 0x3c27c, 0xffffffff, 0x00000100,
1442 0x3c278, 0xffffffff, 0x00000100,
1443 0x3c210, 0xffffffff, 0x06000100,
1444 0x3c290, 0xffffffff, 0x00000100,
1445 0x3c274, 0xffffffff, 0x00000100,
1446 0x3c2b4, 0xffffffff, 0x00000100,
1447 0x3c2b0, 0xffffffff, 0x00000100,
1448 0x3c270, 0xffffffff, 0x00000100,
1449 0x30800, 0xffffffff, 0xe0000000,
1450 0x3c020, 0xffffffff, 0x00010000,
1451 0x3c024, 0xffffffff, 0x00030002,
1452 0x3c028, 0xffffffff, 0x00040007,
1453 0x3c02c, 0xffffffff, 0x00060005,
1454 0x3c030, 0xffffffff, 0x00090008,
1455 0x3c034, 0xffffffff, 0x00010000,
1456 0x3c038, 0xffffffff, 0x00030002,
1457 0x3c03c, 0xffffffff, 0x00040007,
1458 0x3c040, 0xffffffff, 0x00060005,
1459 0x3c044, 0xffffffff, 0x00090008,
1460 0x3c048, 0xffffffff, 0x00010000,
1461 0x3c04c, 0xffffffff, 0x00030002,
1462 0x3c050, 0xffffffff, 0x00040007,
1463 0x3c054, 0xffffffff, 0x00060005,
1464 0x3c058, 0xffffffff, 0x00090008,
1465 0x3c05c, 0xffffffff, 0x00010000,
1466 0x3c060, 0xffffffff, 0x00030002,
1467 0x3c064, 0xffffffff, 0x00040007,
1468 0x3c068, 0xffffffff, 0x00060005,
1469 0x3c06c, 0xffffffff, 0x00090008,
1470 0x3c070, 0xffffffff, 0x00010000,
1471 0x3c074, 0xffffffff, 0x00030002,
1472 0x3c078, 0xffffffff, 0x00040007,
1473 0x3c07c, 0xffffffff, 0x00060005,
1474 0x3c080, 0xffffffff, 0x00090008,
1475 0x3c084, 0xffffffff, 0x00010000,
1476 0x3c088, 0xffffffff, 0x00030002,
1477 0x3c08c, 0xffffffff, 0x00040007,
1478 0x3c090, 0xffffffff, 0x00060005,
1479 0x3c094, 0xffffffff, 0x00090008,
1480 0x3c098, 0xffffffff, 0x00010000,
1481 0x3c09c, 0xffffffff, 0x00030002,
1482 0x3c0a0, 0xffffffff, 0x00040007,
1483 0x3c0a4, 0xffffffff, 0x00060005,
1484 0x3c0a8, 0xffffffff, 0x00090008,
1485 0x3c0ac, 0xffffffff, 0x00010000,
1486 0x3c0b0, 0xffffffff, 0x00030002,
1487 0x3c0b4, 0xffffffff, 0x00040007,
1488 0x3c0b8, 0xffffffff, 0x00060005,
1489 0x3c0bc, 0xffffffff, 0x00090008,
1490 0x3c0c0, 0xffffffff, 0x00010000,
1491 0x3c0c4, 0xffffffff, 0x00030002,
1492 0x3c0c8, 0xffffffff, 0x00040007,
1493 0x3c0cc, 0xffffffff, 0x00060005,
1494 0x3c0d0, 0xffffffff, 0x00090008,
1495 0x3c0d4, 0xffffffff, 0x00010000,
1496 0x3c0d8, 0xffffffff, 0x00030002,
1497 0x3c0dc, 0xffffffff, 0x00040007,
1498 0x3c0e0, 0xffffffff, 0x00060005,
1499 0x3c0e4, 0xffffffff, 0x00090008,
1500 0x3c0e8, 0xffffffff, 0x00010000,
1501 0x3c0ec, 0xffffffff, 0x00030002,
1502 0x3c0f0, 0xffffffff, 0x00040007,
1503 0x3c0f4, 0xffffffff, 0x00060005,
1504 0x3c0f8, 0xffffffff, 0x00090008,
1505 0xc318, 0xffffffff, 0x00020200,
1506 0x3350, 0xffffffff, 0x00000200,
1507 0x15c0, 0xffffffff, 0x00000400,
1508 0x55e8, 0xffffffff, 0x00000000,
1509 0x2f50, 0xffffffff, 0x00000902,
1510 0x3c000, 0xffffffff, 0x96940200,
1511 0x8708, 0xffffffff, 0x00900100,
1512 0xc424, 0xffffffff, 0x0020003f,
1513 0x38, 0xffffffff, 0x0140001c,
1514 0x3c, 0x000f0000, 0x000f0000,
1515 0x220, 0xffffffff, 0xc060000c,
1516 0x224, 0xc0000fff, 0x00000100,
1517 0xf90, 0xffffffff, 0x00000100,
1518 0xf98, 0x00000101, 0x00000000,
1519 0x20a8, 0xffffffff, 0x00000104,
1520 0x55e4, 0xff000fff, 0x00000100,
1521 0x30cc, 0xc0000fff, 0x00000104,
1522 0xc1e4, 0x00000001, 0x00000001,
1523 0xd00c, 0xff000ff0, 0x00000100,
1524 0xd80c, 0xff000ff0, 0x00000100
1525};
1526
f73a9e83
SL		 1527static const u32 godavari_golden_registers[] =
1528{
1529 0x55e4, 0xff607fff, 0xfc000100,
1530 0x6ed8, 0x00010101, 0x00010000,
1531 0x9830, 0xffffffff, 0x00000000,
1532 0x98302, 0xf00fffff, 0x00000400,
1533 0x6130, 0xffffffff, 0x00010000,
1534 0x5bb0, 0x000000f0, 0x00000070,
1535 0x5bc0, 0xf0311fff, 0x80300000,
1536 0x98f8, 0x73773777, 0x12010001,
1537 0x98fc, 0xffffffff, 0x00000010,
1538 0x8030, 0x00001f0f, 0x0000100a,
1539 0x2f48, 0x73773777, 0x12010001,
1540 0x2408, 0x000fffff, 0x000c007f,
1541 0x8a14, 0xf000003f, 0x00000007,
1542 0x8b24, 0xffffffff, 0x00ff0fff,
1543 0x30a04, 0x0000ff0f, 0x00000000,
1544 0x28a4c, 0x07ffffff, 0x06000000,
1545 0x4d8, 0x00000fff, 0x00000100,
1546 0xd014, 0x00010000, 0x00810001,
1547 0xd814, 0x00010000, 0x00810001,
1548 0x3e78, 0x00000001, 0x00000002,
1549 0xc768, 0x00000008, 0x00000008,
1550 0xc770, 0x00000f00, 0x00000800,
1551 0xc774, 0x00000f00, 0x00000800,
1552 0xc798, 0x00ffffff, 0x00ff7fbf,
1553 0xc79c, 0x00ffffff, 0x00ff7faf,
1554 0x8c00, 0x000000ff, 0x00000001,
1555 0x214f8, 0x01ff01ff, 0x00000002,
1556 0x21498, 0x007ff800, 0x00200000,
1557 0x2015c, 0xffffffff, 0x00000f40,
1558 0x88c4, 0x001f3ae3, 0x00000082,
1559 0x88d4, 0x0000001f, 0x00000010,
1560 0x30934, 0xffffffff, 0x00000000
1561};
1562
1563
0aafd313
AD		 1564static void cik_init_golden_registers(struct radeon_device *rdev)
1565{
1566 switch (rdev->family) {
1567 case CHIP_BONAIRE:
1568 radeon_program_register_sequence(rdev,
1569 bonaire_mgcg_cgcg_init,
1570 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571 radeon_program_register_sequence(rdev,
1572 bonaire_golden_registers,
1573 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574 radeon_program_register_sequence(rdev,
1575 bonaire_golden_common_registers,
1576 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577 radeon_program_register_sequence(rdev,
1578 bonaire_golden_spm_registers,
1579 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1580 break;
1581 case CHIP_KABINI:
1582 radeon_program_register_sequence(rdev,
1583 kalindi_mgcg_cgcg_init,
1584 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585 radeon_program_register_sequence(rdev,
1586 kalindi_golden_registers,
1587 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588 radeon_program_register_sequence(rdev,
1589 kalindi_golden_common_registers,
1590 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591 radeon_program_register_sequence(rdev,
1592 kalindi_golden_spm_registers,
1593 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1594 break;
f73a9e83
SL		 1595 case CHIP_MULLINS:
1596 radeon_program_register_sequence(rdev,
1597 kalindi_mgcg_cgcg_init,
1598 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599 radeon_program_register_sequence(rdev,
1600 godavari_golden_registers,
1601 (const u32)ARRAY_SIZE(godavari_golden_registers));
1602 radeon_program_register_sequence(rdev,
1603 kalindi_golden_common_registers,
1604 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605 radeon_program_register_sequence(rdev,
1606 kalindi_golden_spm_registers,
1607 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1608 break;
0aafd313
AD		 1609 case CHIP_KAVERI:
1610 radeon_program_register_sequence(rdev,
1611 spectre_mgcg_cgcg_init,
1612 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613 radeon_program_register_sequence(rdev,
1614 spectre_golden_registers,
1615 (const u32)ARRAY_SIZE(spectre_golden_registers));
1616 radeon_program_register_sequence(rdev,
1617 spectre_golden_common_registers,
1618 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619 radeon_program_register_sequence(rdev,
1620 spectre_golden_spm_registers,
1621 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1622 break;
8efff337
AD		 1623 case CHIP_HAWAII:
1624 radeon_program_register_sequence(rdev,
1625 hawaii_mgcg_cgcg_init,
1626 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627 radeon_program_register_sequence(rdev,
1628 hawaii_golden_registers,
1629 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630 radeon_program_register_sequence(rdev,
1631 hawaii_golden_common_registers,
1632 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633 radeon_program_register_sequence(rdev,
1634 hawaii_golden_spm_registers,
1635 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1636 break;
0aafd313
AD		 1637 default:
1638 break;
1639 }
1640}
1641
2c67912c
AD		 1642/**
1643 * cik_get_xclk - get the xclk
1644 *
1645 * @rdev: radeon_device pointer
1646 *
1647 * Returns the reference clock used by the gfx engine
1648 * (CIK).
1649 */
1650u32 cik_get_xclk(struct radeon_device *rdev)
1651{
1652 u32 reference_clock = rdev->clock.spll.reference_freq;
1653
1654 if (rdev->flags & RADEON_IS_IGP) {
1655 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656 return reference_clock / 2;
1657 } else {
1658 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659 return reference_clock / 4;
1660 }
1661 return reference_clock;
1662}
1663
75efdee1
AD		 1664/**
1665 * cik_mm_rdoorbell - read a doorbell dword
1666 *
1667 * @rdev: radeon_device pointer
d5754ab8 1668 * @index: doorbell index
75efdee1
AD		 1669 *
1670 * Returns the value in the doorbell aperture at the
d5754ab8 1671 * requested doorbell index (CIK).
75efdee1 1672 */
d5754ab8 1673u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1674{
d5754ab8
AL		 1675 if (index < rdev->doorbell.num_doorbells) {
1676 return readl(rdev->doorbell.ptr + index);
75efdee1 1677 } else {
d5754ab8 1678 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1679 return 0;
1680 }
1681}
1682
1683/**
1684 * cik_mm_wdoorbell - write a doorbell dword
1685 *
1686 * @rdev: radeon_device pointer
d5754ab8 1687 * @index: doorbell index
75efdee1
AD		 1688 * @v: value to write
1689 *
1690 * Writes @v to the doorbell aperture at the
d5754ab8 1691 * requested doorbell index (CIK).
75efdee1 1692 */
d5754ab8 1693void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1694{
d5754ab8
AL		 1695 if (index < rdev->doorbell.num_doorbells) {
1696 writel(v, rdev->doorbell.ptr + index);
75efdee1 1697 } else {
d5754ab8 1698 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1699 }
1700}
1701
bc8273fe
AD		 1702#define BONAIRE_IO_MC_REGS_SIZE 36
1703
1704static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1705{
1706 {0x00000070, 0x04400000},
1707 {0x00000071, 0x80c01803},
1708 {0x00000072, 0x00004004},
1709 {0x00000073, 0x00000100},
1710 {0x00000074, 0x00ff0000},
1711 {0x00000075, 0x34000000},
1712 {0x00000076, 0x08000014},
1713 {0x00000077, 0x00cc08ec},
1714 {0x00000078, 0x00000400},
1715 {0x00000079, 0x00000000},
1716 {0x0000007a, 0x04090000},
1717 {0x0000007c, 0x00000000},
1718 {0x0000007e, 0x4408a8e8},
1719 {0x0000007f, 0x00000304},
1720 {0x00000080, 0x00000000},
1721 {0x00000082, 0x00000001},
1722 {0x00000083, 0x00000002},
1723 {0x00000084, 0xf3e4f400},
1724 {0x00000085, 0x052024e3},
1725 {0x00000087, 0x00000000},
1726 {0x00000088, 0x01000000},
1727 {0x0000008a, 0x1c0a0000},
1728 {0x0000008b, 0xff010000},
1729 {0x0000008d, 0xffffefff},
1730 {0x0000008e, 0xfff3efff},
1731 {0x0000008f, 0xfff3efbf},
1732 {0x00000092, 0xf7ffffff},
1733 {0x00000093, 0xffffff7f},
1734 {0x00000095, 0x00101101},
1735 {0x00000096, 0x00000fff},
1736 {0x00000097, 0x00116fff},
1737 {0x00000098, 0x60010000},
1738 {0x00000099, 0x10010000},
1739 {0x0000009a, 0x00006000},
1740 {0x0000009b, 0x00001000},
1741 {0x0000009f, 0x00b48000}
1742};
1743
d4775655
AD		 1744#define HAWAII_IO_MC_REGS_SIZE 22
1745
1746static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1747{
1748 {0x0000007d, 0x40000000},
1749 {0x0000007e, 0x40180304},
1750 {0x0000007f, 0x0000ff00},
1751 {0x00000081, 0x00000000},
1752 {0x00000083, 0x00000800},
1753 {0x00000086, 0x00000000},
1754 {0x00000087, 0x00000100},
1755 {0x00000088, 0x00020100},
1756 {0x00000089, 0x00000000},
1757 {0x0000008b, 0x00040000},
1758 {0x0000008c, 0x00000100},
1759 {0x0000008e, 0xff010000},
1760 {0x00000090, 0xffffefff},
1761 {0x00000091, 0xfff3efff},
1762 {0x00000092, 0xfff3efbf},
1763 {0x00000093, 0xf7ffffff},
1764 {0x00000094, 0xffffff7f},
1765 {0x00000095, 0x00000fff},
1766 {0x00000096, 0x00116fff},
1767 {0x00000097, 0x60010000},
1768 {0x00000098, 0x10010000},
1769 {0x0000009f, 0x00c79000}
1770};
1771
1772
b556b12e
AD		 1773/**
1774 * cik_srbm_select - select specific register instances
1775 *
1776 * @rdev: radeon_device pointer
1777 * @me: selected ME (micro engine)
1778 * @pipe: pipe
1779 * @queue: queue
1780 * @vmid: VMID
1781 *
1782 * Switches the currently active registers instances. Some
1783 * registers are instanced per VMID, others are instanced per
1784 * me/pipe/queue combination.
1785 */
1786static void cik_srbm_select(struct radeon_device *rdev,
1787 u32 me, u32 pipe, u32 queue, u32 vmid)
1788{
1789 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1790 MEID(me & 0x3) |
1791 VMID(vmid & 0xf) |
1792 QUEUEID(queue & 0x7));
1793 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1794}
1795
bc8273fe
AD		 1796/* ucode loading */
1797/**
1798 * ci_mc_load_microcode - load MC ucode into the hw
1799 *
1800 * @rdev: radeon_device pointer
1801 *
1802 * Load the GDDR MC ucode into the hw (CIK).
1803 * Returns 0 on success, error on failure.
1804 */
6c7bccea 1805int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1806{
f2c6b0f4
AD		 1807 const __be32 *fw_data = NULL;
1808 const __le32 *new_fw_data = NULL;
bc8273fe 1809 u32 running, blackout = 0;
f2c6b0f4
AD		 1810 u32 *io_mc_regs = NULL;
1811 const __le32 *new_io_mc_regs = NULL;
bcddee29 1812 int i, regs_size, ucode_size;
bc8273fe
AD		 1813
1814 if (!rdev->mc_fw)
1815 return -EINVAL;
1816
f2c6b0f4
AD		 1817 if (rdev->new_fw) {
1818 const struct mc_firmware_header_v1_0 *hdr =
1819 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1820
f2c6b0f4
AD		 1821 radeon_ucode_print_mc_hdr(&hdr->header);
1822
1823 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824 new_io_mc_regs = (const __le32 *)
1825 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827 new_fw_data = (const __le32 *)
1828 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1829 } else {
1830 ucode_size = rdev->mc_fw->size / 4;
1831
1832 switch (rdev->family) {
1833 case CHIP_BONAIRE:
1834 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1836 break;
1837 case CHIP_HAWAII:
1838 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839 regs_size = HAWAII_IO_MC_REGS_SIZE;
1840 break;
1841 default:
1842 return -EINVAL;
1843 }
1844 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1845 }
1846
1847 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1848
1849 if (running == 0) {
1850 if (running) {
1851 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1853 }
1854
1855 /* reset the engine and set to writable */
1856 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1858
1859 /* load mc io regs */
1860 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1861 if (rdev->new_fw) {
1862 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1864 } else {
1865 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1867 }
bc8273fe
AD		 1868 }
1869 /* load the MC ucode */
f2c6b0f4
AD		 1870 for (i = 0; i < ucode_size; i++) {
1871 if (rdev->new_fw)
1872 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1873 else
1874 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1875 }
bc8273fe
AD		 1876
1877 /* put the engine back into the active state */
1878 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1879 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1880 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1881
1882 /* wait for training to complete */
1883 for (i = 0; i < rdev->usec_timeout; i++) {
1884 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1885 break;
1886 udelay(1);
1887 }
1888 for (i = 0; i < rdev->usec_timeout; i++) {
1889 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1890 break;
1891 udelay(1);
1892 }
1893
1894 if (running)
1895 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1896 }
1897
1898 return 0;
1899}
1900
02c81327
AD		 1901/**
1902 * cik_init_microcode - load ucode images from disk
1903 *
1904 * @rdev: radeon_device pointer
1905 *
1906 * Use the firmware interface to load the ucode images into
1907 * the driver (not loaded into hw).
1908 * Returns 0 on success, error on failure.
1909 */
1910static int cik_init_microcode(struct radeon_device *rdev)
1911{
02c81327 1912 const char *chip_name;
f2c6b0f4 1913 const char *new_chip_name;
02c81327 1914 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1915 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1916 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1917 char fw_name[30];
f2c6b0f4 1918 int new_fw = 0;
02c81327 1919 int err;
f2c6b0f4 1920 int num_fw;
02c81327
AD		 1921
1922 DRM_DEBUG("\n");
1923
02c81327
AD		 1924 switch (rdev->family) {
1925 case CHIP_BONAIRE:
1926 chip_name = "BONAIRE";
f2c6b0f4 1927 new_chip_name = "bonaire";
02c81327
AD		 1928 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1929 me_req_size = CIK_ME_UCODE_SIZE * 4;
1930 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1931 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1932 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 1933 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1934 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 1935 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1936 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 1937 num_fw = 8;
02c81327 1938 break;
d4775655
AD		 1939 case CHIP_HAWAII:
1940 chip_name = "HAWAII";
f2c6b0f4 1941 new_chip_name = "hawaii";
d4775655
AD		 1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 1948 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 1951 num_fw = 8;
d4775655 1952 break;
02c81327
AD		 1953 case CHIP_KAVERI:
1954 chip_name = "KAVERI";
f2c6b0f4 1955 new_chip_name = "kaveri";
02c81327
AD		 1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1961 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1962 num_fw = 7;
02c81327
AD		 1963 break;
1964 case CHIP_KABINI:
1965 chip_name = "KABINI";
f2c6b0f4 1966 new_chip_name = "kabini";
02c81327
AD		 1967 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1968 me_req_size = CIK_ME_UCODE_SIZE * 4;
1969 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1970 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1971 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1972 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1973 num_fw = 6;
02c81327 1974 break;
f73a9e83
SL		 1975 case CHIP_MULLINS:
1976 chip_name = "MULLINS";
f2c6b0f4 1977 new_chip_name = "mullins";
f73a9e83
SL		 1978 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1979 me_req_size = CIK_ME_UCODE_SIZE * 4;
1980 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1981 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1982 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1983 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1984 num_fw = 6;
f73a9e83 1985 break;
02c81327
AD		 1986 default: BUG();
1987 }
1988
f2c6b0f4 1989 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 1990
f2c6b0f4 1991 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 1992 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 1993 if (err) {
1994 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1995 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1996 if (err)
1997 goto out;
1998 if (rdev->pfp_fw->size != pfp_req_size) {
1999 printk(KERN_ERR
2000 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2001 rdev->pfp_fw->size, fw_name);
2002 err = -EINVAL;
2003 goto out;
2004 }
2005 } else {
2006 err = radeon_ucode_validate(rdev->pfp_fw);
2007 if (err) {
2008 printk(KERN_ERR
2009 "cik_fw: validation failed for firmware \"%s\"\n",
2010 fw_name);
2011 goto out;
2012 } else {
2013 new_fw++;
2014 }
02c81327
AD		 2015 }
2016
f2c6b0f4 2017 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2018 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2019 if (err) {
2020 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2021 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2022 if (err)
2023 goto out;
2024 if (rdev->me_fw->size != me_req_size) {
2025 printk(KERN_ERR
2026 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2027 rdev->me_fw->size, fw_name);
2028 err = -EINVAL;
2029 }
2030 } else {
2031 err = radeon_ucode_validate(rdev->me_fw);
2032 if (err) {
2033 printk(KERN_ERR
2034 "cik_fw: validation failed for firmware \"%s\"\n",
2035 fw_name);
2036 goto out;
2037 } else {
2038 new_fw++;
2039 }
02c81327
AD		 2040 }
2041
f2c6b0f4 2042 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2043 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2044 if (err) {
2045 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2046 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2047 if (err)
2048 goto out;
2049 if (rdev->ce_fw->size != ce_req_size) {
2050 printk(KERN_ERR
2051 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2052 rdev->ce_fw->size, fw_name);
2053 err = -EINVAL;
2054 }
2055 } else {
2056 err = radeon_ucode_validate(rdev->ce_fw);
2057 if (err) {
2058 printk(KERN_ERR
2059 "cik_fw: validation failed for firmware \"%s\"\n",
2060 fw_name);
2061 goto out;
2062 } else {
2063 new_fw++;
2064 }
02c81327
AD		 2065 }
2066
f2c6b0f4 2067 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2068 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2069 if (err) {
2070 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2071 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2072 if (err)
2073 goto out;
2074 if (rdev->mec_fw->size != mec_req_size) {
2075 printk(KERN_ERR
2076 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077 rdev->mec_fw->size, fw_name);
2078 err = -EINVAL;
2079 }
2080 } else {
2081 err = radeon_ucode_validate(rdev->mec_fw);
2082 if (err) {
2083 printk(KERN_ERR
2084 "cik_fw: validation failed for firmware \"%s\"\n",
2085 fw_name);
2086 goto out;
2087 } else {
2088 new_fw++;
2089 }
2090 }
2091
2092 if (rdev->family == CHIP_KAVERI) {
2093 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2094 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2095 if (err) {
2096 goto out;
2097 } else {
2098 err = radeon_ucode_validate(rdev->mec2_fw);
2099 if (err) {
2100 goto out;
2101 } else {
2102 new_fw++;
2103 }
2104 }
02c81327
AD		 2105 }
2106
f2c6b0f4 2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2108 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2109 if (err) {
2110 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2111 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2112 if (err)
2113 goto out;
2114 if (rdev->rlc_fw->size != rlc_req_size) {
2115 printk(KERN_ERR
2116 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2117 rdev->rlc_fw->size, fw_name);
2118 err = -EINVAL;
2119 }
2120 } else {
2121 err = radeon_ucode_validate(rdev->rlc_fw);
2122 if (err) {
2123 printk(KERN_ERR
2124 "cik_fw: validation failed for firmware \"%s\"\n",
2125 fw_name);
2126 goto out;
2127 } else {
2128 new_fw++;
2129 }
02c81327
AD		 2130 }
2131
f2c6b0f4 2132 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2133 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2134 if (err) {
2135 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2136 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2137 if (err)
2138 goto out;
2139 if (rdev->sdma_fw->size != sdma_req_size) {
2140 printk(KERN_ERR
2141 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2142 rdev->sdma_fw->size, fw_name);
2143 err = -EINVAL;
2144 }
2145 } else {
2146 err = radeon_ucode_validate(rdev->sdma_fw);
2147 if (err) {
2148 printk(KERN_ERR
2149 "cik_fw: validation failed for firmware \"%s\"\n",
2150 fw_name);
2151 goto out;
2152 } else {
2153 new_fw++;
2154 }
21a93e13
AD		 2155 }
2156
cc8dbbb4 2157 /* No SMC, MC ucode on APUs */
02c81327 2158 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2159 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2160 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2161 if (err) {
f2c6b0f4 2162 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2163 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2164 if (err) {
2165 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2166 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2167 if (err)
2168 goto out;
2169 }
2170 if ((rdev->mc_fw->size != mc_req_size) &&
2171 (rdev->mc_fw->size != mc2_req_size)){
2172 printk(KERN_ERR
2173 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2174 rdev->mc_fw->size, fw_name);
2175 err = -EINVAL;
2176 }
2177 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2178 } else {
2179 err = radeon_ucode_validate(rdev->mc_fw);
2180 if (err) {
2181 printk(KERN_ERR
2182 "cik_fw: validation failed for firmware \"%s\"\n",
2183 fw_name);
277babc3 2184 goto out;
f2c6b0f4
AD		 2185 } else {
2186 new_fw++;
2187 }
277babc3 2188 }
cc8dbbb4 2189
f2c6b0f4 2190 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2191 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2192 if (err) {
f2c6b0f4
AD		 2193 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2194 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2195 if (err) {
2196 printk(KERN_ERR
2197 "smc: error loading firmware \"%s\"\n",
2198 fw_name);
2199 release_firmware(rdev->smc_fw);
2200 rdev->smc_fw = NULL;
2201 err = 0;
2202 } else if (rdev->smc_fw->size != smc_req_size) {
2203 printk(KERN_ERR
2204 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2205 rdev->smc_fw->size, fw_name);
2206 err = -EINVAL;
2207 }
2208 } else {
2209 err = radeon_ucode_validate(rdev->smc_fw);
2210 if (err) {
2211 printk(KERN_ERR
2212 "cik_fw: validation failed for firmware \"%s\"\n",
2213 fw_name);
2214 goto out;
2215 } else {
2216 new_fw++;
2217 }
cc8dbbb4 2218 }
02c81327
AD		 2219 }
2220
f2c6b0f4
AD		 2221 if (new_fw == 0) {
2222 rdev->new_fw = false;
2223 } else if (new_fw < num_fw) {
2224 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2225 err = -EINVAL;
2226 } else {
2227 rdev->new_fw = true;
2228 }
2229
02c81327 2230out:
02c81327
AD		 2231 if (err) {
2232 if (err != -EINVAL)
2233 printk(KERN_ERR
2234 "cik_cp: Failed to load firmware \"%s\"\n",
2235 fw_name);
2236 release_firmware(rdev->pfp_fw);
2237 rdev->pfp_fw = NULL;
2238 release_firmware(rdev->me_fw);
2239 rdev->me_fw = NULL;
2240 release_firmware(rdev->ce_fw);
2241 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2242 release_firmware(rdev->mec_fw);
2243 rdev->mec_fw = NULL;
2244 release_firmware(rdev->mec2_fw);
2245 rdev->mec2_fw = NULL;
02c81327
AD		 2246 release_firmware(rdev->rlc_fw);
2247 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2248 release_firmware(rdev->sdma_fw);
2249 rdev->sdma_fw = NULL;
02c81327
AD		 2250 release_firmware(rdev->mc_fw);
2251 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2252 release_firmware(rdev->smc_fw);
2253 rdev->smc_fw = NULL;
02c81327
AD		 2254 }
2255 return err;
2256}
2257
8cc1a532
AD		 2258/*
2259 * Core functions
2260 */
2261/**
2262 * cik_tiling_mode_table_init - init the hw tiling table
2263 *
2264 * @rdev: radeon_device pointer
2265 *
2266 * Starting with SI, the tiling setup is done globally in a
2267 * set of 32 tiling modes. Rather than selecting each set of
2268 * parameters per surface as on older asics, we just select
2269 * which index in the tiling table we want to use, and the
2270 * surface uses those parameters (CIK).
2271 */
2272static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2273{
2274 const u32 num_tile_mode_states = 32;
2275 const u32 num_secondary_tile_mode_states = 16;
2276 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2277 u32 num_pipe_configs;
2278 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2279 rdev->config.cik.max_shader_engines;
2280
2281 switch (rdev->config.cik.mem_row_size_in_kb) {
2282 case 1:
2283 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2284 break;
2285 case 2:
2286 default:
2287 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2288 break;
2289 case 4:
2290 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2291 break;
2292 }
2293
2294 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2295 if (num_pipe_configs > 8)
21e438af 2296 num_pipe_configs = 16;
8cc1a532 2297
21e438af
AD		 2298 if (num_pipe_configs == 16) {
2299 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2300 switch (reg_offset) {
2301 case 0:
2302 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2303 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2304 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2306 break;
2307 case 1:
2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2311 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2312 break;
2313 case 2:
2314 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2318 break;
2319 case 3:
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2324 break;
2325 case 4:
2326 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329 TILE_SPLIT(split_equal_to_row_size));
2330 break;
2331 case 5:
2332 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2333 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2335 break;
2336 case 6:
2337 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2338 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2339 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2340 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2341 break;
2342 case 7:
2343 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2344 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2345 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346 TILE_SPLIT(split_equal_to_row_size));
2347 break;
2348 case 8:
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2350 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2351 break;
2352 case 9:
2353 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2354 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2355 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2356 break;
2357 case 10:
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2361 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2362 break;
2363 case 11:
2364 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2365 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2367 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2368 break;
2369 case 12:
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2374 break;
2375 case 13:
2376 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2377 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2378 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2379 break;
2380 case 14:
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2385 break;
2386 case 16:
2387 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2389 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2391 break;
2392 case 17:
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397 break;
2398 case 27:
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2401 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2402 break;
2403 case 28:
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2406 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2408 break;
2409 case 29:
2410 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414 break;
2415 case 30:
2416 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420 break;
2421 default:
2422 gb_tile_moden = 0;
2423 break;
2424 }
2425 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2426 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2427 }
2428 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2429 switch (reg_offset) {
2430 case 0:
2431 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2434 NUM_BANKS(ADDR_SURF_16_BANK));
2435 break;
2436 case 1:
2437 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2439 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440 NUM_BANKS(ADDR_SURF_16_BANK));
2441 break;
2442 case 2:
2443 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2446 NUM_BANKS(ADDR_SURF_16_BANK));
2447 break;
2448 case 3:
2449 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452 NUM_BANKS(ADDR_SURF_16_BANK));
2453 break;
2454 case 4:
2455 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458 NUM_BANKS(ADDR_SURF_8_BANK));
2459 break;
2460 case 5:
2461 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 NUM_BANKS(ADDR_SURF_4_BANK));
2465 break;
2466 case 6:
2467 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470 NUM_BANKS(ADDR_SURF_2_BANK));
2471 break;
2472 case 8:
2473 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476 NUM_BANKS(ADDR_SURF_16_BANK));
2477 break;
2478 case 9:
2479 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2482 NUM_BANKS(ADDR_SURF_16_BANK));
2483 break;
2484 case 10:
2485 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488 NUM_BANKS(ADDR_SURF_16_BANK));
2489 break;
2490 case 11:
2491 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494 NUM_BANKS(ADDR_SURF_8_BANK));
2495 break;
2496 case 12:
2497 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500 NUM_BANKS(ADDR_SURF_4_BANK));
2501 break;
2502 case 13:
2503 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506 NUM_BANKS(ADDR_SURF_2_BANK));
2507 break;
2508 case 14:
2509 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512 NUM_BANKS(ADDR_SURF_2_BANK));
2513 break;
2514 default:
2515 gb_tile_moden = 0;
2516 break;
2517 }
1b2c4869 2518 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
21e438af
AD		 2519 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2520 }
2521 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2522 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523 switch (reg_offset) {
2524 case 0:
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2529 break;
2530 case 1:
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2535 break;
2536 case 2:
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2541 break;
2542 case 3:
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2547 break;
2548 case 4:
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552 TILE_SPLIT(split_equal_to_row_size));
2553 break;
2554 case 5:
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2556 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2558 break;
2559 case 6:
2560 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2564 break;
2565 case 7:
2566 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569 TILE_SPLIT(split_equal_to_row_size));
2570 break;
2571 case 8:
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2574 break;
2575 case 9:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2579 break;
2580 case 10:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2585 break;
2586 case 11:
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591 break;
2592 case 12:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2597 break;
2598 case 13:
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2601 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2602 break;
2603 case 14:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2608 break;
2609 case 16:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2613 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614 break;
2615 case 17:
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620 break;
2621 case 27:
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2623 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2624 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2625 break;
2626 case 28:
2627 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2631 break;
2632 case 29:
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2637 break;
2638 case 30:
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 break;
2644 default:
2645 gb_tile_moden = 0;
2646 break;
2647 }
39aee490 2648 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2649 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2650 }
2651 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2652 switch (reg_offset) {
2653 case 0:
2654 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2657 NUM_BANKS(ADDR_SURF_16_BANK));
2658 break;
2659 case 1:
2660 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2663 NUM_BANKS(ADDR_SURF_16_BANK));
2664 break;
2665 case 2:
2666 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2669 NUM_BANKS(ADDR_SURF_16_BANK));
2670 break;
2671 case 3:
2672 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675 NUM_BANKS(ADDR_SURF_16_BANK));
2676 break;
2677 case 4:
2678 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2681 NUM_BANKS(ADDR_SURF_8_BANK));
2682 break;
2683 case 5:
2684 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2687 NUM_BANKS(ADDR_SURF_4_BANK));
2688 break;
2689 case 6:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693 NUM_BANKS(ADDR_SURF_2_BANK));
2694 break;
2695 case 8:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2700 break;
2701 case 9:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2706 break;
2707 case 10:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 11:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 12:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2724 break;
2725 case 13:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2730 break;
2731 case 14:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735 NUM_BANKS(ADDR_SURF_2_BANK));
2736 break;
2737 default:
2738 gb_tile_moden = 0;
2739 break;
2740 }
32f79a8a 2741 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2742 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2743 }
2744 } else if (num_pipe_configs == 4) {
2745 if (num_rbs == 4) {
2746 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2747 switch (reg_offset) {
2748 case 0:
2749 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2753 break;
2754 case 1:
2755 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2759 break;
2760 case 2:
2761 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2765 break;
2766 case 3:
2767 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2771 break;
2772 case 4:
2773 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776 TILE_SPLIT(split_equal_to_row_size));
2777 break;
2778 case 5:
2779 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2780 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2781 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2782 break;
2783 case 6:
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2788 break;
2789 case 7:
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793 TILE_SPLIT(split_equal_to_row_size));
2794 break;
2795 case 8:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2797 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2798 break;
2799 case 9:
2800 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2801 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2803 break;
2804 case 10:
2805 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2809 break;
2810 case 11:
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815 break;
2816 case 12:
2817 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2821 break;
2822 case 13:
2823 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2824 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2825 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2826 break;
2827 case 14:
2828 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2832 break;
2833 case 16:
2834 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2836 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2838 break;
2839 case 17:
2840 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2841 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2844 break;
2845 case 27:
2846 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2847 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2848 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2849 break;
2850 case 28:
2851 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2852 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2853 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2855 break;
2856 case 29:
2857 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2859 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861 break;
2862 case 30:
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867 break;
2868 default:
2869 gb_tile_moden = 0;
2870 break;
2871 }
39aee490 2872 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2873 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2874 }
2875 } else if (num_rbs < 4) {
2876 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877 switch (reg_offset) {
2878 case 0:
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2883 break;
2884 case 1:
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2889 break;
2890 case 2:
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895 break;
2896 case 3:
2897 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2901 break;
2902 case 4:
2903 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906 TILE_SPLIT(split_equal_to_row_size));
2907 break;
2908 case 5:
2909 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2910 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2912 break;
2913 case 6:
2914 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2917 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2918 break;
2919 case 7:
2920 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923 TILE_SPLIT(split_equal_to_row_size));
2924 break;
2925 case 8:
2926 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2928 break;
2929 case 9:
2930 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2931 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2932 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2933 break;
2934 case 10:
2935 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 break;
2940 case 11:
2941 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2945 break;
2946 case 12:
2947 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951 break;
2952 case 13:
2953 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2954 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2955 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2956 break;
2957 case 14:
2958 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2961 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2962 break;
2963 case 16:
2964 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2968 break;
2969 case 17:
2970 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974 break;
2975 case 27:
2976 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2977 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2978 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2979 break;
2980 case 28:
2981 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2984 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2985 break;
2986 case 29:
2987 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2991 break;
2992 case 30:
2993 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2997 break;
2998 default:
2999 gb_tile_moden = 0;
3000 break;
3001 }
39aee490 3002 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3003 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3004 }
3005 }
3006 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3007 switch (reg_offset) {
3008 case 0:
3009 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3010 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3011 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3012 NUM_BANKS(ADDR_SURF_16_BANK));
3013 break;
3014 case 1:
3015 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018 NUM_BANKS(ADDR_SURF_16_BANK));
3019 break;
3020 case 2:
3021 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3024 NUM_BANKS(ADDR_SURF_16_BANK));
3025 break;
3026 case 3:
3027 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030 NUM_BANKS(ADDR_SURF_16_BANK));
3031 break;
3032 case 4:
3033 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036 NUM_BANKS(ADDR_SURF_16_BANK));
3037 break;
3038 case 5:
3039 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042 NUM_BANKS(ADDR_SURF_8_BANK));
3043 break;
3044 case 6:
3045 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3048 NUM_BANKS(ADDR_SURF_4_BANK));
3049 break;
3050 case 8:
3051 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3052 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3053 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3054 NUM_BANKS(ADDR_SURF_16_BANK));
3055 break;
3056 case 9:
3057 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3058 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3059 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3060 NUM_BANKS(ADDR_SURF_16_BANK));
3061 break;
3062 case 10:
3063 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3064 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3065 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066 NUM_BANKS(ADDR_SURF_16_BANK));
3067 break;
3068 case 11:
3069 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072 NUM_BANKS(ADDR_SURF_16_BANK));
3073 break;
3074 case 12:
3075 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3077 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3078 NUM_BANKS(ADDR_SURF_16_BANK));
3079 break;
3080 case 13:
3081 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3084 NUM_BANKS(ADDR_SURF_8_BANK));
3085 break;
3086 case 14:
3087 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3090 NUM_BANKS(ADDR_SURF_4_BANK));
3091 break;
3092 default:
3093 gb_tile_moden = 0;
3094 break;
3095 }
32f79a8a 3096 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3097 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3098 }
3099 } else if (num_pipe_configs == 2) {
3100 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3101 switch (reg_offset) {
3102 case 0:
3103 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3104 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3105 PIPE_CONFIG(ADDR_SURF_P2) |
3106 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3107 break;
3108 case 1:
3109 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3111 PIPE_CONFIG(ADDR_SURF_P2) |
3112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3113 break;
3114 case 2:
3115 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117 PIPE_CONFIG(ADDR_SURF_P2) |
3118 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3119 break;
3120 case 3:
3121 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123 PIPE_CONFIG(ADDR_SURF_P2) |
3124 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3125 break;
3126 case 4:
3127 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129 PIPE_CONFIG(ADDR_SURF_P2) |
3130 TILE_SPLIT(split_equal_to_row_size));
3131 break;
3132 case 5:
3133 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3134 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3135 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3136 break;
3137 case 6:
3138 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3140 PIPE_CONFIG(ADDR_SURF_P2) |
3141 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3142 break;
3143 case 7:
3144 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3145 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3146 PIPE_CONFIG(ADDR_SURF_P2) |
3147 TILE_SPLIT(split_equal_to_row_size));
3148 break;
3149 case 8:
020ff546
MO		 3150 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3151 PIPE_CONFIG(ADDR_SURF_P2);
8cc1a532
AD		 3152 break;
3153 case 9:
3154 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3155 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3156 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3157 break;
3158 case 10:
3159 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3161 PIPE_CONFIG(ADDR_SURF_P2) |
3162 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3163 break;
3164 case 11:
3165 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3166 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3167 PIPE_CONFIG(ADDR_SURF_P2) |
3168 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3169 break;
3170 case 12:
3171 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3172 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173 PIPE_CONFIG(ADDR_SURF_P2) |
3174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3175 break;
3176 case 13:
3177 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3178 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3179 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3180 break;
3181 case 14:
3182 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3183 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3184 PIPE_CONFIG(ADDR_SURF_P2) |
3185 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3186 break;
3187 case 16:
3188 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3189 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3190 PIPE_CONFIG(ADDR_SURF_P2) |
3191 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3192 break;
3193 case 17:
3194 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3195 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196 PIPE_CONFIG(ADDR_SURF_P2) |
3197 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3198 break;
3199 case 27:
3200 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3201 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3202 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3203 break;
3204 case 28:
3205 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3206 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3207 PIPE_CONFIG(ADDR_SURF_P2) |
3208 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3209 break;
3210 case 29:
3211 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3212 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3213 PIPE_CONFIG(ADDR_SURF_P2) |
3214 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3215 break;
3216 case 30:
3217 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219 PIPE_CONFIG(ADDR_SURF_P2) |
3220 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3221 break;
3222 default:
3223 gb_tile_moden = 0;
3224 break;
3225 }
39aee490 3226 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3227 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3228 }
3229 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3230 switch (reg_offset) {
3231 case 0:
3232 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3233 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3234 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3235 NUM_BANKS(ADDR_SURF_16_BANK));
3236 break;
3237 case 1:
3238 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3239 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3240 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3241 NUM_BANKS(ADDR_SURF_16_BANK));
3242 break;
3243 case 2:
3244 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247 NUM_BANKS(ADDR_SURF_16_BANK));
3248 break;
3249 case 3:
3250 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3251 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3252 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253 NUM_BANKS(ADDR_SURF_16_BANK));
3254 break;
3255 case 4:
3256 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259 NUM_BANKS(ADDR_SURF_16_BANK));
3260 break;
3261 case 5:
3262 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265 NUM_BANKS(ADDR_SURF_16_BANK));
3266 break;
3267 case 6:
3268 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3271 NUM_BANKS(ADDR_SURF_8_BANK));
3272 break;
3273 case 8:
3274 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277 NUM_BANKS(ADDR_SURF_16_BANK));
3278 break;
3279 case 9:
3280 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3283 NUM_BANKS(ADDR_SURF_16_BANK));
3284 break;
3285 case 10:
3286 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289 NUM_BANKS(ADDR_SURF_16_BANK));
3290 break;
3291 case 11:
3292 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3293 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3294 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295 NUM_BANKS(ADDR_SURF_16_BANK));
3296 break;
3297 case 12:
3298 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301 NUM_BANKS(ADDR_SURF_16_BANK));
3302 break;
3303 case 13:
3304 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307 NUM_BANKS(ADDR_SURF_16_BANK));
3308 break;
3309 case 14:
3310 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313 NUM_BANKS(ADDR_SURF_8_BANK));
3314 break;
3315 default:
3316 gb_tile_moden = 0;
3317 break;
3318 }
32f79a8a 3319 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3320 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3321 }
3322 } else
3323 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3324}
3325
3326/**
3327 * cik_select_se_sh - select which SE, SH to address
3328 *
3329 * @rdev: radeon_device pointer
3330 * @se_num: shader engine to address
3331 * @sh_num: sh block to address
3332 *
3333 * Select which SE, SH combinations to address. Certain
3334 * registers are instanced per SE or SH. 0xffffffff means
3335 * broadcast to all SEs or SHs (CIK).
3336 */
3337static void cik_select_se_sh(struct radeon_device *rdev,
3338 u32 se_num, u32 sh_num)
3339{
3340 u32 data = INSTANCE_BROADCAST_WRITES;
3341
3342 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3343 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3344 else if (se_num == 0xffffffff)
3345 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3346 else if (sh_num == 0xffffffff)
3347 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3348 else
3349 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3350 WREG32(GRBM_GFX_INDEX, data);
3351}
3352
3353/**
3354 * cik_create_bitmask - create a bitmask
3355 *
3356 * @bit_width: length of the mask
3357 *
3358 * create a variable length bit mask (CIK).
3359 * Returns the bitmask.
3360 */
3361static u32 cik_create_bitmask(u32 bit_width)
3362{
3363 u32 i, mask = 0;
3364
3365 for (i = 0; i < bit_width; i++) {
3366 mask <<= 1;
3367 mask |= 1;
3368 }
3369 return mask;
3370}
3371
3372/**
972c5ddb 3373 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3374 *
3375 * @rdev: radeon_device pointer
3376 * @max_rb_num: max RBs (render backends) for the asic
3377 * @se_num: number of SEs (shader engines) for the asic
3378 * @sh_per_se: number of SH blocks per SE for the asic
3379 *
3380 * Calculates the bitmask of disabled RBs (CIK).
3381 * Returns the disabled RB bitmask.
3382 */
3383static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3384 u32 max_rb_num_per_se,
8cc1a532
AD		 3385 u32 sh_per_se)
3386{
3387 u32 data, mask;
3388
3389 data = RREG32(CC_RB_BACKEND_DISABLE);
3390 if (data & 1)
3391 data &= BACKEND_DISABLE_MASK;
3392 else
3393 data = 0;
3394 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3395
3396 data >>= BACKEND_DISABLE_SHIFT;
3397
9fadb352 3398 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3399
3400 return data & mask;
3401}
3402
3403/**
3404 * cik_setup_rb - setup the RBs on the asic
3405 *
3406 * @rdev: radeon_device pointer
3407 * @se_num: number of SEs (shader engines) for the asic
3408 * @sh_per_se: number of SH blocks per SE for the asic
3409 * @max_rb_num: max RBs (render backends) for the asic
3410 *
3411 * Configures per-SE/SH RB registers (CIK).
3412 */
3413static void cik_setup_rb(struct radeon_device *rdev,
3414 u32 se_num, u32 sh_per_se,
9fadb352 3415 u32 max_rb_num_per_se)
8cc1a532
AD		 3416{
3417 int i, j;
3418 u32 data, mask;
3419 u32 disabled_rbs = 0;
3420 u32 enabled_rbs = 0;
3421
3422 for (i = 0; i < se_num; i++) {
3423 for (j = 0; j < sh_per_se; j++) {
3424 cik_select_se_sh(rdev, i, j);
9fadb352 3425 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3426 if (rdev->family == CHIP_HAWAII)
3427 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3428 else
3429 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3430 }
3431 }
3432 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3433
3434 mask = 1;
9fadb352 3435 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3436 if (!(disabled_rbs & mask))
3437 enabled_rbs |= mask;
3438 mask <<= 1;
3439 }
3440
439a1cff
MO		 3441 rdev->config.cik.backend_enable_mask = enabled_rbs;
3442
8cc1a532
AD		 3443 for (i = 0; i < se_num; i++) {
3444 cik_select_se_sh(rdev, i, 0xffffffff);
3445 data = 0;
3446 for (j = 0; j < sh_per_se; j++) {
3447 switch (enabled_rbs & 3) {
fc821b70
AD		 3448 case 0:
3449 if (j == 0)
3450 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3451 else
3452 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3453 break;
8cc1a532
AD		 3454 case 1:
3455 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3456 break;
3457 case 2:
3458 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3459 break;
3460 case 3:
3461 default:
3462 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3463 break;
3464 }
3465 enabled_rbs >>= 2;
3466 }
3467 WREG32(PA_SC_RASTER_CONFIG, data);
3468 }
3469 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3470}
3471
3472/**
3473 * cik_gpu_init - setup the 3D engine
3474 *
3475 * @rdev: radeon_device pointer
3476 *
3477 * Configures the 3D engine and tiling configuration
3478 * registers so that the 3D engine is usable.
3479 */
3480static void cik_gpu_init(struct radeon_device *rdev)
3481{
3482 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3483 u32 mc_shared_chmap, mc_arb_ramcfg;
3484 u32 hdp_host_path_cntl;
3485 u32 tmp;
65fcf668 3486 int i, j, k;
8cc1a532
AD		 3487
3488 switch (rdev->family) {
3489 case CHIP_BONAIRE:
3490 rdev->config.cik.max_shader_engines = 2;
3491 rdev->config.cik.max_tile_pipes = 4;
3492 rdev->config.cik.max_cu_per_sh = 7;
3493 rdev->config.cik.max_sh_per_se = 1;
3494 rdev->config.cik.max_backends_per_se = 2;
3495 rdev->config.cik.max_texture_channel_caches = 4;
3496 rdev->config.cik.max_gprs = 256;
3497 rdev->config.cik.max_gs_threads = 32;
3498 rdev->config.cik.max_hw_contexts = 8;
3499
3500 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3501 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3502 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3503 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3504 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3505 break;
b496038b
AD		 3506 case CHIP_HAWAII:
3507 rdev->config.cik.max_shader_engines = 4;
3508 rdev->config.cik.max_tile_pipes = 16;
3509 rdev->config.cik.max_cu_per_sh = 11;
3510 rdev->config.cik.max_sh_per_se = 1;
3511 rdev->config.cik.max_backends_per_se = 4;
3512 rdev->config.cik.max_texture_channel_caches = 16;
3513 rdev->config.cik.max_gprs = 256;
3514 rdev->config.cik.max_gs_threads = 32;
3515 rdev->config.cik.max_hw_contexts = 8;
3516
3517 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3518 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3519 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3520 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3521 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3522 break;
8cc1a532 3523 case CHIP_KAVERI:
b2e4c70a
AD		 3524 rdev->config.cik.max_shader_engines = 1;
3525 rdev->config.cik.max_tile_pipes = 4;
3526 if ((rdev->pdev->device == 0x1304) ||
3527 (rdev->pdev->device == 0x1305) ||
3528 (rdev->pdev->device == 0x130C) ||
3529 (rdev->pdev->device == 0x130F) ||
3530 (rdev->pdev->device == 0x1310) ||
3531 (rdev->pdev->device == 0x1311) ||
3532 (rdev->pdev->device == 0x131C)) {
3533 rdev->config.cik.max_cu_per_sh = 8;
3534 rdev->config.cik.max_backends_per_se = 2;
3535 } else if ((rdev->pdev->device == 0x1309) ||
3536 (rdev->pdev->device == 0x130A) ||
3537 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3538 (rdev->pdev->device == 0x1313) ||
3539 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3540 rdev->config.cik.max_cu_per_sh = 6;
3541 rdev->config.cik.max_backends_per_se = 2;
3542 } else if ((rdev->pdev->device == 0x1306) ||
3543 (rdev->pdev->device == 0x1307) ||
3544 (rdev->pdev->device == 0x130B) ||
3545 (rdev->pdev->device == 0x130E) ||
3546 (rdev->pdev->device == 0x1315) ||
3547 (rdev->pdev->device == 0x131B)) {
3548 rdev->config.cik.max_cu_per_sh = 4;
3549 rdev->config.cik.max_backends_per_se = 1;
3550 } else {
3551 rdev->config.cik.max_cu_per_sh = 3;
3552 rdev->config.cik.max_backends_per_se = 1;
3553 }
3554 rdev->config.cik.max_sh_per_se = 1;
3555 rdev->config.cik.max_texture_channel_caches = 4;
3556 rdev->config.cik.max_gprs = 256;
3557 rdev->config.cik.max_gs_threads = 16;
3558 rdev->config.cik.max_hw_contexts = 8;
3559
3560 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3561 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3562 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3563 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3564 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3565 break;
3566 case CHIP_KABINI:
f73a9e83 3567 case CHIP_MULLINS:
8cc1a532
AD		 3568 default:
3569 rdev->config.cik.max_shader_engines = 1;
3570 rdev->config.cik.max_tile_pipes = 2;
3571 rdev->config.cik.max_cu_per_sh = 2;
3572 rdev->config.cik.max_sh_per_se = 1;
3573 rdev->config.cik.max_backends_per_se = 1;
3574 rdev->config.cik.max_texture_channel_caches = 2;
3575 rdev->config.cik.max_gprs = 256;
3576 rdev->config.cik.max_gs_threads = 16;
3577 rdev->config.cik.max_hw_contexts = 8;
3578
3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3584 break;
3585 }
3586
3587 /* Initialize HDP */
3588 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3589 WREG32((0x2c14 + j), 0x00000000);
3590 WREG32((0x2c18 + j), 0x00000000);
3591 WREG32((0x2c1c + j), 0x00000000);
3592 WREG32((0x2c20 + j), 0x00000000);
3593 WREG32((0x2c24 + j), 0x00000000);
3594 }
3595
3596 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3597
3598 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3599
3600 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3601 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3602
3603 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3604 rdev->config.cik.mem_max_burst_length_bytes = 256;
3605 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3606 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3607 if (rdev->config.cik.mem_row_size_in_kb > 4)
3608 rdev->config.cik.mem_row_size_in_kb = 4;
3609 /* XXX use MC settings? */
3610 rdev->config.cik.shader_engine_tile_size = 32;
3611 rdev->config.cik.num_gpus = 1;
3612 rdev->config.cik.multi_gpu_tile_size = 64;
3613
3614 /* fix up row size */
3615 gb_addr_config &= ~ROW_SIZE_MASK;
3616 switch (rdev->config.cik.mem_row_size_in_kb) {
3617 case 1:
3618 default:
3619 gb_addr_config |= ROW_SIZE(0);
3620 break;
3621 case 2:
3622 gb_addr_config |= ROW_SIZE(1);
3623 break;
3624 case 4:
3625 gb_addr_config |= ROW_SIZE(2);
3626 break;
3627 }
3628
3629 /* setup tiling info dword. gb_addr_config is not adequate since it does
3630 * not have bank info, so create a custom tiling dword.
3631 * bits 3:0 num_pipes
3632 * bits 7:4 num_banks
3633 * bits 11:8 group_size
3634 * bits 15:12 row_size
3635 */
3636 rdev->config.cik.tile_config = 0;
3637 switch (rdev->config.cik.num_tile_pipes) {
3638 case 1:
3639 rdev->config.cik.tile_config |= (0 << 0);
3640 break;
3641 case 2:
3642 rdev->config.cik.tile_config |= (1 << 0);
3643 break;
3644 case 4:
3645 rdev->config.cik.tile_config |= (2 << 0);
3646 break;
3647 case 8:
3648 default:
3649 /* XXX what about 12? */
3650 rdev->config.cik.tile_config |= (3 << 0);
3651 break;
3652 }
a537314e
MD		 3653 rdev->config.cik.tile_config |=
3654 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3655 rdev->config.cik.tile_config |=
3656 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3657 rdev->config.cik.tile_config |=
3658 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3659
3660 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3661 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3662 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3663 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3664 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3665 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3666 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3667 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3668
3669 cik_tiling_mode_table_init(rdev);
3670
3671 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3672 rdev->config.cik.max_sh_per_se,
3673 rdev->config.cik.max_backends_per_se);
3674
65fcf668
AD		 3675 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3676 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3677 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k++) {
3678 rdev->config.cik.active_cus +=
3679 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3680 }
3681 }
3682 }
3683
8cc1a532
AD		 3684 /* set HW defaults for 3D engine */
3685 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3686
3687 WREG32(SX_DEBUG_1, 0x20);
3688
3689 WREG32(TA_CNTL_AUX, 0x00010000);
3690
3691 tmp = RREG32(SPI_CONFIG_CNTL);
3692 tmp |= 0x03000000;
3693 WREG32(SPI_CONFIG_CNTL, tmp);
3694
3695 WREG32(SQ_CONFIG, 1);
3696
3697 WREG32(DB_DEBUG, 0);
3698
3699 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3700 tmp |= 0x00000400;
3701 WREG32(DB_DEBUG2, tmp);
3702
3703 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3704 tmp |= 0x00020200;
3705 WREG32(DB_DEBUG3, tmp);
3706
3707 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3708 tmp |= 0x00018208;
3709 WREG32(CB_HW_CONTROL, tmp);
3710
3711 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3712
3713 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3714 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3715 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3716 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3717
3718 WREG32(VGT_NUM_INSTANCES, 1);
3719
3720 WREG32(CP_PERFMON_CNTL, 0);
3721
3722 WREG32(SQ_CONFIG, 0);
3723
3724 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3725 FORCE_EOV_MAX_REZ_CNT(255)));
3726
3727 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3728 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3729
3730 WREG32(VGT_GS_VERTEX_REUSE, 16);
3731 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3732
3733 tmp = RREG32(HDP_MISC_CNTL);
3734 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3735 WREG32(HDP_MISC_CNTL, tmp);
3736
3737 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3738 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3739
3740 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3741 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3742
3743 udelay(50);
3744}
3745
2cae3bc3
AD		 3746/*
3747 * GPU scratch registers helpers function.
3748 */
3749/**
3750 * cik_scratch_init - setup driver info for CP scratch regs
3751 *
3752 * @rdev: radeon_device pointer
3753 *
3754 * Set up the number and offset of the CP scratch registers.
3755 * NOTE: use of CP scratch registers is a legacy inferface and
3756 * is not used by default on newer asics (r6xx+). On newer asics,
3757 * memory buffers are used for fences rather than scratch regs.
3758 */
3759static void cik_scratch_init(struct radeon_device *rdev)
3760{
3761 int i;
3762
3763 rdev->scratch.num_reg = 7;
3764 rdev->scratch.reg_base = SCRATCH_REG0;
3765 for (i = 0; i < rdev->scratch.num_reg; i++) {
3766 rdev->scratch.free[i] = true;
3767 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3768 }
3769}
3770
fbc832c7
AD		 3771/**
3772 * cik_ring_test - basic gfx ring test
3773 *
3774 * @rdev: radeon_device pointer
3775 * @ring: radeon_ring structure holding ring information
3776 *
3777 * Allocate a scratch register and write to it using the gfx ring (CIK).
3778 * Provides a basic gfx ring test to verify that the ring is working.
3779 * Used by cik_cp_gfx_resume();
3780 * Returns 0 on success, error on failure.
3781 */
3782int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3783{
3784 uint32_t scratch;
3785 uint32_t tmp = 0;
3786 unsigned i;
3787 int r;
3788
3789 r = radeon_scratch_get(rdev, &scratch);
3790 if (r) {
3791 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3792 return r;
3793 }
3794 WREG32(scratch, 0xCAFEDEAD);
3795 r = radeon_ring_lock(rdev, ring, 3);
3796 if (r) {
3797 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3798 radeon_scratch_free(rdev, scratch);
3799 return r;
3800 }
3801 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3802 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3803 radeon_ring_write(ring, 0xDEADBEEF);
3804 radeon_ring_unlock_commit(rdev, ring);
963e81f9 3805
fbc832c7
AD		 3806 for (i = 0; i < rdev->usec_timeout; i++) {
3807 tmp = RREG32(scratch);
3808 if (tmp == 0xDEADBEEF)
3809 break;
3810 DRM_UDELAY(1);
3811 }
3812 if (i < rdev->usec_timeout) {
3813 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3814 } else {
3815 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3816 ring->idx, scratch, tmp);
3817 r = -EINVAL;
3818 }
3819 radeon_scratch_free(rdev, scratch);
3820 return r;
3821}
3822
780f5ddd
AD		 3823/**
3824 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3825 *
3826 * @rdev: radeon_device pointer
3827 * @ridx: radeon ring index
3828 *
3829 * Emits an hdp flush on the cp.
3830 */
3831static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3832 int ridx)
3833{
3834 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3835 u32 ref_and_mask;
780f5ddd 3836
5d259067
AD		 3837 switch (ring->idx) {
3838 case CAYMAN_RING_TYPE_CP1_INDEX:
3839 case CAYMAN_RING_TYPE_CP2_INDEX:
3840 default:
3841 switch (ring->me) {
3842 case 0:
3843 ref_and_mask = CP2 << ring->pipe;
3844 break;
3845 case 1:
3846 ref_and_mask = CP6 << ring->pipe;
3847 break;
3848 default:
3849 return;
3850 }
3851 break;
3852 case RADEON_RING_TYPE_GFX_INDEX:
3853 ref_and_mask = CP0;
3854 break;
3855 }
3856
3857 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3858 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3859 WAIT_REG_MEM_FUNCTION(3) | /* == */
3860 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3861 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3862 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3863 radeon_ring_write(ring, ref_and_mask);
3864 radeon_ring_write(ring, ref_and_mask);
3865 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3866}
3867
2cae3bc3 3868/**
b07fdd38 3869 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3870 *
3871 * @rdev: radeon_device pointer
3872 * @fence: radeon fence object
3873 *
3874 * Emits a fence sequnce number on the gfx ring and flushes
3875 * GPU caches.
3876 */
b07fdd38
AD		 3877void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3878 struct radeon_fence *fence)
2cae3bc3
AD		 3879{
3880 struct radeon_ring *ring = &rdev->ring[fence->ring];
3881 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3882
3883 /* EVENT_WRITE_EOP - flush caches, send int */
3884 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3885 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3886 EOP_TC_ACTION_EN |
3887 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3888 EVENT_INDEX(5)));
3889 radeon_ring_write(ring, addr & 0xfffffffc);
3890 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3891 radeon_ring_write(ring, fence->seq);
3892 radeon_ring_write(ring, 0);
3893 /* HDP flush */
780f5ddd 3894 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
2cae3bc3
AD		 3895}
3896
b07fdd38
AD		 3897/**
3898 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3899 *
3900 * @rdev: radeon_device pointer
3901 * @fence: radeon fence object
3902 *
3903 * Emits a fence sequnce number on the compute ring and flushes
3904 * GPU caches.
3905 */
3906void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3907 struct radeon_fence *fence)
3908{
3909 struct radeon_ring *ring = &rdev->ring[fence->ring];
3910 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3911
3912 /* RELEASE_MEM - flush caches, send int */
3913 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3914 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3915 EOP_TC_ACTION_EN |
3916 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3917 EVENT_INDEX(5)));
3918 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3919 radeon_ring_write(ring, addr & 0xfffffffc);
3920 radeon_ring_write(ring, upper_32_bits(addr));
3921 radeon_ring_write(ring, fence->seq);
3922 radeon_ring_write(ring, 0);
3923 /* HDP flush */
780f5ddd 3924 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
b07fdd38
AD		 3925}
3926
1654b817 3927bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3928 struct radeon_ring *ring,
3929 struct radeon_semaphore *semaphore,
3930 bool emit_wait)
3931{
3932 uint64_t addr = semaphore->gpu_addr;
3933 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3934
3935 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 3936 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 3937 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817
CK		 3938
3939 return true;
2cae3bc3
AD		 3940}
3941
c9dbd705
AD		 3942/**
3943 * cik_copy_cpdma - copy pages using the CP DMA engine
3944 *
3945 * @rdev: radeon_device pointer
3946 * @src_offset: src GPU address
3947 * @dst_offset: dst GPU address
3948 * @num_gpu_pages: number of GPU pages to xfer
3949 * @fence: radeon fence object
3950 *
3951 * Copy GPU paging using the CP DMA engine (CIK+).
3952 * Used by the radeon ttm implementation to move pages if
3953 * registered as the asic copy callback.
3954 */
3955int cik_copy_cpdma(struct radeon_device *rdev,
3956 uint64_t src_offset, uint64_t dst_offset,
3957 unsigned num_gpu_pages,
3958 struct radeon_fence **fence)
3959{
3960 struct radeon_semaphore *sem = NULL;
3961 int ring_index = rdev->asic->copy.blit_ring_index;
3962 struct radeon_ring *ring = &rdev->ring[ring_index];
3963 u32 size_in_bytes, cur_size_in_bytes, control;
3964 int i, num_loops;
3965 int r = 0;
3966
3967 r = radeon_semaphore_create(rdev, &sem);
3968 if (r) {
3969 DRM_ERROR("radeon: moving bo (%d).\n", r);
3970 return r;
3971 }
3972
3973 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3974 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3975 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3976 if (r) {
3977 DRM_ERROR("radeon: moving bo (%d).\n", r);
3978 radeon_semaphore_free(rdev, &sem, NULL);
3979 return r;
3980 }
3981
1654b817
CK		 3982 radeon_semaphore_sync_to(sem, *fence);
3983 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
c9dbd705
AD		 3984
3985 for (i = 0; i < num_loops; i++) {
3986 cur_size_in_bytes = size_in_bytes;
3987 if (cur_size_in_bytes > 0x1fffff)
3988 cur_size_in_bytes = 0x1fffff;
3989 size_in_bytes -= cur_size_in_bytes;
3990 control = 0;
3991 if (size_in_bytes == 0)
3992 control |= PACKET3_DMA_DATA_CP_SYNC;
3993 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3994 radeon_ring_write(ring, control);
3995 radeon_ring_write(ring, lower_32_bits(src_offset));
3996 radeon_ring_write(ring, upper_32_bits(src_offset));
3997 radeon_ring_write(ring, lower_32_bits(dst_offset));
3998 radeon_ring_write(ring, upper_32_bits(dst_offset));
3999 radeon_ring_write(ring, cur_size_in_bytes);
4000 src_offset += cur_size_in_bytes;
4001 dst_offset += cur_size_in_bytes;
4002 }
4003
4004 r = radeon_fence_emit(rdev, fence, ring->idx);
4005 if (r) {
4006 radeon_ring_unlock_undo(rdev, ring);
aa4c8b36 4007 radeon_semaphore_free(rdev, &sem, NULL);
c9dbd705
AD		 4008 return r;
4009 }
4010
4011 radeon_ring_unlock_commit(rdev, ring);
4012 radeon_semaphore_free(rdev, &sem, *fence);
4013
4014 return r;
4015}
4016
2cae3bc3
AD		 4017/*
4018 * IB stuff
4019 */
4020/**
4021 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4022 *
4023 * @rdev: radeon_device pointer
4024 * @ib: radeon indirect buffer object
4025 *
4026 * Emits an DE (drawing engine) or CE (constant engine) IB
4027 * on the gfx ring. IBs are usually generated by userspace
4028 * acceleration drivers and submitted to the kernel for
4029 * sheduling on the ring. This function schedules the IB
4030 * on the gfx ring for execution by the GPU.
4031 */
4032void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4033{
4034 struct radeon_ring *ring = &rdev->ring[ib->ring];
4035 u32 header, control = INDIRECT_BUFFER_VALID;
4036
4037 if (ib->is_const_ib) {
4038 /* set switch buffer packet before const IB */
4039 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4040 radeon_ring_write(ring, 0);
4041
4042 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4043 } else {
4044 u32 next_rptr;
4045 if (ring->rptr_save_reg) {
4046 next_rptr = ring->wptr + 3 + 4;
4047 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4048 radeon_ring_write(ring, ((ring->rptr_save_reg -
4049 PACKET3_SET_UCONFIG_REG_START) >> 2));
4050 radeon_ring_write(ring, next_rptr);
4051 } else if (rdev->wb.enabled) {
4052 next_rptr = ring->wptr + 5 + 4;
4053 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4054 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4055 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 4056 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 4057 radeon_ring_write(ring, next_rptr);
4058 }
4059
4060 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4061 }
4062
4063 control |= ib->length_dw |
4064 (ib->vm ? (ib->vm->id << 24) : 0);
4065
4066 radeon_ring_write(ring, header);
4067 radeon_ring_write(ring,
4068#ifdef __BIG_ENDIAN
4069 (2 << 0) |
4070#endif
4071 (ib->gpu_addr & 0xFFFFFFFC));
4072 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4073 radeon_ring_write(ring, control);
4074}
4075
fbc832c7
AD		 4076/**
4077 * cik_ib_test - basic gfx ring IB test
4078 *
4079 * @rdev: radeon_device pointer
4080 * @ring: radeon_ring structure holding ring information
4081 *
4082 * Allocate an IB and execute it on the gfx ring (CIK).
4083 * Provides a basic gfx ring test to verify that IBs are working.
4084 * Returns 0 on success, error on failure.
4085 */
4086int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4087{
4088 struct radeon_ib ib;
4089 uint32_t scratch;
4090 uint32_t tmp = 0;
4091 unsigned i;
4092 int r;
4093
4094 r = radeon_scratch_get(rdev, &scratch);
4095 if (r) {
4096 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4097 return r;
4098 }
4099 WREG32(scratch, 0xCAFEDEAD);
4100 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4101 if (r) {
4102 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 4103 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 4104 return r;
4105 }
4106 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4107 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4108 ib.ptr[2] = 0xDEADBEEF;
4109 ib.length_dw = 3;
4110 r = radeon_ib_schedule(rdev, &ib, NULL);
4111 if (r) {
4112 radeon_scratch_free(rdev, scratch);
4113 radeon_ib_free(rdev, &ib);
4114 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4115 return r;
4116 }
4117 r = radeon_fence_wait(ib.fence, false);
4118 if (r) {
4119 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 4120 radeon_scratch_free(rdev, scratch);
4121 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 4122 return r;
4123 }
4124 for (i = 0; i < rdev->usec_timeout; i++) {
4125 tmp = RREG32(scratch);
4126 if (tmp == 0xDEADBEEF)
4127 break;
4128 DRM_UDELAY(1);
4129 }
4130 if (i < rdev->usec_timeout) {
4131 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4132 } else {
4133 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4134 scratch, tmp);
4135 r = -EINVAL;
4136 }
4137 radeon_scratch_free(rdev, scratch);
4138 radeon_ib_free(rdev, &ib);
4139 return r;
4140}
4141
841cf442
AD		 4142/*
4143 * CP.
4144 * On CIK, gfx and compute now have independant command processors.
4145 *
4146 * GFX
4147 * Gfx consists of a single ring and can process both gfx jobs and
4148 * compute jobs. The gfx CP consists of three microengines (ME):
4149 * PFP - Pre-Fetch Parser
4150 * ME - Micro Engine
4151 * CE - Constant Engine
4152 * The PFP and ME make up what is considered the Drawing Engine (DE).
4153 * The CE is an asynchronous engine used for updating buffer desciptors
4154 * used by the DE so that they can be loaded into cache in parallel
4155 * while the DE is processing state update packets.
4156 *
4157 * Compute
4158 * The compute CP consists of two microengines (ME):
4159 * MEC1 - Compute MicroEngine 1
4160 * MEC2 - Compute MicroEngine 2
4161 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4162 * The queues are exposed to userspace and are programmed directly
4163 * by the compute runtime.
4164 */
4165/**
4166 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4167 *
4168 * @rdev: radeon_device pointer
4169 * @enable: enable or disable the MEs
4170 *
4171 * Halts or unhalts the gfx MEs.
4172 */
4173static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4174{
4175 if (enable)
4176 WREG32(CP_ME_CNTL, 0);
4177 else {
50efa51a
AD		 4178 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4179 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 4180 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4181 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4182 }
4183 udelay(50);
4184}
4185
4186/**
4187 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4188 *
4189 * @rdev: radeon_device pointer
4190 *
4191 * Loads the gfx PFP, ME, and CE ucode.
4192 * Returns 0 for success, -EINVAL if the ucode is not available.
4193 */
4194static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4195{
841cf442
AD		 4196 int i;
4197
4198 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4199 return -EINVAL;
4200
4201 cik_cp_gfx_enable(rdev, false);
4202
f2c6b0f4
AD		 4203 if (rdev->new_fw) {
4204 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4205 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4206 const struct gfx_firmware_header_v1_0 *ce_hdr =
4207 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4208 const struct gfx_firmware_header_v1_0 *me_hdr =
4209 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4210 const __le32 *fw_data;
4211 u32 fw_size;
4212
4213 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4214 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4215 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4216
4217 /* PFP */
4218 fw_data = (const __le32 *)
4219 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4220 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4221 WREG32(CP_PFP_UCODE_ADDR, 0);
4222 for (i = 0; i < fw_size; i++)
4223 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4224 WREG32(CP_PFP_UCODE_ADDR, 0);
4225
4226 /* CE */
4227 fw_data = (const __le32 *)
4228 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4229 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4230 WREG32(CP_CE_UCODE_ADDR, 0);
4231 for (i = 0; i < fw_size; i++)
4232 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4233 WREG32(CP_CE_UCODE_ADDR, 0);
4234
4235 /* ME */
4236 fw_data = (const __be32 *)
4237 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4238 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4239 WREG32(CP_ME_RAM_WADDR, 0);
4240 for (i = 0; i < fw_size; i++)
4241 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4242 WREG32(CP_ME_RAM_WADDR, 0);
4243 } else {
4244 const __be32 *fw_data;
4245
4246 /* PFP */
4247 fw_data = (const __be32 *)rdev->pfp_fw->data;
4248 WREG32(CP_PFP_UCODE_ADDR, 0);
4249 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4250 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4251 WREG32(CP_PFP_UCODE_ADDR, 0);
4252
4253 /* CE */
4254 fw_data = (const __be32 *)rdev->ce_fw->data;
4255 WREG32(CP_CE_UCODE_ADDR, 0);
4256 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4257 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4258 WREG32(CP_CE_UCODE_ADDR, 0);
4259
4260 /* ME */
4261 fw_data = (const __be32 *)rdev->me_fw->data;
4262 WREG32(CP_ME_RAM_WADDR, 0);
4263 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4264 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4265 WREG32(CP_ME_RAM_WADDR, 0);
4266 }
841cf442
AD		 4267
4268 WREG32(CP_PFP_UCODE_ADDR, 0);
4269 WREG32(CP_CE_UCODE_ADDR, 0);
4270 WREG32(CP_ME_RAM_WADDR, 0);
4271 WREG32(CP_ME_RAM_RADDR, 0);
4272 return 0;
4273}
4274
4275/**
4276 * cik_cp_gfx_start - start the gfx ring
4277 *
4278 * @rdev: radeon_device pointer
4279 *
4280 * Enables the ring and loads the clear state context and other
4281 * packets required to init the ring.
4282 * Returns 0 for success, error for failure.
4283 */
4284static int cik_cp_gfx_start(struct radeon_device *rdev)
4285{
4286 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4287 int r, i;
4288
4289 /* init the CP */
4290 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4291 WREG32(CP_ENDIAN_SWAP, 0);
4292 WREG32(CP_DEVICE_ID, 1);
4293
4294 cik_cp_gfx_enable(rdev, true);
4295
4296 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4297 if (r) {
4298 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4299 return r;
4300 }
4301
4302 /* init the CE partitions. CE only used for gfx on CIK */
4303 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4304 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4305 radeon_ring_write(ring, 0xc000);
4306 radeon_ring_write(ring, 0xc000);
4307
4308 /* setup clear context state */
4309 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4310 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4311
4312 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4313 radeon_ring_write(ring, 0x80000000);
4314 radeon_ring_write(ring, 0x80000000);
4315
4316 for (i = 0; i < cik_default_size; i++)
4317 radeon_ring_write(ring, cik_default_state[i]);
4318
4319 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4320 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4321
4322 /* set clear context state */
4323 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4324 radeon_ring_write(ring, 0);
4325
4326 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4327 radeon_ring_write(ring, 0x00000316);
4328 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4329 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4330
4331 radeon_ring_unlock_commit(rdev, ring);
4332
4333 return 0;
4334}
4335
4336/**
4337 * cik_cp_gfx_fini - stop the gfx ring
4338 *
4339 * @rdev: radeon_device pointer
4340 *
4341 * Stop the gfx ring and tear down the driver ring
4342 * info.
4343 */
4344static void cik_cp_gfx_fini(struct radeon_device *rdev)
4345{
4346 cik_cp_gfx_enable(rdev, false);
4347 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4348}
4349
4350/**
4351 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4352 *
4353 * @rdev: radeon_device pointer
4354 *
4355 * Program the location and size of the gfx ring buffer
4356 * and test it to make sure it's working.
4357 * Returns 0 for success, error for failure.
4358 */
4359static int cik_cp_gfx_resume(struct radeon_device *rdev)
4360{
4361 struct radeon_ring *ring;
4362 u32 tmp;
4363 u32 rb_bufsz;
4364 u64 rb_addr;
4365 int r;
4366
4367 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4368 if (rdev->family != CHIP_HAWAII)
4369 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4370
4371 /* Set the write pointer delay */
4372 WREG32(CP_RB_WPTR_DELAY, 0);
4373
4374 /* set the RB to use vmid 0 */
4375 WREG32(CP_RB_VMID, 0);
4376
4377 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4378
4379 /* ring 0 - compute and gfx */
4380 /* Set ring buffer size */
4381 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4382 rb_bufsz = order_base_2(ring->ring_size / 8);
4383 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4384#ifdef __BIG_ENDIAN
4385 tmp |= BUF_SWAP_32BIT;
4386#endif
4387 WREG32(CP_RB0_CNTL, tmp);
4388
4389 /* Initialize the ring buffer's read and write pointers */
4390 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4391 ring->wptr = 0;
4392 WREG32(CP_RB0_WPTR, ring->wptr);
4393
4394 /* set the wb address wether it's enabled or not */
4395 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4396 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4397
4398 /* scratch register shadowing is no longer supported */
4399 WREG32(SCRATCH_UMSK, 0);
4400
4401 if (!rdev->wb.enabled)
4402 tmp |= RB_NO_UPDATE;
4403
4404 mdelay(1);
4405 WREG32(CP_RB0_CNTL, tmp);
4406
4407 rb_addr = ring->gpu_addr >> 8;
4408 WREG32(CP_RB0_BASE, rb_addr);
4409 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4410
841cf442
AD		 4411 /* start the ring */
4412 cik_cp_gfx_start(rdev);
4413 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4414 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4415 if (r) {
4416 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4417 return r;
4418 }
50efa51a
AD		 4419
4420 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4421 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4422
841cf442
AD		 4423 return 0;
4424}
4425
ea31bf69
AD		 4426u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4427 struct radeon_ring *ring)
963e81f9
AD		 4428{
4429 u32 rptr;
4430
ea31bf69
AD		 4431 if (rdev->wb.enabled)
4432 rptr = rdev->wb.wb[ring->rptr_offs/4];
4433 else
4434 rptr = RREG32(CP_RB0_RPTR);
4435
4436 return rptr;
4437}
4438
4439u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4440 struct radeon_ring *ring)
4441{
4442 u32 wptr;
4443
4444 wptr = RREG32(CP_RB0_WPTR);
963e81f9 4445
ea31bf69
AD		 4446 return wptr;
4447}
4448
4449void cik_gfx_set_wptr(struct radeon_device *rdev,
4450 struct radeon_ring *ring)
4451{
4452 WREG32(CP_RB0_WPTR, ring->wptr);
4453 (void)RREG32(CP_RB0_WPTR);
4454}
4455
4456u32 cik_compute_get_rptr(struct radeon_device *rdev,
4457 struct radeon_ring *ring)
4458{
4459 u32 rptr;
963e81f9
AD		 4460
4461 if (rdev->wb.enabled) {
ea31bf69 4462 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4463 } else {
f61d5b46 4464 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4465 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4466 rptr = RREG32(CP_HQD_PQ_RPTR);
4467 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4468 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4469 }
963e81f9
AD		 4470
4471 return rptr;
4472}
4473
ea31bf69
AD		 4474u32 cik_compute_get_wptr(struct radeon_device *rdev,
4475 struct radeon_ring *ring)
963e81f9
AD		 4476{
4477 u32 wptr;
4478
4479 if (rdev->wb.enabled) {
ea31bf69
AD		 4480 /* XXX check if swapping is necessary on BE */
4481 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4482 } else {
f61d5b46 4483 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4484 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4485 wptr = RREG32(CP_HQD_PQ_WPTR);
4486 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4487 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4488 }
963e81f9
AD		 4489
4490 return wptr;
4491}
4492
ea31bf69
AD		 4493void cik_compute_set_wptr(struct radeon_device *rdev,
4494 struct radeon_ring *ring)
963e81f9 4495{
ea31bf69
AD		 4496 /* XXX check if swapping is necessary on BE */
4497 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4498 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4499}
4500
841cf442
AD		 4501/**
4502 * cik_cp_compute_enable - enable/disable the compute CP MEs
4503 *
4504 * @rdev: radeon_device pointer
4505 * @enable: enable or disable the MEs
4506 *
4507 * Halts or unhalts the compute MEs.
4508 */
4509static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4510{
4511 if (enable)
4512 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4513 else {
841cf442 4514 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4515 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4516 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4517 }
841cf442
AD		 4518 udelay(50);
4519}
4520
4521/**
4522 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4523 *
4524 * @rdev: radeon_device pointer
4525 *
4526 * Loads the compute MEC1&2 ucode.
4527 * Returns 0 for success, -EINVAL if the ucode is not available.
4528 */
4529static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4530{
841cf442
AD		 4531 int i;
4532
4533 if (!rdev->mec_fw)
4534 return -EINVAL;
4535
4536 cik_cp_compute_enable(rdev, false);
4537
f2c6b0f4
AD		 4538 if (rdev->new_fw) {
4539 const struct gfx_firmware_header_v1_0 *mec_hdr =
4540 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4541 const __le32 *fw_data;
4542 u32 fw_size;
4543
4544 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4545
4546 /* MEC1 */
4547 fw_data = (const __le32 *)
4548 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4549 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4550 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4551 for (i = 0; i < fw_size; i++)
4552 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4553 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4554
841cf442 4555 /* MEC2 */
f2c6b0f4
AD		 4556 if (rdev->family == CHIP_KAVERI) {
4557 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4558 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4559
4560 fw_data = (const __le32 *)
4561 (rdev->mec2_fw->data +
4562 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4563 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4564 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4565 for (i = 0; i < fw_size; i++)
4566 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4567 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4568 }
4569 } else {
4570 const __be32 *fw_data;
4571
4572 /* MEC1 */
841cf442 4573 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4574 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4575 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4576 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4577 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4578
4579 if (rdev->family == CHIP_KAVERI) {
4580 /* MEC2 */
4581 fw_data = (const __be32 *)rdev->mec_fw->data;
4582 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4583 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4584 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4585 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4586 }
841cf442
AD		 4587 }
4588
4589 return 0;
4590}
4591
4592/**
4593 * cik_cp_compute_start - start the compute queues
4594 *
4595 * @rdev: radeon_device pointer
4596 *
4597 * Enable the compute queues.
4598 * Returns 0 for success, error for failure.
4599 */
4600static int cik_cp_compute_start(struct radeon_device *rdev)
4601{
963e81f9
AD		 4602 cik_cp_compute_enable(rdev, true);
4603
841cf442
AD		 4604 return 0;
4605}
4606
4607/**
4608 * cik_cp_compute_fini - stop the compute queues
4609 *
4610 * @rdev: radeon_device pointer
4611 *
4612 * Stop the compute queues and tear down the driver queue
4613 * info.
4614 */
4615static void cik_cp_compute_fini(struct radeon_device *rdev)
4616{
963e81f9
AD		 4617 int i, idx, r;
4618
841cf442 4619 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4620
4621 for (i = 0; i < 2; i++) {
4622 if (i == 0)
4623 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4624 else
4625 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4626
4627 if (rdev->ring[idx].mqd_obj) {
4628 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4629 if (unlikely(r != 0))
4630 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4631
4632 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4633 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4634
4635 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4636 rdev->ring[idx].mqd_obj = NULL;
4637 }
4638 }
841cf442
AD		 4639}
4640
963e81f9
AD		 4641static void cik_mec_fini(struct radeon_device *rdev)
4642{
4643 int r;
4644
4645 if (rdev->mec.hpd_eop_obj) {
4646 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4647 if (unlikely(r != 0))
4648 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4649 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4650 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4651
4652 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4653 rdev->mec.hpd_eop_obj = NULL;
4654 }
4655}
4656
4657#define MEC_HPD_SIZE 2048
4658
4659static int cik_mec_init(struct radeon_device *rdev)
4660{
4661 int r;
4662 u32 *hpd;
4663
4664 /*
4665 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4666 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4667 */
4668 if (rdev->family == CHIP_KAVERI)
4669 rdev->mec.num_mec = 2;
4670 else
4671 rdev->mec.num_mec = 1;
4672 rdev->mec.num_pipe = 4;
4673 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4674
4675 if (rdev->mec.hpd_eop_obj == NULL) {
4676 r = radeon_bo_create(rdev,
4677 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4678 PAGE_SIZE, true,
02376d82 4679 RADEON_GEM_DOMAIN_GTT, 0, NULL,
963e81f9
AD		 4680 &rdev->mec.hpd_eop_obj);
4681 if (r) {
4682 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4683 return r;
4684 }
4685 }
4686
4687 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4688 if (unlikely(r != 0)) {
4689 cik_mec_fini(rdev);
4690 return r;
4691 }
4692 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4693 &rdev->mec.hpd_eop_gpu_addr);
4694 if (r) {
4695 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4696 cik_mec_fini(rdev);
4697 return r;
4698 }
4699 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4700 if (r) {
4701 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4702 cik_mec_fini(rdev);
4703 return r;
4704 }
4705
4706 /* clear memory. Not sure if this is required or not */
4707 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4708
4709 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4710 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4711
4712 return 0;
4713}
4714
4715struct hqd_registers
4716{
4717 u32 cp_mqd_base_addr;
4718 u32 cp_mqd_base_addr_hi;
4719 u32 cp_hqd_active;
4720 u32 cp_hqd_vmid;
4721 u32 cp_hqd_persistent_state;
4722 u32 cp_hqd_pipe_priority;
4723 u32 cp_hqd_queue_priority;
4724 u32 cp_hqd_quantum;
4725 u32 cp_hqd_pq_base;
4726 u32 cp_hqd_pq_base_hi;
4727 u32 cp_hqd_pq_rptr;
4728 u32 cp_hqd_pq_rptr_report_addr;
4729 u32 cp_hqd_pq_rptr_report_addr_hi;
4730 u32 cp_hqd_pq_wptr_poll_addr;
4731 u32 cp_hqd_pq_wptr_poll_addr_hi;
4732 u32 cp_hqd_pq_doorbell_control;
4733 u32 cp_hqd_pq_wptr;
4734 u32 cp_hqd_pq_control;
4735 u32 cp_hqd_ib_base_addr;
4736 u32 cp_hqd_ib_base_addr_hi;
4737 u32 cp_hqd_ib_rptr;
4738 u32 cp_hqd_ib_control;
4739 u32 cp_hqd_iq_timer;
4740 u32 cp_hqd_iq_rptr;
4741 u32 cp_hqd_dequeue_request;
4742 u32 cp_hqd_dma_offload;
4743 u32 cp_hqd_sema_cmd;
4744 u32 cp_hqd_msg_type;
4745 u32 cp_hqd_atomic0_preop_lo;
4746 u32 cp_hqd_atomic0_preop_hi;
4747 u32 cp_hqd_atomic1_preop_lo;
4748 u32 cp_hqd_atomic1_preop_hi;
4749 u32 cp_hqd_hq_scheduler0;
4750 u32 cp_hqd_hq_scheduler1;
4751 u32 cp_mqd_control;
4752};
4753
4754struct bonaire_mqd
4755{
4756 u32 header;
4757 u32 dispatch_initiator;
4758 u32 dimensions[3];
4759 u32 start_idx[3];
4760 u32 num_threads[3];
4761 u32 pipeline_stat_enable;
4762 u32 perf_counter_enable;
4763 u32 pgm[2];
4764 u32 tba[2];
4765 u32 tma[2];
4766 u32 pgm_rsrc[2];
4767 u32 vmid;
4768 u32 resource_limits;
4769 u32 static_thread_mgmt01[2];
4770 u32 tmp_ring_size;
4771 u32 static_thread_mgmt23[2];
4772 u32 restart[3];
4773 u32 thread_trace_enable;
4774 u32 reserved1;
4775 u32 user_data[16];
4776 u32 vgtcs_invoke_count[2];
4777 struct hqd_registers queue_state;
4778 u32 dequeue_cntr;
4779 u32 interrupt_queue[64];
4780};
4781
841cf442
AD		 4782/**
4783 * cik_cp_compute_resume - setup the compute queue registers
4784 *
4785 * @rdev: radeon_device pointer
4786 *
4787 * Program the compute queues and test them to make sure they
4788 * are working.
4789 * Returns 0 for success, error for failure.
4790 */
4791static int cik_cp_compute_resume(struct radeon_device *rdev)
4792{
963e81f9
AD		 4793 int r, i, idx;
4794 u32 tmp;
4795 bool use_doorbell = true;
4796 u64 hqd_gpu_addr;
4797 u64 mqd_gpu_addr;
4798 u64 eop_gpu_addr;
4799 u64 wb_gpu_addr;
4800 u32 *buf;
4801 struct bonaire_mqd *mqd;
841cf442 4802
841cf442
AD		 4803 r = cik_cp_compute_start(rdev);
4804 if (r)
4805 return r;
963e81f9
AD		 4806
4807 /* fix up chicken bits */
4808 tmp = RREG32(CP_CPF_DEBUG);
4809 tmp |= (1 << 23);
4810 WREG32(CP_CPF_DEBUG, tmp);
4811
4812 /* init the pipes */
f61d5b46 4813 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4814 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4815 int me = (i < 4) ? 1 : 2;
4816 int pipe = (i < 4) ? i : (i - 4);
4817
4818 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4819
4820 cik_srbm_select(rdev, me, pipe, 0, 0);
4821
4822 /* write the EOP addr */
4823 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4824 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4825
4826 /* set the VMID assigned */
4827 WREG32(CP_HPD_EOP_VMID, 0);
4828
4829 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4830 tmp = RREG32(CP_HPD_EOP_CONTROL);
4831 tmp &= ~EOP_SIZE_MASK;
b72a8925 4832 tmp |= order_base_2(MEC_HPD_SIZE / 8);
963e81f9
AD		 4833 WREG32(CP_HPD_EOP_CONTROL, tmp);
4834 }
4835 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4836 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4837
4838 /* init the queues. Just two for now. */
4839 for (i = 0; i < 2; i++) {
4840 if (i == 0)
4841 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4842 else
4843 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4844
4845 if (rdev->ring[idx].mqd_obj == NULL) {
4846 r = radeon_bo_create(rdev,
4847 sizeof(struct bonaire_mqd),
4848 PAGE_SIZE, true,
02376d82 4849 RADEON_GEM_DOMAIN_GTT, 0, NULL,
963e81f9
AD		 4850 &rdev->ring[idx].mqd_obj);
4851 if (r) {
4852 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4853 return r;
4854 }
4855 }
4856
4857 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4858 if (unlikely(r != 0)) {
4859 cik_cp_compute_fini(rdev);
4860 return r;
4861 }
4862 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4863 &mqd_gpu_addr);
4864 if (r) {
4865 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4866 cik_cp_compute_fini(rdev);
4867 return r;
4868 }
4869 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4870 if (r) {
4871 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4872 cik_cp_compute_fini(rdev);
4873 return r;
4874 }
4875
963e81f9
AD		 4876 /* init the mqd struct */
4877 memset(buf, 0, sizeof(struct bonaire_mqd));
4878
4879 mqd = (struct bonaire_mqd *)buf;
4880 mqd->header = 0xC0310800;
4881 mqd->static_thread_mgmt01[0] = 0xffffffff;
4882 mqd->static_thread_mgmt01[1] = 0xffffffff;
4883 mqd->static_thread_mgmt23[0] = 0xffffffff;
4884 mqd->static_thread_mgmt23[1] = 0xffffffff;
4885
f61d5b46 4886 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4887 cik_srbm_select(rdev, rdev->ring[idx].me,
4888 rdev->ring[idx].pipe,
4889 rdev->ring[idx].queue, 0);
4890
4891 /* disable wptr polling */
4892 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4893 tmp &= ~WPTR_POLL_EN;
4894 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4895
4896 /* enable doorbell? */
4897 mqd->queue_state.cp_hqd_pq_doorbell_control =
4898 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4899 if (use_doorbell)
4900 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4901 else
4902 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4903 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4904 mqd->queue_state.cp_hqd_pq_doorbell_control);
4905
4906 /* disable the queue if it's active */
4907 mqd->queue_state.cp_hqd_dequeue_request = 0;
4908 mqd->queue_state.cp_hqd_pq_rptr = 0;
4909 mqd->queue_state.cp_hqd_pq_wptr= 0;
4910 if (RREG32(CP_HQD_ACTIVE) & 1) {
4911 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4912 for (i = 0; i < rdev->usec_timeout; i++) {
4913 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4914 break;
4915 udelay(1);
4916 }
4917 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4918 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4919 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4920 }
4921
4922 /* set the pointer to the MQD */
4923 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4924 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4925 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4926 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4927 /* set MQD vmid to 0 */
4928 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4929 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4930 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4931
4932 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4933 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4934 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4935 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4936 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4937 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4938
4939 /* set up the HQD, this is similar to CP_RB0_CNTL */
4940 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4941 mqd->queue_state.cp_hqd_pq_control &=
4942 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4943
4944 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4945 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4946 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4947 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4948#ifdef __BIG_ENDIAN
4949 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4950#endif
4951 mqd->queue_state.cp_hqd_pq_control &=
4952 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4953 mqd->queue_state.cp_hqd_pq_control |=
4954 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4955 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4956
4957 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4958 if (i == 0)
4959 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4960 else
4961 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4962 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4963 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4964 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4965 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4966 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4967
4968 /* set the wb address wether it's enabled or not */
4969 if (i == 0)
4970 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4971 else
4972 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4973 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4974 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4975 upper_32_bits(wb_gpu_addr) & 0xffff;
4976 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4977 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4978 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4979 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4980
4981 /* enable the doorbell if requested */
4982 if (use_doorbell) {
4983 mqd->queue_state.cp_hqd_pq_doorbell_control =
4984 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4985 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4986 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 4987 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 4988 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4989 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4990 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4991
4992 } else {
4993 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4994 }
4995 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4996 mqd->queue_state.cp_hqd_pq_doorbell_control);
4997
4998 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4999 rdev->ring[idx].wptr = 0;
5000 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5001 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 5002 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 5003
5004 /* set the vmid for the queue */
5005 mqd->queue_state.cp_hqd_vmid = 0;
5006 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5007
5008 /* activate the queue */
5009 mqd->queue_state.cp_hqd_active = 1;
5010 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5011
5012 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5013 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 5014
5015 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5016 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5017
5018 rdev->ring[idx].ready = true;
5019 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5020 if (r)
5021 rdev->ring[idx].ready = false;
5022 }
5023
841cf442
AD		 5024 return 0;
5025}
5026
841cf442
AD		 5027static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5028{
5029 cik_cp_gfx_enable(rdev, enable);
5030 cik_cp_compute_enable(rdev, enable);
5031}
5032
841cf442
AD		 5033static int cik_cp_load_microcode(struct radeon_device *rdev)
5034{
5035 int r;
5036
5037 r = cik_cp_gfx_load_microcode(rdev);
5038 if (r)
5039 return r;
5040 r = cik_cp_compute_load_microcode(rdev);
5041 if (r)
5042 return r;
5043
5044 return 0;
5045}
5046
841cf442
AD		 5047static void cik_cp_fini(struct radeon_device *rdev)
5048{
5049 cik_cp_gfx_fini(rdev);
5050 cik_cp_compute_fini(rdev);
5051}
5052
841cf442
AD		 5053static int cik_cp_resume(struct radeon_device *rdev)
5054{
5055 int r;
5056
4214faf6
AD		 5057 cik_enable_gui_idle_interrupt(rdev, false);
5058
841cf442
AD		 5059 r = cik_cp_load_microcode(rdev);
5060 if (r)
5061 return r;
5062
5063 r = cik_cp_gfx_resume(rdev);
5064 if (r)
5065 return r;
5066 r = cik_cp_compute_resume(rdev);
5067 if (r)
5068 return r;
5069
4214faf6
AD		 5070 cik_enable_gui_idle_interrupt(rdev, true);
5071
841cf442
AD		 5072 return 0;
5073}
5074
cc066715 5075static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 5076{
6f2043ce
AD		 5077 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5078 RREG32(GRBM_STATUS));
5079 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5080 RREG32(GRBM_STATUS2));
5081 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5082 RREG32(GRBM_STATUS_SE0));
5083 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5084 RREG32(GRBM_STATUS_SE1));
5085 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5086 RREG32(GRBM_STATUS_SE2));
5087 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5088 RREG32(GRBM_STATUS_SE3));
5089 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5090 RREG32(SRBM_STATUS));
5091 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5092 RREG32(SRBM_STATUS2));
cc066715
AD		 5093 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5094 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5095 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5096 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 5097 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5098 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5099 RREG32(CP_STALLED_STAT1));
5100 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5101 RREG32(CP_STALLED_STAT2));
5102 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5103 RREG32(CP_STALLED_STAT3));
5104 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5105 RREG32(CP_CPF_BUSY_STAT));
5106 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5107 RREG32(CP_CPF_STALLED_STAT1));
5108 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5109 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5110 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5111 RREG32(CP_CPC_STALLED_STAT1));
5112 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 5113}
6f2043ce 5114
21a93e13 5115/**
cc066715 5116 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 5117 *
5118 * @rdev: radeon_device pointer
21a93e13 5119 *
cc066715
AD		 5120 * Check which blocks are busy and return the relevant reset
5121 * mask to be used by cik_gpu_soft_reset().
5122 * Returns a mask of the blocks to be reset.
21a93e13 5123 */
2483b4ea 5124u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 5125{
cc066715
AD		 5126 u32 reset_mask = 0;
5127 u32 tmp;
21a93e13 5128
cc066715
AD		 5129 /* GRBM_STATUS */
5130 tmp = RREG32(GRBM_STATUS);
5131 if (tmp & (PA_BUSY | SC_BUSY |
5132 BCI_BUSY | SX_BUSY |
5133 TA_BUSY | VGT_BUSY |
5134 DB_BUSY | CB_BUSY |
5135 GDS_BUSY | SPI_BUSY |
5136 IA_BUSY | IA_BUSY_NO_DMA))
5137 reset_mask |= RADEON_RESET_GFX;
21a93e13 5138
cc066715
AD		 5139 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5140 reset_mask |= RADEON_RESET_CP;
21a93e13 5141
cc066715
AD		 5142 /* GRBM_STATUS2 */
5143 tmp = RREG32(GRBM_STATUS2);
5144 if (tmp & RLC_BUSY)
5145 reset_mask |= RADEON_RESET_RLC;
21a93e13 5146
cc066715
AD		 5147 /* SDMA0_STATUS_REG */
5148 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5149 if (!(tmp & SDMA_IDLE))
5150 reset_mask |= RADEON_RESET_DMA;
21a93e13 5151
cc066715
AD		 5152 /* SDMA1_STATUS_REG */
5153 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5154 if (!(tmp & SDMA_IDLE))
5155 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5156
cc066715
AD		 5157 /* SRBM_STATUS2 */
5158 tmp = RREG32(SRBM_STATUS2);
5159 if (tmp & SDMA_BUSY)
5160 reset_mask |= RADEON_RESET_DMA;
21a93e13 5161
cc066715
AD		 5162 if (tmp & SDMA1_BUSY)
5163 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5164
cc066715
AD		 5165 /* SRBM_STATUS */
5166 tmp = RREG32(SRBM_STATUS);
21a93e13 5167
cc066715
AD		 5168 if (tmp & IH_BUSY)
5169 reset_mask |= RADEON_RESET_IH;
21a93e13 5170
cc066715
AD		 5171 if (tmp & SEM_BUSY)
5172 reset_mask |= RADEON_RESET_SEM;
21a93e13 5173
cc066715
AD		 5174 if (tmp & GRBM_RQ_PENDING)
5175 reset_mask |= RADEON_RESET_GRBM;
21a93e13 5176
cc066715
AD		 5177 if (tmp & VMC_BUSY)
5178 reset_mask |= RADEON_RESET_VMC;
21a93e13 5179
cc066715
AD		 5180 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5181 MCC_BUSY | MCD_BUSY))
5182 reset_mask |= RADEON_RESET_MC;
21a93e13 5183
cc066715
AD		 5184 if (evergreen_is_display_hung(rdev))
5185 reset_mask |= RADEON_RESET_DISPLAY;
5186
5187 /* Skip MC reset as it's mostly likely not hung, just busy */
5188 if (reset_mask & RADEON_RESET_MC) {
5189 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5190 reset_mask &= ~RADEON_RESET_MC;
21a93e13 5191 }
cc066715
AD		 5192
5193 return reset_mask;
21a93e13
AD		 5194}
5195
5196/**
cc066715 5197 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 5198 *
5199 * @rdev: radeon_device pointer
cc066715 5200 * @reset_mask: mask of which blocks to reset
21a93e13 5201 *
cc066715 5202 * Soft reset the blocks specified in @reset_mask.
21a93e13 5203 */
cc066715 5204static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 5205{
6f2043ce 5206 struct evergreen_mc_save save;
cc066715
AD		 5207 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5208 u32 tmp;
21a93e13 5209
cc066715
AD		 5210 if (reset_mask == 0)
5211 return;
21a93e13 5212
cc066715 5213 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 5214
cc066715
AD		 5215 cik_print_gpu_status_regs(rdev);
5216 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5217 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5218 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5219 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 5220
fb2c7f4d
AD		 5221 /* disable CG/PG */
5222 cik_fini_pg(rdev);
5223 cik_fini_cg(rdev);
5224
cc066715
AD		 5225 /* stop the rlc */
5226 cik_rlc_stop(rdev);
21a93e13 5227
cc066715
AD		 5228 /* Disable GFX parsing/prefetching */
5229 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 5230
cc066715
AD		 5231 /* Disable MEC parsing/prefetching */
5232 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 5233
cc066715
AD		 5234 if (reset_mask & RADEON_RESET_DMA) {
5235 /* sdma0 */
5236 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5237 tmp |= SDMA_HALT;
5238 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5239 }
5240 if (reset_mask & RADEON_RESET_DMA1) {
5241 /* sdma1 */
5242 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5243 tmp |= SDMA_HALT;
5244 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5245 }
21a93e13 5246
6f2043ce 5247 evergreen_mc_stop(rdev, &save);
cc066715 5248 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 5249 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5250 }
21a93e13 5251
cc066715
AD		 5252 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5253 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 5254
cc066715
AD		 5255 if (reset_mask & RADEON_RESET_CP) {
5256 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 5257
cc066715
AD		 5258 srbm_soft_reset |= SOFT_RESET_GRBM;
5259 }
21a93e13 5260
cc066715
AD		 5261 if (reset_mask & RADEON_RESET_DMA)
5262 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5263
cc066715
AD		 5264 if (reset_mask & RADEON_RESET_DMA1)
5265 srbm_soft_reset |= SOFT_RESET_SDMA1;
5266
5267 if (reset_mask & RADEON_RESET_DISPLAY)
5268 srbm_soft_reset |= SOFT_RESET_DC;
5269
5270 if (reset_mask & RADEON_RESET_RLC)
5271 grbm_soft_reset |= SOFT_RESET_RLC;
5272
5273 if (reset_mask & RADEON_RESET_SEM)
5274 srbm_soft_reset |= SOFT_RESET_SEM;
5275
5276 if (reset_mask & RADEON_RESET_IH)
5277 srbm_soft_reset |= SOFT_RESET_IH;
5278
5279 if (reset_mask & RADEON_RESET_GRBM)
5280 srbm_soft_reset |= SOFT_RESET_GRBM;
5281
5282 if (reset_mask & RADEON_RESET_VMC)
5283 srbm_soft_reset |= SOFT_RESET_VMC;
5284
5285 if (!(rdev->flags & RADEON_IS_IGP)) {
5286 if (reset_mask & RADEON_RESET_MC)
5287 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5288 }
5289
cc066715
AD		 5290 if (grbm_soft_reset) {
5291 tmp = RREG32(GRBM_SOFT_RESET);
5292 tmp |= grbm_soft_reset;
5293 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5294 WREG32(GRBM_SOFT_RESET, tmp);
5295 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5296
cc066715 5297 udelay(50);
21a93e13 5298
cc066715
AD		 5299 tmp &= ~grbm_soft_reset;
5300 WREG32(GRBM_SOFT_RESET, tmp);
5301 tmp = RREG32(GRBM_SOFT_RESET);
5302 }
21a93e13 5303
cc066715
AD		 5304 if (srbm_soft_reset) {
5305 tmp = RREG32(SRBM_SOFT_RESET);
5306 tmp |= srbm_soft_reset;
5307 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5308 WREG32(SRBM_SOFT_RESET, tmp);
5309 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5310
cc066715 5311 udelay(50);
21a93e13 5312
cc066715
AD		 5313 tmp &= ~srbm_soft_reset;
5314 WREG32(SRBM_SOFT_RESET, tmp);
5315 tmp = RREG32(SRBM_SOFT_RESET);
5316 }
21a93e13 5317
6f2043ce
AD		 5318 /* Wait a little for things to settle down */
5319 udelay(50);
21a93e13 5320
6f2043ce 5321 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5322 udelay(50);
5323
5324 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5325}
5326
0279ed19
AD		 5327struct kv_reset_save_regs {
5328 u32 gmcon_reng_execute;
5329 u32 gmcon_misc;
5330 u32 gmcon_misc3;
5331};
5332
5333static void kv_save_regs_for_reset(struct radeon_device *rdev,
5334 struct kv_reset_save_regs *save)
5335{
5336 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5337 save->gmcon_misc = RREG32(GMCON_MISC);
5338 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5339
5340 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5341 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5342 STCTRL_STUTTER_EN));
5343}
5344
5345static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5346 struct kv_reset_save_regs *save)
5347{
5348 int i;
5349
5350 WREG32(GMCON_PGFSM_WRITE, 0);
5351 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5352
5353 for (i = 0; i < 5; i++)
5354 WREG32(GMCON_PGFSM_WRITE, 0);
5355
5356 WREG32(GMCON_PGFSM_WRITE, 0);
5357 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5358
5359 for (i = 0; i < 5; i++)
5360 WREG32(GMCON_PGFSM_WRITE, 0);
5361
5362 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5363 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5364
5365 for (i = 0; i < 5; i++)
5366 WREG32(GMCON_PGFSM_WRITE, 0);
5367
5368 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5369 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5370
5371 for (i = 0; i < 5; i++)
5372 WREG32(GMCON_PGFSM_WRITE, 0);
5373
5374 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5375 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5376
5377 for (i = 0; i < 5; i++)
5378 WREG32(GMCON_PGFSM_WRITE, 0);
5379
5380 WREG32(GMCON_PGFSM_WRITE, 0);
5381 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5382
5383 for (i = 0; i < 5; i++)
5384 WREG32(GMCON_PGFSM_WRITE, 0);
5385
5386 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5387 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5388
5389 for (i = 0; i < 5; i++)
5390 WREG32(GMCON_PGFSM_WRITE, 0);
5391
5392 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5393 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5394
5395 for (i = 0; i < 5; i++)
5396 WREG32(GMCON_PGFSM_WRITE, 0);
5397
5398 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5399 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5400
5401 for (i = 0; i < 5; i++)
5402 WREG32(GMCON_PGFSM_WRITE, 0);
5403
5404 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5405 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5406
5407 for (i = 0; i < 5; i++)
5408 WREG32(GMCON_PGFSM_WRITE, 0);
5409
5410 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5411 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5412
5413 WREG32(GMCON_MISC3, save->gmcon_misc3);
5414 WREG32(GMCON_MISC, save->gmcon_misc);
5415 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5416}
5417
5418static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5419{
5420 struct evergreen_mc_save save;
5421 struct kv_reset_save_regs kv_save = { 0 };
5422 u32 tmp, i;
5423
5424 dev_info(rdev->dev, "GPU pci config reset\n");
5425
5426 /* disable dpm? */
5427
5428 /* disable cg/pg */
5429 cik_fini_pg(rdev);
5430 cik_fini_cg(rdev);
5431
5432 /* Disable GFX parsing/prefetching */
5433 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5434
5435 /* Disable MEC parsing/prefetching */
5436 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5437
5438 /* sdma0 */
5439 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5440 tmp |= SDMA_HALT;
5441 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5442 /* sdma1 */
5443 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5444 tmp |= SDMA_HALT;
5445 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5446 /* XXX other engines? */
5447
5448 /* halt the rlc, disable cp internal ints */
5449 cik_rlc_stop(rdev);
5450
5451 udelay(50);
5452
5453 /* disable mem access */
5454 evergreen_mc_stop(rdev, &save);
5455 if (evergreen_mc_wait_for_idle(rdev)) {
5456 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5457 }
5458
5459 if (rdev->flags & RADEON_IS_IGP)
5460 kv_save_regs_for_reset(rdev, &kv_save);
5461
5462 /* disable BM */
5463 pci_clear_master(rdev->pdev);
5464 /* reset */
5465 radeon_pci_config_reset(rdev);
5466
5467 udelay(100);
5468
5469 /* wait for asic to come out of reset */
5470 for (i = 0; i < rdev->usec_timeout; i++) {
5471 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5472 break;
5473 udelay(1);
5474 }
5475
5476 /* does asic init need to be run first??? */
5477 if (rdev->flags & RADEON_IS_IGP)
5478 kv_restore_regs_for_reset(rdev, &kv_save);
5479}
5480
21a93e13 5481/**
cc066715 5482 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5483 *
5484 * @rdev: radeon_device pointer
5485 *
cc066715
AD		 5486 * Look up which blocks are hung and attempt
5487 * to reset them.
6f2043ce 5488 * Returns 0 for success.
21a93e13 5489 */
6f2043ce 5490int cik_asic_reset(struct radeon_device *rdev)
21a93e13 5491{
cc066715 5492 u32 reset_mask;
21a93e13 5493
cc066715 5494 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5495
cc066715
AD		 5496 if (reset_mask)
5497 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5498
0279ed19 5499 /* try soft reset */
cc066715 5500 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5501
cc066715
AD		 5502 reset_mask = cik_gpu_check_soft_reset(rdev);
5503
0279ed19
AD		 5504 /* try pci config reset */
5505 if (reset_mask && radeon_hard_reset)
5506 cik_gpu_pci_config_reset(rdev);
5507
5508 reset_mask = cik_gpu_check_soft_reset(rdev);
5509
cc066715
AD		 5510 if (!reset_mask)
5511 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5512
5513 return 0;
5514}
5515
5516/**
cc066715 5517 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5518 *
5519 * @rdev: radeon_device pointer
cc066715 5520 * @ring: radeon_ring structure holding ring information
21a93e13 5521 *
cc066715
AD		 5522 * Check if the 3D engine is locked up (CIK).
5523 * Returns true if the engine is locked, false if not.
21a93e13 5524 */
cc066715 5525bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5526{
cc066715 5527 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5528
cc066715
AD		 5529 if (!(reset_mask & (RADEON_RESET_GFX |
5530 RADEON_RESET_COMPUTE |
5531 RADEON_RESET_CP))) {
ff212f25 5532 radeon_ring_lockup_update(rdev, ring);
cc066715 5533 return false;
21a93e13 5534 }
cc066715 5535 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5536}
5537
1c49165d 5538/* MC */
21a93e13 5539/**
1c49165d 5540 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5541 *
5542 * @rdev: radeon_device pointer
21a93e13 5543 *
1c49165d
AD		 5544 * Set the location of vram, gart, and AGP in the GPU's
5545 * physical address space (CIK).
21a93e13 5546 */
1c49165d 5547static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5548{
1c49165d 5549 struct evergreen_mc_save save;
21a93e13 5550 u32 tmp;
1c49165d 5551 int i, j;
21a93e13 5552
1c49165d
AD		 5553 /* Initialize HDP */
5554 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5555 WREG32((0x2c14 + j), 0x00000000);
5556 WREG32((0x2c18 + j), 0x00000000);
5557 WREG32((0x2c1c + j), 0x00000000);
5558 WREG32((0x2c20 + j), 0x00000000);
5559 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5560 }
1c49165d 5561 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5562
1c49165d
AD		 5563 evergreen_mc_stop(rdev, &save);
5564 if (radeon_mc_wait_for_idle(rdev)) {
5565 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5566 }
1c49165d
AD		 5567 /* Lockout access through VGA aperture*/
5568 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5569 /* Update configuration */
5570 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5571 rdev->mc.vram_start >> 12);
5572 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5573 rdev->mc.vram_end >> 12);
5574 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5575 rdev->vram_scratch.gpu_addr >> 12);
5576 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5577 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5578 WREG32(MC_VM_FB_LOCATION, tmp);
5579 /* XXX double check these! */
5580 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5581 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5582 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5583 WREG32(MC_VM_AGP_BASE, 0);
5584 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5585 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5586 if (radeon_mc_wait_for_idle(rdev)) {
5587 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5588 }
1c49165d
AD		 5589 evergreen_mc_resume(rdev, &save);
5590 /* we need to own VRAM, so turn off the VGA renderer here
5591 * to stop it overwriting our objects */
5592 rv515_vga_render_disable(rdev);
21a93e13
AD		 5593}
5594
5595/**
1c49165d 5596 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5597 *
5598 * @rdev: radeon_device pointer
21a93e13 5599 *
1c49165d
AD		 5600 * Look up the amount of vram, vram width, and decide how to place
5601 * vram and gart within the GPU's physical address space (CIK).
5602 * Returns 0 for success.
21a93e13 5603 */
1c49165d 5604static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5605{
1c49165d
AD		 5606 u32 tmp;
5607 int chansize, numchan;
21a93e13 5608
1c49165d
AD		 5609 /* Get VRAM informations */
5610 rdev->mc.vram_is_ddr = true;
5611 tmp = RREG32(MC_ARB_RAMCFG);
5612 if (tmp & CHANSIZE_MASK) {
5613 chansize = 64;
21a93e13 5614 } else {
1c49165d 5615 chansize = 32;
21a93e13 5616 }
1c49165d
AD		 5617 tmp = RREG32(MC_SHARED_CHMAP);
5618 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5619 case 0:
5620 default:
5621 numchan = 1;
5622 break;
5623 case 1:
5624 numchan = 2;
5625 break;
5626 case 2:
5627 numchan = 4;
5628 break;
5629 case 3:
5630 numchan = 8;
5631 break;
5632 case 4:
5633 numchan = 3;
5634 break;
5635 case 5:
5636 numchan = 6;
5637 break;
5638 case 6:
5639 numchan = 10;
5640 break;
5641 case 7:
5642 numchan = 12;
5643 break;
5644 case 8:
5645 numchan = 16;
5646 break;
5647 }
5648 rdev->mc.vram_width = numchan * chansize;
5649 /* Could aper size report 0 ? */
5650 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5651 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5652 /* size in MB on si */
13c5bfda
AD		 5653 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5654 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5655 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5656 si_vram_gtt_location(rdev, &rdev->mc);
5657 radeon_update_bandwidth_info(rdev);
5658
5659 return 0;
5660}
5661
5662/*
5663 * GART
5664 * VMID 0 is the physical GPU addresses as used by the kernel.
5665 * VMIDs 1-15 are used for userspace clients and are handled
5666 * by the radeon vm/hsa code.
5667 */
5668/**
5669 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5670 *
5671 * @rdev: radeon_device pointer
5672 *
5673 * Flush the TLB for the VMID 0 page table (CIK).
5674 */
5675void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5676{
5677 /* flush hdp cache */
5678 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5679
5680 /* bits 0-15 are the VM contexts0-15 */
5681 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5682}
5683
5684/**
5685 * cik_pcie_gart_enable - gart enable
5686 *
5687 * @rdev: radeon_device pointer
5688 *
5689 * This sets up the TLBs, programs the page tables for VMID0,
5690 * sets up the hw for VMIDs 1-15 which are allocated on
5691 * demand, and sets up the global locations for the LDS, GDS,
5692 * and GPUVM for FSA64 clients (CIK).
5693 * Returns 0 for success, errors for failure.
5694 */
5695static int cik_pcie_gart_enable(struct radeon_device *rdev)
5696{
5697 int r, i;
5698
5699 if (rdev->gart.robj == NULL) {
5700 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5701 return -EINVAL;
5702 }
5703 r = radeon_gart_table_vram_pin(rdev);
5704 if (r)
5705 return r;
1c49165d
AD		 5706 /* Setup TLB control */
5707 WREG32(MC_VM_MX_L1_TLB_CNTL,
5708 (0xA << 7) |
5709 ENABLE_L1_TLB |
ec3dbbcb 5710 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5711 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5712 ENABLE_ADVANCED_DRIVER_MODEL |
5713 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5714 /* Setup L2 cache */
5715 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5716 ENABLE_L2_FRAGMENT_PROCESSING |
5717 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5718 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5719 EFFECTIVE_L2_QUEUE_SIZE(7) |
5720 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5721 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5722 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5723 BANK_SELECT(4) |
5724 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5725 /* setup context0 */
5726 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5727 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5728 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5729 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5730 (u32)(rdev->dummy_page.addr >> 12));
5731 WREG32(VM_CONTEXT0_CNTL2, 0);
5732 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5733 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5734
5735 WREG32(0x15D4, 0);
5736 WREG32(0x15D8, 0);
5737 WREG32(0x15DC, 0);
5738
5739 /* empty context1-15 */
5740 /* FIXME start with 4G, once using 2 level pt switch to full
5741 * vm size space
5742 */
5743 /* set vm size, must be a multiple of 4 */
5744 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5745 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5746 for (i = 1; i < 16; i++) {
5747 if (i < 8)
5748 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5749 rdev->gart.table_addr >> 12);
5750 else
5751 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5752 rdev->gart.table_addr >> 12);
5753 }
5754
5755 /* enable context1-15 */
5756 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5757 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5758 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5759 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5760 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5761 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5762 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5763 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5764 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5765 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5766 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5767 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5768 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5769 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5770 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5771 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5772 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5773
1c49165d
AD		 5774 if (rdev->family == CHIP_KAVERI) {
5775 u32 tmp = RREG32(CHUB_CONTROL);
5776 tmp &= ~BYPASS_VM;
5777 WREG32(CHUB_CONTROL, tmp);
5778 }
5779
5780 /* XXX SH_MEM regs */
5781 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5782 mutex_lock(&rdev->srbm_mutex);
1c49165d 5783 for (i = 0; i < 16; i++) {
b556b12e 5784 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5785 /* CP and shaders */
1c49165d
AD		 5786 WREG32(SH_MEM_CONFIG, 0);
5787 WREG32(SH_MEM_APE1_BASE, 1);
5788 WREG32(SH_MEM_APE1_LIMIT, 0);
5789 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5790 /* SDMA GFX */
5791 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5792 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5793 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5794 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5795 /* XXX SDMA RLC - todo */
1c49165d 5796 }
b556b12e 5797 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5798 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 5799
5800 cik_pcie_gart_tlb_flush(rdev);
5801 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5802 (unsigned)(rdev->mc.gtt_size >> 20),
5803 (unsigned long long)rdev->gart.table_addr);
5804 rdev->gart.ready = true;
5805 return 0;
5806}
5807
5808/**
5809 * cik_pcie_gart_disable - gart disable
5810 *
5811 * @rdev: radeon_device pointer
5812 *
5813 * This disables all VM page table (CIK).
5814 */
5815static void cik_pcie_gart_disable(struct radeon_device *rdev)
5816{
5817 /* Disable all tables */
5818 WREG32(VM_CONTEXT0_CNTL, 0);
5819 WREG32(VM_CONTEXT1_CNTL, 0);
5820 /* Setup TLB control */
5821 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5822 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5823 /* Setup L2 cache */
5824 WREG32(VM_L2_CNTL,
5825 ENABLE_L2_FRAGMENT_PROCESSING |
5826 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5827 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5828 EFFECTIVE_L2_QUEUE_SIZE(7) |
5829 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5830 WREG32(VM_L2_CNTL2, 0);
5831 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5832 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5833 radeon_gart_table_vram_unpin(rdev);
5834}
5835
5836/**
5837 * cik_pcie_gart_fini - vm fini callback
5838 *
5839 * @rdev: radeon_device pointer
5840 *
5841 * Tears down the driver GART/VM setup (CIK).
5842 */
5843static void cik_pcie_gart_fini(struct radeon_device *rdev)
5844{
5845 cik_pcie_gart_disable(rdev);
5846 radeon_gart_table_vram_free(rdev);
5847 radeon_gart_fini(rdev);
5848}
5849
5850/* vm parser */
5851/**
5852 * cik_ib_parse - vm ib_parse callback
5853 *
5854 * @rdev: radeon_device pointer
5855 * @ib: indirect buffer pointer
5856 *
5857 * CIK uses hw IB checking so this is a nop (CIK).
5858 */
5859int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5860{
5861 return 0;
5862}
5863
5864/*
5865 * vm
5866 * VMID 0 is the physical GPU addresses as used by the kernel.
5867 * VMIDs 1-15 are used for userspace clients and are handled
5868 * by the radeon vm/hsa code.
5869 */
5870/**
5871 * cik_vm_init - cik vm init callback
5872 *
5873 * @rdev: radeon_device pointer
5874 *
5875 * Inits cik specific vm parameters (number of VMs, base of vram for
5876 * VMIDs 1-15) (CIK).
5877 * Returns 0 for success.
5878 */
5879int cik_vm_init(struct radeon_device *rdev)
5880{
5881 /* number of VMs */
5882 rdev->vm_manager.nvm = 16;
5883 /* base offset of vram pages */
5884 if (rdev->flags & RADEON_IS_IGP) {
5885 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5886 tmp <<= 22;
5887 rdev->vm_manager.vram_base_offset = tmp;
5888 } else
5889 rdev->vm_manager.vram_base_offset = 0;
5890
5891 return 0;
5892}
5893
5894/**
5895 * cik_vm_fini - cik vm fini callback
5896 *
5897 * @rdev: radeon_device pointer
5898 *
5899 * Tear down any asic specific VM setup (CIK).
5900 */
5901void cik_vm_fini(struct radeon_device *rdev)
5902{
5903}
5904
3ec7d11b
AD		 5905/**
5906 * cik_vm_decode_fault - print human readable fault info
5907 *
5908 * @rdev: radeon_device pointer
5909 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5910 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5911 *
5912 * Print human readable fault information (CIK).
5913 */
5914static void cik_vm_decode_fault(struct radeon_device *rdev,
5915 u32 status, u32 addr, u32 mc_client)
5916{
939c0d3c 5917 u32 mc_id;
3ec7d11b
AD		 5918 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5919 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5920 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5921 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5922
939c0d3c
AD		 5923 if (rdev->family == CHIP_HAWAII)
5924 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5925 else
5926 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5927
328a50c7 5928 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5929 protections, vmid, addr,
5930 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5931 block, mc_client, mc_id);
3ec7d11b
AD		 5932}
5933
f96ab484
AD		 5934/**
5935 * cik_vm_flush - cik vm flush using the CP
5936 *
5937 * @rdev: radeon_device pointer
5938 *
5939 * Update the page table base and flush the VM TLB
5940 * using the CP (CIK).
5941 */
5942void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5943{
5944 struct radeon_ring *ring = &rdev->ring[ridx];
5945
5946 if (vm == NULL)
5947 return;
5948
5949 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5950 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5951 WRITE_DATA_DST_SEL(0)));
5952 if (vm->id < 8) {
5953 radeon_ring_write(ring,
5954 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5955 } else {
5956 radeon_ring_write(ring,
5957 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5958 }
5959 radeon_ring_write(ring, 0);
5960 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5961
5962 /* update SH_MEM_* regs */
5963 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5964 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5965 WRITE_DATA_DST_SEL(0)));
5966 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5967 radeon_ring_write(ring, 0);
5968 radeon_ring_write(ring, VMID(vm->id));
5969
5970 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5971 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5972 WRITE_DATA_DST_SEL(0)));
5973 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5974 radeon_ring_write(ring, 0);
5975
5976 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5977 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5978 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5979 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5980
f96ab484
AD		 5981 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5982 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5983 WRITE_DATA_DST_SEL(0)));
5984 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5985 radeon_ring_write(ring, 0);
5986 radeon_ring_write(ring, VMID(0));
6f2043ce 5987
f96ab484 5988 /* HDP flush */
780f5ddd 5989 cik_hdp_flush_cp_ring_emit(rdev, ridx);
f96ab484
AD		 5990
5991 /* bits 0-15 are the VM contexts0-15 */
5992 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5993 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5994 WRITE_DATA_DST_SEL(0)));
5995 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5996 radeon_ring_write(ring, 0);
5997 radeon_ring_write(ring, 1 << vm->id);
5998
b07fdd38
AD		 5999 /* compute doesn't have PFP */
6000 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
6001 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6002 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6003 radeon_ring_write(ring, 0x0);
6004 }
cc066715 6005}
6f2043ce 6006
f6796cae
AD		 6007/*
6008 * RLC
6009 * The RLC is a multi-purpose microengine that handles a
6010 * variety of functions, the most important of which is
6011 * the interrupt controller.
6012 */
866d83de
AD		 6013static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6014 bool enable)
f6796cae 6015{
866d83de 6016 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 6017
866d83de
AD		 6018 if (enable)
6019 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6020 else
6021 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 6022 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 6023}
f6796cae 6024
866d83de 6025static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 6026{
cc066715 6027 u32 tmp;
6f2043ce 6028
866d83de
AD		 6029 tmp = RREG32(RLC_LB_CNTL);
6030 if (enable)
6031 tmp |= LOAD_BALANCE_ENABLE;
6032 else
6033 tmp &= ~LOAD_BALANCE_ENABLE;
6034 WREG32(RLC_LB_CNTL, tmp);
6035}
cc066715 6036
866d83de
AD		 6037static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6038{
6039 u32 i, j, k;
6040 u32 mask;
cc066715 6041
f6796cae
AD		 6042 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6043 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6044 cik_select_se_sh(rdev, i, j);
6045 for (k = 0; k < rdev->usec_timeout; k++) {
6046 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6047 break;
6048 udelay(1);
6049 }
6050 }
6051 }
6052 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 6053
f6796cae
AD		 6054 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6055 for (k = 0; k < rdev->usec_timeout; k++) {
6056 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6057 break;
6058 udelay(1);
6059 }
6060}
cc066715 6061
22c775ce
AD		 6062static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6063{
6064 u32 tmp;
cc066715 6065
22c775ce
AD		 6066 tmp = RREG32(RLC_CNTL);
6067 if (tmp != rlc)
6068 WREG32(RLC_CNTL, rlc);
6069}
cc066715 6070
22c775ce
AD		 6071static u32 cik_halt_rlc(struct radeon_device *rdev)
6072{
6073 u32 data, orig;
cc066715 6074
22c775ce 6075 orig = data = RREG32(RLC_CNTL);
cc066715 6076
22c775ce
AD		 6077 if (data & RLC_ENABLE) {
6078 u32 i;
cc066715 6079
22c775ce
AD		 6080 data &= ~RLC_ENABLE;
6081 WREG32(RLC_CNTL, data);
cc066715 6082
22c775ce
AD		 6083 for (i = 0; i < rdev->usec_timeout; i++) {
6084 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6085 break;
6086 udelay(1);
6087 }
cc066715 6088
22c775ce
AD		 6089 cik_wait_for_rlc_serdes(rdev);
6090 }
cc066715 6091
22c775ce
AD		 6092 return orig;
6093}
cc066715 6094
a412fce0
AD		 6095void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6096{
6097 u32 tmp, i, mask;
6098
6099 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6100 WREG32(RLC_GPR_REG2, tmp);
6101
6102 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6103 for (i = 0; i < rdev->usec_timeout; i++) {
6104 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6105 break;
6106 udelay(1);
6107 }
6108
6109 for (i = 0; i < rdev->usec_timeout; i++) {
6110 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6111 break;
6112 udelay(1);
6113 }
6114}
6115
6116void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6117{
6118 u32 tmp;
6119
6120 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6121 WREG32(RLC_GPR_REG2, tmp);
6122}
6123
866d83de
AD		 6124/**
6125 * cik_rlc_stop - stop the RLC ME
6126 *
6127 * @rdev: radeon_device pointer
6128 *
6129 * Halt the RLC ME (MicroEngine) (CIK).
6130 */
6131static void cik_rlc_stop(struct radeon_device *rdev)
6132{
22c775ce 6133 WREG32(RLC_CNTL, 0);
866d83de
AD		 6134
6135 cik_enable_gui_idle_interrupt(rdev, false);
6136
866d83de
AD		 6137 cik_wait_for_rlc_serdes(rdev);
6138}
6139
f6796cae
AD		 6140/**
6141 * cik_rlc_start - start the RLC ME
6142 *
6143 * @rdev: radeon_device pointer
6144 *
6145 * Unhalt the RLC ME (MicroEngine) (CIK).
6146 */
6147static void cik_rlc_start(struct radeon_device *rdev)
6148{
f6796cae 6149 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 6150
866d83de 6151 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6152
f6796cae 6153 udelay(50);
6f2043ce
AD		 6154}
6155
6156/**
f6796cae 6157 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 6158 *
6159 * @rdev: radeon_device pointer
6160 *
f6796cae
AD		 6161 * Initialize the RLC registers, load the ucode,
6162 * and start the RLC (CIK).
6163 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 6164 */
f6796cae 6165static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 6166{
22c775ce 6167 u32 i, size, tmp;
cc066715 6168
f6796cae
AD		 6169 if (!rdev->rlc_fw)
6170 return -EINVAL;
cc066715 6171
cc066715
AD		 6172 cik_rlc_stop(rdev);
6173
22c775ce
AD		 6174 /* disable CG */
6175 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6176 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 6177
866d83de 6178 si_rlc_reset(rdev);
6f2043ce 6179
22c775ce 6180 cik_init_pg(rdev);
6f2043ce 6181
22c775ce 6182 cik_init_cg(rdev);
cc066715 6183
f6796cae
AD		 6184 WREG32(RLC_LB_CNTR_INIT, 0);
6185 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 6186
f6796cae
AD		 6187 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6188 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6189 WREG32(RLC_LB_PARAMS, 0x00600408);
6190 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 6191
f6796cae
AD		 6192 WREG32(RLC_MC_CNTL, 0);
6193 WREG32(RLC_UCODE_CNTL, 0);
cc066715 6194
f2c6b0f4
AD		 6195 if (rdev->new_fw) {
6196 const struct rlc_firmware_header_v1_0 *hdr =
6197 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6198 const __le32 *fw_data = (const __le32 *)
6199 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6200
6201 radeon_ucode_print_rlc_hdr(&hdr->header);
6202
6203 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6204 WREG32(RLC_GPM_UCODE_ADDR, 0);
6205 for (i = 0; i < size; i++)
6206 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
f6796cae 6207 WREG32(RLC_GPM_UCODE_ADDR, 0);
f2c6b0f4
AD		 6208 } else {
6209 const __be32 *fw_data;
6210
6211 switch (rdev->family) {
6212 case CHIP_BONAIRE:
6213 case CHIP_HAWAII:
6214 default:
6215 size = BONAIRE_RLC_UCODE_SIZE;
6216 break;
6217 case CHIP_KAVERI:
6218 size = KV_RLC_UCODE_SIZE;
6219 break;
6220 case CHIP_KABINI:
6221 size = KB_RLC_UCODE_SIZE;
6222 break;
6223 case CHIP_MULLINS:
6224 size = ML_RLC_UCODE_SIZE;
6225 break;
6226 }
6227
6228 fw_data = (const __be32 *)rdev->rlc_fw->data;
6229 WREG32(RLC_GPM_UCODE_ADDR, 0);
6230 for (i = 0; i < size; i++)
6231 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6232 WREG32(RLC_GPM_UCODE_ADDR, 0);
6233 }
cc066715 6234
866d83de
AD		 6235 /* XXX - find out what chips support lbpw */
6236 cik_enable_lbpw(rdev, false);
cc066715 6237
22c775ce
AD		 6238 if (rdev->family == CHIP_BONAIRE)
6239 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6240
f6796cae 6241 cik_rlc_start(rdev);
cc066715 6242
f6796cae
AD		 6243 return 0;
6244}
cc066715 6245
22c775ce
AD		 6246static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6247{
6248 u32 data, orig, tmp, tmp2;
cc066715 6249
22c775ce 6250 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6251
473359bc 6252 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6253 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6254
22c775ce 6255 tmp = cik_halt_rlc(rdev);
cc066715 6256
22c775ce
AD		 6257 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6258 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6259 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6260 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6261 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 6262
22c775ce 6263 cik_update_rlc(rdev, tmp);
cc066715 6264
22c775ce
AD		 6265 data |= CGCG_EN | CGLS_EN;
6266 } else {
ddc76ff6 6267 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6268
22c775ce
AD		 6269 RREG32(CB_CGTT_SCLK_CTRL);
6270 RREG32(CB_CGTT_SCLK_CTRL);
6271 RREG32(CB_CGTT_SCLK_CTRL);
6272 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6273
22c775ce 6274 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6275 }
6f2043ce 6276
22c775ce
AD		 6277 if (orig != data)
6278 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6279
6f2043ce
AD		 6280}
6281
22c775ce 6282static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6283{
22c775ce
AD		 6284 u32 data, orig, tmp = 0;
6285
473359bc
AD		 6286 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6287 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6288 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6289 orig = data = RREG32(CP_MEM_SLP_CNTL);
6290 data |= CP_MEM_LS_EN;
6291 if (orig != data)
6292 WREG32(CP_MEM_SLP_CNTL, data);
6293 }
6294 }
cc066715 6295
22c775ce
AD		 6296 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6297 data &= 0xfffffffd;
6298 if (orig != data)
6299 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6300
6301 tmp = cik_halt_rlc(rdev);
6302
6303 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6304 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6305 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6306 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6307 WREG32(RLC_SERDES_WR_CTRL, data);
6308
6309 cik_update_rlc(rdev, tmp);
6310
473359bc
AD		 6311 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6312 orig = data = RREG32(CGTS_SM_CTRL_REG);
6313 data &= ~SM_MODE_MASK;
6314 data |= SM_MODE(0x2);
6315 data |= SM_MODE_ENABLE;
6316 data &= ~CGTS_OVERRIDE;
6317 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6318 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6319 data &= ~CGTS_LS_OVERRIDE;
6320 data &= ~ON_MONITOR_ADD_MASK;
6321 data |= ON_MONITOR_ADD_EN;
6322 data |= ON_MONITOR_ADD(0x96);
6323 if (orig != data)
6324 WREG32(CGTS_SM_CTRL_REG, data);
6325 }
22c775ce
AD		 6326 } else {
6327 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6328 data |= 0x00000002;
6329 if (orig != data)
6330 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6331
6332 data = RREG32(RLC_MEM_SLP_CNTL);
6333 if (data & RLC_MEM_LS_EN) {
6334 data &= ~RLC_MEM_LS_EN;
6335 WREG32(RLC_MEM_SLP_CNTL, data);
6336 }
6f2043ce 6337
22c775ce
AD		 6338 data = RREG32(CP_MEM_SLP_CNTL);
6339 if (data & CP_MEM_LS_EN) {
6340 data &= ~CP_MEM_LS_EN;
6341 WREG32(CP_MEM_SLP_CNTL, data);
6342 }
cc066715 6343
22c775ce
AD		 6344 orig = data = RREG32(CGTS_SM_CTRL_REG);
6345 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6346 if (orig != data)
6347 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6348
22c775ce 6349 tmp = cik_halt_rlc(rdev);
cc066715 6350
22c775ce
AD		 6351 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6352 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6353 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6354 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6355 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 6356
22c775ce 6357 cik_update_rlc(rdev, tmp);
cc066715 6358 }
6f2043ce 6359}
1c49165d 6360
22c775ce 6361static const u32 mc_cg_registers[] =
21a93e13 6362{
22c775ce
AD		 6363 MC_HUB_MISC_HUB_CG,
6364 MC_HUB_MISC_SIP_CG,
6365 MC_HUB_MISC_VM_CG,
6366 MC_XPB_CLK_GAT,
6367 ATC_MISC_CG,
6368 MC_CITF_MISC_WR_CG,
6369 MC_CITF_MISC_RD_CG,
6370 MC_CITF_MISC_VM_CG,
6371 VM_L2_CG,
6372};
21a93e13 6373
22c775ce
AD		 6374static void cik_enable_mc_ls(struct radeon_device *rdev,
6375 bool enable)
1c49165d 6376{
22c775ce
AD		 6377 int i;
6378 u32 orig, data;
1c49165d 6379
22c775ce
AD		 6380 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6381 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6382 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6383 data |= MC_LS_ENABLE;
6384 else
6385 data &= ~MC_LS_ENABLE;
6386 if (data != orig)
6387 WREG32(mc_cg_registers[i], data);
1c49165d 6388 }
22c775ce 6389}
1c49165d 6390
22c775ce
AD		 6391static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6392 bool enable)
6393{
6394 int i;
6395 u32 orig, data;
6396
6397 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6398 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6399 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6400 data |= MC_CG_ENABLE;
6401 else
6402 data &= ~MC_CG_ENABLE;
6403 if (data != orig)
6404 WREG32(mc_cg_registers[i], data);
1c49165d 6405 }
1c49165d
AD		 6406}
6407
22c775ce
AD		 6408static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6409 bool enable)
1c49165d 6410{
22c775ce 6411 u32 orig, data;
1c49165d 6412
473359bc 6413 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6414 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6415 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6416 } else {
22c775ce
AD		 6417 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6418 data |= 0xff000000;
6419 if (data != orig)
6420 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6421
22c775ce
AD		 6422 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6423 data |= 0xff000000;
6424 if (data != orig)
6425 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6426 }
1c49165d
AD		 6427}
6428
22c775ce
AD		 6429static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6430 bool enable)
1c49165d 6431{
22c775ce
AD		 6432 u32 orig, data;
6433
473359bc 6434 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6435 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6436 data |= 0x100;
6437 if (orig != data)
6438 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6439
6440 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6441 data |= 0x100;
6442 if (orig != data)
6443 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6444 } else {
6445 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6446 data &= ~0x100;
6447 if (orig != data)
6448 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6449
22c775ce
AD		 6450 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6451 data &= ~0x100;
6452 if (orig != data)
6453 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6454 }
1c49165d
AD		 6455}
6456
22c775ce
AD		 6457static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6458 bool enable)
1c49165d 6459{
22c775ce 6460 u32 orig, data;
1c49165d 6461
473359bc 6462 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6463 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6464 data = 0xfff;
6465 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6466
22c775ce
AD		 6467 orig = data = RREG32(UVD_CGC_CTRL);
6468 data |= DCM;
6469 if (orig != data)
6470 WREG32(UVD_CGC_CTRL, data);
6471 } else {
6472 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6473 data &= ~0xfff;
6474 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6475
22c775ce
AD		 6476 orig = data = RREG32(UVD_CGC_CTRL);
6477 data &= ~DCM;
6478 if (orig != data)
6479 WREG32(UVD_CGC_CTRL, data);
1c49165d 6480 }
22c775ce 6481}
1c49165d 6482
473359bc
AD		 6483static void cik_enable_bif_mgls(struct radeon_device *rdev,
6484 bool enable)
6485{
6486 u32 orig, data;
1c49165d 6487
473359bc 6488 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6489
473359bc
AD		 6490 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6491 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6492 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6493 else
6494 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6495 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6496
473359bc
AD		 6497 if (orig != data)
6498 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6499}
1c49165d 6500
22c775ce
AD		 6501static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6502 bool enable)
6503{
6504 u32 orig, data;
1c49165d 6505
22c775ce 6506 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6507
473359bc 6508 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6509 data &= ~CLOCK_GATING_DIS;
6510 else
6511 data |= CLOCK_GATING_DIS;
6512
6513 if (orig != data)
6514 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6515}
6516
22c775ce
AD		 6517static void cik_enable_hdp_ls(struct radeon_device *rdev,
6518 bool enable)
1c49165d 6519{
22c775ce
AD		 6520 u32 orig, data;
6521
6522 orig = data = RREG32(HDP_MEM_POWER_LS);
6523
473359bc 6524 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6525 data |= HDP_LS_ENABLE;
6526 else
6527 data &= ~HDP_LS_ENABLE;
6528
6529 if (orig != data)
6530 WREG32(HDP_MEM_POWER_LS, data);
6531}
6532
6533void cik_update_cg(struct radeon_device *rdev,
6534 u32 block, bool enable)
6535{
4214faf6 6536
22c775ce 6537 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6538 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6539 /* order matters! */
6540 if (enable) {
6541 cik_enable_mgcg(rdev, true);
6542 cik_enable_cgcg(rdev, true);
6543 } else {
6544 cik_enable_cgcg(rdev, false);
6545 cik_enable_mgcg(rdev, false);
6546 }
4214faf6 6547 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6548 }
6549
6550 if (block & RADEON_CG_BLOCK_MC) {
6551 if (!(rdev->flags & RADEON_IS_IGP)) {
6552 cik_enable_mc_mgcg(rdev, enable);
6553 cik_enable_mc_ls(rdev, enable);
6554 }
6555 }
6556
6557 if (block & RADEON_CG_BLOCK_SDMA) {
6558 cik_enable_sdma_mgcg(rdev, enable);
6559 cik_enable_sdma_mgls(rdev, enable);
6560 }
6561
473359bc
AD		 6562 if (block & RADEON_CG_BLOCK_BIF) {
6563 cik_enable_bif_mgls(rdev, enable);
6564 }
6565
22c775ce
AD		 6566 if (block & RADEON_CG_BLOCK_UVD) {
6567 if (rdev->has_uvd)
6568 cik_enable_uvd_mgcg(rdev, enable);
6569 }
6570
6571 if (block & RADEON_CG_BLOCK_HDP) {
6572 cik_enable_hdp_mgcg(rdev, enable);
6573 cik_enable_hdp_ls(rdev, enable);
6574 }
a1d6f97c
AD		 6575
6576 if (block & RADEON_CG_BLOCK_VCE) {
6577 vce_v2_0_enable_mgcg(rdev, enable);
6578 }
1c49165d
AD		 6579}
6580
22c775ce 6581static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6582{
22c775ce 6583
ddc76ff6 6584 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6585
6586 if (rdev->has_uvd)
6587 si_init_uvd_internal_cg(rdev);
6588
6589 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6590 RADEON_CG_BLOCK_SDMA |
473359bc 6591 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6592 RADEON_CG_BLOCK_UVD |
6593 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6594}
6595
473359bc 6596static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6597{
473359bc
AD		 6598 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6599 RADEON_CG_BLOCK_SDMA |
6600 RADEON_CG_BLOCK_BIF |
6601 RADEON_CG_BLOCK_UVD |
6602 RADEON_CG_BLOCK_HDP), false);
6603
6604 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6605}
6606
22c775ce
AD		 6607static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6608 bool enable)
1c49165d 6609{
22c775ce 6610 u32 data, orig;
1c49165d 6611
22c775ce 6612 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6613 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6614 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6615 else
6616 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6617 if (orig != data)
6618 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6619}
6620
22c775ce
AD		 6621static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6622 bool enable)
1c49165d 6623{
22c775ce
AD		 6624 u32 data, orig;
6625
6626 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6627 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6628 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6629 else
6630 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6631 if (orig != data)
6632 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6633}
6634
22c775ce 6635static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6636{
22c775ce 6637 u32 data, orig;
3ec7d11b 6638
22c775ce 6639 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6640 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6641 data &= ~DISABLE_CP_PG;
6642 else
6643 data |= DISABLE_CP_PG;
6644 if (orig != data)
6645 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6646}
6647
22c775ce 6648static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6649{
22c775ce 6650 u32 data, orig;
f96ab484 6651
22c775ce 6652 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6653 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6654 data &= ~DISABLE_GDS_PG;
6655 else
6656 data |= DISABLE_GDS_PG;
6657 if (orig != data)
6658 WREG32(RLC_PG_CNTL, data);
6659}
6660
6661#define CP_ME_TABLE_SIZE 96
6662#define CP_ME_TABLE_OFFSET 2048
6663#define CP_MEC_TABLE_OFFSET 4096
6664
6665void cik_init_cp_pg_table(struct radeon_device *rdev)
6666{
22c775ce
AD		 6667 volatile u32 *dst_ptr;
6668 int me, i, max_me = 4;
6669 u32 bo_offset = 0;
f2c6b0f4 6670 u32 table_offset, table_size;
22c775ce
AD		 6671
6672 if (rdev->family == CHIP_KAVERI)
6673 max_me = 5;
6674
6675 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6676 return;
6677
22c775ce
AD		 6678 /* write the cp table buffer */
6679 dst_ptr = rdev->rlc.cp_table_ptr;
6680 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6681 if (rdev->new_fw) {
6682 const __le32 *fw_data;
6683 const struct gfx_firmware_header_v1_0 *hdr;
6684
6685 if (me == 0) {
6686 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6687 fw_data = (const __le32 *)
6688 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6689 table_offset = le32_to_cpu(hdr->jt_offset);
6690 table_size = le32_to_cpu(hdr->jt_size);
6691 } else if (me == 1) {
6692 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6693 fw_data = (const __le32 *)
6694 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6695 table_offset = le32_to_cpu(hdr->jt_offset);
6696 table_size = le32_to_cpu(hdr->jt_size);
6697 } else if (me == 2) {
6698 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6699 fw_data = (const __le32 *)
6700 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6701 table_offset = le32_to_cpu(hdr->jt_offset);
6702 table_size = le32_to_cpu(hdr->jt_size);
6703 } else if (me == 3) {
6704 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6705 fw_data = (const __le32 *)
6706 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6707 table_offset = le32_to_cpu(hdr->jt_offset);
6708 table_size = le32_to_cpu(hdr->jt_size);
6709 } else {
6710 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6711 fw_data = (const __le32 *)
6712 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6713 table_offset = le32_to_cpu(hdr->jt_offset);
6714 table_size = le32_to_cpu(hdr->jt_size);
6715 }
6716
6717 for (i = 0; i < table_size; i ++) {
6718 dst_ptr[bo_offset + i] =
6719 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6720 }
6721 bo_offset += table_size;
22c775ce 6722 } else {
f2c6b0f4
AD		 6723 const __be32 *fw_data;
6724 table_size = CP_ME_TABLE_SIZE;
6725
6726 if (me == 0) {
6727 fw_data = (const __be32 *)rdev->ce_fw->data;
6728 table_offset = CP_ME_TABLE_OFFSET;
6729 } else if (me == 1) {
6730 fw_data = (const __be32 *)rdev->pfp_fw->data;
6731 table_offset = CP_ME_TABLE_OFFSET;
6732 } else if (me == 2) {
6733 fw_data = (const __be32 *)rdev->me_fw->data;
6734 table_offset = CP_ME_TABLE_OFFSET;
6735 } else {
6736 fw_data = (const __be32 *)rdev->mec_fw->data;
6737 table_offset = CP_MEC_TABLE_OFFSET;
6738 }
22c775ce 6739
f2c6b0f4
AD		 6740 for (i = 0; i < table_size; i ++) {
6741 dst_ptr[bo_offset + i] =
6742 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6743 }
6744 bo_offset += table_size;
22c775ce 6745 }
f96ab484 6746 }
22c775ce 6747}
f96ab484 6748
22c775ce
AD		 6749static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6750 bool enable)
6751{
6752 u32 data, orig;
6753
2b19d17f 6754 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6755 orig = data = RREG32(RLC_PG_CNTL);
6756 data |= GFX_PG_ENABLE;
6757 if (orig != data)
6758 WREG32(RLC_PG_CNTL, data);
6759
6760 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6761 data |= AUTO_PG_EN;
6762 if (orig != data)
6763 WREG32(RLC_AUTO_PG_CTRL, data);
6764 } else {
6765 orig = data = RREG32(RLC_PG_CNTL);
6766 data &= ~GFX_PG_ENABLE;
6767 if (orig != data)
6768 WREG32(RLC_PG_CNTL, data);
f96ab484 6769
22c775ce
AD		 6770 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6771 data &= ~AUTO_PG_EN;
6772 if (orig != data)
6773 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6774
22c775ce
AD		 6775 data = RREG32(DB_RENDER_CONTROL);
6776 }
6777}
f96ab484 6778
22c775ce
AD		 6779static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6780{
6781 u32 mask = 0, tmp, tmp1;
6782 int i;
f96ab484 6783
22c775ce
AD		 6784 cik_select_se_sh(rdev, se, sh);
6785 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6786 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6787 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 6788
22c775ce 6789 tmp &= 0xffff0000;
f96ab484 6790
22c775ce
AD		 6791 tmp |= tmp1;
6792 tmp >>= 16;
6793
6794 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6795 mask <<= 1;
6796 mask |= 1;
b07fdd38 6797 }
22c775ce
AD		 6798
6799 return (~tmp) & mask;
f96ab484
AD		 6800}
6801
22c775ce 6802static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6803{
22c775ce
AD		 6804 u32 i, j, k, active_cu_number = 0;
6805 u32 mask, counter, cu_bitmap;
6806 u32 tmp = 0;
d0e092d9 6807
22c775ce
AD		 6808 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6809 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6810 mask = 1;
6811 cu_bitmap = 0;
6812 counter = 0;
6813 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6814 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6815 if (counter < 2)
6816 cu_bitmap |= mask;
6817 counter ++;
d0e092d9 6818 }
22c775ce 6819 mask <<= 1;
d0e092d9 6820 }
d0e092d9 6821
22c775ce
AD		 6822 active_cu_number += counter;
6823 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6824 }
d0e092d9 6825 }
22c775ce
AD		 6826
6827 WREG32(RLC_PG_AO_CU_MASK, tmp);
6828
6829 tmp = RREG32(RLC_MAX_PG_CU);
6830 tmp &= ~MAX_PU_CU_MASK;
6831 tmp |= MAX_PU_CU(active_cu_number);
6832 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6833}
6834
22c775ce
AD		 6835static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6836 bool enable)
605de6b9 6837{
22c775ce 6838 u32 data, orig;
605de6b9 6839
22c775ce 6840 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6841 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6842 data |= STATIC_PER_CU_PG_ENABLE;
6843 else
6844 data &= ~STATIC_PER_CU_PG_ENABLE;
6845 if (orig != data)
6846 WREG32(RLC_PG_CNTL, data);
6847}
6848
6849static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6850 bool enable)
6851{
6852 u32 data, orig;
605de6b9 6853
22c775ce 6854 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6855 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6856 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6857 else
22c775ce
AD		 6858 data &= ~DYN_PER_CU_PG_ENABLE;
6859 if (orig != data)
6860 WREG32(RLC_PG_CNTL, data);
6861}
605de6b9 6862
22c775ce
AD		 6863#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6864#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6865
6866static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6867{
6868 u32 data, orig;
6869 u32 i;
6870
6871 if (rdev->rlc.cs_data) {
6872 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6873 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6874 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6875 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6876 } else {
22c775ce
AD		 6877 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6878 for (i = 0; i < 3; i++)
6879 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6880 }
6881 if (rdev->rlc.reg_list) {
6882 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6883 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6884 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6885 }
605de6b9 6886
22c775ce
AD		 6887 orig = data = RREG32(RLC_PG_CNTL);
6888 data |= GFX_PG_SRC;
6889 if (orig != data)
6890 WREG32(RLC_PG_CNTL, data);
605de6b9 6891
22c775ce
AD		 6892 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6893 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6894
22c775ce
AD		 6895 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6896 data &= ~IDLE_POLL_COUNT_MASK;
6897 data |= IDLE_POLL_COUNT(0x60);
6898 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6899
22c775ce
AD		 6900 data = 0x10101010;
6901 WREG32(RLC_PG_DELAY, data);
605de6b9 6902
22c775ce
AD		 6903 data = RREG32(RLC_PG_DELAY_2);
6904 data &= ~0xff;
6905 data |= 0x3;
6906 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6907
22c775ce
AD		 6908 data = RREG32(RLC_AUTO_PG_CTRL);
6909 data &= ~GRBM_REG_SGIT_MASK;
6910 data |= GRBM_REG_SGIT(0x700);
6911 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6912
605de6b9
AD		 6913}
6914
22c775ce 6915static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6916{
473359bc
AD		 6917 cik_enable_gfx_cgpg(rdev, enable);
6918 cik_enable_gfx_static_mgpg(rdev, enable);
6919 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6920}
f6796cae 6921
a0f38609
AD		 6922u32 cik_get_csb_size(struct radeon_device *rdev)
6923{
6924 u32 count = 0;
6925 const struct cs_section_def *sect = NULL;
6926 const struct cs_extent_def *ext = NULL;
f6796cae 6927
a0f38609
AD		 6928 if (rdev->rlc.cs_data == NULL)
6929 return 0;
f6796cae 6930
a0f38609
AD		 6931 /* begin clear state */
6932 count += 2;
6933 /* context control state */
6934 count += 3;
6935
6936 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6937 for (ext = sect->section; ext->extent != NULL; ++ext) {
6938 if (sect->id == SECT_CONTEXT)
6939 count += 2 + ext->reg_count;
6940 else
6941 return 0;
f6796cae
AD		 6942 }
6943 }
a0f38609
AD		 6944 /* pa_sc_raster_config/pa_sc_raster_config1 */
6945 count += 4;
6946 /* end clear state */
6947 count += 2;
6948 /* clear state */
6949 count += 2;
f6796cae 6950
a0f38609 6951 return count;
f6796cae
AD		 6952}
6953
a0f38609 6954void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6955{
a0f38609
AD		 6956 u32 count = 0, i;
6957 const struct cs_section_def *sect = NULL;
6958 const struct cs_extent_def *ext = NULL;
f6796cae 6959
a0f38609
AD		 6960 if (rdev->rlc.cs_data == NULL)
6961 return;
6962 if (buffer == NULL)
6963 return;
f6796cae 6964
6ba81e53
AD		 6965 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6966 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6967
6ba81e53
AD		 6968 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6969 buffer[count++] = cpu_to_le32(0x80000000);
6970 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6971
6972 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6973 for (ext = sect->section; ext->extent != NULL; ++ext) {
6974 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6975 buffer[count++] =
6976 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6977 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6978 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6979 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6980 } else {
6981 return;
6982 }
6983 }
6984 }
f6796cae 6985
6ba81e53
AD		 6986 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6987 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 6988 switch (rdev->family) {
6989 case CHIP_BONAIRE:
6ba81e53
AD		 6990 buffer[count++] = cpu_to_le32(0x16000012);
6991 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6992 break;
6993 case CHIP_KAVERI:
6ba81e53
AD		 6994 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6995 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6996 break;
6997 case CHIP_KABINI:
f73a9e83 6998 case CHIP_MULLINS:
6ba81e53
AD		 6999 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7000 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 7001 break;
bbfe90bd 7002 case CHIP_HAWAII:
a8947f57
AD		 7003 buffer[count++] = cpu_to_le32(0x3a00161a);
7004 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 7005 break;
a0f38609 7006 default:
6ba81e53
AD		 7007 buffer[count++] = cpu_to_le32(0x00000000);
7008 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7009 break;
7010 }
7011
6ba81e53
AD		 7012 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7013 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 7014
6ba81e53
AD		 7015 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7016 buffer[count++] = cpu_to_le32(0);
a0f38609 7017}
f6796cae 7018
473359bc 7019static void cik_init_pg(struct radeon_device *rdev)
22c775ce 7020{
473359bc 7021 if (rdev->pg_flags) {
22c775ce
AD		 7022 cik_enable_sck_slowdown_on_pu(rdev, true);
7023 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 7024 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7025 cik_init_gfx_cgpg(rdev);
7026 cik_enable_cp_pg(rdev, true);
7027 cik_enable_gds_pg(rdev, true);
7028 }
22c775ce
AD		 7029 cik_init_ao_cu_mask(rdev);
7030 cik_update_gfx_pg(rdev, true);
7031 }
7032}
f6796cae 7033
473359bc
AD		 7034static void cik_fini_pg(struct radeon_device *rdev)
7035{
7036 if (rdev->pg_flags) {
7037 cik_update_gfx_pg(rdev, false);
2b19d17f 7038 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7039 cik_enable_cp_pg(rdev, false);
7040 cik_enable_gds_pg(rdev, false);
7041 }
7042 }
f6796cae 7043}
a59781bb
AD		 7044
7045/*
7046 * Interrupts
7047 * Starting with r6xx, interrupts are handled via a ring buffer.
7048 * Ring buffers are areas of GPU accessible memory that the GPU
7049 * writes interrupt vectors into and the host reads vectors out of.
7050 * There is a rptr (read pointer) that determines where the
7051 * host is currently reading, and a wptr (write pointer)
7052 * which determines where the GPU has written. When the
7053 * pointers are equal, the ring is idle. When the GPU
7054 * writes vectors to the ring buffer, it increments the
7055 * wptr. When there is an interrupt, the host then starts
7056 * fetching commands and processing them until the pointers are
7057 * equal again at which point it updates the rptr.
7058 */
7059
7060/**
7061 * cik_enable_interrupts - Enable the interrupt ring buffer
7062 *
7063 * @rdev: radeon_device pointer
7064 *
7065 * Enable the interrupt ring buffer (CIK).
7066 */
7067static void cik_enable_interrupts(struct radeon_device *rdev)
7068{
7069 u32 ih_cntl = RREG32(IH_CNTL);
7070 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7071
7072 ih_cntl |= ENABLE_INTR;
7073 ih_rb_cntl |= IH_RB_ENABLE;
7074 WREG32(IH_CNTL, ih_cntl);
7075 WREG32(IH_RB_CNTL, ih_rb_cntl);
7076 rdev->ih.enabled = true;
7077}
7078
7079/**
7080 * cik_disable_interrupts - Disable the interrupt ring buffer
7081 *
7082 * @rdev: radeon_device pointer
7083 *
7084 * Disable the interrupt ring buffer (CIK).
7085 */
7086static void cik_disable_interrupts(struct radeon_device *rdev)
7087{
7088 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7089 u32 ih_cntl = RREG32(IH_CNTL);
7090
7091 ih_rb_cntl &= ~IH_RB_ENABLE;
7092 ih_cntl &= ~ENABLE_INTR;
7093 WREG32(IH_RB_CNTL, ih_rb_cntl);
7094 WREG32(IH_CNTL, ih_cntl);
7095 /* set rptr, wptr to 0 */
7096 WREG32(IH_RB_RPTR, 0);
7097 WREG32(IH_RB_WPTR, 0);
7098 rdev->ih.enabled = false;
7099 rdev->ih.rptr = 0;
7100}
7101
7102/**
7103 * cik_disable_interrupt_state - Disable all interrupt sources
7104 *
7105 * @rdev: radeon_device pointer
7106 *
7107 * Clear all interrupt enable bits used by the driver (CIK).
7108 */
7109static void cik_disable_interrupt_state(struct radeon_device *rdev)
7110{
7111 u32 tmp;
7112
7113 /* gfx ring */
4214faf6
AD		 7114 tmp = RREG32(CP_INT_CNTL_RING0) &
7115 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7116 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 7117 /* sdma */
7118 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7119 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7120 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7121 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 7122 /* compute queues */
7123 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7124 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7125 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7126 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7127 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7128 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7129 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7130 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7131 /* grbm */
7132 WREG32(GRBM_INT_CNTL, 0);
7133 /* vline/vblank, etc. */
7134 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7135 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7136 if (rdev->num_crtc >= 4) {
7137 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7138 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7139 }
7140 if (rdev->num_crtc >= 6) {
7141 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7142 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7143 }
f5d636d2
CK		 7144 /* pflip */
7145 if (rdev->num_crtc >= 2) {
7146 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7147 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7148 }
7149 if (rdev->num_crtc >= 4) {
7150 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7151 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7152 }
7153 if (rdev->num_crtc >= 6) {
7154 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7155 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7156 }
a59781bb
AD		 7157
7158 /* dac hotplug */
7159 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7160
7161 /* digital hotplug */
7162 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7163 WREG32(DC_HPD1_INT_CONTROL, tmp);
7164 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7165 WREG32(DC_HPD2_INT_CONTROL, tmp);
7166 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7167 WREG32(DC_HPD3_INT_CONTROL, tmp);
7168 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7169 WREG32(DC_HPD4_INT_CONTROL, tmp);
7170 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7171 WREG32(DC_HPD5_INT_CONTROL, tmp);
7172 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7173 WREG32(DC_HPD6_INT_CONTROL, tmp);
7174
7175}
7176
7177/**
7178 * cik_irq_init - init and enable the interrupt ring
7179 *
7180 * @rdev: radeon_device pointer
7181 *
7182 * Allocate a ring buffer for the interrupt controller,
7183 * enable the RLC, disable interrupts, enable the IH
7184 * ring buffer and enable it (CIK).
7185 * Called at device load and reume.
7186 * Returns 0 for success, errors for failure.
7187 */
7188static int cik_irq_init(struct radeon_device *rdev)
7189{
7190 int ret = 0;
7191 int rb_bufsz;
7192 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7193
7194 /* allocate ring */
7195 ret = r600_ih_ring_alloc(rdev);
7196 if (ret)
7197 return ret;
7198
7199 /* disable irqs */
7200 cik_disable_interrupts(rdev);
7201
7202 /* init rlc */
7203 ret = cik_rlc_resume(rdev);
7204 if (ret) {
7205 r600_ih_ring_fini(rdev);
7206 return ret;
7207 }
7208
7209 /* setup interrupt control */
7210 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7211 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7212 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7213 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7214 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7215 */
7216 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7217 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7218 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7219 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7220
7221 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7222 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7223
7224 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7225 IH_WPTR_OVERFLOW_CLEAR |
7226 (rb_bufsz << 1));
7227
7228 if (rdev->wb.enabled)
7229 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7230
7231 /* set the writeback address whether it's enabled or not */
7232 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7233 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7234
7235 WREG32(IH_RB_CNTL, ih_rb_cntl);
7236
7237 /* set rptr, wptr to 0 */
7238 WREG32(IH_RB_RPTR, 0);
7239 WREG32(IH_RB_WPTR, 0);
7240
7241 /* Default settings for IH_CNTL (disabled at first) */
7242 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7243 /* RPTR_REARM only works if msi's are enabled */
7244 if (rdev->msi_enabled)
7245 ih_cntl |= RPTR_REARM;
7246 WREG32(IH_CNTL, ih_cntl);
7247
7248 /* force the active interrupt state to all disabled */
7249 cik_disable_interrupt_state(rdev);
7250
7251 pci_set_master(rdev->pdev);
7252
7253 /* enable irqs */
7254 cik_enable_interrupts(rdev);
7255
7256 return ret;
7257}
7258
7259/**
7260 * cik_irq_set - enable/disable interrupt sources
7261 *
7262 * @rdev: radeon_device pointer
7263 *
7264 * Enable interrupt sources on the GPU (vblanks, hpd,
7265 * etc.) (CIK).
7266 * Returns 0 for success, errors for failure.
7267 */
7268int cik_irq_set(struct radeon_device *rdev)
7269{
4214faf6 7270 u32 cp_int_cntl;
2b0781a6
AD		 7271 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7272 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 7273 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7274 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7275 u32 grbm_int_cntl = 0;
21a93e13 7276 u32 dma_cntl, dma_cntl1;
41a524ab 7277 u32 thermal_int;
a59781bb
AD		 7278
7279 if (!rdev->irq.installed) {
7280 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7281 return -EINVAL;
7282 }
7283 /* don't enable anything if the ih is disabled */
7284 if (!rdev->ih.enabled) {
7285 cik_disable_interrupts(rdev);
7286 /* force the active interrupt state to all disabled */
7287 cik_disable_interrupt_state(rdev);
7288 return 0;
7289 }
7290
4214faf6
AD		 7291 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7292 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7293 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7294
a59781bb
AD		 7295 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7296 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7297 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7298 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7299 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7300 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7301
21a93e13
AD		 7302 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7303 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7304
2b0781a6
AD		 7305 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7306 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7307 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7308 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7309 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7310 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7311 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7312 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7313
cc8dbbb4
AD		 7314 if (rdev->flags & RADEON_IS_IGP)
7315 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7316 ~(THERM_INTH_MASK | THERM_INTL_MASK);
7317 else
7318 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7319 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 7320
a59781bb
AD		 7321 /* enable CP interrupts on all rings */
7322 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7323 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7324 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7325 }
2b0781a6
AD		 7326 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7327 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7328 DRM_DEBUG("si_irq_set: sw int cp1\n");
7329 if (ring->me == 1) {
7330 switch (ring->pipe) {
7331 case 0:
7332 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7333 break;
7334 case 1:
7335 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7336 break;
7337 case 2:
7338 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7339 break;
7340 case 3:
7341 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7342 break;
7343 default:
7344 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7345 break;
7346 }
7347 } else if (ring->me == 2) {
7348 switch (ring->pipe) {
7349 case 0:
7350 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7351 break;
7352 case 1:
7353 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7354 break;
7355 case 2:
7356 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7357 break;
7358 case 3:
7359 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7360 break;
7361 default:
7362 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7363 break;
7364 }
7365 } else {
7366 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7367 }
7368 }
7369 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7370 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7371 DRM_DEBUG("si_irq_set: sw int cp2\n");
7372 if (ring->me == 1) {
7373 switch (ring->pipe) {
7374 case 0:
7375 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7376 break;
7377 case 1:
7378 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7379 break;
7380 case 2:
7381 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7382 break;
7383 case 3:
7384 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7385 break;
7386 default:
7387 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7388 break;
7389 }
7390 } else if (ring->me == 2) {
7391 switch (ring->pipe) {
7392 case 0:
7393 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7394 break;
7395 case 1:
7396 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7397 break;
7398 case 2:
7399 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7400 break;
7401 case 3:
7402 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7403 break;
7404 default:
7405 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7406 break;
7407 }
7408 } else {
7409 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7410 }
7411 }
a59781bb 7412
21a93e13
AD		 7413 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7414 DRM_DEBUG("cik_irq_set: sw int dma\n");
7415 dma_cntl |= TRAP_ENABLE;
7416 }
7417
7418 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7419 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7420 dma_cntl1 |= TRAP_ENABLE;
7421 }
7422
a59781bb
AD		 7423 if (rdev->irq.crtc_vblank_int[0] ||
7424 atomic_read(&rdev->irq.pflip[0])) {
7425 DRM_DEBUG("cik_irq_set: vblank 0\n");
7426 crtc1 |= VBLANK_INTERRUPT_MASK;
7427 }
7428 if (rdev->irq.crtc_vblank_int[1] ||
7429 atomic_read(&rdev->irq.pflip[1])) {
7430 DRM_DEBUG("cik_irq_set: vblank 1\n");
7431 crtc2 |= VBLANK_INTERRUPT_MASK;
7432 }
7433 if (rdev->irq.crtc_vblank_int[2] ||
7434 atomic_read(&rdev->irq.pflip[2])) {
7435 DRM_DEBUG("cik_irq_set: vblank 2\n");
7436 crtc3 |= VBLANK_INTERRUPT_MASK;
7437 }
7438 if (rdev->irq.crtc_vblank_int[3] ||
7439 atomic_read(&rdev->irq.pflip[3])) {
7440 DRM_DEBUG("cik_irq_set: vblank 3\n");
7441 crtc4 |= VBLANK_INTERRUPT_MASK;
7442 }
7443 if (rdev->irq.crtc_vblank_int[4] ||
7444 atomic_read(&rdev->irq.pflip[4])) {
7445 DRM_DEBUG("cik_irq_set: vblank 4\n");
7446 crtc5 |= VBLANK_INTERRUPT_MASK;
7447 }
7448 if (rdev->irq.crtc_vblank_int[5] ||
7449 atomic_read(&rdev->irq.pflip[5])) {
7450 DRM_DEBUG("cik_irq_set: vblank 5\n");
7451 crtc6 |= VBLANK_INTERRUPT_MASK;
7452 }
7453 if (rdev->irq.hpd[0]) {
7454 DRM_DEBUG("cik_irq_set: hpd 1\n");
7455 hpd1 |= DC_HPDx_INT_EN;
7456 }
7457 if (rdev->irq.hpd[1]) {
7458 DRM_DEBUG("cik_irq_set: hpd 2\n");
7459 hpd2 |= DC_HPDx_INT_EN;
7460 }
7461 if (rdev->irq.hpd[2]) {
7462 DRM_DEBUG("cik_irq_set: hpd 3\n");
7463 hpd3 |= DC_HPDx_INT_EN;
7464 }
7465 if (rdev->irq.hpd[3]) {
7466 DRM_DEBUG("cik_irq_set: hpd 4\n");
7467 hpd4 |= DC_HPDx_INT_EN;
7468 }
7469 if (rdev->irq.hpd[4]) {
7470 DRM_DEBUG("cik_irq_set: hpd 5\n");
7471 hpd5 |= DC_HPDx_INT_EN;
7472 }
7473 if (rdev->irq.hpd[5]) {
7474 DRM_DEBUG("cik_irq_set: hpd 6\n");
7475 hpd6 |= DC_HPDx_INT_EN;
7476 }
7477
41a524ab
AD		 7478 if (rdev->irq.dpm_thermal) {
7479 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 7480 if (rdev->flags & RADEON_IS_IGP)
7481 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7482 else
7483 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 7484 }
7485
a59781bb
AD		 7486 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7487
21a93e13
AD		 7488 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7489 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7490
2b0781a6
AD		 7491 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7492 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7493 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7494 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7495 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7496 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7497 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7498 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7499
a59781bb
AD		 7500 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7501
7502 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7503 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7504 if (rdev->num_crtc >= 4) {
7505 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7506 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7507 }
7508 if (rdev->num_crtc >= 6) {
7509 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7510 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7511 }
7512
f5d636d2
CK		 7513 if (rdev->num_crtc >= 2) {
7514 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7515 GRPH_PFLIP_INT_MASK);
7516 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7517 GRPH_PFLIP_INT_MASK);
7518 }
7519 if (rdev->num_crtc >= 4) {
7520 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7521 GRPH_PFLIP_INT_MASK);
7522 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7523 GRPH_PFLIP_INT_MASK);
7524 }
7525 if (rdev->num_crtc >= 6) {
7526 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7527 GRPH_PFLIP_INT_MASK);
7528 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7529 GRPH_PFLIP_INT_MASK);
7530 }
7531
a59781bb
AD		 7532 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7533 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7534 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7535 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7536 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7537 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7538
cc8dbbb4
AD		 7539 if (rdev->flags & RADEON_IS_IGP)
7540 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7541 else
7542 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 7543
a59781bb
AD		 7544 return 0;
7545}
7546
7547/**
7548 * cik_irq_ack - ack interrupt sources
7549 *
7550 * @rdev: radeon_device pointer
7551 *
7552 * Ack interrupt sources on the GPU (vblanks, hpd,
7553 * etc.) (CIK). Certain interrupts sources are sw
7554 * generated and do not require an explicit ack.
7555 */
7556static inline void cik_irq_ack(struct radeon_device *rdev)
7557{
7558 u32 tmp;
7559
7560 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7561 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7562 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7563 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7564 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7565 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7566 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7567
f5d636d2
CK		 7568 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7569 EVERGREEN_CRTC0_REGISTER_OFFSET);
7570 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7571 EVERGREEN_CRTC1_REGISTER_OFFSET);
7572 if (rdev->num_crtc >= 4) {
7573 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7574 EVERGREEN_CRTC2_REGISTER_OFFSET);
7575 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7576 EVERGREEN_CRTC3_REGISTER_OFFSET);
7577 }
7578 if (rdev->num_crtc >= 6) {
7579 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7580 EVERGREEN_CRTC4_REGISTER_OFFSET);
7581 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7582 EVERGREEN_CRTC5_REGISTER_OFFSET);
7583 }
7584
7585 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7586 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7587 GRPH_PFLIP_INT_CLEAR);
7588 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7589 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7590 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7591 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7592 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7593 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7594 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7595 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7596 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7597 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7598 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7599
7600 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7601 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7602 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7603 GRPH_PFLIP_INT_CLEAR);
7604 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7605 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7606 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7607 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7608 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7609 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7610 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7611 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7612 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7613 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7614 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7615 }
7616
7617 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7618 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7619 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7620 GRPH_PFLIP_INT_CLEAR);
7621 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7622 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7623 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7624 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7625 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7626 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7627 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7628 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7629 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7630 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7631 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7632 }
7633
7634 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7635 tmp = RREG32(DC_HPD1_INT_CONTROL);
7636 tmp |= DC_HPDx_INT_ACK;
7637 WREG32(DC_HPD1_INT_CONTROL, tmp);
7638 }
7639 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7640 tmp = RREG32(DC_HPD2_INT_CONTROL);
7641 tmp |= DC_HPDx_INT_ACK;
7642 WREG32(DC_HPD2_INT_CONTROL, tmp);
7643 }
7644 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7645 tmp = RREG32(DC_HPD3_INT_CONTROL);
7646 tmp |= DC_HPDx_INT_ACK;
7647 WREG32(DC_HPD3_INT_CONTROL, tmp);
7648 }
7649 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7650 tmp = RREG32(DC_HPD4_INT_CONTROL);
7651 tmp |= DC_HPDx_INT_ACK;
7652 WREG32(DC_HPD4_INT_CONTROL, tmp);
7653 }
7654 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7655 tmp = RREG32(DC_HPD5_INT_CONTROL);
7656 tmp |= DC_HPDx_INT_ACK;
7657 WREG32(DC_HPD5_INT_CONTROL, tmp);
7658 }
7659 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7660 tmp = RREG32(DC_HPD5_INT_CONTROL);
7661 tmp |= DC_HPDx_INT_ACK;
7662 WREG32(DC_HPD6_INT_CONTROL, tmp);
7663 }
7664}
7665
7666/**
7667 * cik_irq_disable - disable interrupts
7668 *
7669 * @rdev: radeon_device pointer
7670 *
7671 * Disable interrupts on the hw (CIK).
7672 */
7673static void cik_irq_disable(struct radeon_device *rdev)
7674{
7675 cik_disable_interrupts(rdev);
7676 /* Wait and acknowledge irq */
7677 mdelay(1);
7678 cik_irq_ack(rdev);
7679 cik_disable_interrupt_state(rdev);
7680}
7681
7682/**
7683 * cik_irq_disable - disable interrupts for suspend
7684 *
7685 * @rdev: radeon_device pointer
7686 *
7687 * Disable interrupts and stop the RLC (CIK).
7688 * Used for suspend.
7689 */
7690static void cik_irq_suspend(struct radeon_device *rdev)
7691{
7692 cik_irq_disable(rdev);
7693 cik_rlc_stop(rdev);
7694}
7695
7696/**
7697 * cik_irq_fini - tear down interrupt support
7698 *
7699 * @rdev: radeon_device pointer
7700 *
7701 * Disable interrupts on the hw and free the IH ring
7702 * buffer (CIK).
7703 * Used for driver unload.
7704 */
7705static void cik_irq_fini(struct radeon_device *rdev)
7706{
7707 cik_irq_suspend(rdev);
7708 r600_ih_ring_fini(rdev);
7709}
7710
7711/**
7712 * cik_get_ih_wptr - get the IH ring buffer wptr
7713 *
7714 * @rdev: radeon_device pointer
7715 *
7716 * Get the IH ring buffer wptr from either the register
7717 * or the writeback memory buffer (CIK). Also check for
7718 * ring buffer overflow and deal with it.
7719 * Used by cik_irq_process().
7720 * Returns the value of the wptr.
7721 */
7722static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7723{
7724 u32 wptr, tmp;
7725
7726 if (rdev->wb.enabled)
7727 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7728 else
7729 wptr = RREG32(IH_RB_WPTR);
7730
7731 if (wptr & RB_OVERFLOW) {
7732 /* When a ring buffer overflow happen start parsing interrupt
7733 * from the last not overwritten vector (wptr + 16). Hopefully
7734 * this should allow us to catchup.
7735 */
7736 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7737 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7738 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7739 tmp = RREG32(IH_RB_CNTL);
7740 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7741 WREG32(IH_RB_CNTL, tmp);
e8c214d2 7742 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7743 }
7744 return (wptr & rdev->ih.ptr_mask);
7745}
7746
7747/* CIK IV Ring
7748 * Each IV ring entry is 128 bits:
7749 * [7:0] - interrupt source id
7750 * [31:8] - reserved
7751 * [59:32] - interrupt source data
7752 * [63:60] - reserved
21a93e13
AD		 7753 * [71:64] - RINGID
7754 * CP:
7755 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7756 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7757 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7758 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7759 * PIPE_ID - ME0 0=3D
7760 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7761 * SDMA:
7762 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7763 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7764 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7765 * [79:72] - VMID
7766 * [95:80] - PASID
7767 * [127:96] - reserved
7768 */
7769/**
7770 * cik_irq_process - interrupt handler
7771 *
7772 * @rdev: radeon_device pointer
7773 *
7774 * Interrupt hander (CIK). Walk the IH ring,
7775 * ack interrupts and schedule work to handle
7776 * interrupt events.
7777 * Returns irq process return code.
7778 */
7779int cik_irq_process(struct radeon_device *rdev)
7780{
2b0781a6
AD		 7781 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7782 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7783 u32 wptr;
7784 u32 rptr;
7785 u32 src_id, src_data, ring_id;
7786 u8 me_id, pipe_id, queue_id;
7787 u32 ring_index;
7788 bool queue_hotplug = false;
7789 bool queue_reset = false;
3ec7d11b 7790 u32 addr, status, mc_client;
41a524ab 7791 bool queue_thermal = false;
a59781bb
AD		 7792
7793 if (!rdev->ih.enabled || rdev->shutdown)
7794 return IRQ_NONE;
7795
7796 wptr = cik_get_ih_wptr(rdev);
7797
7798restart_ih:
7799 /* is somebody else already processing irqs? */
7800 if (atomic_xchg(&rdev->ih.lock, 1))
7801 return IRQ_NONE;
7802
7803 rptr = rdev->ih.rptr;
7804 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7805
7806 /* Order reading of wptr vs. reading of IH ring data */
7807 rmb();
7808
7809 /* display interrupts */
7810 cik_irq_ack(rdev);
7811
7812 while (rptr != wptr) {
7813 /* wptr/rptr are in bytes! */
7814 ring_index = rptr / 4;
7815 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7816 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7817 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7818
7819 switch (src_id) {
7820 case 1: /* D1 vblank/vline */
7821 switch (src_data) {
7822 case 0: /* D1 vblank */
7823 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7824 if (rdev->irq.crtc_vblank_int[0]) {
7825 drm_handle_vblank(rdev->ddev, 0);
7826 rdev->pm.vblank_sync = true;
7827 wake_up(&rdev->irq.vblank_queue);
7828 }
7829 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 7830 radeon_crtc_handle_vblank(rdev, 0);
a59781bb
AD		 7831 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7832 DRM_DEBUG("IH: D1 vblank\n");
7833 }
7834 break;
7835 case 1: /* D1 vline */
7836 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7837 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7838 DRM_DEBUG("IH: D1 vline\n");
7839 }
7840 break;
7841 default:
7842 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7843 break;
7844 }
7845 break;
7846 case 2: /* D2 vblank/vline */
7847 switch (src_data) {
7848 case 0: /* D2 vblank */
7849 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7850 if (rdev->irq.crtc_vblank_int[1]) {
7851 drm_handle_vblank(rdev->ddev, 1);
7852 rdev->pm.vblank_sync = true;
7853 wake_up(&rdev->irq.vblank_queue);
7854 }
7855 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 7856 radeon_crtc_handle_vblank(rdev, 1);
a59781bb
AD		 7857 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7858 DRM_DEBUG("IH: D2 vblank\n");
7859 }
7860 break;
7861 case 1: /* D2 vline */
7862 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7863 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7864 DRM_DEBUG("IH: D2 vline\n");
7865 }
7866 break;
7867 default:
7868 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7869 break;
7870 }
7871 break;
7872 case 3: /* D3 vblank/vline */
7873 switch (src_data) {
7874 case 0: /* D3 vblank */
7875 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7876 if (rdev->irq.crtc_vblank_int[2]) {
7877 drm_handle_vblank(rdev->ddev, 2);
7878 rdev->pm.vblank_sync = true;
7879 wake_up(&rdev->irq.vblank_queue);
7880 }
7881 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 7882 radeon_crtc_handle_vblank(rdev, 2);
a59781bb
AD		 7883 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7884 DRM_DEBUG("IH: D3 vblank\n");
7885 }
7886 break;
7887 case 1: /* D3 vline */
7888 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7889 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7890 DRM_DEBUG("IH: D3 vline\n");
7891 }
7892 break;
7893 default:
7894 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7895 break;
7896 }
7897 break;
7898 case 4: /* D4 vblank/vline */
7899 switch (src_data) {
7900 case 0: /* D4 vblank */
7901 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7902 if (rdev->irq.crtc_vblank_int[3]) {
7903 drm_handle_vblank(rdev->ddev, 3);
7904 rdev->pm.vblank_sync = true;
7905 wake_up(&rdev->irq.vblank_queue);
7906 }
7907 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 7908 radeon_crtc_handle_vblank(rdev, 3);
a59781bb
AD		 7909 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7910 DRM_DEBUG("IH: D4 vblank\n");
7911 }
7912 break;
7913 case 1: /* D4 vline */
7914 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7915 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7916 DRM_DEBUG("IH: D4 vline\n");
7917 }
7918 break;
7919 default:
7920 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7921 break;
7922 }
7923 break;
7924 case 5: /* D5 vblank/vline */
7925 switch (src_data) {
7926 case 0: /* D5 vblank */
7927 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7928 if (rdev->irq.crtc_vblank_int[4]) {
7929 drm_handle_vblank(rdev->ddev, 4);
7930 rdev->pm.vblank_sync = true;
7931 wake_up(&rdev->irq.vblank_queue);
7932 }
7933 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 7934 radeon_crtc_handle_vblank(rdev, 4);
a59781bb
AD		 7935 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7936 DRM_DEBUG("IH: D5 vblank\n");
7937 }
7938 break;
7939 case 1: /* D5 vline */
7940 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7941 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7942 DRM_DEBUG("IH: D5 vline\n");
7943 }
7944 break;
7945 default:
7946 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7947 break;
7948 }
7949 break;
7950 case 6: /* D6 vblank/vline */
7951 switch (src_data) {
7952 case 0: /* D6 vblank */
7953 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7954 if (rdev->irq.crtc_vblank_int[5]) {
7955 drm_handle_vblank(rdev->ddev, 5);
7956 rdev->pm.vblank_sync = true;
7957 wake_up(&rdev->irq.vblank_queue);
7958 }
7959 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 7960 radeon_crtc_handle_vblank(rdev, 5);
a59781bb
AD		 7961 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7962 DRM_DEBUG("IH: D6 vblank\n");
7963 }
7964 break;
7965 case 1: /* D6 vline */
7966 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7967 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7968 DRM_DEBUG("IH: D6 vline\n");
7969 }
7970 break;
7971 default:
7972 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7973 break;
7974 }
7975 break;
f5d636d2
CK		 7976 case 8: /* D1 page flip */
7977 case 10: /* D2 page flip */
7978 case 12: /* D3 page flip */
7979 case 14: /* D4 page flip */
7980 case 16: /* D5 page flip */
7981 case 18: /* D6 page flip */
7982 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7983 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7984 break;
a59781bb
AD		 7985 case 42: /* HPD hotplug */
7986 switch (src_data) {
7987 case 0:
7988 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7989 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7990 queue_hotplug = true;
7991 DRM_DEBUG("IH: HPD1\n");
7992 }
7993 break;
7994 case 1:
7995 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7996 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7997 queue_hotplug = true;
7998 DRM_DEBUG("IH: HPD2\n");
7999 }
8000 break;
8001 case 2:
8002 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8003 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8004 queue_hotplug = true;
8005 DRM_DEBUG("IH: HPD3\n");
8006 }
8007 break;
8008 case 3:
8009 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8010 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8011 queue_hotplug = true;
8012 DRM_DEBUG("IH: HPD4\n");
8013 }
8014 break;
8015 case 4:
8016 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8017 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8018 queue_hotplug = true;
8019 DRM_DEBUG("IH: HPD5\n");
8020 }
8021 break;
8022 case 5:
8023 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8024 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8025 queue_hotplug = true;
8026 DRM_DEBUG("IH: HPD6\n");
8027 }
8028 break;
8029 default:
8030 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8031 break;
8032 }
8033 break;
6a3808b8
CK		 8034 case 124: /* UVD */
8035 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8036 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8037 break;
9d97c99b
AD		 8038 case 146:
8039 case 147:
3ec7d11b
AD		 8040 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8041 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8042 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 8043 /* reset addr and status */
8044 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8045 if (addr == 0x0 && status == 0x0)
8046 break;
9d97c99b
AD		 8047 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8048 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 8049 addr);
9d97c99b 8050 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 8051 status);
8052 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 8053 break;
d93f7937
CK		 8054 case 167: /* VCE */
8055 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8056 switch (src_data) {
8057 case 0:
8058 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8059 break;
8060 case 1:
8061 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8062 break;
8063 default:
8064 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8065 break;
8066 }
8067 break;
a59781bb
AD		 8068 case 176: /* GFX RB CP_INT */
8069 case 177: /* GFX IB CP_INT */
8070 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8071 break;
8072 case 181: /* CP EOP event */
8073 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 8074 /* XXX check the bitfield order! */
8075 me_id = (ring_id & 0x60) >> 5;
8076 pipe_id = (ring_id & 0x18) >> 3;
8077 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8078 switch (me_id) {
8079 case 0:
8080 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8081 break;
8082 case 1:
a59781bb 8083 case 2:
2b0781a6
AD		 8084 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8085 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8086 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8087 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 8088 break;
8089 }
8090 break;
8091 case 184: /* CP Privileged reg access */
8092 DRM_ERROR("Illegal register access in command stream\n");
8093 /* XXX check the bitfield order! */
8094 me_id = (ring_id & 0x60) >> 5;
8095 pipe_id = (ring_id & 0x18) >> 3;
8096 queue_id = (ring_id & 0x7) >> 0;
8097 switch (me_id) {
8098 case 0:
8099 /* This results in a full GPU reset, but all we need to do is soft
8100 * reset the CP for gfx
8101 */
8102 queue_reset = true;
8103 break;
8104 case 1:
8105 /* XXX compute */
2b0781a6 8106 queue_reset = true;
a59781bb
AD		 8107 break;
8108 case 2:
8109 /* XXX compute */
2b0781a6 8110 queue_reset = true;
a59781bb
AD		 8111 break;
8112 }
8113 break;
8114 case 185: /* CP Privileged inst */
8115 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 8116 /* XXX check the bitfield order! */
8117 me_id = (ring_id & 0x60) >> 5;
8118 pipe_id = (ring_id & 0x18) >> 3;
8119 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8120 switch (me_id) {
8121 case 0:
8122 /* This results in a full GPU reset, but all we need to do is soft
8123 * reset the CP for gfx
8124 */
8125 queue_reset = true;
8126 break;
8127 case 1:
8128 /* XXX compute */
2b0781a6 8129 queue_reset = true;
a59781bb
AD		 8130 break;
8131 case 2:
8132 /* XXX compute */
2b0781a6 8133 queue_reset = true;
a59781bb
AD		 8134 break;
8135 }
8136 break;
21a93e13
AD		 8137 case 224: /* SDMA trap event */
8138 /* XXX check the bitfield order! */
8139 me_id = (ring_id & 0x3) >> 0;
8140 queue_id = (ring_id & 0xc) >> 2;
8141 DRM_DEBUG("IH: SDMA trap\n");
8142 switch (me_id) {
8143 case 0:
8144 switch (queue_id) {
8145 case 0:
8146 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8147 break;
8148 case 1:
8149 /* XXX compute */
8150 break;
8151 case 2:
8152 /* XXX compute */
8153 break;
8154 }
8155 break;
8156 case 1:
8157 switch (queue_id) {
8158 case 0:
8159 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8160 break;
8161 case 1:
8162 /* XXX compute */
8163 break;
8164 case 2:
8165 /* XXX compute */
8166 break;
8167 }
8168 break;
8169 }
8170 break;
41a524ab
AD		 8171 case 230: /* thermal low to high */
8172 DRM_DEBUG("IH: thermal low to high\n");
8173 rdev->pm.dpm.thermal.high_to_low = false;
8174 queue_thermal = true;
8175 break;
8176 case 231: /* thermal high to low */
8177 DRM_DEBUG("IH: thermal high to low\n");
8178 rdev->pm.dpm.thermal.high_to_low = true;
8179 queue_thermal = true;
8180 break;
8181 case 233: /* GUI IDLE */
8182 DRM_DEBUG("IH: GUI idle\n");
8183 break;
21a93e13
AD		 8184 case 241: /* SDMA Privileged inst */
8185 case 247: /* SDMA Privileged inst */
8186 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8187 /* XXX check the bitfield order! */
8188 me_id = (ring_id & 0x3) >> 0;
8189 queue_id = (ring_id & 0xc) >> 2;
8190 switch (me_id) {
8191 case 0:
8192 switch (queue_id) {
8193 case 0:
8194 queue_reset = true;
8195 break;
8196 case 1:
8197 /* XXX compute */
8198 queue_reset = true;
8199 break;
8200 case 2:
8201 /* XXX compute */
8202 queue_reset = true;
8203 break;
8204 }
8205 break;
8206 case 1:
8207 switch (queue_id) {
8208 case 0:
8209 queue_reset = true;
8210 break;
8211 case 1:
8212 /* XXX compute */
8213 queue_reset = true;
8214 break;
8215 case 2:
8216 /* XXX compute */
8217 queue_reset = true;
8218 break;
8219 }
8220 break;
8221 }
8222 break;
a59781bb
AD		 8223 default:
8224 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8225 break;
8226 }
8227
8228 /* wptr/rptr are in bytes! */
8229 rptr += 16;
8230 rptr &= rdev->ih.ptr_mask;
8231 }
8232 if (queue_hotplug)
8233 schedule_work(&rdev->hotplug_work);
8234 if (queue_reset)
8235 schedule_work(&rdev->reset_work);
41a524ab
AD		 8236 if (queue_thermal)
8237 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb
AD		 8238 rdev->ih.rptr = rptr;
8239 WREG32(IH_RB_RPTR, rdev->ih.rptr);
8240 atomic_set(&rdev->ih.lock, 0);
8241
8242 /* make sure wptr hasn't changed while processing */
8243 wptr = cik_get_ih_wptr(rdev);
8244 if (wptr != rptr)
8245 goto restart_ih;
8246
8247 return IRQ_HANDLED;
8248}
7bf94a2c
AD		 8249
8250/*
8251 * startup/shutdown callbacks
8252 */
8253/**
8254 * cik_startup - program the asic to a functional state
8255 *
8256 * @rdev: radeon_device pointer
8257 *
8258 * Programs the asic to a functional state (CIK).
8259 * Called by cik_init() and cik_resume().
8260 * Returns 0 for success, error for failure.
8261 */
8262static int cik_startup(struct radeon_device *rdev)
8263{
8264 struct radeon_ring *ring;
8265 int r;
8266
8a7cd276
AD		 8267 /* enable pcie gen2/3 link */
8268 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8269 /* enable aspm */
8270 cik_program_aspm(rdev);
8a7cd276 8271
e5903d39
AD		 8272 /* scratch needs to be initialized before MC */
8273 r = r600_vram_scratch_init(rdev);
8274 if (r)
8275 return r;
8276
6fab3feb
AD		 8277 cik_mc_program(rdev);
8278
6c7bccea 8279 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8280 r = ci_mc_load_microcode(rdev);
8281 if (r) {
8282 DRM_ERROR("Failed to load MC firmware!\n");
8283 return r;
8284 }
8285 }
8286
7bf94a2c
AD		 8287 r = cik_pcie_gart_enable(rdev);
8288 if (r)
8289 return r;
8290 cik_gpu_init(rdev);
8291
8292 /* allocate rlc buffers */
22c775ce
AD		 8293 if (rdev->flags & RADEON_IS_IGP) {
8294 if (rdev->family == CHIP_KAVERI) {
8295 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8296 rdev->rlc.reg_list_size =
8297 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8298 } else {
8299 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8300 rdev->rlc.reg_list_size =
8301 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8302 }
8303 }
8304 rdev->rlc.cs_data = ci_cs_data;
8305 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 8306 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8307 if (r) {
8308 DRM_ERROR("Failed to init rlc BOs!\n");
8309 return r;
8310 }
8311
8312 /* allocate wb buffer */
8313 r = radeon_wb_init(rdev);
8314 if (r)
8315 return r;
8316
963e81f9
AD		 8317 /* allocate mec buffers */
8318 r = cik_mec_init(rdev);
8319 if (r) {
8320 DRM_ERROR("Failed to init MEC BOs!\n");
8321 return r;
8322 }
8323
7bf94a2c
AD		 8324 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8325 if (r) {
8326 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8327 return r;
8328 }
8329
963e81f9
AD		 8330 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8331 if (r) {
8332 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8333 return r;
8334 }
8335
8336 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8337 if (r) {
8338 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8339 return r;
8340 }
8341
7bf94a2c
AD		 8342 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8343 if (r) {
8344 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8345 return r;
8346 }
8347
8348 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8349 if (r) {
8350 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8351 return r;
8352 }
8353
2ce529da 8354 r = radeon_uvd_resume(rdev);
87167bb1 8355 if (!r) {
2ce529da
AD		 8356 r = uvd_v4_2_resume(rdev);
8357 if (!r) {
8358 r = radeon_fence_driver_start_ring(rdev,
8359 R600_RING_TYPE_UVD_INDEX);
8360 if (r)
8361 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8362 }
87167bb1
CK		 8363 }
8364 if (r)
8365 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8366
d93f7937
CK		 8367 r = radeon_vce_resume(rdev);
8368 if (!r) {
8369 r = vce_v2_0_resume(rdev);
8370 if (!r)
8371 r = radeon_fence_driver_start_ring(rdev,
8372 TN_RING_TYPE_VCE1_INDEX);
8373 if (!r)
8374 r = radeon_fence_driver_start_ring(rdev,
8375 TN_RING_TYPE_VCE2_INDEX);
8376 }
8377 if (r) {
8378 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8379 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8380 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8381 }
8382
7bf94a2c
AD		 8383 /* Enable IRQ */
8384 if (!rdev->irq.installed) {
8385 r = radeon_irq_kms_init(rdev);
8386 if (r)
8387 return r;
8388 }
8389
8390 r = cik_irq_init(rdev);
8391 if (r) {
8392 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8393 radeon_irq_kms_fini(rdev);
8394 return r;
8395 }
8396 cik_irq_set(rdev);
8397
8398 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8399 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1dac28eb 8400 PACKET3(PACKET3_NOP, 0x3FFF));
7bf94a2c
AD		 8401 if (r)
8402 return r;
8403
963e81f9 8404 /* set up the compute queues */
2615b53a 8405 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8406 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8407 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
2e1e6dad 8408 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 8409 if (r)
8410 return r;
8411 ring->me = 1; /* first MEC */
8412 ring->pipe = 0; /* first pipe */
8413 ring->queue = 0; /* first queue */
8414 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8415
2615b53a 8416 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8417 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8418 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
2e1e6dad 8419 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 8420 if (r)
8421 return r;
8422 /* dGPU only have 1 MEC */
8423 ring->me = 1; /* first MEC */
8424 ring->pipe = 0; /* first pipe */
8425 ring->queue = 1; /* second queue */
8426 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8427
7bf94a2c
AD		 8428 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8429 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8430 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8431 if (r)
8432 return r;
8433
8434 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8435 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8436 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8437 if (r)
8438 return r;
8439
8440 r = cik_cp_resume(rdev);
8441 if (r)
8442 return r;
8443
8444 r = cik_sdma_resume(rdev);
8445 if (r)
8446 return r;
8447
87167bb1
CK		 8448 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8449 if (ring->ring_size) {
02c9f7fa 8450 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 8451 RADEON_CP_PACKET2);
87167bb1 8452 if (!r)
e409b128 8453 r = uvd_v1_0_init(rdev);
87167bb1
CK		 8454 if (r)
8455 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8456 }
8457
d93f7937
CK		 8458 r = -ENOENT;
8459
8460 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8461 if (ring->ring_size)
8462 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8463 VCE_CMD_NO_OP);
8464
8465 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8466 if (ring->ring_size)
8467 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8468 VCE_CMD_NO_OP);
8469
8470 if (!r)
8471 r = vce_v1_0_init(rdev);
8472 else if (r != -ENOENT)
8473 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8474
7bf94a2c
AD		 8475 r = radeon_ib_pool_init(rdev);
8476 if (r) {
8477 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8478 return r;
8479 }
8480
8481 r = radeon_vm_manager_init(rdev);
8482 if (r) {
8483 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8484 return r;
8485 }
8486
b530602f
AD		 8487 r = dce6_audio_init(rdev);
8488 if (r)
8489 return r;
8490
7bf94a2c
AD		 8491 return 0;
8492}
8493
8494/**
8495 * cik_resume - resume the asic to a functional state
8496 *
8497 * @rdev: radeon_device pointer
8498 *
8499 * Programs the asic to a functional state (CIK).
8500 * Called at resume.
8501 * Returns 0 for success, error for failure.
8502 */
8503int cik_resume(struct radeon_device *rdev)
8504{
8505 int r;
8506
8507 /* post card */
8508 atom_asic_init(rdev->mode_info.atom_context);
8509
0aafd313
AD		 8510 /* init golden registers */
8511 cik_init_golden_registers(rdev);
8512
bc6a6295
AD		 8513 if (rdev->pm.pm_method == PM_METHOD_DPM)
8514 radeon_pm_resume(rdev);
6c7bccea 8515
7bf94a2c
AD		 8516 rdev->accel_working = true;
8517 r = cik_startup(rdev);
8518 if (r) {
8519 DRM_ERROR("cik startup failed on resume\n");
8520 rdev->accel_working = false;
8521 return r;
8522 }
8523
8524 return r;
8525
8526}
8527
8528/**
8529 * cik_suspend - suspend the asic
8530 *
8531 * @rdev: radeon_device pointer
8532 *
8533 * Bring the chip into a state suitable for suspend (CIK).
8534 * Called at suspend.
8535 * Returns 0 for success.
8536 */
8537int cik_suspend(struct radeon_device *rdev)
8538{
6c7bccea 8539 radeon_pm_suspend(rdev);
b530602f 8540 dce6_audio_fini(rdev);
7bf94a2c
AD		 8541 radeon_vm_manager_fini(rdev);
8542 cik_cp_enable(rdev, false);
8543 cik_sdma_enable(rdev, false);
e409b128 8544 uvd_v1_0_fini(rdev);
87167bb1 8545 radeon_uvd_suspend(rdev);
d93f7937 8546 radeon_vce_suspend(rdev);
473359bc
AD		 8547 cik_fini_pg(rdev);
8548 cik_fini_cg(rdev);
7bf94a2c
AD		 8549 cik_irq_suspend(rdev);
8550 radeon_wb_disable(rdev);
8551 cik_pcie_gart_disable(rdev);
8552 return 0;
8553}
8554
8555/* Plan is to move initialization in that function and use
8556 * helper function so that radeon_device_init pretty much
8557 * do nothing more than calling asic specific function. This
8558 * should also allow to remove a bunch of callback function
8559 * like vram_info.
8560 */
8561/**
8562 * cik_init - asic specific driver and hw init
8563 *
8564 * @rdev: radeon_device pointer
8565 *
8566 * Setup asic specific driver variables and program the hw
8567 * to a functional state (CIK).
8568 * Called at driver startup.
8569 * Returns 0 for success, errors for failure.
8570 */
8571int cik_init(struct radeon_device *rdev)
8572{
8573 struct radeon_ring *ring;
8574 int r;
8575
8576 /* Read BIOS */
8577 if (!radeon_get_bios(rdev)) {
8578 if (ASIC_IS_AVIVO(rdev))
8579 return -EINVAL;
8580 }
8581 /* Must be an ATOMBIOS */
8582 if (!rdev->is_atom_bios) {
8583 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8584 return -EINVAL;
8585 }
8586 r = radeon_atombios_init(rdev);
8587 if (r)
8588 return r;
8589
8590 /* Post card if necessary */
8591 if (!radeon_card_posted(rdev)) {
8592 if (!rdev->bios) {
8593 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8594 return -EINVAL;
8595 }
8596 DRM_INFO("GPU not posted. posting now...\n");
8597 atom_asic_init(rdev->mode_info.atom_context);
8598 }
0aafd313
AD		 8599 /* init golden registers */
8600 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8601 /* Initialize scratch registers */
8602 cik_scratch_init(rdev);
8603 /* Initialize surface registers */
8604 radeon_surface_init(rdev);
8605 /* Initialize clocks */
8606 radeon_get_clock_info(rdev->ddev);
8607
8608 /* Fence driver */
8609 r = radeon_fence_driver_init(rdev);
8610 if (r)
8611 return r;
8612
8613 /* initialize memory controller */
8614 r = cik_mc_init(rdev);
8615 if (r)
8616 return r;
8617 /* Memory manager */
8618 r = radeon_bo_init(rdev);
8619 if (r)
8620 return r;
8621
01ac8794
AD		 8622 if (rdev->flags & RADEON_IS_IGP) {
8623 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8624 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8625 r = cik_init_microcode(rdev);
8626 if (r) {
8627 DRM_ERROR("Failed to load firmware!\n");
8628 return r;
8629 }
8630 }
8631 } else {
8632 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8633 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8634 !rdev->mc_fw) {
8635 r = cik_init_microcode(rdev);
8636 if (r) {
8637 DRM_ERROR("Failed to load firmware!\n");
8638 return r;
8639 }
8640 }
8641 }
8642
6c7bccea
AD		 8643 /* Initialize power management */
8644 radeon_pm_init(rdev);
8645
7bf94a2c
AD		 8646 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8647 ring->ring_obj = NULL;
8648 r600_ring_init(rdev, ring, 1024 * 1024);
8649
963e81f9
AD		 8650 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8651 ring->ring_obj = NULL;
8652 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8653 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8654 if (r)
8655 return r;
8656
8657 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8658 ring->ring_obj = NULL;
8659 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8660 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8661 if (r)
8662 return r;
8663
7bf94a2c
AD		 8664 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8665 ring->ring_obj = NULL;
8666 r600_ring_init(rdev, ring, 256 * 1024);
8667
8668 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8669 ring->ring_obj = NULL;
8670 r600_ring_init(rdev, ring, 256 * 1024);
8671
87167bb1
CK		 8672 r = radeon_uvd_init(rdev);
8673 if (!r) {
8674 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8675 ring->ring_obj = NULL;
8676 r600_ring_init(rdev, ring, 4096);
8677 }
8678
d93f7937
CK		 8679 r = radeon_vce_init(rdev);
8680 if (!r) {
8681 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8682 ring->ring_obj = NULL;
8683 r600_ring_init(rdev, ring, 4096);
8684
8685 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8686 ring->ring_obj = NULL;
8687 r600_ring_init(rdev, ring, 4096);
8688 }
8689
7bf94a2c
AD		 8690 rdev->ih.ring_obj = NULL;
8691 r600_ih_ring_init(rdev, 64 * 1024);
8692
8693 r = r600_pcie_gart_init(rdev);
8694 if (r)
8695 return r;
8696
8697 rdev->accel_working = true;
8698 r = cik_startup(rdev);
8699 if (r) {
8700 dev_err(rdev->dev, "disabling GPU acceleration\n");
8701 cik_cp_fini(rdev);
8702 cik_sdma_fini(rdev);
8703 cik_irq_fini(rdev);
1fd11777 8704 sumo_rlc_fini(rdev);
963e81f9 8705 cik_mec_fini(rdev);
7bf94a2c
AD		 8706 radeon_wb_fini(rdev);
8707 radeon_ib_pool_fini(rdev);
8708 radeon_vm_manager_fini(rdev);
8709 radeon_irq_kms_fini(rdev);
8710 cik_pcie_gart_fini(rdev);
8711 rdev->accel_working = false;
8712 }
8713
8714 /* Don't start up if the MC ucode is missing.
8715 * The default clocks and voltages before the MC ucode
8716 * is loaded are not suffient for advanced operations.
8717 */
8718 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8719 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8720 return -EINVAL;
8721 }
8722
8723 return 0;
8724}
8725
8726/**
8727 * cik_fini - asic specific driver and hw fini
8728 *
8729 * @rdev: radeon_device pointer
8730 *
8731 * Tear down the asic specific driver variables and program the hw
8732 * to an idle state (CIK).
8733 * Called at driver unload.
8734 */
8735void cik_fini(struct radeon_device *rdev)
8736{
6c7bccea 8737 radeon_pm_fini(rdev);
7bf94a2c
AD		 8738 cik_cp_fini(rdev);
8739 cik_sdma_fini(rdev);
473359bc
AD		 8740 cik_fini_pg(rdev);
8741 cik_fini_cg(rdev);
7bf94a2c 8742 cik_irq_fini(rdev);
1fd11777 8743 sumo_rlc_fini(rdev);
963e81f9 8744 cik_mec_fini(rdev);
7bf94a2c
AD		 8745 radeon_wb_fini(rdev);
8746 radeon_vm_manager_fini(rdev);
8747 radeon_ib_pool_fini(rdev);
8748 radeon_irq_kms_fini(rdev);
e409b128 8749 uvd_v1_0_fini(rdev);
87167bb1 8750 radeon_uvd_fini(rdev);
d93f7937 8751 radeon_vce_fini(rdev);
7bf94a2c
AD		 8752 cik_pcie_gart_fini(rdev);
8753 r600_vram_scratch_fini(rdev);
8754 radeon_gem_fini(rdev);
8755 radeon_fence_driver_fini(rdev);
8756 radeon_bo_fini(rdev);
8757 radeon_atombios_fini(rdev);
8758 kfree(rdev->bios);
8759 rdev->bios = NULL;
8760}
cd84a27d 8761
134b480f
AD		 8762void dce8_program_fmt(struct drm_encoder *encoder)
8763{
8764 struct drm_device *dev = encoder->dev;
8765 struct radeon_device *rdev = dev->dev_private;
8766 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8767 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8768 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8769 int bpc = 0;
8770 u32 tmp = 0;
6214bb74 8771 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8772
6214bb74
AD		 8773 if (connector) {
8774 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8775 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8776 dither = radeon_connector->dither;
8777 }
134b480f
AD		 8778
8779 /* LVDS/eDP FMT is set up by atom */
8780 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8781 return;
8782
8783 /* not needed for analog */
8784 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8785 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8786 return;
8787
8788 if (bpc == 0)
8789 return;
8790
8791 switch (bpc) {
8792 case 6:
6214bb74 8793 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8794 /* XXX sort out optimal dither settings */
8795 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8796 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8797 else
8798 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8799 break;
8800 case 8:
6214bb74 8801 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8802 /* XXX sort out optimal dither settings */
8803 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8804 FMT_RGB_RANDOM_ENABLE |
8805 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8806 else
8807 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8808 break;
8809 case 10:
6214bb74 8810 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8811 /* XXX sort out optimal dither settings */
8812 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8813 FMT_RGB_RANDOM_ENABLE |
8814 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8815 else
8816 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8817 break;
8818 default:
8819 /* not needed */
8820 break;
8821 }
8822
8823 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8824}
8825
cd84a27d
AD		 8826/* display watermark setup */
8827/**
8828 * dce8_line_buffer_adjust - Set up the line buffer
8829 *
8830 * @rdev: radeon_device pointer
8831 * @radeon_crtc: the selected display controller
8832 * @mode: the current display mode on the selected display
8833 * controller
8834 *
8835 * Setup up the line buffer allocation for
8836 * the selected display controller (CIK).
8837 * Returns the line buffer size in pixels.
8838 */
8839static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8840 struct radeon_crtc *radeon_crtc,
8841 struct drm_display_mode *mode)
8842{
bc01a8c7
AD		 8843 u32 tmp, buffer_alloc, i;
8844 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8845 /*
8846 * Line Buffer Setup
8847 * There are 6 line buffers, one for each display controllers.
8848 * There are 3 partitions per LB. Select the number of partitions
8849 * to enable based on the display width. For display widths larger
8850 * than 4096, you need use to use 2 display controllers and combine
8851 * them using the stereo blender.
8852 */
8853 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8854 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8855 tmp = 1;
bc01a8c7
AD		 8856 buffer_alloc = 2;
8857 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8858 tmp = 2;
bc01a8c7
AD		 8859 buffer_alloc = 2;
8860 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8861 tmp = 0;
bc01a8c7
AD		 8862 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8863 } else {
cd84a27d
AD		 8864 DRM_DEBUG_KMS("Mode too big for LB!\n");
8865 tmp = 0;
bc01a8c7 8866 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8867 }
bc01a8c7 8868 } else {
cd84a27d 8869 tmp = 1;
bc01a8c7
AD		 8870 buffer_alloc = 0;
8871 }
cd84a27d
AD		 8872
8873 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8874 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8875
bc01a8c7
AD		 8876 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8877 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8878 for (i = 0; i < rdev->usec_timeout; i++) {
8879 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8880 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8881 break;
8882 udelay(1);
8883 }
8884
cd84a27d
AD		 8885 if (radeon_crtc->base.enabled && mode) {
8886 switch (tmp) {
8887 case 0:
8888 default:
8889 return 4096 * 2;
8890 case 1:
8891 return 1920 * 2;
8892 case 2:
8893 return 2560 * 2;
8894 }
8895 }
8896
8897 /* controller not enabled, so no lb used */
8898 return 0;
8899}
8900
8901/**
8902 * cik_get_number_of_dram_channels - get the number of dram channels
8903 *
8904 * @rdev: radeon_device pointer
8905 *
8906 * Look up the number of video ram channels (CIK).
8907 * Used for display watermark bandwidth calculations
8908 * Returns the number of dram channels
8909 */
8910static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8911{
8912 u32 tmp = RREG32(MC_SHARED_CHMAP);
8913
8914 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8915 case 0:
8916 default:
8917 return 1;
8918 case 1:
8919 return 2;
8920 case 2:
8921 return 4;
8922 case 3:
8923 return 8;
8924 case 4:
8925 return 3;
8926 case 5:
8927 return 6;
8928 case 6:
8929 return 10;
8930 case 7:
8931 return 12;
8932 case 8:
8933 return 16;
8934 }
8935}
8936
8937struct dce8_wm_params {
8938 u32 dram_channels; /* number of dram channels */
8939 u32 yclk; /* bandwidth per dram data pin in kHz */
8940 u32 sclk; /* engine clock in kHz */
8941 u32 disp_clk; /* display clock in kHz */
8942 u32 src_width; /* viewport width */
8943 u32 active_time; /* active display time in ns */
8944 u32 blank_time; /* blank time in ns */
8945 bool interlaced; /* mode is interlaced */
8946 fixed20_12 vsc; /* vertical scale ratio */
8947 u32 num_heads; /* number of active crtcs */
8948 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8949 u32 lb_size; /* line buffer allocated to pipe */
8950 u32 vtaps; /* vertical scaler taps */
8951};
8952
8953/**
8954 * dce8_dram_bandwidth - get the dram bandwidth
8955 *
8956 * @wm: watermark calculation data
8957 *
8958 * Calculate the raw dram bandwidth (CIK).
8959 * Used for display watermark bandwidth calculations
8960 * Returns the dram bandwidth in MBytes/s
8961 */
8962static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8963{
8964 /* Calculate raw DRAM Bandwidth */
8965 fixed20_12 dram_efficiency; /* 0.7 */
8966 fixed20_12 yclk, dram_channels, bandwidth;
8967 fixed20_12 a;
8968
8969 a.full = dfixed_const(1000);
8970 yclk.full = dfixed_const(wm->yclk);
8971 yclk.full = dfixed_div(yclk, a);
8972 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8973 a.full = dfixed_const(10);
8974 dram_efficiency.full = dfixed_const(7);
8975 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8976 bandwidth.full = dfixed_mul(dram_channels, yclk);
8977 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8978
8979 return dfixed_trunc(bandwidth);
8980}
8981
8982/**
8983 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8984 *
8985 * @wm: watermark calculation data
8986 *
8987 * Calculate the dram bandwidth used for display (CIK).
8988 * Used for display watermark bandwidth calculations
8989 * Returns the dram bandwidth for display in MBytes/s
8990 */
8991static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8992{
8993 /* Calculate DRAM Bandwidth and the part allocated to display. */
8994 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8995 fixed20_12 yclk, dram_channels, bandwidth;
8996 fixed20_12 a;
8997
8998 a.full = dfixed_const(1000);
8999 yclk.full = dfixed_const(wm->yclk);
9000 yclk.full = dfixed_div(yclk, a);
9001 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9002 a.full = dfixed_const(10);
9003 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9004 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9005 bandwidth.full = dfixed_mul(dram_channels, yclk);
9006 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9007
9008 return dfixed_trunc(bandwidth);
9009}
9010
9011/**
9012 * dce8_data_return_bandwidth - get the data return bandwidth
9013 *
9014 * @wm: watermark calculation data
9015 *
9016 * Calculate the data return bandwidth used for display (CIK).
9017 * Used for display watermark bandwidth calculations
9018 * Returns the data return bandwidth in MBytes/s
9019 */
9020static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9021{
9022 /* Calculate the display Data return Bandwidth */
9023 fixed20_12 return_efficiency; /* 0.8 */
9024 fixed20_12 sclk, bandwidth;
9025 fixed20_12 a;
9026
9027 a.full = dfixed_const(1000);
9028 sclk.full = dfixed_const(wm->sclk);
9029 sclk.full = dfixed_div(sclk, a);
9030 a.full = dfixed_const(10);
9031 return_efficiency.full = dfixed_const(8);
9032 return_efficiency.full = dfixed_div(return_efficiency, a);
9033 a.full = dfixed_const(32);
9034 bandwidth.full = dfixed_mul(a, sclk);
9035 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9036
9037 return dfixed_trunc(bandwidth);
9038}
9039
9040/**
9041 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9042 *
9043 * @wm: watermark calculation data
9044 *
9045 * Calculate the dmif bandwidth used for display (CIK).
9046 * Used for display watermark bandwidth calculations
9047 * Returns the dmif bandwidth in MBytes/s
9048 */
9049static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9050{
9051 /* Calculate the DMIF Request Bandwidth */
9052 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9053 fixed20_12 disp_clk, bandwidth;
9054 fixed20_12 a, b;
9055
9056 a.full = dfixed_const(1000);
9057 disp_clk.full = dfixed_const(wm->disp_clk);
9058 disp_clk.full = dfixed_div(disp_clk, a);
9059 a.full = dfixed_const(32);
9060 b.full = dfixed_mul(a, disp_clk);
9061
9062 a.full = dfixed_const(10);
9063 disp_clk_request_efficiency.full = dfixed_const(8);
9064 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9065
9066 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9067
9068 return dfixed_trunc(bandwidth);
9069}
9070
9071/**
9072 * dce8_available_bandwidth - get the min available bandwidth
9073 *
9074 * @wm: watermark calculation data
9075 *
9076 * Calculate the min available bandwidth used for display (CIK).
9077 * Used for display watermark bandwidth calculations
9078 * Returns the min available bandwidth in MBytes/s
9079 */
9080static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9081{
9082 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9083 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9084 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9085 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9086
9087 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9088}
9089
9090/**
9091 * dce8_average_bandwidth - get the average available bandwidth
9092 *
9093 * @wm: watermark calculation data
9094 *
9095 * Calculate the average available bandwidth used for display (CIK).
9096 * Used for display watermark bandwidth calculations
9097 * Returns the average available bandwidth in MBytes/s
9098 */
9099static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9100{
9101 /* Calculate the display mode Average Bandwidth
9102 * DisplayMode should contain the source and destination dimensions,
9103 * timing, etc.
9104 */
9105 fixed20_12 bpp;
9106 fixed20_12 line_time;
9107 fixed20_12 src_width;
9108 fixed20_12 bandwidth;
9109 fixed20_12 a;
9110
9111 a.full = dfixed_const(1000);
9112 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9113 line_time.full = dfixed_div(line_time, a);
9114 bpp.full = dfixed_const(wm->bytes_per_pixel);
9115 src_width.full = dfixed_const(wm->src_width);
9116 bandwidth.full = dfixed_mul(src_width, bpp);
9117 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9118 bandwidth.full = dfixed_div(bandwidth, line_time);
9119
9120 return dfixed_trunc(bandwidth);
9121}
9122
9123/**
9124 * dce8_latency_watermark - get the latency watermark
9125 *
9126 * @wm: watermark calculation data
9127 *
9128 * Calculate the latency watermark (CIK).
9129 * Used for display watermark bandwidth calculations
9130 * Returns the latency watermark in ns
9131 */
9132static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9133{
9134 /* First calculate the latency in ns */
9135 u32 mc_latency = 2000; /* 2000 ns. */
9136 u32 available_bandwidth = dce8_available_bandwidth(wm);
9137 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9138 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9139 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9140 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9141 (wm->num_heads * cursor_line_pair_return_time);
9142 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9143 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9144 u32 tmp, dmif_size = 12288;
9145 fixed20_12 a, b, c;
9146
9147 if (wm->num_heads == 0)
9148 return 0;
9149
9150 a.full = dfixed_const(2);
9151 b.full = dfixed_const(1);
9152 if ((wm->vsc.full > a.full) ||
9153 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9154 (wm->vtaps >= 5) ||
9155 ((wm->vsc.full >= a.full) && wm->interlaced))
9156 max_src_lines_per_dst_line = 4;
9157 else
9158 max_src_lines_per_dst_line = 2;
9159
9160 a.full = dfixed_const(available_bandwidth);
9161 b.full = dfixed_const(wm->num_heads);
9162 a.full = dfixed_div(a, b);
9163
9164 b.full = dfixed_const(mc_latency + 512);
9165 c.full = dfixed_const(wm->disp_clk);
9166 b.full = dfixed_div(b, c);
9167
9168 c.full = dfixed_const(dmif_size);
9169 b.full = dfixed_div(c, b);
9170
9171 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9172
9173 b.full = dfixed_const(1000);
9174 c.full = dfixed_const(wm->disp_clk);
9175 b.full = dfixed_div(c, b);
9176 c.full = dfixed_const(wm->bytes_per_pixel);
9177 b.full = dfixed_mul(b, c);
9178
9179 lb_fill_bw = min(tmp, dfixed_trunc(b));
9180
9181 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9182 b.full = dfixed_const(1000);
9183 c.full = dfixed_const(lb_fill_bw);
9184 b.full = dfixed_div(c, b);
9185 a.full = dfixed_div(a, b);
9186 line_fill_time = dfixed_trunc(a);
9187
9188 if (line_fill_time < wm->active_time)
9189 return latency;
9190 else
9191 return latency + (line_fill_time - wm->active_time);
9192
9193}
9194
9195/**
9196 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9197 * average and available dram bandwidth
9198 *
9199 * @wm: watermark calculation data
9200 *
9201 * Check if the display average bandwidth fits in the display
9202 * dram bandwidth (CIK).
9203 * Used for display watermark bandwidth calculations
9204 * Returns true if the display fits, false if not.
9205 */
9206static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9207{
9208 if (dce8_average_bandwidth(wm) <=
9209 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9210 return true;
9211 else
9212 return false;
9213}
9214
9215/**
9216 * dce8_average_bandwidth_vs_available_bandwidth - check
9217 * average and available bandwidth
9218 *
9219 * @wm: watermark calculation data
9220 *
9221 * Check if the display average bandwidth fits in the display
9222 * available bandwidth (CIK).
9223 * Used for display watermark bandwidth calculations
9224 * Returns true if the display fits, false if not.
9225 */
9226static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9227{
9228 if (dce8_average_bandwidth(wm) <=
9229 (dce8_available_bandwidth(wm) / wm->num_heads))
9230 return true;
9231 else
9232 return false;
9233}
9234
9235/**
9236 * dce8_check_latency_hiding - check latency hiding
9237 *
9238 * @wm: watermark calculation data
9239 *
9240 * Check latency hiding (CIK).
9241 * Used for display watermark bandwidth calculations
9242 * Returns true if the display fits, false if not.
9243 */
9244static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9245{
9246 u32 lb_partitions = wm->lb_size / wm->src_width;
9247 u32 line_time = wm->active_time + wm->blank_time;
9248 u32 latency_tolerant_lines;
9249 u32 latency_hiding;
9250 fixed20_12 a;
9251
9252 a.full = dfixed_const(1);
9253 if (wm->vsc.full > a.full)
9254 latency_tolerant_lines = 1;
9255 else {
9256 if (lb_partitions <= (wm->vtaps + 1))
9257 latency_tolerant_lines = 1;
9258 else
9259 latency_tolerant_lines = 2;
9260 }
9261
9262 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9263
9264 if (dce8_latency_watermark(wm) <= latency_hiding)
9265 return true;
9266 else
9267 return false;
9268}
9269
9270/**
9271 * dce8_program_watermarks - program display watermarks
9272 *
9273 * @rdev: radeon_device pointer
9274 * @radeon_crtc: the selected display controller
9275 * @lb_size: line buffer size
9276 * @num_heads: number of display controllers in use
9277 *
9278 * Calculate and program the display watermarks for the
9279 * selected display controller (CIK).
9280 */
9281static void dce8_program_watermarks(struct radeon_device *rdev,
9282 struct radeon_crtc *radeon_crtc,
9283 u32 lb_size, u32 num_heads)
9284{
9285 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9286 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 9287 u32 pixel_period;
9288 u32 line_time = 0;
9289 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9290 u32 tmp, wm_mask;
9291
9292 if (radeon_crtc->base.enabled && num_heads && mode) {
9293 pixel_period = 1000000 / (u32)mode->clock;
9294 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9295
58ea2dea
AD		 9296 /* watermark for high clocks */
9297 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9298 rdev->pm.dpm_enabled) {
9299 wm_high.yclk =
9300 radeon_dpm_get_mclk(rdev, false) * 10;
9301 wm_high.sclk =
9302 radeon_dpm_get_sclk(rdev, false) * 10;
9303 } else {
9304 wm_high.yclk = rdev->pm.current_mclk * 10;
9305 wm_high.sclk = rdev->pm.current_sclk * 10;
9306 }
9307
9308 wm_high.disp_clk = mode->clock;
9309 wm_high.src_width = mode->crtc_hdisplay;
9310 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9311 wm_high.blank_time = line_time - wm_high.active_time;
9312 wm_high.interlaced = false;
cd84a27d 9313 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9314 wm_high.interlaced = true;
9315 wm_high.vsc = radeon_crtc->vsc;
9316 wm_high.vtaps = 1;
cd84a27d 9317 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9318 wm_high.vtaps = 2;
9319 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9320 wm_high.lb_size = lb_size;
9321 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9322 wm_high.num_heads = num_heads;
cd84a27d
AD		 9323
9324 /* set for high clocks */
58ea2dea
AD		 9325 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9326
9327 /* possibly force display priority to high */
9328 /* should really do this at mode validation time... */
9329 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9330 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9331 !dce8_check_latency_hiding(&wm_high) ||
9332 (rdev->disp_priority == 2)) {
9333 DRM_DEBUG_KMS("force priority to high\n");
9334 }
9335
9336 /* watermark for low clocks */
9337 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9338 rdev->pm.dpm_enabled) {
9339 wm_low.yclk =
9340 radeon_dpm_get_mclk(rdev, true) * 10;
9341 wm_low.sclk =
9342 radeon_dpm_get_sclk(rdev, true) * 10;
9343 } else {
9344 wm_low.yclk = rdev->pm.current_mclk * 10;
9345 wm_low.sclk = rdev->pm.current_sclk * 10;
9346 }
9347
9348 wm_low.disp_clk = mode->clock;
9349 wm_low.src_width = mode->crtc_hdisplay;
9350 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9351 wm_low.blank_time = line_time - wm_low.active_time;
9352 wm_low.interlaced = false;
9353 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9354 wm_low.interlaced = true;
9355 wm_low.vsc = radeon_crtc->vsc;
9356 wm_low.vtaps = 1;
9357 if (radeon_crtc->rmx_type != RMX_OFF)
9358 wm_low.vtaps = 2;
9359 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9360 wm_low.lb_size = lb_size;
9361 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9362 wm_low.num_heads = num_heads;
9363
cd84a27d 9364 /* set for low clocks */
58ea2dea 9365 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9366
9367 /* possibly force display priority to high */
9368 /* should really do this at mode validation time... */
58ea2dea
AD		 9369 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9370 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9371 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9372 (rdev->disp_priority == 2)) {
9373 DRM_DEBUG_KMS("force priority to high\n");
9374 }
9375 }
9376
9377 /* select wm A */
9378 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9379 tmp = wm_mask;
9380 tmp &= ~LATENCY_WATERMARK_MASK(3);
9381 tmp |= LATENCY_WATERMARK_MASK(1);
9382 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9383 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9384 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9385 LATENCY_HIGH_WATERMARK(line_time)));
9386 /* select wm B */
9387 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9388 tmp &= ~LATENCY_WATERMARK_MASK(3);
9389 tmp |= LATENCY_WATERMARK_MASK(2);
9390 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9391 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9392 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9393 LATENCY_HIGH_WATERMARK(line_time)));
9394 /* restore original selection */
9395 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9396
9397 /* save values for DPM */
9398 radeon_crtc->line_time = line_time;
9399 radeon_crtc->wm_high = latency_watermark_a;
9400 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9401}
9402
9403/**
9404 * dce8_bandwidth_update - program display watermarks
9405 *
9406 * @rdev: radeon_device pointer
9407 *
9408 * Calculate and program the display watermarks and line
9409 * buffer allocation (CIK).
9410 */
9411void dce8_bandwidth_update(struct radeon_device *rdev)
9412{
9413 struct drm_display_mode *mode = NULL;
9414 u32 num_heads = 0, lb_size;
9415 int i;
9416
9417 radeon_update_display_priority(rdev);
9418
9419 for (i = 0; i < rdev->num_crtc; i++) {
9420 if (rdev->mode_info.crtcs[i]->base.enabled)
9421 num_heads++;
9422 }
9423 for (i = 0; i < rdev->num_crtc; i++) {
9424 mode = &rdev->mode_info.crtcs[i]->base.mode;
9425 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9426 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9427 }
9428}
44fa346f
AD		 9429
9430/**
9431 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9432 *
9433 * @rdev: radeon_device pointer
9434 *
9435 * Fetches a GPU clock counter snapshot (SI).
9436 * Returns the 64 bit clock counter snapshot.
9437 */
9438uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9439{
9440 uint64_t clock;
9441
9442 mutex_lock(&rdev->gpu_clock_mutex);
9443 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9444 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9445 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9446 mutex_unlock(&rdev->gpu_clock_mutex);
9447 return clock;
9448}
9449
87167bb1
CK		 9450static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9451 u32 cntl_reg, u32 status_reg)
9452{
9453 int r, i;
9454 struct atom_clock_dividers dividers;
9455 uint32_t tmp;
9456
9457 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9458 clock, false, &dividers);
9459 if (r)
9460 return r;
9461
9462 tmp = RREG32_SMC(cntl_reg);
9463 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9464 tmp |= dividers.post_divider;
9465 WREG32_SMC(cntl_reg, tmp);
9466
9467 for (i = 0; i < 100; i++) {
9468 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9469 break;
9470 mdelay(10);
9471 }
9472 if (i == 100)
9473 return -ETIMEDOUT;
9474
9475 return 0;
9476}
9477
9478int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9479{
9480 int r = 0;
9481
9482 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9483 if (r)
9484 return r;
9485
9486 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9487 return r;
9488}
9489
5ad6bf91
AD		 9490int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9491{
9492 int r, i;
9493 struct atom_clock_dividers dividers;
9494 u32 tmp;
9495
9496 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9497 ecclk, false, &dividers);
9498 if (r)
9499 return r;
9500
9501 for (i = 0; i < 100; i++) {
9502 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9503 break;
9504 mdelay(10);
9505 }
9506 if (i == 100)
9507 return -ETIMEDOUT;
9508
9509 tmp = RREG32_SMC(CG_ECLK_CNTL);
9510 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9511 tmp |= dividers.post_divider;
9512 WREG32_SMC(CG_ECLK_CNTL, tmp);
9513
9514 for (i = 0; i < 100; i++) {
9515 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9516 break;
9517 mdelay(10);
9518 }
9519 if (i == 100)
9520 return -ETIMEDOUT;
9521
9522 return 0;
9523}
9524
8a7cd276 9525static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9526{
8a7cd276
AD		 9527 struct pci_dev *root = rdev->pdev->bus->self;
9528 int bridge_pos, gpu_pos;
9529 u32 speed_cntl, mask, current_data_rate;
9530 int ret, i;
9531 u16 tmp16;
87167bb1 9532
8a7cd276
AD		 9533 if (radeon_pcie_gen2 == 0)
9534 return;
87167bb1 9535
8a7cd276
AD		 9536 if (rdev->flags & RADEON_IS_IGP)
9537 return;
87167bb1 9538
8a7cd276
AD		 9539 if (!(rdev->flags & RADEON_IS_PCIE))
9540 return;
87167bb1 9541
8a7cd276
AD		 9542 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9543 if (ret != 0)
9544 return;
87167bb1 9545
8a7cd276
AD		 9546 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9547 return;
87167bb1 9548
8a7cd276
AD		 9549 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9550 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9551 LC_CURRENT_DATA_RATE_SHIFT;
9552 if (mask & DRM_PCIE_SPEED_80) {
9553 if (current_data_rate == 2) {
9554 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9555 return;
9556 }
9557 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9558 } else if (mask & DRM_PCIE_SPEED_50) {
9559 if (current_data_rate == 1) {
9560 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9561 return;
9562 }
9563 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9564 }
87167bb1 9565
8a7cd276
AD		 9566 bridge_pos = pci_pcie_cap(root);
9567 if (!bridge_pos)
9568 return;
9569
9570 gpu_pos = pci_pcie_cap(rdev->pdev);
9571 if (!gpu_pos)
9572 return;
9573
9574 if (mask & DRM_PCIE_SPEED_80) {
9575 /* re-try equalization if gen3 is not already enabled */
9576 if (current_data_rate != 2) {
9577 u16 bridge_cfg, gpu_cfg;
9578 u16 bridge_cfg2, gpu_cfg2;
9579 u32 max_lw, current_lw, tmp;
9580
9581 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9582 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9583
9584 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9585 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9586
9587 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9588 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9589
9590 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9591 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9592 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9593
9594 if (current_lw < max_lw) {
9595 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9596 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9597 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9598 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9599 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9600 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9601 }
9602 }
9603
9604 for (i = 0; i < 10; i++) {
9605 /* check status */
9606 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9607 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9608 break;
9609
9610 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9611 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9612
9613 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9614 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9615
9616 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9617 tmp |= LC_SET_QUIESCE;
9618 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9619
9620 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9621 tmp |= LC_REDO_EQ;
9622 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9623
9624 mdelay(100);
9625
9626 /* linkctl */
9627 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9628 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9629 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9630 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9631
9632 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9633 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9634 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9635 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9636
9637 /* linkctl2 */
9638 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9639 tmp16 &= ~((1 << 4) | (7 << 9));
9640 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9641 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9642
9643 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9644 tmp16 &= ~((1 << 4) | (7 << 9));
9645 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9646 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9647
9648 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9649 tmp &= ~LC_SET_QUIESCE;
9650 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9651 }
9652 }
9653 }
9654
9655 /* set the link speed */
9656 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9657 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9658 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9659
9660 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9661 tmp16 &= ~0xf;
9662 if (mask & DRM_PCIE_SPEED_80)
9663 tmp16 |= 3; /* gen3 */
9664 else if (mask & DRM_PCIE_SPEED_50)
9665 tmp16 |= 2; /* gen2 */
9666 else
9667 tmp16 |= 1; /* gen1 */
9668 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9669
9670 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9671 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9672 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9673
9674 for (i = 0; i < rdev->usec_timeout; i++) {
9675 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9676 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9677 break;
9678 udelay(1);
9679 }
9680}
7235711a
AD		 9681
9682static void cik_program_aspm(struct radeon_device *rdev)
9683{
9684 u32 data, orig;
9685 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9686 bool disable_clkreq = false;
9687
9688 if (radeon_aspm == 0)
9689 return;
9690
9691 /* XXX double check IGPs */
9692 if (rdev->flags & RADEON_IS_IGP)
9693 return;
9694
9695 if (!(rdev->flags & RADEON_IS_PCIE))
9696 return;
9697
9698 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9699 data &= ~LC_XMIT_N_FTS_MASK;
9700 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9701 if (orig != data)
9702 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9703
9704 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9705 data |= LC_GO_TO_RECOVERY;
9706 if (orig != data)
9707 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9708
9709 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9710 data |= P_IGNORE_EDB_ERR;
9711 if (orig != data)
9712 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9713
9714 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9715 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9716 data |= LC_PMI_TO_L1_DIS;
9717 if (!disable_l0s)
9718 data |= LC_L0S_INACTIVITY(7);
9719
9720 if (!disable_l1) {
9721 data |= LC_L1_INACTIVITY(7);
9722 data &= ~LC_PMI_TO_L1_DIS;
9723 if (orig != data)
9724 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9725
9726 if (!disable_plloff_in_l1) {
9727 bool clk_req_support;
9728
9729 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9730 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9731 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9732 if (orig != data)
9733 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9734
9735 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9736 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9737 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9738 if (orig != data)
9739 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9740
9741 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9742 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9743 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9744 if (orig != data)
9745 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9746
9747 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9748 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9749 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9750 if (orig != data)
9751 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9752
9753 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9754 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9755 data |= LC_DYN_LANES_PWR_STATE(3);
9756 if (orig != data)
9757 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9758
9759 if (!disable_clkreq) {
9760 struct pci_dev *root = rdev->pdev->bus->self;
9761 u32 lnkcap;
9762
9763 clk_req_support = false;
9764 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9765 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9766 clk_req_support = true;
9767 } else {
9768 clk_req_support = false;
9769 }
9770
9771 if (clk_req_support) {
9772 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9773 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9774 if (orig != data)
9775 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9776
9777 orig = data = RREG32_SMC(THM_CLK_CNTL);
9778 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9779 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9780 if (orig != data)
9781 WREG32_SMC(THM_CLK_CNTL, data);
9782
9783 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9784 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9785 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9786 if (orig != data)
9787 WREG32_SMC(MISC_CLK_CTRL, data);
9788
9789 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9790 data &= ~BCLK_AS_XCLK;
9791 if (orig != data)
9792 WREG32_SMC(CG_CLKPIN_CNTL, data);
9793
9794 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9795 data &= ~FORCE_BIF_REFCLK_EN;
9796 if (orig != data)
9797 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9798
9799 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9800 data &= ~MPLL_CLKOUT_SEL_MASK;
9801 data |= MPLL_CLKOUT_SEL(4);
9802 if (orig != data)
9803 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9804 }
9805 }
9806 } else {
9807 if (orig != data)
9808 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9809 }
9810
9811 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9812 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9813 if (orig != data)
9814 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9815
9816 if (!disable_l0s) {
9817 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9818 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9819 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9820 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9821 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9822 data &= ~LC_L0S_INACTIVITY_MASK;
9823 if (orig != data)
9824 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9825 }
9826 }
9827 }
87167bb1 9828}
Linux kernel - Deliverables
This page took 0.659353 seconds and 5 git commands to generate.