projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'drm-etnaviv-fixes' of git://git.pengutronix.de/git/lst/linux into drm...
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
5a0e3ad6 25#include <linux/slab.h>
760285e7 26#include <drm/drmP.h>
bcc1c2a1 27#include "radeon.h"
e6990375 28#include "radeon_asic.h"
bfc1f97d 29#include "radeon_audio.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
9e5acbc2
DV		 38/*
39 * Indirect registers accessor
40 */
41u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
42{
43 unsigned long flags;
44 u32 r;
45
46 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
47 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
48 r = RREG32(EVERGREEN_CG_IND_DATA);
49 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
50 return r;
51}
52
53void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
54{
55 unsigned long flags;
56
57 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
58 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
59 WREG32(EVERGREEN_CG_IND_DATA, (v));
60 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
61}
62
63u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
64{
65 unsigned long flags;
66 u32 r;
67
68 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
69 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
70 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
71 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
72 return r;
73}
74
75void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
76{
77 unsigned long flags;
78
79 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
80 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
81 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
82 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
83}
84
85u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
86{
87 unsigned long flags;
88 u32 r;
89
90 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
91 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
92 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
93 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
94 return r;
95}
96
97void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
98{
99 unsigned long flags;
100
101 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
102 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
103 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
104 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
105}
106
4a15903d
AD		 107static const u32 crtc_offsets[6] =
108{
109 EVERGREEN_CRTC0_REGISTER_OFFSET,
110 EVERGREEN_CRTC1_REGISTER_OFFSET,
111 EVERGREEN_CRTC2_REGISTER_OFFSET,
112 EVERGREEN_CRTC3_REGISTER_OFFSET,
113 EVERGREEN_CRTC4_REGISTER_OFFSET,
114 EVERGREEN_CRTC5_REGISTER_OFFSET
115};
116
2948f5e6
AD		 117#include "clearstate_evergreen.h"
118
1fd11777 119static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 120{
121 0x98fc,
122 0x9830,
123 0x9834,
124 0x9838,
125 0x9870,
126 0x9874,
127 0x8a14,
128 0x8b24,
129 0x8bcc,
130 0x8b10,
131 0x8d00,
132 0x8d04,
133 0x8c00,
134 0x8c04,
135 0x8c08,
136 0x8c0c,
137 0x8d8c,
138 0x8c20,
139 0x8c24,
140 0x8c28,
141 0x8c18,
142 0x8c1c,
143 0x8cf0,
144 0x8e2c,
145 0x8e38,
146 0x8c30,
147 0x9508,
148 0x9688,
149 0x9608,
150 0x960c,
151 0x9610,
152 0x9614,
153 0x88c4,
154 0x88d4,
155 0xa008,
156 0x900c,
157 0x9100,
158 0x913c,
159 0x98f8,
160 0x98f4,
161 0x9b7c,
162 0x3f8c,
163 0x8950,
164 0x8954,
165 0x8a18,
166 0x8b28,
167 0x9144,
168 0x9148,
169 0x914c,
170 0x3f90,
171 0x3f94,
172 0x915c,
173 0x9160,
174 0x9178,
175 0x917c,
176 0x9180,
177 0x918c,
178 0x9190,
179 0x9194,
180 0x9198,
181 0x919c,
182 0x91a8,
183 0x91ac,
184 0x91b0,
185 0x91b4,
186 0x91b8,
187 0x91c4,
188 0x91c8,
189 0x91cc,
190 0x91d0,
191 0x91d4,
192 0x91e0,
193 0x91e4,
194 0x91ec,
195 0x91f0,
196 0x91f4,
197 0x9200,
198 0x9204,
199 0x929c,
200 0x9150,
201 0x802c,
202};
2948f5e6 203
bcc1c2a1
AD		 204static void evergreen_gpu_init(struct radeon_device *rdev);
205void evergreen_fini(struct radeon_device *rdev);
b07759bf 206void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 207void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 208extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
209 int ring, u32 cp_int_cntl);
54e2e49c
AD		 210extern void cayman_vm_decode_fault(struct radeon_device *rdev,
211 u32 status, u32 addr);
22c775ce 212void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 213
59a82d0e
AD		 214extern u32 si_get_csb_size(struct radeon_device *rdev);
215extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 216extern u32 cik_get_csb_size(struct radeon_device *rdev);
217extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
b5470b03 218extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
bcc1c2a1 219
d4788db3
AD		 220static const u32 evergreen_golden_registers[] =
221{
222 0x3f90, 0xffff0000, 0xff000000,
223 0x9148, 0xffff0000, 0xff000000,
224 0x3f94, 0xffff0000, 0xff000000,
225 0x914c, 0xffff0000, 0xff000000,
226 0x9b7c, 0xffffffff, 0x00000000,
227 0x8a14, 0xffffffff, 0x00000007,
228 0x8b10, 0xffffffff, 0x00000000,
229 0x960c, 0xffffffff, 0x54763210,
230 0x88c4, 0xffffffff, 0x000000c2,
231 0x88d4, 0xffffffff, 0x00000010,
232 0x8974, 0xffffffff, 0x00000000,
233 0xc78, 0x00000080, 0x00000080,
234 0x5eb4, 0xffffffff, 0x00000002,
235 0x5e78, 0xffffffff, 0x001000f0,
236 0x6104, 0x01000300, 0x00000000,
237 0x5bc0, 0x00300000, 0x00000000,
238 0x7030, 0xffffffff, 0x00000011,
239 0x7c30, 0xffffffff, 0x00000011,
240 0x10830, 0xffffffff, 0x00000011,
241 0x11430, 0xffffffff, 0x00000011,
242 0x12030, 0xffffffff, 0x00000011,
243 0x12c30, 0xffffffff, 0x00000011,
244 0xd02c, 0xffffffff, 0x08421000,
245 0x240c, 0xffffffff, 0x00000380,
246 0x8b24, 0xffffffff, 0x00ff0fff,
247 0x28a4c, 0x06000000, 0x06000000,
248 0x10c, 0x00000001, 0x00000001,
249 0x8d00, 0xffffffff, 0x100e4848,
250 0x8d04, 0xffffffff, 0x00164745,
251 0x8c00, 0xffffffff, 0xe4000003,
252 0x8c04, 0xffffffff, 0x40600060,
253 0x8c08, 0xffffffff, 0x001c001c,
254 0x8cf0, 0xffffffff, 0x08e00620,
255 0x8c20, 0xffffffff, 0x00800080,
256 0x8c24, 0xffffffff, 0x00800080,
257 0x8c18, 0xffffffff, 0x20202078,
258 0x8c1c, 0xffffffff, 0x00001010,
259 0x28350, 0xffffffff, 0x00000000,
260 0xa008, 0xffffffff, 0x00010000,
6abafb78 261 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 262 0x9508, 0xffffffff, 0x00000002,
263 0x913c, 0x0000000f, 0x0000000a
264};
265
266static const u32 evergreen_golden_registers2[] =
267{
268 0x2f4c, 0xffffffff, 0x00000000,
269 0x54f4, 0xffffffff, 0x00000000,
270 0x54f0, 0xffffffff, 0x00000000,
271 0x5498, 0xffffffff, 0x00000000,
272 0x549c, 0xffffffff, 0x00000000,
273 0x5494, 0xffffffff, 0x00000000,
274 0x53cc, 0xffffffff, 0x00000000,
275 0x53c8, 0xffffffff, 0x00000000,
276 0x53c4, 0xffffffff, 0x00000000,
277 0x53c0, 0xffffffff, 0x00000000,
278 0x53bc, 0xffffffff, 0x00000000,
279 0x53b8, 0xffffffff, 0x00000000,
280 0x53b4, 0xffffffff, 0x00000000,
281 0x53b0, 0xffffffff, 0x00000000
282};
283
284static const u32 cypress_mgcg_init[] =
285{
286 0x802c, 0xffffffff, 0xc0000000,
287 0x5448, 0xffffffff, 0x00000100,
288 0x55e4, 0xffffffff, 0x00000100,
289 0x160c, 0xffffffff, 0x00000100,
290 0x5644, 0xffffffff, 0x00000100,
291 0xc164, 0xffffffff, 0x00000100,
292 0x8a18, 0xffffffff, 0x00000100,
293 0x897c, 0xffffffff, 0x06000100,
294 0x8b28, 0xffffffff, 0x00000100,
295 0x9144, 0xffffffff, 0x00000100,
296 0x9a60, 0xffffffff, 0x00000100,
297 0x9868, 0xffffffff, 0x00000100,
298 0x8d58, 0xffffffff, 0x00000100,
299 0x9510, 0xffffffff, 0x00000100,
300 0x949c, 0xffffffff, 0x00000100,
301 0x9654, 0xffffffff, 0x00000100,
302 0x9030, 0xffffffff, 0x00000100,
303 0x9034, 0xffffffff, 0x00000100,
304 0x9038, 0xffffffff, 0x00000100,
305 0x903c, 0xffffffff, 0x00000100,
306 0x9040, 0xffffffff, 0x00000100,
307 0xa200, 0xffffffff, 0x00000100,
308 0xa204, 0xffffffff, 0x00000100,
309 0xa208, 0xffffffff, 0x00000100,
310 0xa20c, 0xffffffff, 0x00000100,
311 0x971c, 0xffffffff, 0x00000100,
312 0x977c, 0xffffffff, 0x00000100,
313 0x3f80, 0xffffffff, 0x00000100,
314 0xa210, 0xffffffff, 0x00000100,
315 0xa214, 0xffffffff, 0x00000100,
316 0x4d8, 0xffffffff, 0x00000100,
317 0x9784, 0xffffffff, 0x00000100,
318 0x9698, 0xffffffff, 0x00000100,
319 0x4d4, 0xffffffff, 0x00000200,
320 0x30cc, 0xffffffff, 0x00000100,
321 0xd0c0, 0xffffffff, 0xff000100,
322 0x802c, 0xffffffff, 0x40000000,
323 0x915c, 0xffffffff, 0x00010000,
324 0x9160, 0xffffffff, 0x00030002,
325 0x9178, 0xffffffff, 0x00070000,
326 0x917c, 0xffffffff, 0x00030002,
327 0x9180, 0xffffffff, 0x00050004,
328 0x918c, 0xffffffff, 0x00010006,
329 0x9190, 0xffffffff, 0x00090008,
330 0x9194, 0xffffffff, 0x00070000,
331 0x9198, 0xffffffff, 0x00030002,
332 0x919c, 0xffffffff, 0x00050004,
333 0x91a8, 0xffffffff, 0x00010006,
334 0x91ac, 0xffffffff, 0x00090008,
335 0x91b0, 0xffffffff, 0x00070000,
336 0x91b4, 0xffffffff, 0x00030002,
337 0x91b8, 0xffffffff, 0x00050004,
338 0x91c4, 0xffffffff, 0x00010006,
339 0x91c8, 0xffffffff, 0x00090008,
340 0x91cc, 0xffffffff, 0x00070000,
341 0x91d0, 0xffffffff, 0x00030002,
342 0x91d4, 0xffffffff, 0x00050004,
343 0x91e0, 0xffffffff, 0x00010006,
344 0x91e4, 0xffffffff, 0x00090008,
345 0x91e8, 0xffffffff, 0x00000000,
346 0x91ec, 0xffffffff, 0x00070000,
347 0x91f0, 0xffffffff, 0x00030002,
348 0x91f4, 0xffffffff, 0x00050004,
349 0x9200, 0xffffffff, 0x00010006,
350 0x9204, 0xffffffff, 0x00090008,
351 0x9208, 0xffffffff, 0x00070000,
352 0x920c, 0xffffffff, 0x00030002,
353 0x9210, 0xffffffff, 0x00050004,
354 0x921c, 0xffffffff, 0x00010006,
355 0x9220, 0xffffffff, 0x00090008,
356 0x9224, 0xffffffff, 0x00070000,
357 0x9228, 0xffffffff, 0x00030002,
358 0x922c, 0xffffffff, 0x00050004,
359 0x9238, 0xffffffff, 0x00010006,
360 0x923c, 0xffffffff, 0x00090008,
361 0x9240, 0xffffffff, 0x00070000,
362 0x9244, 0xffffffff, 0x00030002,
363 0x9248, 0xffffffff, 0x00050004,
364 0x9254, 0xffffffff, 0x00010006,
365 0x9258, 0xffffffff, 0x00090008,
366 0x925c, 0xffffffff, 0x00070000,
367 0x9260, 0xffffffff, 0x00030002,
368 0x9264, 0xffffffff, 0x00050004,
369 0x9270, 0xffffffff, 0x00010006,
370 0x9274, 0xffffffff, 0x00090008,
371 0x9278, 0xffffffff, 0x00070000,
372 0x927c, 0xffffffff, 0x00030002,
373 0x9280, 0xffffffff, 0x00050004,
374 0x928c, 0xffffffff, 0x00010006,
375 0x9290, 0xffffffff, 0x00090008,
376 0x9294, 0xffffffff, 0x00000000,
377 0x929c, 0xffffffff, 0x00000001,
378 0x802c, 0xffffffff, 0x40010000,
379 0x915c, 0xffffffff, 0x00010000,
380 0x9160, 0xffffffff, 0x00030002,
381 0x9178, 0xffffffff, 0x00070000,
382 0x917c, 0xffffffff, 0x00030002,
383 0x9180, 0xffffffff, 0x00050004,
384 0x918c, 0xffffffff, 0x00010006,
385 0x9190, 0xffffffff, 0x00090008,
386 0x9194, 0xffffffff, 0x00070000,
387 0x9198, 0xffffffff, 0x00030002,
388 0x919c, 0xffffffff, 0x00050004,
389 0x91a8, 0xffffffff, 0x00010006,
390 0x91ac, 0xffffffff, 0x00090008,
391 0x91b0, 0xffffffff, 0x00070000,
392 0x91b4, 0xffffffff, 0x00030002,
393 0x91b8, 0xffffffff, 0x00050004,
394 0x91c4, 0xffffffff, 0x00010006,
395 0x91c8, 0xffffffff, 0x00090008,
396 0x91cc, 0xffffffff, 0x00070000,
397 0x91d0, 0xffffffff, 0x00030002,
398 0x91d4, 0xffffffff, 0x00050004,
399 0x91e0, 0xffffffff, 0x00010006,
400 0x91e4, 0xffffffff, 0x00090008,
401 0x91e8, 0xffffffff, 0x00000000,
402 0x91ec, 0xffffffff, 0x00070000,
403 0x91f0, 0xffffffff, 0x00030002,
404 0x91f4, 0xffffffff, 0x00050004,
405 0x9200, 0xffffffff, 0x00010006,
406 0x9204, 0xffffffff, 0x00090008,
407 0x9208, 0xffffffff, 0x00070000,
408 0x920c, 0xffffffff, 0x00030002,
409 0x9210, 0xffffffff, 0x00050004,
410 0x921c, 0xffffffff, 0x00010006,
411 0x9220, 0xffffffff, 0x00090008,
412 0x9224, 0xffffffff, 0x00070000,
413 0x9228, 0xffffffff, 0x00030002,
414 0x922c, 0xffffffff, 0x00050004,
415 0x9238, 0xffffffff, 0x00010006,
416 0x923c, 0xffffffff, 0x00090008,
417 0x9240, 0xffffffff, 0x00070000,
418 0x9244, 0xffffffff, 0x00030002,
419 0x9248, 0xffffffff, 0x00050004,
420 0x9254, 0xffffffff, 0x00010006,
421 0x9258, 0xffffffff, 0x00090008,
422 0x925c, 0xffffffff, 0x00070000,
423 0x9260, 0xffffffff, 0x00030002,
424 0x9264, 0xffffffff, 0x00050004,
425 0x9270, 0xffffffff, 0x00010006,
426 0x9274, 0xffffffff, 0x00090008,
427 0x9278, 0xffffffff, 0x00070000,
428 0x927c, 0xffffffff, 0x00030002,
429 0x9280, 0xffffffff, 0x00050004,
430 0x928c, 0xffffffff, 0x00010006,
431 0x9290, 0xffffffff, 0x00090008,
432 0x9294, 0xffffffff, 0x00000000,
433 0x929c, 0xffffffff, 0x00000001,
434 0x802c, 0xffffffff, 0xc0000000
435};
436
437static const u32 redwood_mgcg_init[] =
438{
439 0x802c, 0xffffffff, 0xc0000000,
440 0x5448, 0xffffffff, 0x00000100,
441 0x55e4, 0xffffffff, 0x00000100,
442 0x160c, 0xffffffff, 0x00000100,
443 0x5644, 0xffffffff, 0x00000100,
444 0xc164, 0xffffffff, 0x00000100,
445 0x8a18, 0xffffffff, 0x00000100,
446 0x897c, 0xffffffff, 0x06000100,
447 0x8b28, 0xffffffff, 0x00000100,
448 0x9144, 0xffffffff, 0x00000100,
449 0x9a60, 0xffffffff, 0x00000100,
450 0x9868, 0xffffffff, 0x00000100,
451 0x8d58, 0xffffffff, 0x00000100,
452 0x9510, 0xffffffff, 0x00000100,
453 0x949c, 0xffffffff, 0x00000100,
454 0x9654, 0xffffffff, 0x00000100,
455 0x9030, 0xffffffff, 0x00000100,
456 0x9034, 0xffffffff, 0x00000100,
457 0x9038, 0xffffffff, 0x00000100,
458 0x903c, 0xffffffff, 0x00000100,
459 0x9040, 0xffffffff, 0x00000100,
460 0xa200, 0xffffffff, 0x00000100,
461 0xa204, 0xffffffff, 0x00000100,
462 0xa208, 0xffffffff, 0x00000100,
463 0xa20c, 0xffffffff, 0x00000100,
464 0x971c, 0xffffffff, 0x00000100,
465 0x977c, 0xffffffff, 0x00000100,
466 0x3f80, 0xffffffff, 0x00000100,
467 0xa210, 0xffffffff, 0x00000100,
468 0xa214, 0xffffffff, 0x00000100,
469 0x4d8, 0xffffffff, 0x00000100,
470 0x9784, 0xffffffff, 0x00000100,
471 0x9698, 0xffffffff, 0x00000100,
472 0x4d4, 0xffffffff, 0x00000200,
473 0x30cc, 0xffffffff, 0x00000100,
474 0xd0c0, 0xffffffff, 0xff000100,
475 0x802c, 0xffffffff, 0x40000000,
476 0x915c, 0xffffffff, 0x00010000,
477 0x9160, 0xffffffff, 0x00030002,
478 0x9178, 0xffffffff, 0x00070000,
479 0x917c, 0xffffffff, 0x00030002,
480 0x9180, 0xffffffff, 0x00050004,
481 0x918c, 0xffffffff, 0x00010006,
482 0x9190, 0xffffffff, 0x00090008,
483 0x9194, 0xffffffff, 0x00070000,
484 0x9198, 0xffffffff, 0x00030002,
485 0x919c, 0xffffffff, 0x00050004,
486 0x91a8, 0xffffffff, 0x00010006,
487 0x91ac, 0xffffffff, 0x00090008,
488 0x91b0, 0xffffffff, 0x00070000,
489 0x91b4, 0xffffffff, 0x00030002,
490 0x91b8, 0xffffffff, 0x00050004,
491 0x91c4, 0xffffffff, 0x00010006,
492 0x91c8, 0xffffffff, 0x00090008,
493 0x91cc, 0xffffffff, 0x00070000,
494 0x91d0, 0xffffffff, 0x00030002,
495 0x91d4, 0xffffffff, 0x00050004,
496 0x91e0, 0xffffffff, 0x00010006,
497 0x91e4, 0xffffffff, 0x00090008,
498 0x91e8, 0xffffffff, 0x00000000,
499 0x91ec, 0xffffffff, 0x00070000,
500 0x91f0, 0xffffffff, 0x00030002,
501 0x91f4, 0xffffffff, 0x00050004,
502 0x9200, 0xffffffff, 0x00010006,
503 0x9204, 0xffffffff, 0x00090008,
504 0x9294, 0xffffffff, 0x00000000,
505 0x929c, 0xffffffff, 0x00000001,
506 0x802c, 0xffffffff, 0xc0000000
507};
508
509static const u32 cedar_golden_registers[] =
510{
511 0x3f90, 0xffff0000, 0xff000000,
512 0x9148, 0xffff0000, 0xff000000,
513 0x3f94, 0xffff0000, 0xff000000,
514 0x914c, 0xffff0000, 0xff000000,
515 0x9b7c, 0xffffffff, 0x00000000,
516 0x8a14, 0xffffffff, 0x00000007,
517 0x8b10, 0xffffffff, 0x00000000,
518 0x960c, 0xffffffff, 0x54763210,
519 0x88c4, 0xffffffff, 0x000000c2,
520 0x88d4, 0xffffffff, 0x00000000,
521 0x8974, 0xffffffff, 0x00000000,
522 0xc78, 0x00000080, 0x00000080,
523 0x5eb4, 0xffffffff, 0x00000002,
524 0x5e78, 0xffffffff, 0x001000f0,
525 0x6104, 0x01000300, 0x00000000,
526 0x5bc0, 0x00300000, 0x00000000,
527 0x7030, 0xffffffff, 0x00000011,
528 0x7c30, 0xffffffff, 0x00000011,
529 0x10830, 0xffffffff, 0x00000011,
530 0x11430, 0xffffffff, 0x00000011,
531 0xd02c, 0xffffffff, 0x08421000,
532 0x240c, 0xffffffff, 0x00000380,
533 0x8b24, 0xffffffff, 0x00ff0fff,
534 0x28a4c, 0x06000000, 0x06000000,
535 0x10c, 0x00000001, 0x00000001,
536 0x8d00, 0xffffffff, 0x100e4848,
537 0x8d04, 0xffffffff, 0x00164745,
538 0x8c00, 0xffffffff, 0xe4000003,
539 0x8c04, 0xffffffff, 0x40600060,
540 0x8c08, 0xffffffff, 0x001c001c,
541 0x8cf0, 0xffffffff, 0x08e00410,
542 0x8c20, 0xffffffff, 0x00800080,
543 0x8c24, 0xffffffff, 0x00800080,
544 0x8c18, 0xffffffff, 0x20202078,
545 0x8c1c, 0xffffffff, 0x00001010,
546 0x28350, 0xffffffff, 0x00000000,
547 0xa008, 0xffffffff, 0x00010000,
6abafb78 548 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 549 0x9508, 0xffffffff, 0x00000002
550};
551
552static const u32 cedar_mgcg_init[] =
553{
554 0x802c, 0xffffffff, 0xc0000000,
555 0x5448, 0xffffffff, 0x00000100,
556 0x55e4, 0xffffffff, 0x00000100,
557 0x160c, 0xffffffff, 0x00000100,
558 0x5644, 0xffffffff, 0x00000100,
559 0xc164, 0xffffffff, 0x00000100,
560 0x8a18, 0xffffffff, 0x00000100,
561 0x897c, 0xffffffff, 0x06000100,
562 0x8b28, 0xffffffff, 0x00000100,
563 0x9144, 0xffffffff, 0x00000100,
564 0x9a60, 0xffffffff, 0x00000100,
565 0x9868, 0xffffffff, 0x00000100,
566 0x8d58, 0xffffffff, 0x00000100,
567 0x9510, 0xffffffff, 0x00000100,
568 0x949c, 0xffffffff, 0x00000100,
569 0x9654, 0xffffffff, 0x00000100,
570 0x9030, 0xffffffff, 0x00000100,
571 0x9034, 0xffffffff, 0x00000100,
572 0x9038, 0xffffffff, 0x00000100,
573 0x903c, 0xffffffff, 0x00000100,
574 0x9040, 0xffffffff, 0x00000100,
575 0xa200, 0xffffffff, 0x00000100,
576 0xa204, 0xffffffff, 0x00000100,
577 0xa208, 0xffffffff, 0x00000100,
578 0xa20c, 0xffffffff, 0x00000100,
579 0x971c, 0xffffffff, 0x00000100,
580 0x977c, 0xffffffff, 0x00000100,
581 0x3f80, 0xffffffff, 0x00000100,
582 0xa210, 0xffffffff, 0x00000100,
583 0xa214, 0xffffffff, 0x00000100,
584 0x4d8, 0xffffffff, 0x00000100,
585 0x9784, 0xffffffff, 0x00000100,
586 0x9698, 0xffffffff, 0x00000100,
587 0x4d4, 0xffffffff, 0x00000200,
588 0x30cc, 0xffffffff, 0x00000100,
589 0xd0c0, 0xffffffff, 0xff000100,
590 0x802c, 0xffffffff, 0x40000000,
591 0x915c, 0xffffffff, 0x00010000,
592 0x9178, 0xffffffff, 0x00050000,
593 0x917c, 0xffffffff, 0x00030002,
594 0x918c, 0xffffffff, 0x00010004,
595 0x9190, 0xffffffff, 0x00070006,
596 0x9194, 0xffffffff, 0x00050000,
597 0x9198, 0xffffffff, 0x00030002,
598 0x91a8, 0xffffffff, 0x00010004,
599 0x91ac, 0xffffffff, 0x00070006,
600 0x91e8, 0xffffffff, 0x00000000,
601 0x9294, 0xffffffff, 0x00000000,
602 0x929c, 0xffffffff, 0x00000001,
603 0x802c, 0xffffffff, 0xc0000000
604};
605
606static const u32 juniper_mgcg_init[] =
607{
608 0x802c, 0xffffffff, 0xc0000000,
609 0x5448, 0xffffffff, 0x00000100,
610 0x55e4, 0xffffffff, 0x00000100,
611 0x160c, 0xffffffff, 0x00000100,
612 0x5644, 0xffffffff, 0x00000100,
613 0xc164, 0xffffffff, 0x00000100,
614 0x8a18, 0xffffffff, 0x00000100,
615 0x897c, 0xffffffff, 0x06000100,
616 0x8b28, 0xffffffff, 0x00000100,
617 0x9144, 0xffffffff, 0x00000100,
618 0x9a60, 0xffffffff, 0x00000100,
619 0x9868, 0xffffffff, 0x00000100,
620 0x8d58, 0xffffffff, 0x00000100,
621 0x9510, 0xffffffff, 0x00000100,
622 0x949c, 0xffffffff, 0x00000100,
623 0x9654, 0xffffffff, 0x00000100,
624 0x9030, 0xffffffff, 0x00000100,
625 0x9034, 0xffffffff, 0x00000100,
626 0x9038, 0xffffffff, 0x00000100,
627 0x903c, 0xffffffff, 0x00000100,
628 0x9040, 0xffffffff, 0x00000100,
629 0xa200, 0xffffffff, 0x00000100,
630 0xa204, 0xffffffff, 0x00000100,
631 0xa208, 0xffffffff, 0x00000100,
632 0xa20c, 0xffffffff, 0x00000100,
633 0x971c, 0xffffffff, 0x00000100,
634 0xd0c0, 0xffffffff, 0xff000100,
635 0x802c, 0xffffffff, 0x40000000,
636 0x915c, 0xffffffff, 0x00010000,
637 0x9160, 0xffffffff, 0x00030002,
638 0x9178, 0xffffffff, 0x00070000,
639 0x917c, 0xffffffff, 0x00030002,
640 0x9180, 0xffffffff, 0x00050004,
641 0x918c, 0xffffffff, 0x00010006,
642 0x9190, 0xffffffff, 0x00090008,
643 0x9194, 0xffffffff, 0x00070000,
644 0x9198, 0xffffffff, 0x00030002,
645 0x919c, 0xffffffff, 0x00050004,
646 0x91a8, 0xffffffff, 0x00010006,
647 0x91ac, 0xffffffff, 0x00090008,
648 0x91b0, 0xffffffff, 0x00070000,
649 0x91b4, 0xffffffff, 0x00030002,
650 0x91b8, 0xffffffff, 0x00050004,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91c8, 0xffffffff, 0x00090008,
653 0x91cc, 0xffffffff, 0x00070000,
654 0x91d0, 0xffffffff, 0x00030002,
655 0x91d4, 0xffffffff, 0x00050004,
656 0x91e0, 0xffffffff, 0x00010006,
657 0x91e4, 0xffffffff, 0x00090008,
658 0x91e8, 0xffffffff, 0x00000000,
659 0x91ec, 0xffffffff, 0x00070000,
660 0x91f0, 0xffffffff, 0x00030002,
661 0x91f4, 0xffffffff, 0x00050004,
662 0x9200, 0xffffffff, 0x00010006,
663 0x9204, 0xffffffff, 0x00090008,
664 0x9208, 0xffffffff, 0x00070000,
665 0x920c, 0xffffffff, 0x00030002,
666 0x9210, 0xffffffff, 0x00050004,
667 0x921c, 0xffffffff, 0x00010006,
668 0x9220, 0xffffffff, 0x00090008,
669 0x9224, 0xffffffff, 0x00070000,
670 0x9228, 0xffffffff, 0x00030002,
671 0x922c, 0xffffffff, 0x00050004,
672 0x9238, 0xffffffff, 0x00010006,
673 0x923c, 0xffffffff, 0x00090008,
674 0x9240, 0xffffffff, 0x00070000,
675 0x9244, 0xffffffff, 0x00030002,
676 0x9248, 0xffffffff, 0x00050004,
677 0x9254, 0xffffffff, 0x00010006,
678 0x9258, 0xffffffff, 0x00090008,
679 0x925c, 0xffffffff, 0x00070000,
680 0x9260, 0xffffffff, 0x00030002,
681 0x9264, 0xffffffff, 0x00050004,
682 0x9270, 0xffffffff, 0x00010006,
683 0x9274, 0xffffffff, 0x00090008,
684 0x9278, 0xffffffff, 0x00070000,
685 0x927c, 0xffffffff, 0x00030002,
686 0x9280, 0xffffffff, 0x00050004,
687 0x928c, 0xffffffff, 0x00010006,
688 0x9290, 0xffffffff, 0x00090008,
689 0x9294, 0xffffffff, 0x00000000,
690 0x929c, 0xffffffff, 0x00000001,
691 0x802c, 0xffffffff, 0xc0000000,
692 0x977c, 0xffffffff, 0x00000100,
693 0x3f80, 0xffffffff, 0x00000100,
694 0xa210, 0xffffffff, 0x00000100,
695 0xa214, 0xffffffff, 0x00000100,
696 0x4d8, 0xffffffff, 0x00000100,
697 0x9784, 0xffffffff, 0x00000100,
698 0x9698, 0xffffffff, 0x00000100,
699 0x4d4, 0xffffffff, 0x00000200,
700 0x30cc, 0xffffffff, 0x00000100,
701 0x802c, 0xffffffff, 0xc0000000
702};
703
704static const u32 supersumo_golden_registers[] =
705{
706 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 707 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 708 0x7030, 0xffffffff, 0x00000011,
709 0x7c30, 0xffffffff, 0x00000011,
710 0x6104, 0x01000300, 0x00000000,
711 0x5bc0, 0x00300000, 0x00000000,
712 0x8c04, 0xffffffff, 0x40600060,
713 0x8c08, 0xffffffff, 0x001c001c,
714 0x8c20, 0xffffffff, 0x00800080,
715 0x8c24, 0xffffffff, 0x00800080,
716 0x8c18, 0xffffffff, 0x20202078,
717 0x8c1c, 0xffffffff, 0x00001010,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x91c4, 0xffffffff, 0x00010006,
721 0x91e0, 0xffffffff, 0x00010006,
722 0x9200, 0xffffffff, 0x00010006,
723 0x9150, 0xffffffff, 0x6e944040,
724 0x917c, 0xffffffff, 0x00030002,
725 0x9180, 0xffffffff, 0x00050004,
726 0x9198, 0xffffffff, 0x00030002,
727 0x919c, 0xffffffff, 0x00050004,
728 0x91b4, 0xffffffff, 0x00030002,
729 0x91b8, 0xffffffff, 0x00050004,
730 0x91d0, 0xffffffff, 0x00030002,
731 0x91d4, 0xffffffff, 0x00050004,
732 0x91f0, 0xffffffff, 0x00030002,
733 0x91f4, 0xffffffff, 0x00050004,
734 0x915c, 0xffffffff, 0x00010000,
735 0x9160, 0xffffffff, 0x00030002,
736 0x3f90, 0xffff0000, 0xff000000,
737 0x9178, 0xffffffff, 0x00070000,
738 0x9194, 0xffffffff, 0x00070000,
739 0x91b0, 0xffffffff, 0x00070000,
740 0x91cc, 0xffffffff, 0x00070000,
741 0x91ec, 0xffffffff, 0x00070000,
742 0x9148, 0xffff0000, 0xff000000,
743 0x9190, 0xffffffff, 0x00090008,
744 0x91ac, 0xffffffff, 0x00090008,
745 0x91c8, 0xffffffff, 0x00090008,
746 0x91e4, 0xffffffff, 0x00090008,
747 0x9204, 0xffffffff, 0x00090008,
748 0x3f94, 0xffff0000, 0xff000000,
749 0x914c, 0xffff0000, 0xff000000,
750 0x929c, 0xffffffff, 0x00000001,
751 0x8a18, 0xffffffff, 0x00000100,
752 0x8b28, 0xffffffff, 0x00000100,
753 0x9144, 0xffffffff, 0x00000100,
754 0x5644, 0xffffffff, 0x00000100,
755 0x9b7c, 0xffffffff, 0x00000000,
756 0x8030, 0xffffffff, 0x0000100a,
757 0x8a14, 0xffffffff, 0x00000007,
758 0x8b24, 0xffffffff, 0x00ff0fff,
759 0x8b10, 0xffffffff, 0x00000000,
760 0x28a4c, 0x06000000, 0x06000000,
761 0x4d8, 0xffffffff, 0x00000100,
762 0x913c, 0xffff000f, 0x0100000a,
763 0x960c, 0xffffffff, 0x54763210,
764 0x88c4, 0xffffffff, 0x000000c2,
765 0x88d4, 0xffffffff, 0x00000010,
766 0x8974, 0xffffffff, 0x00000000,
767 0xc78, 0x00000080, 0x00000080,
768 0x5e78, 0xffffffff, 0x001000f0,
769 0xd02c, 0xffffffff, 0x08421000,
770 0xa008, 0xffffffff, 0x00010000,
771 0x8d00, 0xffffffff, 0x100e4848,
772 0x8d04, 0xffffffff, 0x00164745,
773 0x8c00, 0xffffffff, 0xe4000003,
774 0x8cf0, 0x1fffffff, 0x08e00620,
775 0x28350, 0xffffffff, 0x00000000,
776 0x9508, 0xffffffff, 0x00000002
777};
778
779static const u32 sumo_golden_registers[] =
780{
781 0x900c, 0x00ffffff, 0x0017071f,
782 0x8c18, 0xffffffff, 0x10101060,
783 0x8c1c, 0xffffffff, 0x00001010,
784 0x8c30, 0x0000000f, 0x00000005,
785 0x9688, 0x0000000f, 0x00000007
786};
787
788static const u32 wrestler_golden_registers[] =
789{
790 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 791 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 792 0x7030, 0xffffffff, 0x00000011,
793 0x7c30, 0xffffffff, 0x00000011,
794 0x6104, 0x01000300, 0x00000000,
795 0x5bc0, 0x00300000, 0x00000000,
796 0x918c, 0xffffffff, 0x00010006,
797 0x91a8, 0xffffffff, 0x00010006,
798 0x9150, 0xffffffff, 0x6e944040,
799 0x917c, 0xffffffff, 0x00030002,
800 0x9198, 0xffffffff, 0x00030002,
801 0x915c, 0xffffffff, 0x00010000,
802 0x3f90, 0xffff0000, 0xff000000,
803 0x9178, 0xffffffff, 0x00070000,
804 0x9194, 0xffffffff, 0x00070000,
805 0x9148, 0xffff0000, 0xff000000,
806 0x9190, 0xffffffff, 0x00090008,
807 0x91ac, 0xffffffff, 0x00090008,
808 0x3f94, 0xffff0000, 0xff000000,
809 0x914c, 0xffff0000, 0xff000000,
810 0x929c, 0xffffffff, 0x00000001,
811 0x8a18, 0xffffffff, 0x00000100,
812 0x8b28, 0xffffffff, 0x00000100,
813 0x9144, 0xffffffff, 0x00000100,
814 0x9b7c, 0xffffffff, 0x00000000,
815 0x8030, 0xffffffff, 0x0000100a,
816 0x8a14, 0xffffffff, 0x00000001,
817 0x8b24, 0xffffffff, 0x00ff0fff,
818 0x8b10, 0xffffffff, 0x00000000,
819 0x28a4c, 0x06000000, 0x06000000,
820 0x4d8, 0xffffffff, 0x00000100,
821 0x913c, 0xffff000f, 0x0100000a,
822 0x960c, 0xffffffff, 0x54763210,
823 0x88c4, 0xffffffff, 0x000000c2,
824 0x88d4, 0xffffffff, 0x00000010,
825 0x8974, 0xffffffff, 0x00000000,
826 0xc78, 0x00000080, 0x00000080,
827 0x5e78, 0xffffffff, 0x001000f0,
828 0xd02c, 0xffffffff, 0x08421000,
829 0xa008, 0xffffffff, 0x00010000,
830 0x8d00, 0xffffffff, 0x100e4848,
831 0x8d04, 0xffffffff, 0x00164745,
832 0x8c00, 0xffffffff, 0xe4000003,
833 0x8cf0, 0x1fffffff, 0x08e00410,
834 0x28350, 0xffffffff, 0x00000000,
835 0x9508, 0xffffffff, 0x00000002,
836 0x900c, 0xffffffff, 0x0017071f,
837 0x8c18, 0xffffffff, 0x10101060,
838 0x8c1c, 0xffffffff, 0x00001010
839};
840
841static const u32 barts_golden_registers[] =
842{
843 0x5eb4, 0xffffffff, 0x00000002,
844 0x5e78, 0x8f311ff1, 0x001000f0,
845 0x3f90, 0xffff0000, 0xff000000,
846 0x9148, 0xffff0000, 0xff000000,
847 0x3f94, 0xffff0000, 0xff000000,
848 0x914c, 0xffff0000, 0xff000000,
849 0xc78, 0x00000080, 0x00000080,
850 0xbd4, 0x70073777, 0x00010001,
851 0xd02c, 0xbfffff1f, 0x08421000,
852 0xd0b8, 0x03773777, 0x02011003,
853 0x5bc0, 0x00200000, 0x50100000,
854 0x98f8, 0x33773777, 0x02011003,
855 0x98fc, 0xffffffff, 0x76543210,
856 0x7030, 0x31000311, 0x00000011,
857 0x2f48, 0x00000007, 0x02011003,
858 0x6b28, 0x00000010, 0x00000012,
859 0x7728, 0x00000010, 0x00000012,
860 0x10328, 0x00000010, 0x00000012,
861 0x10f28, 0x00000010, 0x00000012,
862 0x11b28, 0x00000010, 0x00000012,
863 0x12728, 0x00000010, 0x00000012,
864 0x240c, 0x000007ff, 0x00000380,
865 0x8a14, 0xf000001f, 0x00000007,
866 0x8b24, 0x3fff3fff, 0x00ff0fff,
867 0x8b10, 0x0000ff0f, 0x00000000,
868 0x28a4c, 0x07ffffff, 0x06000000,
869 0x10c, 0x00000001, 0x00010003,
870 0xa02c, 0xffffffff, 0x0000009b,
871 0x913c, 0x0000000f, 0x0100000a,
872 0x8d00, 0xffff7f7f, 0x100e4848,
873 0x8d04, 0x00ffffff, 0x00164745,
874 0x8c00, 0xfffc0003, 0xe4000003,
875 0x8c04, 0xf8ff00ff, 0x40600060,
876 0x8c08, 0x00ff00ff, 0x001c001c,
877 0x8cf0, 0x1fff1fff, 0x08e00620,
878 0x8c20, 0x0fff0fff, 0x00800080,
879 0x8c24, 0x0fff0fff, 0x00800080,
880 0x8c18, 0xffffffff, 0x20202078,
881 0x8c1c, 0x0000ffff, 0x00001010,
882 0x28350, 0x00000f01, 0x00000000,
883 0x9508, 0x3700001f, 0x00000002,
884 0x960c, 0xffffffff, 0x54763210,
885 0x88c4, 0x001f3ae3, 0x000000c2,
886 0x88d4, 0x0000001f, 0x00000010,
887 0x8974, 0xffffffff, 0x00000000
888};
889
890static const u32 turks_golden_registers[] =
891{
892 0x5eb4, 0xffffffff, 0x00000002,
893 0x5e78, 0x8f311ff1, 0x001000f0,
894 0x8c8, 0x00003000, 0x00001070,
895 0x8cc, 0x000fffff, 0x00040035,
896 0x3f90, 0xffff0000, 0xfff00000,
897 0x9148, 0xffff0000, 0xfff00000,
898 0x3f94, 0xffff0000, 0xfff00000,
899 0x914c, 0xffff0000, 0xfff00000,
900 0xc78, 0x00000080, 0x00000080,
901 0xbd4, 0x00073007, 0x00010002,
902 0xd02c, 0xbfffff1f, 0x08421000,
903 0xd0b8, 0x03773777, 0x02010002,
904 0x5bc0, 0x00200000, 0x50100000,
905 0x98f8, 0x33773777, 0x00010002,
906 0x98fc, 0xffffffff, 0x33221100,
907 0x7030, 0x31000311, 0x00000011,
908 0x2f48, 0x33773777, 0x00010002,
909 0x6b28, 0x00000010, 0x00000012,
910 0x7728, 0x00000010, 0x00000012,
911 0x10328, 0x00000010, 0x00000012,
912 0x10f28, 0x00000010, 0x00000012,
913 0x11b28, 0x00000010, 0x00000012,
914 0x12728, 0x00000010, 0x00000012,
915 0x240c, 0x000007ff, 0x00000380,
916 0x8a14, 0xf000001f, 0x00000007,
917 0x8b24, 0x3fff3fff, 0x00ff0fff,
918 0x8b10, 0x0000ff0f, 0x00000000,
919 0x28a4c, 0x07ffffff, 0x06000000,
920 0x10c, 0x00000001, 0x00010003,
921 0xa02c, 0xffffffff, 0x0000009b,
922 0x913c, 0x0000000f, 0x0100000a,
923 0x8d00, 0xffff7f7f, 0x100e4848,
924 0x8d04, 0x00ffffff, 0x00164745,
925 0x8c00, 0xfffc0003, 0xe4000003,
926 0x8c04, 0xf8ff00ff, 0x40600060,
927 0x8c08, 0x00ff00ff, 0x001c001c,
928 0x8cf0, 0x1fff1fff, 0x08e00410,
929 0x8c20, 0x0fff0fff, 0x00800080,
930 0x8c24, 0x0fff0fff, 0x00800080,
931 0x8c18, 0xffffffff, 0x20202078,
932 0x8c1c, 0x0000ffff, 0x00001010,
933 0x28350, 0x00000f01, 0x00000000,
934 0x9508, 0x3700001f, 0x00000002,
935 0x960c, 0xffffffff, 0x54763210,
936 0x88c4, 0x001f3ae3, 0x000000c2,
937 0x88d4, 0x0000001f, 0x00000010,
938 0x8974, 0xffffffff, 0x00000000
939};
940
941static const u32 caicos_golden_registers[] =
942{
943 0x5eb4, 0xffffffff, 0x00000002,
944 0x5e78, 0x8f311ff1, 0x001000f0,
945 0x8c8, 0x00003420, 0x00001450,
946 0x8cc, 0x000fffff, 0x00040035,
947 0x3f90, 0xffff0000, 0xfffc0000,
948 0x9148, 0xffff0000, 0xfffc0000,
949 0x3f94, 0xffff0000, 0xfffc0000,
950 0x914c, 0xffff0000, 0xfffc0000,
951 0xc78, 0x00000080, 0x00000080,
952 0xbd4, 0x00073007, 0x00010001,
953 0xd02c, 0xbfffff1f, 0x08421000,
954 0xd0b8, 0x03773777, 0x02010001,
955 0x5bc0, 0x00200000, 0x50100000,
956 0x98f8, 0x33773777, 0x02010001,
957 0x98fc, 0xffffffff, 0x33221100,
958 0x7030, 0x31000311, 0x00000011,
959 0x2f48, 0x33773777, 0x02010001,
960 0x6b28, 0x00000010, 0x00000012,
961 0x7728, 0x00000010, 0x00000012,
962 0x10328, 0x00000010, 0x00000012,
963 0x10f28, 0x00000010, 0x00000012,
964 0x11b28, 0x00000010, 0x00000012,
965 0x12728, 0x00000010, 0x00000012,
966 0x240c, 0x000007ff, 0x00000380,
967 0x8a14, 0xf000001f, 0x00000001,
968 0x8b24, 0x3fff3fff, 0x00ff0fff,
969 0x8b10, 0x0000ff0f, 0x00000000,
970 0x28a4c, 0x07ffffff, 0x06000000,
971 0x10c, 0x00000001, 0x00010003,
972 0xa02c, 0xffffffff, 0x0000009b,
973 0x913c, 0x0000000f, 0x0100000a,
974 0x8d00, 0xffff7f7f, 0x100e4848,
975 0x8d04, 0x00ffffff, 0x00164745,
976 0x8c00, 0xfffc0003, 0xe4000003,
977 0x8c04, 0xf8ff00ff, 0x40600060,
978 0x8c08, 0x00ff00ff, 0x001c001c,
979 0x8cf0, 0x1fff1fff, 0x08e00410,
980 0x8c20, 0x0fff0fff, 0x00800080,
981 0x8c24, 0x0fff0fff, 0x00800080,
982 0x8c18, 0xffffffff, 0x20202078,
983 0x8c1c, 0x0000ffff, 0x00001010,
984 0x28350, 0x00000f01, 0x00000000,
985 0x9508, 0x3700001f, 0x00000002,
986 0x960c, 0xffffffff, 0x54763210,
987 0x88c4, 0x001f3ae3, 0x000000c2,
988 0x88d4, 0x0000001f, 0x00000010,
989 0x8974, 0xffffffff, 0x00000000
990};
991
992static void evergreen_init_golden_registers(struct radeon_device *rdev)
993{
994 switch (rdev->family) {
995 case CHIP_CYPRESS:
996 case CHIP_HEMLOCK:
997 radeon_program_register_sequence(rdev,
998 evergreen_golden_registers,
999 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1000 radeon_program_register_sequence(rdev,
1001 evergreen_golden_registers2,
1002 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1003 radeon_program_register_sequence(rdev,
1004 cypress_mgcg_init,
1005 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1006 break;
1007 case CHIP_JUNIPER:
1008 radeon_program_register_sequence(rdev,
1009 evergreen_golden_registers,
1010 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1011 radeon_program_register_sequence(rdev,
1012 evergreen_golden_registers2,
1013 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1014 radeon_program_register_sequence(rdev,
1015 juniper_mgcg_init,
1016 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1017 break;
1018 case CHIP_REDWOOD:
1019 radeon_program_register_sequence(rdev,
1020 evergreen_golden_registers,
1021 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1022 radeon_program_register_sequence(rdev,
1023 evergreen_golden_registers2,
1024 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1025 radeon_program_register_sequence(rdev,
1026 redwood_mgcg_init,
1027 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1028 break;
1029 case CHIP_CEDAR:
1030 radeon_program_register_sequence(rdev,
1031 cedar_golden_registers,
1032 (const u32)ARRAY_SIZE(cedar_golden_registers));
1033 radeon_program_register_sequence(rdev,
1034 evergreen_golden_registers2,
1035 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1036 radeon_program_register_sequence(rdev,
1037 cedar_mgcg_init,
1038 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1039 break;
1040 case CHIP_PALM:
1041 radeon_program_register_sequence(rdev,
1042 wrestler_golden_registers,
1043 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1044 break;
1045 case CHIP_SUMO:
1046 radeon_program_register_sequence(rdev,
1047 supersumo_golden_registers,
1048 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1049 break;
1050 case CHIP_SUMO2:
1051 radeon_program_register_sequence(rdev,
1052 supersumo_golden_registers,
1053 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054 radeon_program_register_sequence(rdev,
1055 sumo_golden_registers,
1056 (const u32)ARRAY_SIZE(sumo_golden_registers));
1057 break;
1058 case CHIP_BARTS:
1059 radeon_program_register_sequence(rdev,
1060 barts_golden_registers,
1061 (const u32)ARRAY_SIZE(barts_golden_registers));
1062 break;
1063 case CHIP_TURKS:
1064 radeon_program_register_sequence(rdev,
1065 turks_golden_registers,
1066 (const u32)ARRAY_SIZE(turks_golden_registers));
1067 break;
1068 case CHIP_CAICOS:
1069 radeon_program_register_sequence(rdev,
1070 caicos_golden_registers,
1071 (const u32)ARRAY_SIZE(caicos_golden_registers));
1072 break;
1073 default:
1074 break;
1075 }
1076}
1077
ff609975
AD		 1078/**
1079 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1080 *
1081 * @rdev: radeon_device pointer
1082 * @reg: register offset in bytes
1083 * @val: register value
1084 *
1085 * Returns 0 for success or -EINVAL for an invalid register
1086 *
1087 */
1088int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1089 u32 reg, u32 *val)
1090{
1091 switch (reg) {
1092 case GRBM_STATUS:
1093 case GRBM_STATUS_SE0:
1094 case GRBM_STATUS_SE1:
1095 case SRBM_STATUS:
1096 case SRBM_STATUS2:
1097 case DMA_STATUS_REG:
1098 case UVD_STATUS:
1099 *val = RREG32(reg);
1100 return 0;
1101 default:
1102 return -EINVAL;
1103 }
1104}
1105
285484e2
JG		 1106void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1107 unsigned *bankh, unsigned *mtaspect,
1108 unsigned *tile_split)
1109{
1110 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1111 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1112 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1113 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1114 switch (*bankw) {
1115 default:
1116 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1117 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1118 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1119 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1120 }
1121 switch (*bankh) {
1122 default:
1123 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1124 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1125 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1126 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1127 }
1128 switch (*mtaspect) {
1129 default:
1130 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1131 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1132 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1133 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1134 }
1135}
1136
23d33ba3
AD		 1137static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1138 u32 cntl_reg, u32 status_reg)
1139{
1140 int r, i;
1141 struct atom_clock_dividers dividers;
1142
3cf8bb1a 1143 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
23d33ba3
AD		 1144 clock, false, &dividers);
1145 if (r)
1146 return r;
1147
1148 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1149
1150 for (i = 0; i < 100; i++) {
1151 if (RREG32(status_reg) & DCLK_STATUS)
1152 break;
1153 mdelay(10);
1154 }
1155 if (i == 100)
1156 return -ETIMEDOUT;
1157
1158 return 0;
1159}
1160
1161int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1162{
1163 int r = 0;
1164 u32 cg_scratch = RREG32(CG_SCRATCH1);
1165
1166 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1167 if (r)
1168 goto done;
1169 cg_scratch &= 0xffff0000;
1170 cg_scratch |= vclk / 100; /* Mhz */
1171
1172 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1173 if (r)
1174 goto done;
1175 cg_scratch &= 0x0000ffff;
1176 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1177
1178done:
1179 WREG32(CG_SCRATCH1, cg_scratch);
1180
1181 return r;
1182}
1183
a8b4925c
AD		 1184int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1185{
1186 /* start off with something large */
facd112d 1187 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1188 int r;
1189
4ed10835
CK		 1190 /* bypass vclk and dclk with bclk */
1191 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1192 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1193 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1194
1195 /* put PLL in bypass mode */
1196 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1197
1198 if (!vclk || !dclk) {
1199 /* keep the Bypass mode, put PLL to sleep */
1200 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1201 return 0;
1202 }
1203
facd112d
CK		 1204 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1205 16384, 0x03FFFFFF, 0, 128, 5,
1206 &fb_div, &vclk_div, &dclk_div);
1207 if (r)
1208 return r;
a8b4925c
AD		 1209
1210 /* set VCO_MODE to 1 */
1211 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1212
1213 /* toggle UPLL_SLEEP to 1 then back to 0 */
1214 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1215 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1216
1217 /* deassert UPLL_RESET */
1218 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1219
1220 mdelay(1);
1221
facd112d 1222 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1223 if (r)
1224 return r;
1225
1226 /* assert UPLL_RESET again */
1227 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1228
1229 /* disable spread spectrum. */
1230 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1231
1232 /* set feedback divider */
facd112d 1233 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1234
1235 /* set ref divider to 0 */
1236 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1237
facd112d 1238 if (fb_div < 307200)
a8b4925c
AD		 1239 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1240 else
1241 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1242
1243 /* set PDIV_A and PDIV_B */
1244 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1245 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1246 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1247
1248 /* give the PLL some time to settle */
1249 mdelay(15);
1250
1251 /* deassert PLL_RESET */
1252 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1253
1254 mdelay(15);
1255
1256 /* switch from bypass mode to normal mode */
1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1258
facd112d 1259 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1260 if (r)
1261 return r;
1262
1263 /* switch VCLK and DCLK selection */
1264 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1265 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1266 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1267
1268 mdelay(100);
1269
1270 return 0;
1271}
1272
d054ac16
AD		 1273void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1274{
c11592fe
YW		 1275 int readrq;
1276 u16 v;
d054ac16 1277
c11592fe
YW		 1278 readrq = pcie_get_readrq(rdev->pdev);
1279 v = ffs(readrq) - 8;
d054ac16
AD		 1280 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1281 * to avoid hangs or perfomance issues
1282 */
c11592fe
YW		 1283 if ((v == 0) || (v == 6) || (v == 7))
1284 pcie_set_readrq(rdev->pdev, 512);
d054ac16
AD		 1285}
1286
134b480f
AD		 1287void dce4_program_fmt(struct drm_encoder *encoder)
1288{
1289 struct drm_device *dev = encoder->dev;
1290 struct radeon_device *rdev = dev->dev_private;
1291 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1292 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1293 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1294 int bpc = 0;
1295 u32 tmp = 0;
6214bb74 1296 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1297
6214bb74
AD		 1298 if (connector) {
1299 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1300 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1301 dither = radeon_connector->dither;
1302 }
134b480f
AD		 1303
1304 /* LVDS/eDP FMT is set up by atom */
1305 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1306 return;
1307
1308 /* not needed for analog */
1309 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1310 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1311 return;
1312
1313 if (bpc == 0)
1314 return;
1315
1316 switch (bpc) {
1317 case 6:
6214bb74 1318 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1319 /* XXX sort out optimal dither settings */
1320 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1321 FMT_SPATIAL_DITHER_EN);
1322 else
1323 tmp |= FMT_TRUNCATE_EN;
1324 break;
1325 case 8:
6214bb74 1326 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1327 /* XXX sort out optimal dither settings */
1328 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329 FMT_RGB_RANDOM_ENABLE |
1330 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1331 else
1332 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1333 break;
1334 case 10:
1335 default:
1336 /* not needed */
1337 break;
1338 }
1339
1340 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1341}
1342
10257a6d
AD		 1343static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1344{
1345 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1346 return true;
1347 else
1348 return false;
1349}
1350
1351static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1352{
1353 u32 pos1, pos2;
1354
1355 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1356 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1357
1358 if (pos1 != pos2)
1359 return true;
1360 else
1361 return false;
1362}
1363
377edc8b
AD		 1364/**
1365 * dce4_wait_for_vblank - vblank wait asic callback.
1366 *
1367 * @rdev: radeon_device pointer
1368 * @crtc: crtc to wait for vblank on
1369 *
1370 * Wait for vblank on the requested crtc (evergreen+).
1371 */
3ae19b75
AD		 1372void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1373{
10257a6d 1374 unsigned i = 0;
3ae19b75 1375
4a15903d
AD		 1376 if (crtc >= rdev->num_crtc)
1377 return;
1378
10257a6d
AD		 1379 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1380 return;
1381
1382 /* depending on when we hit vblank, we may be close to active; if so,
1383 * wait for another frame.
1384 */
1385 while (dce4_is_in_vblank(rdev, crtc)) {
1386 if (i++ % 100 == 0) {
1387 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1388 break;
3ae19b75 1389 }
10257a6d
AD		 1390 }
1391
1392 while (!dce4_is_in_vblank(rdev, crtc)) {
1393 if (i++ % 100 == 0) {
1394 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1395 break;
3ae19b75
AD		 1396 }
1397 }
1398}
1399
377edc8b
AD		 1400/**
1401 * evergreen_page_flip - pageflip callback.
1402 *
1403 * @rdev: radeon_device pointer
1404 * @crtc_id: crtc to cleanup pageflip on
1405 * @crtc_base: new address of the crtc (GPU MC address)
1406 *
cbd3e249
AD		 1407 * Triggers the actual pageflip by updating the primary
1408 * surface base address (evergreen+).
377edc8b 1409 */
c63dd758
MD		 1410void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1411 bool async)
6f34be50
AD		 1412{
1413 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1414
1415 /* update the scanout addresses */
c63dd758
MD		 1416 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1417 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
6f34be50
AD		 1418 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1419 upper_32_bits(crtc_base));
1420 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1421 (u32)crtc_base);
cbd3e249
AD		 1422 /* post the write */
1423 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
157fa14d
CK		 1424}
1425
1426/**
1427 * evergreen_page_flip_pending - check if page flip is still pending
1428 *
1429 * @rdev: radeon_device pointer
1430 * @crtc_id: crtc to check
1431 *
1432 * Returns the current update pending status.
1433 */
1434bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1435{
1436 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1437
1438 /* Return current update_pending status: */
157fa14d
CK		 1439 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1440 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
6f34be50
AD		 1441}
1442
21a8122a 1443/* get temperature in millidegrees */
20d391d7 1444int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1445{
1c88d74f
AD		 1446 u32 temp, toffset;
1447 int actual_temp = 0;
67b3f823
AD		 1448
1449 if (rdev->family == CHIP_JUNIPER) {
1450 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1451 TOFFSET_SHIFT;
1452 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1453 TS0_ADC_DOUT_SHIFT;
1454
1455 if (toffset & 0x100)
1456 actual_temp = temp / 2 - (0x200 - toffset);
1457 else
1458 actual_temp = temp / 2 + toffset;
1459
1460 actual_temp = actual_temp * 1000;
1461
1462 } else {
1463 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1464 ASIC_T_SHIFT;
1465
1466 if (temp & 0x400)
1467 actual_temp = -256;
1468 else if (temp & 0x200)
1469 actual_temp = 255;
1470 else if (temp & 0x100) {
1471 actual_temp = temp & 0x1ff;
1472 actual_temp |= ~0x1ff;
1473 } else
1474 actual_temp = temp & 0xff;
1475
1476 actual_temp = (actual_temp * 1000) / 2;
1477 }
21a8122a 1478
67b3f823 1479 return actual_temp;
21a8122a
AD		 1480}
1481
20d391d7 1482int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1483{
1484 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1485 int actual_temp = temp - 49;
e33df25f
AD		 1486
1487 return actual_temp * 1000;
1488}
1489
377edc8b
AD		 1490/**
1491 * sumo_pm_init_profile - Initialize power profiles callback.
1492 *
1493 * @rdev: radeon_device pointer
1494 *
1495 * Initialize the power states used in profile mode
1496 * (sumo, trinity, SI).
1497 * Used for profile mode only.
1498 */
a4c9e2ee
AD		 1499void sumo_pm_init_profile(struct radeon_device *rdev)
1500{
1501 int idx;
1502
1503 /* default */
1504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1505 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1506 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1508
1509 /* low,mid sh/mh */
1510 if (rdev->flags & RADEON_IS_MOBILITY)
1511 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1512 else
1513 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1514
1515 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1516 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1517 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1518 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1519
1520 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1521 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1523 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1524
1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1526 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1528 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1529
1530 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1531 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1532 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1533 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1534
1535 /* high sh/mh */
1536 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1537 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1538 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1539 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1540 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1541 rdev->pm.power_state[idx].num_clock_modes - 1;
1542
1543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1544 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1545 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1546 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1547 rdev->pm.power_state[idx].num_clock_modes - 1;
1548}
1549
27810fb2
AD		 1550/**
1551 * btc_pm_init_profile - Initialize power profiles callback.
1552 *
1553 * @rdev: radeon_device pointer
1554 *
1555 * Initialize the power states used in profile mode
1556 * (BTC, cayman).
1557 * Used for profile mode only.
1558 */
1559void btc_pm_init_profile(struct radeon_device *rdev)
1560{
1561 int idx;
1562
1563 /* default */
1564 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1565 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1566 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1567 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1568 /* starting with BTC, there is one state that is used for both
1569 * MH and SH. Difference is that we always use the high clock index for
1570 * mclk.
1571 */
1572 if (rdev->flags & RADEON_IS_MOBILITY)
1573 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1574 else
1575 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1576 /* low sh */
1577 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1578 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1579 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1580 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1581 /* mid sh */
1582 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1583 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1584 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1585 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1586 /* high sh */
1587 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1588 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1589 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1590 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1591 /* low mh */
1592 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1593 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1594 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1595 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1596 /* mid mh */
1597 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1598 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1599 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1600 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1601 /* high mh */
1602 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1603 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1604 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1605 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1606}
1607
377edc8b
AD		 1608/**
1609 * evergreen_pm_misc - set additional pm hw parameters callback.
1610 *
1611 * @rdev: radeon_device pointer
1612 *
1613 * Set non-clock parameters associated with a power state
1614 * (voltage, etc.) (evergreen+).
1615 */
49e02b73
AD		 1616void evergreen_pm_misc(struct radeon_device *rdev)
1617{
a081a9d6
RM		 1618 int req_ps_idx = rdev->pm.requested_power_state_index;
1619 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1620 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1621 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1622
2feea49a 1623 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1624 /* 0xff0x are flags rather then an actual voltage */
1625 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1626 return;
2feea49a 1627 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1628 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1629 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1630 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1631 }
7ae764b1
AD		 1632
1633 /* starting with BTC, there is one state that is used for both
1634 * MH and SH. Difference is that we always use the high clock index for
1635 * mclk and vddci.
1636 */
1637 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1638 (rdev->family >= CHIP_BARTS) &&
1639 rdev->pm.active_crtc_count &&
1640 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1641 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1642 voltage = &rdev->pm.power_state[req_ps_idx].
1643 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1644
c6cf7777
AD		 1645 /* 0xff0x are flags rather then an actual voltage */
1646 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1647 return;
2feea49a
AD		 1648 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1649 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1650 rdev->pm.current_vddci = voltage->vddci;
1651 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1652 }
1653 }
49e02b73
AD		 1654}
1655
377edc8b
AD		 1656/**
1657 * evergreen_pm_prepare - pre-power state change callback.
1658 *
1659 * @rdev: radeon_device pointer
1660 *
1661 * Prepare for a power state change (evergreen+).
1662 */
49e02b73
AD		 1663void evergreen_pm_prepare(struct radeon_device *rdev)
1664{
1665 struct drm_device *ddev = rdev->ddev;
1666 struct drm_crtc *crtc;
1667 struct radeon_crtc *radeon_crtc;
1668 u32 tmp;
1669
1670 /* disable any active CRTCs */
1671 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1672 radeon_crtc = to_radeon_crtc(crtc);
1673 if (radeon_crtc->enabled) {
1674 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1675 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1676 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1677 }
1678 }
1679}
1680
377edc8b
AD		 1681/**
1682 * evergreen_pm_finish - post-power state change callback.
1683 *
1684 * @rdev: radeon_device pointer
1685 *
1686 * Clean up after a power state change (evergreen+).
1687 */
49e02b73
AD		 1688void evergreen_pm_finish(struct radeon_device *rdev)
1689{
1690 struct drm_device *ddev = rdev->ddev;
1691 struct drm_crtc *crtc;
1692 struct radeon_crtc *radeon_crtc;
1693 u32 tmp;
1694
1695 /* enable any active CRTCs */
1696 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1697 radeon_crtc = to_radeon_crtc(crtc);
1698 if (radeon_crtc->enabled) {
1699 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1700 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1701 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1702 }
1703 }
1704}
1705
377edc8b
AD		 1706/**
1707 * evergreen_hpd_sense - hpd sense callback.
1708 *
1709 * @rdev: radeon_device pointer
1710 * @hpd: hpd (hotplug detect) pin
1711 *
1712 * Checks if a digital monitor is connected (evergreen+).
1713 * Returns true if connected, false if not connected.
1714 */
bcc1c2a1
AD		 1715bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1716{
1717 bool connected = false;
0ca2ab52
AD		 1718
1719 switch (hpd) {
1720 case RADEON_HPD_1:
1721 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1722 connected = true;
1723 break;
1724 case RADEON_HPD_2:
1725 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1726 connected = true;
1727 break;
1728 case RADEON_HPD_3:
1729 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1730 connected = true;
1731 break;
1732 case RADEON_HPD_4:
1733 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1734 connected = true;
1735 break;
1736 case RADEON_HPD_5:
1737 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1738 connected = true;
1739 break;
1740 case RADEON_HPD_6:
1741 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1742 connected = true;
ebc54ffe 1743 break;
0ca2ab52
AD		 1744 default:
1745 break;
1746 }
1747
bcc1c2a1
AD		 1748 return connected;
1749}
1750
377edc8b
AD		 1751/**
1752 * evergreen_hpd_set_polarity - hpd set polarity callback.
1753 *
1754 * @rdev: radeon_device pointer
1755 * @hpd: hpd (hotplug detect) pin
1756 *
1757 * Set the polarity of the hpd pin (evergreen+).
1758 */
bcc1c2a1
AD		 1759void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1760 enum radeon_hpd_id hpd)
1761{
0ca2ab52
AD		 1762 u32 tmp;
1763 bool connected = evergreen_hpd_sense(rdev, hpd);
1764
1765 switch (hpd) {
1766 case RADEON_HPD_1:
1767 tmp = RREG32(DC_HPD1_INT_CONTROL);
1768 if (connected)
1769 tmp &= ~DC_HPDx_INT_POLARITY;
1770 else
1771 tmp |= DC_HPDx_INT_POLARITY;
1772 WREG32(DC_HPD1_INT_CONTROL, tmp);
1773 break;
1774 case RADEON_HPD_2:
1775 tmp = RREG32(DC_HPD2_INT_CONTROL);
1776 if (connected)
1777 tmp &= ~DC_HPDx_INT_POLARITY;
1778 else
1779 tmp |= DC_HPDx_INT_POLARITY;
1780 WREG32(DC_HPD2_INT_CONTROL, tmp);
1781 break;
1782 case RADEON_HPD_3:
1783 tmp = RREG32(DC_HPD3_INT_CONTROL);
1784 if (connected)
1785 tmp &= ~DC_HPDx_INT_POLARITY;
1786 else
1787 tmp |= DC_HPDx_INT_POLARITY;
1788 WREG32(DC_HPD3_INT_CONTROL, tmp);
1789 break;
1790 case RADEON_HPD_4:
1791 tmp = RREG32(DC_HPD4_INT_CONTROL);
1792 if (connected)
1793 tmp &= ~DC_HPDx_INT_POLARITY;
1794 else
1795 tmp |= DC_HPDx_INT_POLARITY;
1796 WREG32(DC_HPD4_INT_CONTROL, tmp);
1797 break;
1798 case RADEON_HPD_5:
1799 tmp = RREG32(DC_HPD5_INT_CONTROL);
1800 if (connected)
1801 tmp &= ~DC_HPDx_INT_POLARITY;
1802 else
1803 tmp |= DC_HPDx_INT_POLARITY;
1804 WREG32(DC_HPD5_INT_CONTROL, tmp);
1805 break;
1806 case RADEON_HPD_6:
1807 tmp = RREG32(DC_HPD6_INT_CONTROL);
1808 if (connected)
1809 tmp &= ~DC_HPDx_INT_POLARITY;
1810 else
1811 tmp |= DC_HPDx_INT_POLARITY;
1812 WREG32(DC_HPD6_INT_CONTROL, tmp);
1813 break;
1814 default:
1815 break;
1816 }
bcc1c2a1
AD		 1817}
1818
377edc8b
AD		 1819/**
1820 * evergreen_hpd_init - hpd setup callback.
1821 *
1822 * @rdev: radeon_device pointer
1823 *
1824 * Setup the hpd pins used by the card (evergreen+).
1825 * Enable the pin, set the polarity, and enable the hpd interrupts.
1826 */
bcc1c2a1
AD		 1827void evergreen_hpd_init(struct radeon_device *rdev)
1828{
0ca2ab52
AD		 1829 struct drm_device *dev = rdev->ddev;
1830 struct drm_connector *connector;
fb98257a 1831 unsigned enabled = 0;
0ca2ab52
AD		 1832 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1833 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1834
0ca2ab52
AD		 1835 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1836 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1837
1838 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1839 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1840 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1841 * aux dp channel on imac and help (but not completely fix)
1842 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1843 * also avoid interrupt storms during dpms.
1844 */
1845 continue;
1846 }
0ca2ab52
AD		 1847 switch (radeon_connector->hpd.hpd) {
1848 case RADEON_HPD_1:
1849 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1850 break;
1851 case RADEON_HPD_2:
1852 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1853 break;
1854 case RADEON_HPD_3:
1855 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1856 break;
1857 case RADEON_HPD_4:
1858 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1859 break;
1860 case RADEON_HPD_5:
1861 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1862 break;
1863 case RADEON_HPD_6:
1864 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1865 break;
1866 default:
1867 break;
1868 }
64912e99 1869 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
b2c0cbd6
NS		 1870 if (radeon_connector->hpd.hpd != RADEON_HPD_NONE)
1871 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1872 }
fb98257a 1873 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1874}
1875
377edc8b
AD		 1876/**
1877 * evergreen_hpd_fini - hpd tear down callback.
1878 *
1879 * @rdev: radeon_device pointer
1880 *
1881 * Tear down the hpd pins used by the card (evergreen+).
1882 * Disable the hpd interrupts.
1883 */
0ca2ab52 1884void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1885{
0ca2ab52
AD		 1886 struct drm_device *dev = rdev->ddev;
1887 struct drm_connector *connector;
fb98257a 1888 unsigned disabled = 0;
0ca2ab52
AD		 1889
1890 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1891 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1892 switch (radeon_connector->hpd.hpd) {
1893 case RADEON_HPD_1:
1894 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1895 break;
1896 case RADEON_HPD_2:
1897 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1898 break;
1899 case RADEON_HPD_3:
1900 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1901 break;
1902 case RADEON_HPD_4:
1903 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1904 break;
1905 case RADEON_HPD_5:
1906 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1907 break;
1908 case RADEON_HPD_6:
1909 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1910 break;
1911 default:
1912 break;
1913 }
b2c0cbd6
NS		 1914 if (radeon_connector->hpd.hpd != RADEON_HPD_NONE)
1915 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1916 }
fb98257a 1917 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1918}
1919
f9d9c362
AD		 1920/* watermark setup */
1921
1922static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1923 struct radeon_crtc *radeon_crtc,
1924 struct drm_display_mode *mode,
1925 struct drm_display_mode *other_mode)
1926{
0b31e023
AD		 1927 u32 tmp, buffer_alloc, i;
1928 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1929 /*
1930 * Line Buffer Setup
1931 * There are 3 line buffers, each one shared by 2 display controllers.
1932 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1933 * the display controllers. The paritioning is done via one of four
1934 * preset allocations specified in bits 2:0:
1935 * first display controller
1936 * 0 - first half of lb (3840 * 2)
1937 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1938 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1939 * 3 - first 1/4 of lb (1920 * 2)
1940 * second display controller
1941 * 4 - second half of lb (3840 * 2)
1942 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1943 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1944 * 7 - last 1/4 of lb (1920 * 2)
1945 */
12dfc843
AD		 1946 /* this can get tricky if we have two large displays on a paired group
1947 * of crtcs. Ideally for multiple large displays we'd assign them to
1948 * non-linked crtcs for maximum line buffer allocation.
1949 */
1950 if (radeon_crtc->base.enabled && mode) {
0b31e023 1951 if (other_mode) {
f9d9c362 1952 tmp = 0; /* 1/2 */
0b31e023
AD		 1953 buffer_alloc = 1;
1954 } else {
12dfc843 1955 tmp = 2; /* whole */
0b31e023
AD		 1956 buffer_alloc = 2;
1957 }
1958 } else {
12dfc843 1959 tmp = 0;
0b31e023
AD		 1960 buffer_alloc = 0;
1961 }
f9d9c362
AD		 1962
1963 /* second controller of the pair uses second half of the lb */
1964 if (radeon_crtc->crtc_id % 2)
1965 tmp += 4;
1966 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1967
0b31e023
AD		 1968 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1969 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1970 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1971 for (i = 0; i < rdev->usec_timeout; i++) {
1972 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1973 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1974 break;
1975 udelay(1);
1976 }
1977 }
1978
12dfc843
AD		 1979 if (radeon_crtc->base.enabled && mode) {
1980 switch (tmp) {
1981 case 0:
1982 case 4:
1983 default:
1984 if (ASIC_IS_DCE5(rdev))
1985 return 4096 * 2;
1986 else
1987 return 3840 * 2;
1988 case 1:
1989 case 5:
1990 if (ASIC_IS_DCE5(rdev))
1991 return 6144 * 2;
1992 else
1993 return 5760 * 2;
1994 case 2:
1995 case 6:
1996 if (ASIC_IS_DCE5(rdev))
1997 return 8192 * 2;
1998 else
1999 return 7680 * 2;
2000 case 3:
2001 case 7:
2002 if (ASIC_IS_DCE5(rdev))
2003 return 2048 * 2;
2004 else
2005 return 1920 * 2;
2006 }
f9d9c362 2007 }
12dfc843
AD		 2008
2009 /* controller not enabled, so no lb used */
2010 return 0;
f9d9c362
AD		 2011}
2012
ca7db22b 2013u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 2014{
2015 u32 tmp = RREG32(MC_SHARED_CHMAP);
2016
2017 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2018 case 0:
2019 default:
2020 return 1;
2021 case 1:
2022 return 2;
2023 case 2:
2024 return 4;
2025 case 3:
2026 return 8;
2027 }
2028}
2029
2030struct evergreen_wm_params {
2031 u32 dram_channels; /* number of dram channels */
2032 u32 yclk; /* bandwidth per dram data pin in kHz */
2033 u32 sclk; /* engine clock in kHz */
2034 u32 disp_clk; /* display clock in kHz */
2035 u32 src_width; /* viewport width */
2036 u32 active_time; /* active display time in ns */
2037 u32 blank_time; /* blank time in ns */
2038 bool interlaced; /* mode is interlaced */
2039 fixed20_12 vsc; /* vertical scale ratio */
2040 u32 num_heads; /* number of active crtcs */
2041 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2042 u32 lb_size; /* line buffer allocated to pipe */
2043 u32 vtaps; /* vertical scaler taps */
2044};
2045
2046static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
2047{
2048 /* Calculate DRAM Bandwidth and the part allocated to display. */
2049 fixed20_12 dram_efficiency; /* 0.7 */
2050 fixed20_12 yclk, dram_channels, bandwidth;
2051 fixed20_12 a;
2052
2053 a.full = dfixed_const(1000);
2054 yclk.full = dfixed_const(wm->yclk);
2055 yclk.full = dfixed_div(yclk, a);
2056 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2057 a.full = dfixed_const(10);
2058 dram_efficiency.full = dfixed_const(7);
2059 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2060 bandwidth.full = dfixed_mul(dram_channels, yclk);
2061 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2062
2063 return dfixed_trunc(bandwidth);
2064}
2065
2066static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2067{
2068 /* Calculate DRAM Bandwidth and the part allocated to display. */
2069 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2070 fixed20_12 yclk, dram_channels, bandwidth;
2071 fixed20_12 a;
2072
2073 a.full = dfixed_const(1000);
2074 yclk.full = dfixed_const(wm->yclk);
2075 yclk.full = dfixed_div(yclk, a);
2076 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2077 a.full = dfixed_const(10);
2078 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2079 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2080 bandwidth.full = dfixed_mul(dram_channels, yclk);
2081 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2082
2083 return dfixed_trunc(bandwidth);
2084}
2085
2086static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2087{
2088 /* Calculate the display Data return Bandwidth */
2089 fixed20_12 return_efficiency; /* 0.8 */
2090 fixed20_12 sclk, bandwidth;
2091 fixed20_12 a;
2092
2093 a.full = dfixed_const(1000);
2094 sclk.full = dfixed_const(wm->sclk);
2095 sclk.full = dfixed_div(sclk, a);
2096 a.full = dfixed_const(10);
2097 return_efficiency.full = dfixed_const(8);
2098 return_efficiency.full = dfixed_div(return_efficiency, a);
2099 a.full = dfixed_const(32);
2100 bandwidth.full = dfixed_mul(a, sclk);
2101 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2102
2103 return dfixed_trunc(bandwidth);
2104}
2105
2106static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2107{
2108 /* Calculate the DMIF Request Bandwidth */
2109 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2110 fixed20_12 disp_clk, bandwidth;
2111 fixed20_12 a;
2112
2113 a.full = dfixed_const(1000);
2114 disp_clk.full = dfixed_const(wm->disp_clk);
2115 disp_clk.full = dfixed_div(disp_clk, a);
2116 a.full = dfixed_const(10);
2117 disp_clk_request_efficiency.full = dfixed_const(8);
2118 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2119 a.full = dfixed_const(32);
2120 bandwidth.full = dfixed_mul(a, disp_clk);
2121 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2122
2123 return dfixed_trunc(bandwidth);
2124}
2125
2126static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2127{
2128 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2129 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2130 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2131 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2132
2133 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2134}
2135
2136static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2137{
2138 /* Calculate the display mode Average Bandwidth
2139 * DisplayMode should contain the source and destination dimensions,
2140 * timing, etc.
2141 */
2142 fixed20_12 bpp;
2143 fixed20_12 line_time;
2144 fixed20_12 src_width;
2145 fixed20_12 bandwidth;
2146 fixed20_12 a;
2147
2148 a.full = dfixed_const(1000);
2149 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2150 line_time.full = dfixed_div(line_time, a);
2151 bpp.full = dfixed_const(wm->bytes_per_pixel);
2152 src_width.full = dfixed_const(wm->src_width);
2153 bandwidth.full = dfixed_mul(src_width, bpp);
2154 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2155 bandwidth.full = dfixed_div(bandwidth, line_time);
2156
2157 return dfixed_trunc(bandwidth);
2158}
2159
2160static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2161{
2162 /* First calcualte the latency in ns */
2163 u32 mc_latency = 2000; /* 2000 ns. */
2164 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2165 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2166 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2167 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2168 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2169 (wm->num_heads * cursor_line_pair_return_time);
2170 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2171 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2172 fixed20_12 a, b, c;
2173
2174 if (wm->num_heads == 0)
2175 return 0;
2176
2177 a.full = dfixed_const(2);
2178 b.full = dfixed_const(1);
2179 if ((wm->vsc.full > a.full) ||
2180 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2181 (wm->vtaps >= 5) ||
2182 ((wm->vsc.full >= a.full) && wm->interlaced))
2183 max_src_lines_per_dst_line = 4;
2184 else
2185 max_src_lines_per_dst_line = 2;
2186
2187 a.full = dfixed_const(available_bandwidth);
2188 b.full = dfixed_const(wm->num_heads);
2189 a.full = dfixed_div(a, b);
2190
2191 b.full = dfixed_const(1000);
2192 c.full = dfixed_const(wm->disp_clk);
2193 b.full = dfixed_div(c, b);
2194 c.full = dfixed_const(wm->bytes_per_pixel);
2195 b.full = dfixed_mul(b, c);
2196
2197 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2198
2199 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2200 b.full = dfixed_const(1000);
2201 c.full = dfixed_const(lb_fill_bw);
2202 b.full = dfixed_div(c, b);
2203 a.full = dfixed_div(a, b);
2204 line_fill_time = dfixed_trunc(a);
2205
2206 if (line_fill_time < wm->active_time)
2207 return latency;
2208 else
2209 return latency + (line_fill_time - wm->active_time);
2210
2211}
2212
2213static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2214{
2215 if (evergreen_average_bandwidth(wm) <=
2216 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2217 return true;
2218 else
2219 return false;
2220};
2221
2222static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2223{
2224 if (evergreen_average_bandwidth(wm) <=
2225 (evergreen_available_bandwidth(wm) / wm->num_heads))
2226 return true;
2227 else
2228 return false;
2229};
2230
2231static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2232{
2233 u32 lb_partitions = wm->lb_size / wm->src_width;
2234 u32 line_time = wm->active_time + wm->blank_time;
2235 u32 latency_tolerant_lines;
2236 u32 latency_hiding;
2237 fixed20_12 a;
2238
2239 a.full = dfixed_const(1);
2240 if (wm->vsc.full > a.full)
2241 latency_tolerant_lines = 1;
2242 else {
2243 if (lb_partitions <= (wm->vtaps + 1))
2244 latency_tolerant_lines = 1;
2245 else
2246 latency_tolerant_lines = 2;
2247 }
2248
2249 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2250
2251 if (evergreen_latency_watermark(wm) <= latency_hiding)
2252 return true;
2253 else
2254 return false;
2255}
2256
2257static void evergreen_program_watermarks(struct radeon_device *rdev,
2258 struct radeon_crtc *radeon_crtc,
2259 u32 lb_size, u32 num_heads)
2260{
2261 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2262 struct evergreen_wm_params wm_low, wm_high;
2263 u32 dram_channels;
f9d9c362
AD		 2264 u32 pixel_period;
2265 u32 line_time = 0;
2266 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2267 u32 priority_a_mark = 0, priority_b_mark = 0;
2268 u32 priority_a_cnt = PRIORITY_OFF;
2269 u32 priority_b_cnt = PRIORITY_OFF;
2270 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2271 u32 tmp, arb_control3;
2272 fixed20_12 a, b, c;
2273
2274 if (radeon_crtc->base.enabled && num_heads && mode) {
2275 pixel_period = 1000000 / (u32)mode->clock;
2276 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2277 priority_a_cnt = 0;
2278 priority_b_cnt = 0;
cf0cfdd7
AD		 2279 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2280
2281 /* watermark for high clocks */
2282 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2283 wm_high.yclk =
2284 radeon_dpm_get_mclk(rdev, false) * 10;
2285 wm_high.sclk =
2286 radeon_dpm_get_sclk(rdev, false) * 10;
2287 } else {
2288 wm_high.yclk = rdev->pm.current_mclk * 10;
2289 wm_high.sclk = rdev->pm.current_sclk * 10;
2290 }
f9d9c362 2291
cf0cfdd7
AD		 2292 wm_high.disp_clk = mode->clock;
2293 wm_high.src_width = mode->crtc_hdisplay;
2294 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2295 wm_high.blank_time = line_time - wm_high.active_time;
2296 wm_high.interlaced = false;
f9d9c362 2297 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2298 wm_high.interlaced = true;
2299 wm_high.vsc = radeon_crtc->vsc;
2300 wm_high.vtaps = 1;
f9d9c362 2301 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2302 wm_high.vtaps = 2;
2303 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2304 wm_high.lb_size = lb_size;
2305 wm_high.dram_channels = dram_channels;
2306 wm_high.num_heads = num_heads;
2307
2308 /* watermark for low clocks */
2309 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2310 wm_low.yclk =
2311 radeon_dpm_get_mclk(rdev, true) * 10;
2312 wm_low.sclk =
2313 radeon_dpm_get_sclk(rdev, true) * 10;
2314 } else {
2315 wm_low.yclk = rdev->pm.current_mclk * 10;
2316 wm_low.sclk = rdev->pm.current_sclk * 10;
2317 }
2318
2319 wm_low.disp_clk = mode->clock;
2320 wm_low.src_width = mode->crtc_hdisplay;
2321 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2322 wm_low.blank_time = line_time - wm_low.active_time;
2323 wm_low.interlaced = false;
2324 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2325 wm_low.interlaced = true;
2326 wm_low.vsc = radeon_crtc->vsc;
2327 wm_low.vtaps = 1;
2328 if (radeon_crtc->rmx_type != RMX_OFF)
2329 wm_low.vtaps = 2;
2330 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2331 wm_low.lb_size = lb_size;
2332 wm_low.dram_channels = dram_channels;
2333 wm_low.num_heads = num_heads;
f9d9c362
AD		 2334
2335 /* set for high clocks */
cf0cfdd7 2336 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2337 /* set for low clocks */
cf0cfdd7 2338 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2339
2340 /* possibly force display priority to high */
2341 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2342 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2343 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2344 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2345 (rdev->disp_priority == 2)) {
cf0cfdd7 2346 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2347 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2348 }
2349 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2350 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2351 !evergreen_check_latency_hiding(&wm_low) ||
2352 (rdev->disp_priority == 2)) {
2353 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2354 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2355 }
2356
2357 a.full = dfixed_const(1000);
2358 b.full = dfixed_const(mode->clock);
2359 b.full = dfixed_div(b, a);
2360 c.full = dfixed_const(latency_watermark_a);
2361 c.full = dfixed_mul(c, b);
2362 c.full = dfixed_mul(c, radeon_crtc->hsc);
2363 c.full = dfixed_div(c, a);
2364 a.full = dfixed_const(16);
2365 c.full = dfixed_div(c, a);
2366 priority_a_mark = dfixed_trunc(c);
2367 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2368
2369 a.full = dfixed_const(1000);
2370 b.full = dfixed_const(mode->clock);
2371 b.full = dfixed_div(b, a);
2372 c.full = dfixed_const(latency_watermark_b);
2373 c.full = dfixed_mul(c, b);
2374 c.full = dfixed_mul(c, radeon_crtc->hsc);
2375 c.full = dfixed_div(c, a);
2376 a.full = dfixed_const(16);
2377 c.full = dfixed_div(c, a);
2378 priority_b_mark = dfixed_trunc(c);
2379 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
5b5561b3
MK		 2380
2381 /* Save number of lines the linebuffer leads before the scanout */
2382 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
f9d9c362
AD		 2383 }
2384
2385 /* select wm A */
2386 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2387 tmp = arb_control3;
2388 tmp &= ~LATENCY_WATERMARK_MASK(3);
2389 tmp |= LATENCY_WATERMARK_MASK(1);
2390 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2391 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2392 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2393 LATENCY_HIGH_WATERMARK(line_time)));
2394 /* select wm B */
2395 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2396 tmp &= ~LATENCY_WATERMARK_MASK(3);
2397 tmp |= LATENCY_WATERMARK_MASK(2);
2398 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2399 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2400 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2401 LATENCY_HIGH_WATERMARK(line_time)));
2402 /* restore original selection */
2403 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2404
2405 /* write the priority marks */
2406 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2407 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2408
7178d2a6
AD		 2409 /* save values for DPM */
2410 radeon_crtc->line_time = line_time;
2411 radeon_crtc->wm_high = latency_watermark_a;
2412 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2413}
2414
377edc8b
AD		 2415/**
2416 * evergreen_bandwidth_update - update display watermarks callback.
2417 *
2418 * @rdev: radeon_device pointer
2419 *
2420 * Update the display watermarks based on the requested mode(s)
2421 * (evergreen+).
2422 */
0ca2ab52 2423void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2424{
f9d9c362
AD		 2425 struct drm_display_mode *mode0 = NULL;
2426 struct drm_display_mode *mode1 = NULL;
2427 u32 num_heads = 0, lb_size;
2428 int i;
2429
8efe82ca
AD		 2430 if (!rdev->mode_info.mode_config_initialized)
2431 return;
2432
f9d9c362
AD		 2433 radeon_update_display_priority(rdev);
2434
2435 for (i = 0; i < rdev->num_crtc; i++) {
2436 if (rdev->mode_info.crtcs[i]->base.enabled)
2437 num_heads++;
2438 }
2439 for (i = 0; i < rdev->num_crtc; i += 2) {
2440 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2441 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2442 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2443 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2444 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2445 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2446 }
bcc1c2a1
AD		 2447}
2448
377edc8b
AD		 2449/**
2450 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2451 *
2452 * @rdev: radeon_device pointer
2453 *
2454 * Wait for the MC (memory controller) to be idle.
2455 * (evergreen+).
2456 * Returns 0 if the MC is idle, -1 if not.
2457 */
b9952a8a 2458int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2459{
2460 unsigned i;
2461 u32 tmp;
2462
2463 for (i = 0; i < rdev->usec_timeout; i++) {
2464 /* read MC_STATUS */
2465 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2466 if (!tmp)
2467 return 0;
2468 udelay(1);
2469 }
2470 return -1;
2471}
2472
2473/*
2474 * GART
2475 */
0fcdb61e
AD		 2476void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2477{
2478 unsigned i;
2479 u32 tmp;
2480
6f2f48a9
AD		 2481 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2482
0fcdb61e
AD		 2483 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2484 for (i = 0; i < rdev->usec_timeout; i++) {
2485 /* read MC_STATUS */
2486 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2487 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2488 if (tmp == 2) {
2489 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2490 return;
2491 }
2492 if (tmp) {
2493 return;
2494 }
2495 udelay(1);
2496 }
2497}
2498
1109ca09 2499static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2500{
2501 u32 tmp;
0fcdb61e 2502 int r;
bcc1c2a1 2503
c9a1be96 2504 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2505 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2506 return -EINVAL;
2507 }
2508 r = radeon_gart_table_vram_pin(rdev);
2509 if (r)
2510 return r;
2511 /* Setup L2 cache */
2512 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2513 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2514 EFFECTIVE_L2_QUEUE_SIZE(7));
2515 WREG32(VM_L2_CNTL2, 0);
2516 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2517 /* Setup TLB control */
2518 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2519 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2520 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2521 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2522 if (rdev->flags & RADEON_IS_IGP) {
2523 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2524 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2525 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2526 } else {
2527 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2528 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2529 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2530 if ((rdev->family == CHIP_JUNIPER) ||
2531 (rdev->family == CHIP_CYPRESS) ||
2532 (rdev->family == CHIP_HEMLOCK) ||
2533 (rdev->family == CHIP_BARTS))
2534 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2535 }
bcc1c2a1
AD		 2536 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2537 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2538 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2539 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2540 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 2541 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
bcc1c2a1
AD		 2542 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2543 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2544 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2545 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2546 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2547 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2548
0fcdb61e 2549 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2550 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2551 (unsigned)(rdev->mc.gtt_size >> 20),
2552 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2553 rdev->gart.ready = true;
2554 return 0;
2555}
2556
1109ca09 2557static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2558{
2559 u32 tmp;
bcc1c2a1
AD		 2560
2561 /* Disable all tables */
0fcdb61e
AD		 2562 WREG32(VM_CONTEXT0_CNTL, 0);
2563 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2564
2565 /* Setup L2 cache */
2566 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2567 EFFECTIVE_L2_QUEUE_SIZE(7));
2568 WREG32(VM_L2_CNTL2, 0);
2569 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2570 /* Setup TLB control */
2571 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2572 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2573 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2574 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2575 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2576 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2577 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2578 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2579 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2580}
2581
1109ca09 2582static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2583{
2584 evergreen_pcie_gart_disable(rdev);
2585 radeon_gart_table_vram_free(rdev);
2586 radeon_gart_fini(rdev);
2587}
2588
2589
1109ca09 2590static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2591{
2592 u32 tmp;
bcc1c2a1
AD		 2593
2594 /* Setup L2 cache */
2595 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2596 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2597 EFFECTIVE_L2_QUEUE_SIZE(7));
2598 WREG32(VM_L2_CNTL2, 0);
2599 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2600 /* Setup TLB control */
2601 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2602 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2603 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2604 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2605 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2606 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2607 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2608 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2609 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2610 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2611 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2612 WREG32(VM_CONTEXT0_CNTL, 0);
2613 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2614}
2615
5d5b7803
VP		 2616static const unsigned ni_dig_offsets[] =
2617{
2618 NI_DIG0_REGISTER_OFFSET,
2619 NI_DIG1_REGISTER_OFFSET,
2620 NI_DIG2_REGISTER_OFFSET,
2621 NI_DIG3_REGISTER_OFFSET,
2622 NI_DIG4_REGISTER_OFFSET,
2623 NI_DIG5_REGISTER_OFFSET
2624};
2625
2626static const unsigned ni_tx_offsets[] =
2627{
2628 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2629 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2630 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2631 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2632 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2633 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2634};
2635
2636static const unsigned evergreen_dp_offsets[] =
2637{
2638 EVERGREEN_DP0_REGISTER_OFFSET,
2639 EVERGREEN_DP1_REGISTER_OFFSET,
2640 EVERGREEN_DP2_REGISTER_OFFSET,
2641 EVERGREEN_DP3_REGISTER_OFFSET,
2642 EVERGREEN_DP4_REGISTER_OFFSET,
2643 EVERGREEN_DP5_REGISTER_OFFSET
2644};
2645
2646
2647/*
2648 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2649 * We go from crtc to connector and it is not relible since it
2650 * should be an opposite direction .If crtc is enable then
2651 * find the dig_fe which selects this crtc and insure that it enable.
2652 * if such dig_fe is found then find dig_be which selects found dig_be and
2653 * insure that it enable and in DP_SST mode.
2654 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2655 * from dp symbols clocks .
2656 */
2657static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2658 unsigned crtc_id, unsigned *ret_dig_fe)
2659{
2660 unsigned i;
2661 unsigned dig_fe;
2662 unsigned dig_be;
2663 unsigned dig_en_be;
2664 unsigned uniphy_pll;
2665 unsigned digs_fe_selected;
2666 unsigned dig_be_mode;
2667 unsigned dig_fe_mask;
2668 bool is_enabled = false;
2669 bool found_crtc = false;
2670
2671 /* loop through all running dig_fe to find selected crtc */
2672 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2673 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2674 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2675 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2676 /* found running pipe */
2677 found_crtc = true;
2678 dig_fe_mask = 1 << i;
2679 dig_fe = i;
2680 break;
2681 }
2682 }
2683
2684 if (found_crtc) {
2685 /* loop through all running dig_be to find selected dig_fe */
2686 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2687 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2688 /* if dig_fe_selected by dig_be? */
2689 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2690 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2691 if (dig_fe_mask & digs_fe_selected &&
2692 /* if dig_be in sst mode? */
2693 dig_be_mode == NI_DIG_BE_DPSST) {
2694 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2695 ni_dig_offsets[i]);
2696 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2697 ni_tx_offsets[i]);
2698 /* dig_be enable and tx is running */
2699 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2700 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2701 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2702 is_enabled = true;
2703 *ret_dig_fe = dig_fe;
2704 break;
2705 }
2706 }
2707 }
2708 }
2709
2710 return is_enabled;
2711}
2712
2713/*
2714 * Blank dig when in dp sst mode
2715 * Dig ignores crtc timing
2716 */
2717static void evergreen_blank_dp_output(struct radeon_device *rdev,
2718 unsigned dig_fe)
2719{
2720 unsigned stream_ctrl;
2721 unsigned fifo_ctrl;
2722 unsigned counter = 0;
2723
2724 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2725 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2726 return;
2727 }
2728
2729 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2730 evergreen_dp_offsets[dig_fe]);
2731 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2732 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2733 return;
2734 }
2735
2736 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2737 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2738 evergreen_dp_offsets[dig_fe], stream_ctrl);
2739
2740 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2741 evergreen_dp_offsets[dig_fe]);
2742 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2743 msleep(1);
2744 counter++;
2745 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2746 evergreen_dp_offsets[dig_fe]);
2747 }
2748 if (counter >= 32 )
2749 DRM_ERROR("counter exceeds %d\n", counter);
2750
2751 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2752 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2753 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2754
2755}
2756
b9952a8a 2757void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2758{
62444b74
AD		 2759 u32 crtc_enabled, tmp, frame_count, blackout;
2760 int i, j;
5d5b7803 2761 unsigned dig_fe;
62444b74 2762
5153550a
AD		 2763 if (!ASIC_IS_NODCE(rdev)) {
2764 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2765 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2766
5153550a
AD		 2767 /* disable VGA render */
2768 WREG32(VGA_RENDER_CONTROL, 0);
2769 }
62444b74
AD		 2770 /* blank the display controllers */
2771 for (i = 0; i < rdev->num_crtc; i++) {
2772 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2773 if (crtc_enabled) {
2774 save->crtc_enabled[i] = true;
2775 if (ASIC_IS_DCE6(rdev)) {
2776 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2777 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2778 radeon_wait_for_vblank(rdev, i);
abf1457b 2779 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2780 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2781 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
f0d7bfb9 2782 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2783 }
2784 } else {
2785 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2786 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2787 radeon_wait_for_vblank(rdev, i);
abf1457b 2788 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2789 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2790 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2791 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2792 }
2793 }
2794 /* wait for the next frame */
2795 frame_count = radeon_get_vblank_counter(rdev, i);
2796 for (j = 0; j < rdev->usec_timeout; j++) {
2797 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2798 break;
2799 udelay(1);
2800 }
5d5b7803
VP		 2801 /*we should disable dig if it drives dp sst*/
2802 /*but we are in radeon_device_init and the topology is unknown*/
2803 /*and it is available after radeon_modeset_init*/
2804 /*the following method radeon_atom_encoder_dpms_dig*/
2805 /*does the job if we initialize it properly*/
2806 /*for now we do it this manually*/
2807 /**/
2808 if (ASIC_IS_DCE5(rdev) &&
2809 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2810 evergreen_blank_dp_output(rdev, dig_fe);
2811 /*we could remove 6 lines below*/
abf1457b
AD		 2812 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2813 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2814 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2815 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2816 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2817 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2818 save->crtc_enabled[i] = false;
2819 /* ***** */
804cc4a0
AD		 2820 } else {
2821 save->crtc_enabled[i] = false;
62444b74 2822 }
18007401 2823 }
bcc1c2a1 2824
62444b74
AD		 2825 radeon_mc_wait_for_idle(rdev);
2826
2827 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2828 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2829 /* Block CPU access */
2830 WREG32(BIF_FB_EN, 0);
2831 /* blackout the MC */
2832 blackout &= ~BLACKOUT_MODE_MASK;
2833 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2834 }
ed39fadd
AD		 2835 /* wait for the MC to settle */
2836 udelay(100);
968c0166
AD		 2837
2838 /* lock double buffered regs */
2839 for (i = 0; i < rdev->num_crtc; i++) {
2840 if (save->crtc_enabled[i]) {
2841 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2842 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2843 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2844 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2845 }
2846 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2847 if (!(tmp & 1)) {
2848 tmp |= 1;
2849 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2850 }
2851 }
2852 }
bcc1c2a1
AD		 2853}
2854
b9952a8a 2855void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2856{
62444b74
AD		 2857 u32 tmp, frame_count;
2858 int i, j;
18007401 2859
62444b74
AD		 2860 /* update crtc base addresses */
2861 for (i = 0; i < rdev->num_crtc; i++) {
2862 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2863 upper_32_bits(rdev->mc.vram_start));
62444b74 2864 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2865 upper_32_bits(rdev->mc.vram_start));
62444b74 2866 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2867 (u32)rdev->mc.vram_start);
62444b74 2868 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2869 (u32)rdev->mc.vram_start);
2870 }
5153550a
AD		 2871
2872 if (!ASIC_IS_NODCE(rdev)) {
2873 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2874 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2875 }
62444b74 2876
968c0166
AD		 2877 /* unlock regs and wait for update */
2878 for (i = 0; i < rdev->num_crtc; i++) {
2879 if (save->crtc_enabled[i]) {
2880 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
f53f81b2
MK		 2881 if ((tmp & 0x7) != 3) {
2882 tmp &= ~0x7;
2883 tmp |= 0x3;
968c0166
AD		 2884 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2885 }
2886 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2887 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2888 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2889 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2890 }
2891 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2892 if (tmp & 1) {
2893 tmp &= ~1;
2894 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2895 }
2896 for (j = 0; j < rdev->usec_timeout; j++) {
2897 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2898 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2899 break;
2900 udelay(1);
2901 }
2902 }
2903 }
2904
62444b74
AD		 2905 /* unblackout the MC */
2906 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2907 tmp &= ~BLACKOUT_MODE_MASK;
2908 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2909 /* allow CPU access */
2910 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2911
2912 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2913 if (save->crtc_enabled[i]) {
62444b74
AD		 2914 if (ASIC_IS_DCE6(rdev)) {
2915 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
3157c589 2916 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2917 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2918 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2919 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2920 } else {
2921 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2922 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2923 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2924 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2925 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2926 }
2927 /* wait for the next frame */
2928 frame_count = radeon_get_vblank_counter(rdev, i);
2929 for (j = 0; j < rdev->usec_timeout; j++) {
2930 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2931 break;
2932 udelay(1);
2933 }
2934 }
2935 }
5153550a
AD		 2936 if (!ASIC_IS_NODCE(rdev)) {
2937 /* Unlock vga access */
2938 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2939 mdelay(1);
2940 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2941 }
bcc1c2a1
AD		 2942}
2943
755d819e 2944void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2945{
2946 struct evergreen_mc_save save;
2947 u32 tmp;
2948 int i, j;
2949
2950 /* Initialize HDP */
2951 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2952 WREG32((0x2c14 + j), 0x00000000);
2953 WREG32((0x2c18 + j), 0x00000000);
2954 WREG32((0x2c1c + j), 0x00000000);
2955 WREG32((0x2c20 + j), 0x00000000);
2956 WREG32((0x2c24 + j), 0x00000000);
2957 }
2958 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2959
2960 evergreen_mc_stop(rdev, &save);
2961 if (evergreen_mc_wait_for_idle(rdev)) {
2962 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2963 }
2964 /* Lockout access through VGA aperture*/
2965 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2966 /* Update configuration */
2967 if (rdev->flags & RADEON_IS_AGP) {
2968 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2969 /* VRAM before AGP */
2970 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2971 rdev->mc.vram_start >> 12);
2972 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2973 rdev->mc.gtt_end >> 12);
2974 } else {
2975 /* VRAM after AGP */
2976 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2977 rdev->mc.gtt_start >> 12);
2978 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2979 rdev->mc.vram_end >> 12);
2980 }
2981 } else {
2982 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2983 rdev->mc.vram_start >> 12);
2984 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2985 rdev->mc.vram_end >> 12);
2986 }
3b9832f6 2987 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2988 /* llano/ontario only */
2989 if ((rdev->family == CHIP_PALM) ||
2990 (rdev->family == CHIP_SUMO) ||
2991 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2992 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2993 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2994 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2995 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2996 }
bcc1c2a1
AD		 2997 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2998 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2999 WREG32(MC_VM_FB_LOCATION, tmp);
3000 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 3001 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 3002 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 3003 if (rdev->flags & RADEON_IS_AGP) {
3004 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
3005 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
3006 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
3007 } else {
3008 WREG32(MC_VM_AGP_BASE, 0);
3009 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3010 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3011 }
3012 if (evergreen_mc_wait_for_idle(rdev)) {
3013 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3014 }
3015 evergreen_mc_resume(rdev, &save);
3016 /* we need to own VRAM, so turn off the VGA renderer here
3017 * to stop it overwriting our objects */
3018 rv515_vga_render_disable(rdev);
3019}
3020
bcc1c2a1
AD		 3021/*
3022 * CP.
3023 */
12920591
AD		 3024void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3025{
876dc9f3 3026 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 3027 u32 next_rptr;
7b1f2485 3028
12920591 3029 /* set to DX10/11 mode */
e32eb50d
CK		 3030 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
3031 radeon_ring_write(ring, 1);
45df6803
CK		 3032
3033 if (ring->rptr_save_reg) {
89d35807 3034 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 3035 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3036 radeon_ring_write(ring, ((ring->rptr_save_reg -
3037 PACKET3_SET_CONFIG_REG_START) >> 2));
3038 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3039 } else if (rdev->wb.enabled) {
3040 next_rptr = ring->wptr + 5 + 4;
3041 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3042 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3043 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3044 radeon_ring_write(ring, next_rptr);
3045 radeon_ring_write(ring, 0);
45df6803
CK		 3046 }
3047
e32eb50d
CK		 3048 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3049 radeon_ring_write(ring,
0f234f5f
AD		 3050#ifdef __BIG_ENDIAN
3051 (2 << 0) |
3052#endif
3053 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 3054 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3055 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 3056}
3057
bcc1c2a1
AD		 3058
3059static int evergreen_cp_load_microcode(struct radeon_device *rdev)
3060{
fe251e2f
AD		 3061 const __be32 *fw_data;
3062 int i;
3063
3064 if (!rdev->me_fw || !rdev->pfp_fw)
3065 return -EINVAL;
bcc1c2a1 3066
fe251e2f 3067 r700_cp_stop(rdev);
0f234f5f
AD		 3068 WREG32(CP_RB_CNTL,
3069#ifdef __BIG_ENDIAN
3070 BUF_SWAP_32BIT |
3071#endif
3072 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 3073
3074 fw_data = (const __be32 *)rdev->pfp_fw->data;
3075 WREG32(CP_PFP_UCODE_ADDR, 0);
3076 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
3077 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3078 WREG32(CP_PFP_UCODE_ADDR, 0);
3079
3080 fw_data = (const __be32 *)rdev->me_fw->data;
3081 WREG32(CP_ME_RAM_WADDR, 0);
3082 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
3083 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3084
3085 WREG32(CP_PFP_UCODE_ADDR, 0);
3086 WREG32(CP_ME_RAM_WADDR, 0);
3087 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 3088 return 0;
3089}
3090
7e7b41d2
AD		 3091static int evergreen_cp_start(struct radeon_device *rdev)
3092{
e32eb50d 3093 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 3094 int r, i;
7e7b41d2
AD		 3095 uint32_t cp_me;
3096
e32eb50d 3097 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 3098 if (r) {
3099 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3100 return r;
3101 }
e32eb50d
CK		 3102 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3103 radeon_ring_write(ring, 0x1);
3104 radeon_ring_write(ring, 0x0);
3105 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3106 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3107 radeon_ring_write(ring, 0);
3108 radeon_ring_write(ring, 0);
1538a9e0 3109 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 3110
3111 cp_me = 0xff;
3112 WREG32(CP_ME_CNTL, cp_me);
3113
e32eb50d 3114 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 3115 if (r) {
3116 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3117 return r;
3118 }
2281a378
AD		 3119
3120 /* setup clear context state */
e32eb50d
CK		 3121 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3122 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 3123
3124 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 3125 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 3126
e32eb50d
CK		 3127 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3128 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 3129
3130 /* set clear context state */
e32eb50d
CK		 3131 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3132 radeon_ring_write(ring, 0);
2281a378
AD		 3133
3134 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 3135 radeon_ring_write(ring, 0xc0026f00);
3136 radeon_ring_write(ring, 0x00000000);
3137 radeon_ring_write(ring, 0x00000000);
3138 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 3139
3140 /* Clear consts */
e32eb50d
CK		 3141 radeon_ring_write(ring, 0xc0036f00);
3142 radeon_ring_write(ring, 0x00000bc4);
3143 radeon_ring_write(ring, 0xffffffff);
3144 radeon_ring_write(ring, 0xffffffff);
3145 radeon_ring_write(ring, 0xffffffff);
2281a378 3146
e32eb50d
CK		 3147 radeon_ring_write(ring, 0xc0026900);
3148 radeon_ring_write(ring, 0x00000316);
3149 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3150 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 3151
1538a9e0 3152 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 3153
3154 return 0;
3155}
3156
1109ca09 3157static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 3158{
e32eb50d 3159 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 3160 u32 tmp;
3161 u32 rb_bufsz;
3162 int r;
3163
3164 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3165 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3166 SOFT_RESET_PA |
3167 SOFT_RESET_SH |
3168 SOFT_RESET_VGT |
a49a50da 3169 SOFT_RESET_SPI |
fe251e2f
AD		 3170 SOFT_RESET_SX));
3171 RREG32(GRBM_SOFT_RESET);
3172 mdelay(15);
3173 WREG32(GRBM_SOFT_RESET, 0);
3174 RREG32(GRBM_SOFT_RESET);
3175
3176 /* Set ring buffer size */
b72a8925
DV		 3177 rb_bufsz = order_base_2(ring->ring_size / 8);
3178 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 3179#ifdef __BIG_ENDIAN
3180 tmp |= BUF_SWAP_32BIT;
32fcdbf4 3181#endif
fe251e2f 3182 WREG32(CP_RB_CNTL, tmp);
15d3332f 3183 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 3184 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 3185
3186 /* Set the write pointer delay */
3187 WREG32(CP_RB_WPTR_DELAY, 0);
3188
3189 /* Initialize the ring buffer's read and write pointers */
3190 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3191 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 3192 ring->wptr = 0;
3193 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 3194
48fc7f7e 3195 /* set the wb address whether it's enabled or not */
0f234f5f 3196 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 3197 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 3198 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3199 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3200
3201 if (rdev->wb.enabled)
3202 WREG32(SCRATCH_UMSK, 0xff);
3203 else {
3204 tmp |= RB_NO_UPDATE;
3205 WREG32(SCRATCH_UMSK, 0);
3206 }
3207
fe251e2f
AD		 3208 mdelay(1);
3209 WREG32(CP_RB_CNTL, tmp);
3210
e32eb50d 3211 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 3212 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3213
7e7b41d2 3214 evergreen_cp_start(rdev);
e32eb50d 3215 ring->ready = true;
f712812e 3216 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 3217 if (r) {
e32eb50d 3218 ring->ready = false;
fe251e2f
AD		 3219 return r;
3220 }
3221 return 0;
3222}
bcc1c2a1
AD		 3223
3224/*
3225 * Core functions
3226 */
bcc1c2a1
AD		 3227static void evergreen_gpu_init(struct radeon_device *rdev)
3228{
416a2bd2 3229 u32 gb_addr_config;
32fcdbf4 3230 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 3231 u32 sx_debug_1;
3232 u32 smx_dc_ctl0;
3233 u32 sq_config;
3234 u32 sq_lds_resource_mgmt;
3235 u32 sq_gpr_resource_mgmt_1;
3236 u32 sq_gpr_resource_mgmt_2;
3237 u32 sq_gpr_resource_mgmt_3;
3238 u32 sq_thread_resource_mgmt;
3239 u32 sq_thread_resource_mgmt_2;
3240 u32 sq_stack_resource_mgmt_1;
3241 u32 sq_stack_resource_mgmt_2;
3242 u32 sq_stack_resource_mgmt_3;
3243 u32 vgt_cache_invalidation;
f25a5c63 3244 u32 hdp_host_path_cntl, tmp;
416a2bd2 3245 u32 disabled_rb_mask;
072c44bf 3246 int i, j, ps_thread_count;
32fcdbf4
AD		 3247
3248 switch (rdev->family) {
3249 case CHIP_CYPRESS:
3250 case CHIP_HEMLOCK:
3251 rdev->config.evergreen.num_ses = 2;
3252 rdev->config.evergreen.max_pipes = 4;
3253 rdev->config.evergreen.max_tile_pipes = 8;
3254 rdev->config.evergreen.max_simds = 10;
3255 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3256 rdev->config.evergreen.max_gprs = 256;
3257 rdev->config.evergreen.max_threads = 248;
3258 rdev->config.evergreen.max_gs_threads = 32;
3259 rdev->config.evergreen.max_stack_entries = 512;
3260 rdev->config.evergreen.sx_num_of_sets = 4;
3261 rdev->config.evergreen.sx_max_export_size = 256;
3262 rdev->config.evergreen.sx_max_export_pos_size = 64;
3263 rdev->config.evergreen.sx_max_export_smx_size = 192;
3264 rdev->config.evergreen.max_hw_contexts = 8;
3265 rdev->config.evergreen.sq_num_cf_insts = 2;
3266
3267 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3268 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3269 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3270 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3271 break;
3272 case CHIP_JUNIPER:
3273 rdev->config.evergreen.num_ses = 1;
3274 rdev->config.evergreen.max_pipes = 4;
3275 rdev->config.evergreen.max_tile_pipes = 4;
3276 rdev->config.evergreen.max_simds = 10;
3277 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3278 rdev->config.evergreen.max_gprs = 256;
3279 rdev->config.evergreen.max_threads = 248;
3280 rdev->config.evergreen.max_gs_threads = 32;
3281 rdev->config.evergreen.max_stack_entries = 512;
3282 rdev->config.evergreen.sx_num_of_sets = 4;
3283 rdev->config.evergreen.sx_max_export_size = 256;
3284 rdev->config.evergreen.sx_max_export_pos_size = 64;
3285 rdev->config.evergreen.sx_max_export_smx_size = 192;
3286 rdev->config.evergreen.max_hw_contexts = 8;
3287 rdev->config.evergreen.sq_num_cf_insts = 2;
3288
3289 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3290 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3291 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3292 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3293 break;
3294 case CHIP_REDWOOD:
3295 rdev->config.evergreen.num_ses = 1;
3296 rdev->config.evergreen.max_pipes = 4;
3297 rdev->config.evergreen.max_tile_pipes = 4;
3298 rdev->config.evergreen.max_simds = 5;
3299 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3300 rdev->config.evergreen.max_gprs = 256;
3301 rdev->config.evergreen.max_threads = 248;
3302 rdev->config.evergreen.max_gs_threads = 32;
3303 rdev->config.evergreen.max_stack_entries = 256;
3304 rdev->config.evergreen.sx_num_of_sets = 4;
3305 rdev->config.evergreen.sx_max_export_size = 256;
3306 rdev->config.evergreen.sx_max_export_pos_size = 64;
3307 rdev->config.evergreen.sx_max_export_smx_size = 192;
3308 rdev->config.evergreen.max_hw_contexts = 8;
3309 rdev->config.evergreen.sq_num_cf_insts = 2;
3310
3311 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3312 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3313 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3314 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3315 break;
3316 case CHIP_CEDAR:
3317 default:
3318 rdev->config.evergreen.num_ses = 1;
3319 rdev->config.evergreen.max_pipes = 2;
3320 rdev->config.evergreen.max_tile_pipes = 2;
3321 rdev->config.evergreen.max_simds = 2;
3322 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3323 rdev->config.evergreen.max_gprs = 256;
3324 rdev->config.evergreen.max_threads = 192;
3325 rdev->config.evergreen.max_gs_threads = 16;
3326 rdev->config.evergreen.max_stack_entries = 256;
3327 rdev->config.evergreen.sx_num_of_sets = 4;
3328 rdev->config.evergreen.sx_max_export_size = 128;
3329 rdev->config.evergreen.sx_max_export_pos_size = 32;
3330 rdev->config.evergreen.sx_max_export_smx_size = 96;
3331 rdev->config.evergreen.max_hw_contexts = 4;
3332 rdev->config.evergreen.sq_num_cf_insts = 1;
3333
d5e455e4
AD		 3334 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3335 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3336 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3337 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3338 break;
3339 case CHIP_PALM:
3340 rdev->config.evergreen.num_ses = 1;
3341 rdev->config.evergreen.max_pipes = 2;
3342 rdev->config.evergreen.max_tile_pipes = 2;
3343 rdev->config.evergreen.max_simds = 2;
3344 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3345 rdev->config.evergreen.max_gprs = 256;
3346 rdev->config.evergreen.max_threads = 192;
3347 rdev->config.evergreen.max_gs_threads = 16;
3348 rdev->config.evergreen.max_stack_entries = 256;
3349 rdev->config.evergreen.sx_num_of_sets = 4;
3350 rdev->config.evergreen.sx_max_export_size = 128;
3351 rdev->config.evergreen.sx_max_export_pos_size = 32;
3352 rdev->config.evergreen.sx_max_export_smx_size = 96;
3353 rdev->config.evergreen.max_hw_contexts = 4;
3354 rdev->config.evergreen.sq_num_cf_insts = 1;
3355
d5c5a72f
AD		 3356 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3357 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3358 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3359 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3360 break;
3361 case CHIP_SUMO:
3362 rdev->config.evergreen.num_ses = 1;
3363 rdev->config.evergreen.max_pipes = 4;
bd25f078 3364 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3365 if (rdev->pdev->device == 0x9648)
3366 rdev->config.evergreen.max_simds = 3;
3367 else if ((rdev->pdev->device == 0x9647) ||
3368 (rdev->pdev->device == 0x964a))
3369 rdev->config.evergreen.max_simds = 4;
3370 else
3371 rdev->config.evergreen.max_simds = 5;
3372 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3373 rdev->config.evergreen.max_gprs = 256;
3374 rdev->config.evergreen.max_threads = 248;
3375 rdev->config.evergreen.max_gs_threads = 32;
3376 rdev->config.evergreen.max_stack_entries = 256;
3377 rdev->config.evergreen.sx_num_of_sets = 4;
3378 rdev->config.evergreen.sx_max_export_size = 256;
3379 rdev->config.evergreen.sx_max_export_pos_size = 64;
3380 rdev->config.evergreen.sx_max_export_smx_size = 192;
3381 rdev->config.evergreen.max_hw_contexts = 8;
3382 rdev->config.evergreen.sq_num_cf_insts = 2;
3383
3384 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3385 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3386 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3387 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3388 break;
3389 case CHIP_SUMO2:
3390 rdev->config.evergreen.num_ses = 1;
3391 rdev->config.evergreen.max_pipes = 4;
3392 rdev->config.evergreen.max_tile_pipes = 4;
3393 rdev->config.evergreen.max_simds = 2;
3394 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3395 rdev->config.evergreen.max_gprs = 256;
3396 rdev->config.evergreen.max_threads = 248;
3397 rdev->config.evergreen.max_gs_threads = 32;
3398 rdev->config.evergreen.max_stack_entries = 512;
3399 rdev->config.evergreen.sx_num_of_sets = 4;
3400 rdev->config.evergreen.sx_max_export_size = 256;
3401 rdev->config.evergreen.sx_max_export_pos_size = 64;
3402 rdev->config.evergreen.sx_max_export_smx_size = 192;
50b8f5ae 3403 rdev->config.evergreen.max_hw_contexts = 4;
d5c5a72f
AD		 3404 rdev->config.evergreen.sq_num_cf_insts = 2;
3405
adb68fa2
AD		 3406 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3407 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3408 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3409 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3410 break;
3411 case CHIP_BARTS:
3412 rdev->config.evergreen.num_ses = 2;
3413 rdev->config.evergreen.max_pipes = 4;
3414 rdev->config.evergreen.max_tile_pipes = 8;
3415 rdev->config.evergreen.max_simds = 7;
3416 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3417 rdev->config.evergreen.max_gprs = 256;
3418 rdev->config.evergreen.max_threads = 248;
3419 rdev->config.evergreen.max_gs_threads = 32;
3420 rdev->config.evergreen.max_stack_entries = 512;
3421 rdev->config.evergreen.sx_num_of_sets = 4;
3422 rdev->config.evergreen.sx_max_export_size = 256;
3423 rdev->config.evergreen.sx_max_export_pos_size = 64;
3424 rdev->config.evergreen.sx_max_export_smx_size = 192;
3425 rdev->config.evergreen.max_hw_contexts = 8;
3426 rdev->config.evergreen.sq_num_cf_insts = 2;
3427
3428 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3429 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3430 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3431 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3432 break;
3433 case CHIP_TURKS:
3434 rdev->config.evergreen.num_ses = 1;
3435 rdev->config.evergreen.max_pipes = 4;
3436 rdev->config.evergreen.max_tile_pipes = 4;
3437 rdev->config.evergreen.max_simds = 6;
3438 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3439 rdev->config.evergreen.max_gprs = 256;
3440 rdev->config.evergreen.max_threads = 248;
3441 rdev->config.evergreen.max_gs_threads = 32;
3442 rdev->config.evergreen.max_stack_entries = 256;
3443 rdev->config.evergreen.sx_num_of_sets = 4;
3444 rdev->config.evergreen.sx_max_export_size = 256;
3445 rdev->config.evergreen.sx_max_export_pos_size = 64;
3446 rdev->config.evergreen.sx_max_export_smx_size = 192;
3447 rdev->config.evergreen.max_hw_contexts = 8;
3448 rdev->config.evergreen.sq_num_cf_insts = 2;
3449
3450 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3451 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3452 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3453 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3454 break;
3455 case CHIP_CAICOS:
3456 rdev->config.evergreen.num_ses = 1;
bd25f078 3457 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3458 rdev->config.evergreen.max_tile_pipes = 2;
3459 rdev->config.evergreen.max_simds = 2;
3460 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3461 rdev->config.evergreen.max_gprs = 256;
3462 rdev->config.evergreen.max_threads = 192;
3463 rdev->config.evergreen.max_gs_threads = 16;
3464 rdev->config.evergreen.max_stack_entries = 256;
3465 rdev->config.evergreen.sx_num_of_sets = 4;
3466 rdev->config.evergreen.sx_max_export_size = 128;
3467 rdev->config.evergreen.sx_max_export_pos_size = 32;
3468 rdev->config.evergreen.sx_max_export_smx_size = 96;
3469 rdev->config.evergreen.max_hw_contexts = 4;
3470 rdev->config.evergreen.sq_num_cf_insts = 1;
3471
32fcdbf4
AD		 3472 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3473 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3474 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3475 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3476 break;
3477 }
3478
3479 /* Initialize HDP */
3480 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3481 WREG32((0x2c14 + j), 0x00000000);
3482 WREG32((0x2c18 + j), 0x00000000);
3483 WREG32((0x2c1c + j), 0x00000000);
3484 WREG32((0x2c20 + j), 0x00000000);
3485 WREG32((0x2c24 + j), 0x00000000);
3486 }
3487
3488 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
acc1522a
CK		 3489 WREG32(SRBM_INT_CNTL, 0x1);
3490 WREG32(SRBM_INT_ACK, 0x1);
32fcdbf4 3491
d054ac16
AD		 3492 evergreen_fix_pci_max_read_req_size(rdev);
3493
32fcdbf4 3494 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3495 if ((rdev->family == CHIP_PALM) ||
3496 (rdev->family == CHIP_SUMO) ||
3497 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3498 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3499 else
3500 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3501
1aa52bd3
AD		 3502 /* setup tiling info dword. gb_addr_config is not adequate since it does
3503 * not have bank info, so create a custom tiling dword.
3504 * bits 3:0 num_pipes
3505 * bits 7:4 num_banks
3506 * bits 11:8 group_size
3507 * bits 15:12 row_size
3508 */
3509 rdev->config.evergreen.tile_config = 0;
3510 switch (rdev->config.evergreen.max_tile_pipes) {
3511 case 1:
3512 default:
3513 rdev->config.evergreen.tile_config |= (0 << 0);
3514 break;
3515 case 2:
3516 rdev->config.evergreen.tile_config |= (1 << 0);
3517 break;
3518 case 4:
3519 rdev->config.evergreen.tile_config |= (2 << 0);
3520 break;
3521 case 8:
3522 rdev->config.evergreen.tile_config |= (3 << 0);
3523 break;
3524 }
d698a34d 3525 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3526 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3527 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3528 else {
c8d15edc
AD		 3529 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3530 case 0: /* four banks */
29d65406 3531 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3532 break;
3533 case 1: /* eight banks */
3534 rdev->config.evergreen.tile_config |= 1 << 4;
3535 break;
3536 case 2: /* sixteen banks */
3537 default:
3538 rdev->config.evergreen.tile_config |= 2 << 4;
3539 break;
3540 }
29d65406 3541 }
416a2bd2 3542 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3543 rdev->config.evergreen.tile_config |=
3544 ((gb_addr_config & 0x30000000) >> 28) << 12;
3545
416a2bd2
AD		 3546 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3547 u32 efuse_straps_4;
3548 u32 efuse_straps_3;
32fcdbf4 3549
ff82bbc4
AD		 3550 efuse_straps_4 = RREG32_RCU(0x204);
3551 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3552 tmp = (((efuse_straps_4 & 0xf) << 4) |
3553 ((efuse_straps_3 & 0xf0000000) >> 28));
3554 } else {
3555 tmp = 0;
3556 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3557 u32 rb_disable_bitmap;
3558
3559 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3560 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3561 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3562 tmp <<= 4;
3563 tmp |= rb_disable_bitmap;
32fcdbf4 3564 }
416a2bd2
AD		 3565 }
3566 /* enabled rb are just the one not disabled :) */
3567 disabled_rb_mask = tmp;
cedb655a
AD		 3568 tmp = 0;
3569 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3570 tmp |= (1 << i);
3571 /* if all the backends are disabled, fix it up here */
3572 if ((disabled_rb_mask & tmp) == tmp) {
3573 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3574 disabled_rb_mask &= ~(1 << i);
3575 }
32fcdbf4 3576
65fcf668
AD		 3577 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3578 u32 simd_disable_bitmap;
3579
3580 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3581 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3582 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3583 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3584 tmp <<= 16;
3585 tmp |= simd_disable_bitmap;
3586 }
3587 rdev->config.evergreen.active_simds = hweight32(~tmp);
3588
416a2bd2
AD		 3589 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3590 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3591
416a2bd2
AD		 3592 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3593 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3594 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3595 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3596 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3597 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3598 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3599
f7eb9730
AD		 3600 if ((rdev->config.evergreen.max_backends == 1) &&
3601 (rdev->flags & RADEON_IS_IGP)) {
3602 if ((disabled_rb_mask & 3) == 1) {
3603 /* RB0 disabled, RB1 enabled */
3604 tmp = 0x11111111;
3605 } else {
3606 /* RB1 disabled, RB0 enabled */
3607 tmp = 0x00000000;
3608 }
3609 } else {
3610 tmp = gb_addr_config & NUM_PIPES_MASK;
3611 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3612 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3613 }
416a2bd2 3614 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3615
3616 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3617 WREG32(CGTS_TCC_DISABLE, 0);
3618 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3619 WREG32(CGTS_USER_TCC_DISABLE, 0);
3620
3621 /* set HW defaults for 3D engine */
3622 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3623 ROQ_IB2_START(0x2b)));
3624
3625 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3626
3627 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3628 SYNC_GRADIENT |
3629 SYNC_WALKER |
3630 SYNC_ALIGNER));
3631
3632 sx_debug_1 = RREG32(SX_DEBUG_1);
3633 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3634 WREG32(SX_DEBUG_1, sx_debug_1);
3635
3636
3637 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3638 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3639 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3640 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3641
b866d133
AD		 3642 if (rdev->family <= CHIP_SUMO2)
3643 WREG32(SMX_SAR_CTL0, 0x00010000);
3644
32fcdbf4
AD		 3645 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3646 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3647 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3648
3649 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3650 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3651 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3652
3653 WREG32(VGT_NUM_INSTANCES, 1);
3654 WREG32(SPI_CONFIG_CNTL, 0);
3655 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3656 WREG32(CP_PERFMON_CNTL, 0);
3657
3658 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3659 FETCH_FIFO_HIWATER(0x4) |
3660 DONE_FIFO_HIWATER(0xe0) |
3661 ALU_UPDATE_FIFO_HIWATER(0x8)));
3662
3663 sq_config = RREG32(SQ_CONFIG);
3664 sq_config &= ~(PS_PRIO(3) |
3665 VS_PRIO(3) |
3666 GS_PRIO(3) |
3667 ES_PRIO(3));
3668 sq_config |= (VC_ENABLE |
3669 EXPORT_SRC_C |
3670 PS_PRIO(0) |
3671 VS_PRIO(1) |
3672 GS_PRIO(2) |
3673 ES_PRIO(3));
3674
d5e455e4
AD		 3675 switch (rdev->family) {
3676 case CHIP_CEDAR:
3677 case CHIP_PALM:
d5c5a72f
AD		 3678 case CHIP_SUMO:
3679 case CHIP_SUMO2:
adb68fa2 3680 case CHIP_CAICOS:
32fcdbf4
AD		 3681 /* no vertex cache */
3682 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3683 break;
3684 default:
3685 break;
3686 }
32fcdbf4
AD		 3687
3688 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3689
3690 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3691 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3692 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3693 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3694 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3695 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3696 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3697
d5e455e4
AD		 3698 switch (rdev->family) {
3699 case CHIP_CEDAR:
3700 case CHIP_PALM:
d5c5a72f
AD		 3701 case CHIP_SUMO:
3702 case CHIP_SUMO2:
32fcdbf4 3703 ps_thread_count = 96;
d5e455e4
AD		 3704 break;
3705 default:
32fcdbf4 3706 ps_thread_count = 128;
d5e455e4
AD		 3707 break;
3708 }
32fcdbf4
AD		 3709
3710 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3711 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3712 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3713 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3714 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3715 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3716
3717 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3718 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3719 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3720 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3721 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3722 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3723
3724 WREG32(SQ_CONFIG, sq_config);
3725 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3726 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3727 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3728 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3729 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3730 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3731 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3732 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3733 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3734 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3735
3736 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3737 FORCE_EOV_MAX_REZ_CNT(255)));
3738
d5e455e4
AD		 3739 switch (rdev->family) {
3740 case CHIP_CEDAR:
3741 case CHIP_PALM:
d5c5a72f
AD		 3742 case CHIP_SUMO:
3743 case CHIP_SUMO2:
adb68fa2 3744 case CHIP_CAICOS:
32fcdbf4 3745 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3746 break;
3747 default:
32fcdbf4 3748 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3749 break;
3750 }
32fcdbf4
AD		 3751 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3752 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3753
3754 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3755 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3756 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3757
60a4a3e0
AD		 3758 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3759 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3760
32fcdbf4
AD		 3761 WREG32(CB_PERF_CTR0_SEL_0, 0);
3762 WREG32(CB_PERF_CTR0_SEL_1, 0);
3763 WREG32(CB_PERF_CTR1_SEL_0, 0);
3764 WREG32(CB_PERF_CTR1_SEL_1, 0);
3765 WREG32(CB_PERF_CTR2_SEL_0, 0);
3766 WREG32(CB_PERF_CTR2_SEL_1, 0);
3767 WREG32(CB_PERF_CTR3_SEL_0, 0);
3768 WREG32(CB_PERF_CTR3_SEL_1, 0);
3769
60a4a3e0
AD		 3770 /* clear render buffer base addresses */
3771 WREG32(CB_COLOR0_BASE, 0);
3772 WREG32(CB_COLOR1_BASE, 0);
3773 WREG32(CB_COLOR2_BASE, 0);
3774 WREG32(CB_COLOR3_BASE, 0);
3775 WREG32(CB_COLOR4_BASE, 0);
3776 WREG32(CB_COLOR5_BASE, 0);
3777 WREG32(CB_COLOR6_BASE, 0);
3778 WREG32(CB_COLOR7_BASE, 0);
3779 WREG32(CB_COLOR8_BASE, 0);
3780 WREG32(CB_COLOR9_BASE, 0);
3781 WREG32(CB_COLOR10_BASE, 0);
3782 WREG32(CB_COLOR11_BASE, 0);
3783
3784 /* set the shader const cache sizes to 0 */
3785 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3786 WREG32(i, 0);
3787 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3788 WREG32(i, 0);
3789
f25a5c63
AD		 3790 tmp = RREG32(HDP_MISC_CNTL);
3791 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3792 WREG32(HDP_MISC_CNTL, tmp);
3793
32fcdbf4
AD		 3794 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3795 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3796
3797 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3798
3799 udelay(50);
3800
bcc1c2a1
AD		 3801}
3802
3803int evergreen_mc_init(struct radeon_device *rdev)
3804{
bcc1c2a1
AD		 3805 u32 tmp;
3806 int chansize, numchan;
bcc1c2a1
AD		 3807
3808 /* Get VRAM informations */
3809 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3810 if ((rdev->family == CHIP_PALM) ||
3811 (rdev->family == CHIP_SUMO) ||
3812 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3813 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3814 else
3815 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3816 if (tmp & CHANSIZE_OVERRIDE) {
3817 chansize = 16;
3818 } else if (tmp & CHANSIZE_MASK) {
3819 chansize = 64;
3820 } else {
3821 chansize = 32;
3822 }
3823 tmp = RREG32(MC_SHARED_CHMAP);
3824 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3825 case 0:
3826 default:
3827 numchan = 1;
3828 break;
3829 case 1:
3830 numchan = 2;
3831 break;
3832 case 2:
3833 numchan = 4;
3834 break;
3835 case 3:
3836 numchan = 8;
3837 break;
3838 }
3839 rdev->mc.vram_width = numchan * chansize;
3840 /* Could aper size report 0 ? */
01d73a69
JC		 3841 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3842 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3843 /* Setup GPU memory space */
05b3ef69
AD		 3844 if ((rdev->family == CHIP_PALM) ||
3845 (rdev->family == CHIP_SUMO) ||
3846 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3847 /* size in bytes on fusion */
3848 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3849 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3850 } else {
05b3ef69 3851 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3852 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3853 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3854 }
51e5fcd3 3855 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3856 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3857 radeon_update_bandwidth_info(rdev);
3858
bcc1c2a1
AD		 3859 return 0;
3860}
d594e46a 3861
187e3593 3862void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3863{
64c56e8c 3864 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3865 RREG32(GRBM_STATUS));
64c56e8c 3866 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3867 RREG32(GRBM_STATUS_SE0));
64c56e8c 3868 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3869 RREG32(GRBM_STATUS_SE1));
64c56e8c 3870 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3871 RREG32(SRBM_STATUS));
a65a4369
AD		 3872 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3873 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3874 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3875 RREG32(CP_STALLED_STAT1));
3876 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3877 RREG32(CP_STALLED_STAT2));
3878 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3879 RREG32(CP_BUSY_STAT));
3880 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3881 RREG32(CP_STAT));
eaaa6983
JG		 3882 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3883 RREG32(DMA_STATUS_REG));
168757ea
AD		 3884 if (rdev->family >= CHIP_CAYMAN) {
3885 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3886 RREG32(DMA_STATUS_REG + 0x800));
3887 }
0ecebb9e
AD		 3888}
3889
168757ea 3890bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3891{
a65a4369
AD		 3892 u32 crtc_hung = 0;
3893 u32 crtc_status[6];
3894 u32 i, j, tmp;
3895
3896 for (i = 0; i < rdev->num_crtc; i++) {
3897 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3898 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3899 crtc_hung |= (1 << i);
3900 }
3901 }
3902
3903 for (j = 0; j < 10; j++) {
3904 for (i = 0; i < rdev->num_crtc; i++) {
3905 if (crtc_hung & (1 << i)) {
3906 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3907 if (tmp != crtc_status[i])
3908 crtc_hung &= ~(1 << i);
3909 }
3910 }
3911 if (crtc_hung == 0)
3912 return false;
3913 udelay(100);
3914 }
3915
3916 return true;
3917}
3918
2483b4ea 3919u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3920{
3921 u32 reset_mask = 0;
b7630473 3922 u32 tmp;
0ecebb9e 3923
a65a4369
AD		 3924 /* GRBM_STATUS */
3925 tmp = RREG32(GRBM_STATUS);
3926 if (tmp & (PA_BUSY | SC_BUSY |
3927 SH_BUSY | SX_BUSY |
3928 TA_BUSY | VGT_BUSY |
3929 DB_BUSY | CB_BUSY |
3930 SPI_BUSY | VGT_BUSY_NO_DMA))
3931 reset_mask |= RADEON_RESET_GFX;
3932
3933 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3934 CP_BUSY | CP_COHERENCY_BUSY))
3935 reset_mask |= RADEON_RESET_CP;
3936
3937 if (tmp & GRBM_EE_BUSY)
3938 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3939
a65a4369
AD		 3940 /* DMA_STATUS_REG */
3941 tmp = RREG32(DMA_STATUS_REG);
3942 if (!(tmp & DMA_IDLE))
3943 reset_mask |= RADEON_RESET_DMA;
3944
3945 /* SRBM_STATUS2 */
3946 tmp = RREG32(SRBM_STATUS2);
3947 if (tmp & DMA_BUSY)
3948 reset_mask |= RADEON_RESET_DMA;
3949
3950 /* SRBM_STATUS */
3951 tmp = RREG32(SRBM_STATUS);
3952 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3953 reset_mask |= RADEON_RESET_RLC;
3954
3955 if (tmp & IH_BUSY)
3956 reset_mask |= RADEON_RESET_IH;
3957
3958 if (tmp & SEM_BUSY)
3959 reset_mask |= RADEON_RESET_SEM;
3960
3961 if (tmp & GRBM_RQ_PENDING)
3962 reset_mask |= RADEON_RESET_GRBM;
3963
3964 if (tmp & VMC_BUSY)
3965 reset_mask |= RADEON_RESET_VMC;
3966
3967 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3968 MCC_BUSY | MCD_BUSY))
3969 reset_mask |= RADEON_RESET_MC;
3970
3971 if (evergreen_is_display_hung(rdev))
3972 reset_mask |= RADEON_RESET_DISPLAY;
3973
3974 /* VM_L2_STATUS */
3975 tmp = RREG32(VM_L2_STATUS);
3976 if (tmp & L2_BUSY)
3977 reset_mask |= RADEON_RESET_VMC;
3978
d808fc88
AD		 3979 /* Skip MC reset as it's mostly likely not hung, just busy */
3980 if (reset_mask & RADEON_RESET_MC) {
3981 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3982 reset_mask &= ~RADEON_RESET_MC;
3983 }
3984
a65a4369
AD		 3985 return reset_mask;
3986}
3987
3988static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3989{
3990 struct evergreen_mc_save save;
3991 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3992 u32 tmp;
19fc42ed 3993
0ecebb9e 3994 if (reset_mask == 0)
a65a4369 3995 return;
0ecebb9e
AD		 3996
3997 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3998
b7630473
AD		 3999 evergreen_print_gpu_status_regs(rdev);
4000
b7630473
AD		 4001 /* Disable CP parsing/prefetching */
4002 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4003
4004 if (reset_mask & RADEON_RESET_DMA) {
4005 /* Disable DMA */
4006 tmp = RREG32(DMA_RB_CNTL);
4007 tmp &= ~DMA_RB_ENABLE;
4008 WREG32(DMA_RB_CNTL, tmp);
4009 }
4010
b21b6e7a
AD		 4011 udelay(50);
4012
4013 evergreen_mc_stop(rdev, &save);
4014 if (evergreen_mc_wait_for_idle(rdev)) {
4015 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4016 }
4017
b7630473
AD		 4018 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
4019 grbm_soft_reset |= SOFT_RESET_DB |
4020 SOFT_RESET_CB |
4021 SOFT_RESET_PA |
4022 SOFT_RESET_SC |
4023 SOFT_RESET_SPI |
4024 SOFT_RESET_SX |
4025 SOFT_RESET_SH |
4026 SOFT_RESET_TC |
4027 SOFT_RESET_TA |
4028 SOFT_RESET_VC |
4029 SOFT_RESET_VGT;
4030 }
4031
4032 if (reset_mask & RADEON_RESET_CP) {
4033 grbm_soft_reset |= SOFT_RESET_CP |
4034 SOFT_RESET_VGT;
4035
4036 srbm_soft_reset |= SOFT_RESET_GRBM;
4037 }
0ecebb9e
AD		 4038
4039 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 4040 srbm_soft_reset |= SOFT_RESET_DMA;
4041
a65a4369
AD		 4042 if (reset_mask & RADEON_RESET_DISPLAY)
4043 srbm_soft_reset |= SOFT_RESET_DC;
4044
4045 if (reset_mask & RADEON_RESET_RLC)
4046 srbm_soft_reset |= SOFT_RESET_RLC;
4047
4048 if (reset_mask & RADEON_RESET_SEM)
4049 srbm_soft_reset |= SOFT_RESET_SEM;
4050
4051 if (reset_mask & RADEON_RESET_IH)
4052 srbm_soft_reset |= SOFT_RESET_IH;
4053
4054 if (reset_mask & RADEON_RESET_GRBM)
4055 srbm_soft_reset |= SOFT_RESET_GRBM;
4056
4057 if (reset_mask & RADEON_RESET_VMC)
4058 srbm_soft_reset |= SOFT_RESET_VMC;
4059
24178ec4
AD		 4060 if (!(rdev->flags & RADEON_IS_IGP)) {
4061 if (reset_mask & RADEON_RESET_MC)
4062 srbm_soft_reset |= SOFT_RESET_MC;
4063 }
a65a4369 4064
b7630473
AD		 4065 if (grbm_soft_reset) {
4066 tmp = RREG32(GRBM_SOFT_RESET);
4067 tmp |= grbm_soft_reset;
4068 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4069 WREG32(GRBM_SOFT_RESET, tmp);
4070 tmp = RREG32(GRBM_SOFT_RESET);
4071
4072 udelay(50);
4073
4074 tmp &= ~grbm_soft_reset;
4075 WREG32(GRBM_SOFT_RESET, tmp);
4076 tmp = RREG32(GRBM_SOFT_RESET);
4077 }
4078
4079 if (srbm_soft_reset) {
4080 tmp = RREG32(SRBM_SOFT_RESET);
4081 tmp |= srbm_soft_reset;
4082 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4083 WREG32(SRBM_SOFT_RESET, tmp);
4084 tmp = RREG32(SRBM_SOFT_RESET);
4085
4086 udelay(50);
4087
4088 tmp &= ~srbm_soft_reset;
4089 WREG32(SRBM_SOFT_RESET, tmp);
4090 tmp = RREG32(SRBM_SOFT_RESET);
4091 }
0ecebb9e
AD		 4092
4093 /* Wait a little for things to settle down */
4094 udelay(50);
4095
747943ea 4096 evergreen_mc_resume(rdev, &save);
b7630473
AD		 4097 udelay(50);
4098
b7630473 4099 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 4100}
4101
b5470b03
AD		 4102void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4103{
4104 struct evergreen_mc_save save;
4105 u32 tmp, i;
4106
4107 dev_info(rdev->dev, "GPU pci config reset\n");
4108
4109 /* disable dpm? */
4110
4111 /* Disable CP parsing/prefetching */
4112 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4113 udelay(50);
4114 /* Disable DMA */
4115 tmp = RREG32(DMA_RB_CNTL);
4116 tmp &= ~DMA_RB_ENABLE;
4117 WREG32(DMA_RB_CNTL, tmp);
4118 /* XXX other engines? */
4119
4120 /* halt the rlc */
4121 r600_rlc_stop(rdev);
4122
4123 udelay(50);
4124
4125 /* set mclk/sclk to bypass */
4126 rv770_set_clk_bypass_mode(rdev);
4127 /* disable BM */
4128 pci_clear_master(rdev->pdev);
4129 /* disable mem access */
4130 evergreen_mc_stop(rdev, &save);
4131 if (evergreen_mc_wait_for_idle(rdev)) {
4132 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4133 }
4134 /* reset */
4135 radeon_pci_config_reset(rdev);
4136 /* wait for asic to come out of reset */
4137 for (i = 0; i < rdev->usec_timeout; i++) {
4138 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4139 break;
4140 udelay(1);
4141 }
4142}
4143
71fe2899 4144int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
bcc1c2a1 4145{
a65a4369
AD		 4146 u32 reset_mask;
4147
71fe2899
JG		 4148 if (hard) {
4149 evergreen_gpu_pci_config_reset(rdev);
4150 return 0;
4151 }
4152
a65a4369
AD		 4153 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4154
4155 if (reset_mask)
4156 r600_set_bios_scratch_engine_hung(rdev, true);
4157
b5470b03 4158 /* try soft reset */
a65a4369
AD		 4159 evergreen_gpu_soft_reset(rdev, reset_mask);
4160
4161 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4162
b5470b03
AD		 4163 /* try pci config reset */
4164 if (reset_mask && radeon_hard_reset)
4165 evergreen_gpu_pci_config_reset(rdev);
4166
4167 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4168
a65a4369
AD		 4169 if (!reset_mask)
4170 r600_set_bios_scratch_engine_hung(rdev, false);
4171
4172 return 0;
747943ea
AD		 4173}
4174
123bc183
AD		 4175/**
4176 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4177 *
4178 * @rdev: radeon_device pointer
4179 * @ring: radeon_ring structure holding ring information
4180 *
4181 * Check if the GFX engine is locked up.
4182 * Returns true if the engine appears to be locked up, false if not.
4183 */
4184bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4185{
4186 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4187
4188 if (!(reset_mask & (RADEON_RESET_GFX |
4189 RADEON_RESET_COMPUTE |
4190 RADEON_RESET_CP))) {
ff212f25 4191 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 4192 return false;
4193 }
123bc183
AD		 4194 return radeon_ring_test_lockup(rdev, ring);
4195}
4196
2948f5e6
AD		 4197/*
4198 * RLC
4199 */
4200#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4201#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4202
4203void sumo_rlc_fini(struct radeon_device *rdev)
4204{
4205 int r;
4206
4207 /* save restore block */
4208 if (rdev->rlc.save_restore_obj) {
4209 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4210 if (unlikely(r != 0))
4211 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4212 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4213 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4214
4215 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4216 rdev->rlc.save_restore_obj = NULL;
4217 }
4218
4219 /* clear state block */
4220 if (rdev->rlc.clear_state_obj) {
4221 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4222 if (unlikely(r != 0))
4223 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4224 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4225 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4226
4227 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4228 rdev->rlc.clear_state_obj = NULL;
4229 }
22c775ce
AD		 4230
4231 /* clear state block */
4232 if (rdev->rlc.cp_table_obj) {
4233 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4234 if (unlikely(r != 0))
4235 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4236 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4237 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4238
4239 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4240 rdev->rlc.cp_table_obj = NULL;
4241 }
2948f5e6
AD		 4242}
4243
22c775ce
AD		 4244#define CP_ME_TABLE_SIZE 96
4245
2948f5e6
AD		 4246int sumo_rlc_init(struct radeon_device *rdev)
4247{
1fd11777 4248 const u32 *src_ptr;
2948f5e6
AD		 4249 volatile u32 *dst_ptr;
4250 u32 dws, data, i, j, k, reg_num;
59a82d0e 4251 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 4252 u64 reg_list_mc_addr;
1fd11777 4253 const struct cs_section_def *cs_data;
2948f5e6
AD		 4254 int r;
4255
4256 src_ptr = rdev->rlc.reg_list;
4257 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 4258 if (rdev->family >= CHIP_BONAIRE) {
4259 dws += (5 * 16) + 48 + 48 + 64;
4260 }
2948f5e6
AD		 4261 cs_data = rdev->rlc.cs_data;
4262
10b7ca7e
AD		 4263 if (src_ptr) {
4264 /* save restore block */
4265 if (rdev->rlc.save_restore_obj == NULL) {
4266 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4267 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4268 NULL, &rdev->rlc.save_restore_obj);
10b7ca7e
AD		 4269 if (r) {
4270 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4271 return r;
4272 }
4273 }
4274
4275 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4276 if (unlikely(r != 0)) {
4277 sumo_rlc_fini(rdev);
4278 return r;
4279 }
4280 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4281 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4282 if (r) {
10b7ca7e
AD		 4283 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4284 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4285 sumo_rlc_fini(rdev);
2948f5e6
AD		 4286 return r;
4287 }
2948f5e6 4288
10b7ca7e
AD		 4289 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4290 if (r) {
4291 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4292 sumo_rlc_fini(rdev);
4293 return r;
4294 }
4295 /* write the sr buffer */
4296 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4297 if (rdev->family >= CHIP_TAHITI) {
4298 /* SI */
59a82d0e 4299 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6ba81e53 4300 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
1fd11777
AD		 4301 } else {
4302 /* ON/LN/TN */
4303 /* format:
4304 * dw0: (reg2 << 16) | reg1
4305 * dw1: reg1 save space
4306 * dw2: reg2 save space
4307 */
4308 for (i = 0; i < dws; i++) {
4309 data = src_ptr[i] >> 2;
4310 i++;
4311 if (i < dws)
4312 data |= (src_ptr[i] >> 2) << 16;
4313 j = (((i - 1) * 3) / 2);
6ba81e53 4314 dst_ptr[j] = cpu_to_le32(data);
1fd11777
AD		 4315 }
4316 j = ((i * 3) / 2);
6ba81e53 4317 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
10b7ca7e 4318 }
10b7ca7e 4319 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4320 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4321 }
2948f5e6 4322
10b7ca7e
AD		 4323 if (cs_data) {
4324 /* clear state block */
a0f38609
AD		 4325 if (rdev->family >= CHIP_BONAIRE) {
4326 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4327 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4328 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4329 dws = rdev->rlc.clear_state_size + (256 / 4);
4330 } else {
4331 reg_list_num = 0;
4332 dws = 0;
4333 for (i = 0; cs_data[i].section != NULL; i++) {
4334 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4335 reg_list_num++;
4336 dws += cs_data[i].section[j].reg_count;
4337 }
10b7ca7e 4338 }
59a82d0e
AD		 4339 reg_list_blk_index = (3 * reg_list_num + 2);
4340 dws += reg_list_blk_index;
4341 rdev->rlc.clear_state_size = dws;
2948f5e6 4342 }
2948f5e6 4343
10b7ca7e 4344 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4345 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4346 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4347 NULL, &rdev->rlc.clear_state_obj);
10b7ca7e
AD		 4348 if (r) {
4349 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4350 sumo_rlc_fini(rdev);
4351 return r;
4352 }
4353 }
4354 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4355 if (unlikely(r != 0)) {
4356 sumo_rlc_fini(rdev);
4357 return r;
4358 }
4359 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4360 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4361 if (r) {
10b7ca7e
AD		 4362 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4363 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4364 sumo_rlc_fini(rdev);
4365 return r;
4366 }
2948f5e6 4367
10b7ca7e
AD		 4368 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4369 if (r) {
4370 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4371 sumo_rlc_fini(rdev);
4372 return r;
4373 }
4374 /* set up the cs buffer */
4375 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4376 if (rdev->family >= CHIP_BONAIRE) {
4377 cik_get_csb_buffer(rdev, dst_ptr);
4378 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e 4379 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
6ba81e53
AD		 4380 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4381 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4382 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
59a82d0e
AD		 4383 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4384 } else {
4385 reg_list_hdr_blk_index = 0;
4386 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4387 data = upper_32_bits(reg_list_mc_addr);
6ba81e53 4388 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
2948f5e6 4389 reg_list_hdr_blk_index++;
59a82d0e
AD		 4390 for (i = 0; cs_data[i].section != NULL; i++) {
4391 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4392 reg_num = cs_data[i].section[j].reg_count;
4393 data = reg_list_mc_addr & 0xffffffff;
6ba81e53 4394 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4395 reg_list_hdr_blk_index++;
4396
4397 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
6ba81e53 4398 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4399 reg_list_hdr_blk_index++;
4400
4401 data = 0x08000000 | (reg_num * 4);
6ba81e53 4402 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4403 reg_list_hdr_blk_index++;
4404
4405 for (k = 0; k < reg_num; k++) {
4406 data = cs_data[i].section[j].extent[k];
6ba81e53 4407 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
59a82d0e
AD		 4408 }
4409 reg_list_mc_addr += reg_num * 4;
4410 reg_list_blk_index += reg_num;
10b7ca7e 4411 }
2948f5e6 4412 }
6ba81e53 4413 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
2948f5e6 4414 }
10b7ca7e
AD		 4415 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4416 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4417 }
2948f5e6 4418
22c775ce
AD		 4419 if (rdev->rlc.cp_table_size) {
4420 if (rdev->rlc.cp_table_obj == NULL) {
02376d82
MD		 4421 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4422 PAGE_SIZE, true,
4423 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4424 NULL, &rdev->rlc.cp_table_obj);
22c775ce
AD		 4425 if (r) {
4426 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4427 sumo_rlc_fini(rdev);
4428 return r;
4429 }
4430 }
4431
4432 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4433 if (unlikely(r != 0)) {
4434 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4435 sumo_rlc_fini(rdev);
4436 return r;
4437 }
4438 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4439 &rdev->rlc.cp_table_gpu_addr);
4440 if (r) {
4441 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4442 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4443 sumo_rlc_fini(rdev);
4444 return r;
4445 }
4446 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4447 if (r) {
4448 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4449 sumo_rlc_fini(rdev);
4450 return r;
4451 }
4452
4453 cik_init_cp_pg_table(rdev);
4454
4455 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4456 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4457
4458 }
2948f5e6
AD		 4459
4460 return 0;
4461}
4462
4463static void evergreen_rlc_start(struct radeon_device *rdev)
4464{
8ba10463
AD		 4465 u32 mask = RLC_ENABLE;
4466
4467 if (rdev->flags & RADEON_IS_IGP) {
4468 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4469 }
4470
4471 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4472}
4473
4474int evergreen_rlc_resume(struct radeon_device *rdev)
4475{
4476 u32 i;
4477 const __be32 *fw_data;
4478
4479 if (!rdev->rlc_fw)
4480 return -EINVAL;
4481
4482 r600_rlc_stop(rdev);
4483
4484 WREG32(RLC_HB_CNTL, 0);
4485
4486 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4487 if (rdev->family == CHIP_ARUBA) {
4488 u32 always_on_bitmap =
4489 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4490 /* find out the number of active simds */
4491 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4492 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4493 tmp = hweight32(~tmp);
4494 if (tmp == rdev->config.cayman.max_simds_per_se) {
4495 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4496 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4497 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4498 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4499 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4500 }
4501 } else {
4502 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4503 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4504 }
2948f5e6
AD		 4505 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4506 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4507 } else {
4508 WREG32(RLC_HB_BASE, 0);
4509 WREG32(RLC_HB_RPTR, 0);
4510 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4511 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4512 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4513 }
2948f5e6
AD		 4514 WREG32(RLC_MC_CNTL, 0);
4515 WREG32(RLC_UCODE_CNTL, 0);
4516
4517 fw_data = (const __be32 *)rdev->rlc_fw->data;
4518 if (rdev->family >= CHIP_ARUBA) {
4519 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4520 WREG32(RLC_UCODE_ADDR, i);
4521 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4522 }
4523 } else if (rdev->family >= CHIP_CAYMAN) {
4524 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4525 WREG32(RLC_UCODE_ADDR, i);
4526 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4527 }
4528 } else {
4529 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4530 WREG32(RLC_UCODE_ADDR, i);
4531 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4532 }
4533 }
4534 WREG32(RLC_UCODE_ADDR, 0);
4535
4536 evergreen_rlc_start(rdev);
4537
4538 return 0;
4539}
4540
45f9a39b
AD		 4541/* Interrupts */
4542
4543u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4544{
46437057 4545 if (crtc >= rdev->num_crtc)
45f9a39b 4546 return 0;
46437057
AD		 4547 else
4548 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4549}
4550
4551void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4552{
4553 u32 tmp;
4554
1b37078b
AD		 4555 if (rdev->family >= CHIP_CAYMAN) {
4556 cayman_cp_int_cntl_setup(rdev, 0,
4557 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4558 cayman_cp_int_cntl_setup(rdev, 1, 0);
4559 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4560 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4561 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4562 } else
4563 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4564 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4565 WREG32(DMA_CNTL, tmp);
45f9a39b 4566 WREG32(GRBM_INT_CNTL, 0);
acc1522a 4567 WREG32(SRBM_INT_CNTL, 0);
45f9a39b
AD		 4568 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4569 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4570 if (rdev->num_crtc >= 4) {
18007401
AD		 4571 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4572 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4573 }
4574 if (rdev->num_crtc >= 6) {
18007401
AD		 4575 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4576 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4577 }
45f9a39b
AD		 4578
4579 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4580 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4581 if (rdev->num_crtc >= 4) {
18007401
AD		 4582 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4583 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4584 }
4585 if (rdev->num_crtc >= 6) {
18007401
AD		 4586 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4587 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4588 }
45f9a39b 4589
e9a321c6
AD		 4590 /* only one DAC on DCE5 */
4591 if (!ASIC_IS_DCE5(rdev))
05b3ef69 4592 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4593 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4594
4595 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4596 WREG32(DC_HPD1_INT_CONTROL, tmp);
4597 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4598 WREG32(DC_HPD2_INT_CONTROL, tmp);
4599 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4600 WREG32(DC_HPD3_INT_CONTROL, tmp);
4601 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4602 WREG32(DC_HPD4_INT_CONTROL, tmp);
4603 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4604 WREG32(DC_HPD5_INT_CONTROL, tmp);
4605 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4606 WREG32(DC_HPD6_INT_CONTROL, tmp);
4607
4608}
4609
4610int evergreen_irq_set(struct radeon_device *rdev)
4611{
4612 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4613 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4614 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4615 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4616 u32 grbm_int_cntl = 0;
f122c610 4617 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4618 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4619 u32 thermal_int = 0;
45f9a39b
AD		 4620
4621 if (!rdev->irq.installed) {
fce7d61b 4622 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4623 return -EINVAL;
4624 }
4625 /* don't enable anything if the ih is disabled */
4626 if (!rdev->ih.enabled) {
4627 r600_disable_interrupts(rdev);
4628 /* force the active interrupt state to all disabled */
4629 evergreen_disable_interrupt_state(rdev);
4630 return 0;
4631 }
4632
2bc67b4d
DA		 4633 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4634 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4635 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4636 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4637 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4638 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
d70229f7
AD		 4639 if (rdev->family == CHIP_ARUBA)
4640 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4641 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4642 else
4643 thermal_int = RREG32(CG_THERMAL_INT) &
4644 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4645
f122c610
AD		 4646 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4647 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4648 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4649 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4650 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4651 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4652
233d1ad5
AD		 4653 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4654
1b37078b
AD		 4655 if (rdev->family >= CHIP_CAYMAN) {
4656 /* enable CP interrupts on all rings */
736fc37f 4657 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4658 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4659 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4660 }
736fc37f 4661 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4662 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4663 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4664 }
736fc37f 4665 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4666 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4667 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4668 }
4669 } else {
736fc37f 4670 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4671 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4672 cp_int_cntl |= RB_INT_ENABLE;
4673 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4674 }
45f9a39b 4675 }
1b37078b 4676
233d1ad5
AD		 4677 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4678 DRM_DEBUG("r600_irq_set: sw int dma\n");
4679 dma_cntl |= TRAP_ENABLE;
4680 }
4681
f60cbd11
AD		 4682 if (rdev->family >= CHIP_CAYMAN) {
4683 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4684 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4685 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4686 dma_cntl1 |= TRAP_ENABLE;
4687 }
4688 }
4689
dc50ba7f
AD		 4690 if (rdev->irq.dpm_thermal) {
4691 DRM_DEBUG("dpm thermal\n");
4692 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4693 }
4694
6f34be50 4695 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4696 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4697 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4698 crtc1 |= VBLANK_INT_MASK;
4699 }
6f34be50 4700 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4701 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4702 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4703 crtc2 |= VBLANK_INT_MASK;
4704 }
6f34be50 4705 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4706 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4707 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4708 crtc3 |= VBLANK_INT_MASK;
4709 }
6f34be50 4710 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4711 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4712 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4713 crtc4 |= VBLANK_INT_MASK;
4714 }
6f34be50 4715 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4716 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4717 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4718 crtc5 |= VBLANK_INT_MASK;
4719 }
6f34be50 4720 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4721 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4722 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4723 crtc6 |= VBLANK_INT_MASK;
4724 }
4725 if (rdev->irq.hpd[0]) {
4726 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2bc67b4d 4727 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4728 }
4729 if (rdev->irq.hpd[1]) {
4730 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2bc67b4d 4731 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4732 }
4733 if (rdev->irq.hpd[2]) {
4734 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2bc67b4d 4735 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4736 }
4737 if (rdev->irq.hpd[3]) {
4738 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2bc67b4d 4739 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4740 }
4741 if (rdev->irq.hpd[4]) {
4742 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2bc67b4d 4743 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4744 }
4745 if (rdev->irq.hpd[5]) {
4746 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2bc67b4d 4747 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b 4748 }
f122c610
AD		 4749 if (rdev->irq.afmt[0]) {
4750 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4751 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4752 }
4753 if (rdev->irq.afmt[1]) {
4754 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4755 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4756 }
4757 if (rdev->irq.afmt[2]) {
4758 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4759 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4760 }
4761 if (rdev->irq.afmt[3]) {
4762 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4763 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4764 }
4765 if (rdev->irq.afmt[4]) {
4766 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4767 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4768 }
4769 if (rdev->irq.afmt[5]) {
4770 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4771 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4772 }
45f9a39b 4773
1b37078b
AD		 4774 if (rdev->family >= CHIP_CAYMAN) {
4775 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4776 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4777 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4778 } else
4779 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4780
4781 WREG32(DMA_CNTL, dma_cntl);
4782
f60cbd11
AD		 4783 if (rdev->family >= CHIP_CAYMAN)
4784 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4785
2031f77c 4786 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4787
4788 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4789 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4790 if (rdev->num_crtc >= 4) {
18007401
AD		 4791 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4792 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4793 }
4794 if (rdev->num_crtc >= 6) {
18007401
AD		 4795 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4796 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4797 }
45f9a39b 4798
f5d636d2
CK		 4799 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4800 GRPH_PFLIP_INT_MASK);
4801 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4802 GRPH_PFLIP_INT_MASK);
b7eff394 4803 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 4804 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4805 GRPH_PFLIP_INT_MASK);
4806 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4807 GRPH_PFLIP_INT_MASK);
b7eff394
AD		 4808 }
4809 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 4810 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4811 GRPH_PFLIP_INT_MASK);
4812 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4813 GRPH_PFLIP_INT_MASK);
b7eff394 4814 }
6f34be50 4815
45f9a39b
AD		 4816 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4817 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4818 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4819 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4820 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4821 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4822 if (rdev->family == CHIP_ARUBA)
4823 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4824 else
4825 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4826
f122c610
AD		 4827 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4828 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4829 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4830 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4831 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4832 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4833
c320bb5f
AD		 4834 /* posting read */
4835 RREG32(SRBM_STATUS);
4836
bcc1c2a1
AD		 4837 return 0;
4838}
4839
cbdd4501 4840static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4841{
4842 u32 tmp;
4843
6f34be50
AD		 4844 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4845 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4846 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4847 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4848 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4849 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4850 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4851 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4852 if (rdev->num_crtc >= 4) {
4853 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4854 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4855 }
4856 if (rdev->num_crtc >= 6) {
4857 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4858 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4859 }
6f34be50 4860
f122c610
AD		 4861 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4862 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4863 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4864 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4865 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4866 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4867
6f34be50
AD		 4868 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4869 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4870 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4871 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4872 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4873 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4874 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4875 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4876 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4877 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4878 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4879 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4880
b7eff394
AD		 4881 if (rdev->num_crtc >= 4) {
4882 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4883 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4884 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4885 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4886 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4887 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4888 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4889 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4890 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4891 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4892 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4893 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4894 }
4895
4896 if (rdev->num_crtc >= 6) {
4897 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4898 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4899 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4900 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4901 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4902 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4903 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4904 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4905 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4906 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4907 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4908 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4909 }
45f9a39b 4910
6f34be50 4911 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4912 tmp = RREG32(DC_HPD1_INT_CONTROL);
4913 tmp |= DC_HPDx_INT_ACK;
4914 WREG32(DC_HPD1_INT_CONTROL, tmp);
4915 }
6f34be50 4916 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4917 tmp = RREG32(DC_HPD2_INT_CONTROL);
4918 tmp |= DC_HPDx_INT_ACK;
4919 WREG32(DC_HPD2_INT_CONTROL, tmp);
4920 }
6f34be50 4921 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4922 tmp = RREG32(DC_HPD3_INT_CONTROL);
4923 tmp |= DC_HPDx_INT_ACK;
4924 WREG32(DC_HPD3_INT_CONTROL, tmp);
4925 }
6f34be50 4926 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4927 tmp = RREG32(DC_HPD4_INT_CONTROL);
4928 tmp |= DC_HPDx_INT_ACK;
4929 WREG32(DC_HPD4_INT_CONTROL, tmp);
4930 }
6f34be50 4931 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4932 tmp = RREG32(DC_HPD5_INT_CONTROL);
4933 tmp |= DC_HPDx_INT_ACK;
4934 WREG32(DC_HPD5_INT_CONTROL, tmp);
4935 }
6f34be50 4936 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4937 tmp = RREG32(DC_HPD5_INT_CONTROL);
4938 tmp |= DC_HPDx_INT_ACK;
4939 WREG32(DC_HPD6_INT_CONTROL, tmp);
4940 }
2bc67b4d
DA		 4941
4942 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4943 tmp = RREG32(DC_HPD1_INT_CONTROL);
4944 tmp |= DC_HPDx_RX_INT_ACK;
4945 WREG32(DC_HPD1_INT_CONTROL, tmp);
4946 }
4947 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4948 tmp = RREG32(DC_HPD2_INT_CONTROL);
4949 tmp |= DC_HPDx_RX_INT_ACK;
4950 WREG32(DC_HPD2_INT_CONTROL, tmp);
4951 }
4952 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4953 tmp = RREG32(DC_HPD3_INT_CONTROL);
4954 tmp |= DC_HPDx_RX_INT_ACK;
4955 WREG32(DC_HPD3_INT_CONTROL, tmp);
4956 }
4957 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4958 tmp = RREG32(DC_HPD4_INT_CONTROL);
4959 tmp |= DC_HPDx_RX_INT_ACK;
4960 WREG32(DC_HPD4_INT_CONTROL, tmp);
4961 }
4962 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4963 tmp = RREG32(DC_HPD5_INT_CONTROL);
4964 tmp |= DC_HPDx_RX_INT_ACK;
4965 WREG32(DC_HPD5_INT_CONTROL, tmp);
4966 }
4967 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4968 tmp = RREG32(DC_HPD5_INT_CONTROL);
4969 tmp |= DC_HPDx_RX_INT_ACK;
4970 WREG32(DC_HPD6_INT_CONTROL, tmp);
4971 }
4972
f122c610
AD		 4973 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4974 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4975 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4976 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4977 }
4978 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4979 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4980 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4981 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4982 }
4983 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4984 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4985 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4986 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4987 }
4988 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4989 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4990 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4991 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4992 }
4993 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4994 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4995 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4996 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4997 }
4998 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4999 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
5000 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
5001 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
5002 }
45f9a39b
AD		 5003}
5004
1109ca09 5005static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 5006{
45f9a39b
AD		 5007 r600_disable_interrupts(rdev);
5008 /* Wait and acknowledge irq */
5009 mdelay(1);
6f34be50 5010 evergreen_irq_ack(rdev);
45f9a39b
AD		 5011 evergreen_disable_interrupt_state(rdev);
5012}
5013
755d819e 5014void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 5015{
5016 evergreen_irq_disable(rdev);
5017 r600_rlc_stop(rdev);
5018}
5019
cbdd4501 5020static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 5021{
5022 u32 wptr, tmp;
5023
724c80e1 5024 if (rdev->wb.enabled)
204ae24d 5025 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 5026 else
5027 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 5028
5029 if (wptr & RB_OVERFLOW) {
11bab0ae 5030 wptr &= ~RB_OVERFLOW;
45f9a39b
AD		 5031 /* When a ring buffer overflow happen start parsing interrupt
5032 * from the last not overwritten vector (wptr + 16). Hopefully
5033 * this should allow us to catchup.
5034 */
6cc2fda2
MD		 5035 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
5036 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
45f9a39b
AD		 5037 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5038 tmp = RREG32(IH_RB_CNTL);
5039 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5040 WREG32(IH_RB_CNTL, tmp);
5041 }
5042 return (wptr & rdev->ih.ptr_mask);
5043}
5044
5045int evergreen_irq_process(struct radeon_device *rdev)
5046{
682f1a54
DA		 5047 u32 wptr;
5048 u32 rptr;
45f9a39b
AD		 5049 u32 src_id, src_data;
5050 u32 ring_index;
45f9a39b 5051 bool queue_hotplug = false;
f122c610 5052 bool queue_hdmi = false;
2bc67b4d 5053 bool queue_dp = false;
dc50ba7f 5054 bool queue_thermal = false;
54e2e49c 5055 u32 status, addr;
45f9a39b 5056
682f1a54 5057 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 5058 return IRQ_NONE;
5059
682f1a54 5060 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 5061
5062restart_ih:
5063 /* is somebody else already processing irqs? */
5064 if (atomic_xchg(&rdev->ih.lock, 1))
5065 return IRQ_NONE;
5066
682f1a54 5067 rptr = rdev->ih.rptr;
07f18f0b 5068 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 5069
964f6645
BH		 5070 /* Order reading of wptr vs. reading of IH ring data */
5071 rmb();
5072
45f9a39b 5073 /* display interrupts */
6f34be50 5074 evergreen_irq_ack(rdev);
45f9a39b 5075
45f9a39b
AD		 5076 while (rptr != wptr) {
5077 /* wptr/rptr are in bytes! */
5078 ring_index = rptr / 4;
0f234f5f
AD		 5079 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5080 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 5081
5082 switch (src_id) {
5083 case 1: /* D1 vblank/vline */
5084 switch (src_data) {
5085 case 0: /* D1 vblank */
07f18f0b
MK		 5086 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
5087 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
5088
5089 if (rdev->irq.crtc_vblank_int[0]) {
5090 drm_handle_vblank(rdev->ddev, 0);
5091 rdev->pm.vblank_sync = true;
5092 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5093 }
07f18f0b
MK		 5094 if (atomic_read(&rdev->irq.pflip[0]))
5095 radeon_crtc_handle_vblank(rdev, 0);
5096 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5097 DRM_DEBUG("IH: D1 vblank\n");
5098
45f9a39b
AD		 5099 break;
5100 case 1: /* D1 vline */
07f18f0b
MK		 5101 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
5102 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
5103
5104 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5105 DRM_DEBUG("IH: D1 vline\n");
5106
45f9a39b
AD		 5107 break;
5108 default:
5109 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5110 break;
5111 }
5112 break;
5113 case 2: /* D2 vblank/vline */
5114 switch (src_data) {
5115 case 0: /* D2 vblank */
07f18f0b
MK		 5116 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
5117 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
5118
5119 if (rdev->irq.crtc_vblank_int[1]) {
5120 drm_handle_vblank(rdev->ddev, 1);
5121 rdev->pm.vblank_sync = true;
5122 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5123 }
07f18f0b
MK		 5124 if (atomic_read(&rdev->irq.pflip[1]))
5125 radeon_crtc_handle_vblank(rdev, 1);
5126 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5127 DRM_DEBUG("IH: D2 vblank\n");
5128
45f9a39b
AD		 5129 break;
5130 case 1: /* D2 vline */
07f18f0b
MK		 5131 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
5132 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
5133
5134 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5135 DRM_DEBUG("IH: D2 vline\n");
5136
45f9a39b
AD		 5137 break;
5138 default:
5139 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5140 break;
5141 }
5142 break;
5143 case 3: /* D3 vblank/vline */
5144 switch (src_data) {
5145 case 0: /* D3 vblank */
07f18f0b
MK		 5146 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
5147 DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
5148
5149 if (rdev->irq.crtc_vblank_int[2]) {
5150 drm_handle_vblank(rdev->ddev, 2);
5151 rdev->pm.vblank_sync = true;
5152 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5153 }
07f18f0b
MK		 5154 if (atomic_read(&rdev->irq.pflip[2]))
5155 radeon_crtc_handle_vblank(rdev, 2);
5156 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5157 DRM_DEBUG("IH: D3 vblank\n");
5158
45f9a39b
AD		 5159 break;
5160 case 1: /* D3 vline */
07f18f0b
MK		 5161 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
5162 DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
5163
5164 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5165 DRM_DEBUG("IH: D3 vline\n");
5166
45f9a39b
AD		 5167 break;
5168 default:
5169 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5170 break;
5171 }
5172 break;
5173 case 4: /* D4 vblank/vline */
5174 switch (src_data) {
5175 case 0: /* D4 vblank */
07f18f0b
MK		 5176 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5177 DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5178
5179 if (rdev->irq.crtc_vblank_int[3]) {
5180 drm_handle_vblank(rdev->ddev, 3);
5181 rdev->pm.vblank_sync = true;
5182 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5183 }
07f18f0b
MK		 5184 if (atomic_read(&rdev->irq.pflip[3]))
5185 radeon_crtc_handle_vblank(rdev, 3);
5186 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5187 DRM_DEBUG("IH: D4 vblank\n");
5188
45f9a39b
AD		 5189 break;
5190 case 1: /* D4 vline */
07f18f0b
MK		 5191 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5192 DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5193
5194 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5195 DRM_DEBUG("IH: D4 vline\n");
5196
45f9a39b
AD		 5197 break;
5198 default:
5199 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5200 break;
5201 }
5202 break;
5203 case 5: /* D5 vblank/vline */
5204 switch (src_data) {
5205 case 0: /* D5 vblank */
07f18f0b
MK		 5206 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5207 DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5208
5209 if (rdev->irq.crtc_vblank_int[4]) {
5210 drm_handle_vblank(rdev->ddev, 4);
5211 rdev->pm.vblank_sync = true;
5212 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5213 }
07f18f0b
MK		 5214 if (atomic_read(&rdev->irq.pflip[4]))
5215 radeon_crtc_handle_vblank(rdev, 4);
5216 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5217 DRM_DEBUG("IH: D5 vblank\n");
5218
45f9a39b
AD		 5219 break;
5220 case 1: /* D5 vline */
07f18f0b
MK		 5221 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5222 DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5223
5224 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5225 DRM_DEBUG("IH: D5 vline\n");
5226
45f9a39b
AD		 5227 break;
5228 default:
5229 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5230 break;
5231 }
5232 break;
5233 case 6: /* D6 vblank/vline */
5234 switch (src_data) {
5235 case 0: /* D6 vblank */
07f18f0b
MK		 5236 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5237 DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5238
5239 if (rdev->irq.crtc_vblank_int[5]) {
5240 drm_handle_vblank(rdev->ddev, 5);
5241 rdev->pm.vblank_sync = true;
5242 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5243 }
07f18f0b
MK		 5244 if (atomic_read(&rdev->irq.pflip[5]))
5245 radeon_crtc_handle_vblank(rdev, 5);
5246 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5247 DRM_DEBUG("IH: D6 vblank\n");
5248
45f9a39b
AD		 5249 break;
5250 case 1: /* D6 vline */
07f18f0b
MK		 5251 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5252 DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5253
5254 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5255 DRM_DEBUG("IH: D6 vline\n");
5256
45f9a39b
AD		 5257 break;
5258 default:
5259 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5260 break;
5261 }
5262 break;
f5d636d2
CK		 5263 case 8: /* D1 page flip */
5264 case 10: /* D2 page flip */
5265 case 12: /* D3 page flip */
5266 case 14: /* D4 page flip */
5267 case 16: /* D5 page flip */
5268 case 18: /* D6 page flip */
5269 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 5270 if (radeon_use_pflipirq > 0)
5271 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 5272 break;
45f9a39b
AD		 5273 case 42: /* HPD hotplug */
5274 switch (src_data) {
5275 case 0:
07f18f0b
MK		 5276 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5277 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5278
5279 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5280 queue_hotplug = true;
5281 DRM_DEBUG("IH: HPD1\n");
45f9a39b
AD		 5282 break;
5283 case 1:
07f18f0b
MK		 5284 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5285 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5286
5287 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5288 queue_hotplug = true;
5289 DRM_DEBUG("IH: HPD2\n");
45f9a39b
AD		 5290 break;
5291 case 2:
07f18f0b
MK		 5292 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5293 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5294
5295 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5296 queue_hotplug = true;
5297 DRM_DEBUG("IH: HPD3\n");
45f9a39b
AD		 5298 break;
5299 case 3:
07f18f0b
MK		 5300 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5301 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5302
5303 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5304 queue_hotplug = true;
5305 DRM_DEBUG("IH: HPD4\n");
45f9a39b
AD		 5306 break;
5307 case 4:
07f18f0b
MK		 5308 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5309 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5310
5311 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5312 queue_hotplug = true;
5313 DRM_DEBUG("IH: HPD5\n");
45f9a39b
AD		 5314 break;
5315 case 5:
07f18f0b
MK		 5316 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5317 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5318
5319 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5320 queue_hotplug = true;
5321 DRM_DEBUG("IH: HPD6\n");
45f9a39b 5322 break;
2bc67b4d 5323 case 6:
07f18f0b
MK		 5324 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5325 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5326
5327 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5328 queue_dp = true;
5329 DRM_DEBUG("IH: HPD_RX 1\n");
2bc67b4d
DA		 5330 break;
5331 case 7:
07f18f0b
MK		 5332 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5333 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5334
5335 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5336 queue_dp = true;
5337 DRM_DEBUG("IH: HPD_RX 2\n");
2bc67b4d
DA		 5338 break;
5339 case 8:
07f18f0b
MK		 5340 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5341 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5342
5343 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5344 queue_dp = true;
5345 DRM_DEBUG("IH: HPD_RX 3\n");
2bc67b4d
DA		 5346 break;
5347 case 9:
07f18f0b
MK		 5348 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5349 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5350
5351 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5352 queue_dp = true;
5353 DRM_DEBUG("IH: HPD_RX 4\n");
2bc67b4d
DA		 5354 break;
5355 case 10:
07f18f0b
MK		 5356 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5357 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5358
5359 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5360 queue_dp = true;
5361 DRM_DEBUG("IH: HPD_RX 5\n");
2bc67b4d
DA		 5362 break;
5363 case 11:
07f18f0b
MK		 5364 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5365 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5366
5367 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5368 queue_dp = true;
5369 DRM_DEBUG("IH: HPD_RX 6\n");
2bc67b4d 5370 break;
45f9a39b
AD		 5371 default:
5372 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5373 break;
5374 }
5375 break;
f122c610
AD		 5376 case 44: /* hdmi */
5377 switch (src_data) {
5378 case 0:
07f18f0b
MK		 5379 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5380 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5381
5382 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5383 queue_hdmi = true;
5384 DRM_DEBUG("IH: HDMI0\n");
f122c610
AD		 5385 break;
5386 case 1:
07f18f0b
MK		 5387 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5388 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5389
5390 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5391 queue_hdmi = true;
5392 DRM_DEBUG("IH: HDMI1\n");
f122c610
AD		 5393 break;
5394 case 2:
07f18f0b
MK		 5395 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5396 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5397
5398 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5399 queue_hdmi = true;
5400 DRM_DEBUG("IH: HDMI2\n");
f122c610
AD		 5401 break;
5402 case 3:
07f18f0b
MK		 5403 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5404 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5405
5406 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5407 queue_hdmi = true;
5408 DRM_DEBUG("IH: HDMI3\n");
f122c610
AD		 5409 break;
5410 case 4:
07f18f0b
MK		 5411 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5412 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5413
5414 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5415 queue_hdmi = true;
5416 DRM_DEBUG("IH: HDMI4\n");
f122c610
AD		 5417 break;
5418 case 5:
07f18f0b
MK		 5419 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5420 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5421
5422 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5423 queue_hdmi = true;
5424 DRM_DEBUG("IH: HDMI5\n");
f122c610
AD		 5425 break;
5426 default:
5427 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5428 break;
5429 }
acc1522a
CK		 5430 case 96:
5431 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5432 WREG32(SRBM_INT_ACK, 0x1);
5433 break;
f2ba57b5
CK		 5434 case 124: /* UVD */
5435 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5436 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5437 break;
ae133a11
CK		 5438 case 146:
5439 case 147:
54e2e49c
AD		 5440 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5441 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 5442 /* reset addr and status */
5443 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5444 if (addr == 0x0 && status == 0x0)
5445 break;
ae133a11
CK		 5446 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5447 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5448 addr);
ae133a11 5449 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5450 status);
5451 cayman_vm_decode_fault(rdev, status, addr);
ae133a11 5452 break;
45f9a39b
AD		 5453 case 176: /* CP_INT in ring buffer */
5454 case 177: /* CP_INT in IB1 */
5455 case 178: /* CP_INT in IB2 */
5456 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5457 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5458 break;
5459 case 181: /* CP EOP event */
5460 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5461 if (rdev->family >= CHIP_CAYMAN) {
5462 switch (src_data) {
5463 case 0:
5464 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5465 break;
5466 case 1:
5467 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5468 break;
5469 case 2:
5470 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5471 break;
5472 }
5473 } else
5474 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5475 break;
233d1ad5
AD		 5476 case 224: /* DMA trap event */
5477 DRM_DEBUG("IH: DMA trap\n");
5478 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5479 break;
dc50ba7f
AD		 5480 case 230: /* thermal low to high */
5481 DRM_DEBUG("IH: thermal low to high\n");
5482 rdev->pm.dpm.thermal.high_to_low = false;
5483 queue_thermal = true;
5484 break;
5485 case 231: /* thermal high to low */
5486 DRM_DEBUG("IH: thermal high to low\n");
5487 rdev->pm.dpm.thermal.high_to_low = true;
5488 queue_thermal = true;
5489 break;
2031f77c 5490 case 233: /* GUI IDLE */
303c805c 5491 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5492 break;
f60cbd11
AD		 5493 case 244: /* DMA trap event */
5494 if (rdev->family >= CHIP_CAYMAN) {
5495 DRM_DEBUG("IH: DMA1 trap\n");
5496 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5497 }
5498 break;
45f9a39b
AD		 5499 default:
5500 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5501 break;
5502 }
5503
5504 /* wptr/rptr are in bytes! */
5505 rptr += 16;
5506 rptr &= rdev->ih.ptr_mask;
f55e03b9 5507 WREG32(IH_RB_RPTR, rptr);
45f9a39b 5508 }
2bc67b4d
DA		 5509 if (queue_dp)
5510 schedule_work(&rdev->dp_work);
45f9a39b 5511 if (queue_hotplug)
cb5d4166 5512 schedule_delayed_work(&rdev->hotplug_work, 0);
f122c610
AD		 5513 if (queue_hdmi)
5514 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5515 if (queue_thermal && rdev->pm.dpm_enabled)
5516 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b 5517 rdev->ih.rptr = rptr;
c20dc369
CK		 5518 atomic_set(&rdev->ih.lock, 0);
5519
5520 /* make sure wptr hasn't changed while processing */
5521 wptr = evergreen_get_ih_wptr(rdev);
5522 if (wptr != rptr)
5523 goto restart_ih;
5524
45f9a39b
AD		 5525 return IRQ_HANDLED;
5526}
5527
d78d6f39
JG		 5528static void evergreen_uvd_init(struct radeon_device *rdev)
5529{
5530 int r;
5531
5532 if (!rdev->has_uvd)
5533 return;
5534
5535 r = radeon_uvd_init(rdev);
5536 if (r) {
5537 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
5538 /*
5539 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
5540 * to early fails uvd_v2_2_resume() and thus nothing happens
5541 * there. So it is pointless to try to go through that code
5542 * hence why we disable uvd here.
5543 */
5544 rdev->has_uvd = 0;
5545 return;
5546 }
5547 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5548 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
5549}
5550
5551static void evergreen_uvd_start(struct radeon_device *rdev)
5552{
5553 int r;
5554
5555 if (!rdev->has_uvd)
5556 return;
5557
5558 r = uvd_v2_2_resume(rdev);
5559 if (r) {
5560 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
5561 goto error;
5562 }
5563 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
5564 if (r) {
5565 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
5566 goto error;
5567 }
5568 return;
5569
5570error:
5571 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5572}
5573
5574static void evergreen_uvd_resume(struct radeon_device *rdev)
5575{
5576 struct radeon_ring *ring;
5577 int r;
5578
5579 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
5580 return;
5581
5582 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5583 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
5584 if (r) {
5585 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
5586 return;
5587 }
5588 r = uvd_v1_0_init(rdev);
5589 if (r) {
5590 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
5591 return;
5592 }
5593}
5594
bcc1c2a1
AD		 5595static int evergreen_startup(struct radeon_device *rdev)
5596{
f2ba57b5 5597 struct radeon_ring *ring;
bcc1c2a1
AD		 5598 int r;
5599
9e46a48d 5600 /* enable pcie gen2 link */
cd54033a 5601 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5602 /* enable aspm */
5603 evergreen_program_aspm(rdev);
9e46a48d 5604
e5903d39
AD		 5605 /* scratch needs to be initialized before MC */
5606 r = r600_vram_scratch_init(rdev);
5607 if (r)
5608 return r;
5609
6fab3feb
AD		 5610 evergreen_mc_program(rdev);
5611
6c7bccea 5612 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
755d819e 5613 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5614 if (r) {
0af62b01 5615 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5616 return r;
5617 }
5618 }
fe251e2f 5619
bcc1c2a1 5620 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5621 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5622 } else {
5623 r = evergreen_pcie_gart_enable(rdev);
5624 if (r)
5625 return r;
5626 }
bcc1c2a1 5627 evergreen_gpu_init(rdev);
bcc1c2a1 5628
2948f5e6
AD		 5629 /* allocate rlc buffers */
5630 if (rdev->flags & RADEON_IS_IGP) {
5631 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5632 rdev->rlc.reg_list_size =
5633 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5634 rdev->rlc.cs_data = evergreen_cs_data;
5635 r = sumo_rlc_init(rdev);
5636 if (r) {
5637 DRM_ERROR("Failed to init rlc BOs!\n");
5638 return r;
5639 }
5640 }
5641
724c80e1
AD		 5642 /* allocate wb buffer */
5643 r = radeon_wb_init(rdev);
5644 if (r)
5645 return r;
5646
30eb77f4
JG		 5647 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5648 if (r) {
5649 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5650 return r;
5651 }
5652
233d1ad5
AD		 5653 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5654 if (r) {
5655 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5656 return r;
5657 }
5658
d78d6f39 5659 evergreen_uvd_start(rdev);
f2ba57b5 5660
bcc1c2a1 5661 /* Enable IRQ */
e49f3959
AH		 5662 if (!rdev->irq.installed) {
5663 r = radeon_irq_kms_init(rdev);
5664 if (r)
5665 return r;
5666 }
5667
bcc1c2a1
AD		 5668 r = r600_irq_init(rdev);
5669 if (r) {
5670 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5671 radeon_irq_kms_fini(rdev);
5672 return r;
5673 }
45f9a39b 5674 evergreen_irq_set(rdev);
bcc1c2a1 5675
f2ba57b5 5676 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5677 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 5678 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5679 if (r)
5680 return r;
233d1ad5
AD		 5681
5682 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5683 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 5684 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5685 if (r)
5686 return r;
5687
bcc1c2a1
AD		 5688 r = evergreen_cp_load_microcode(rdev);
5689 if (r)
5690 return r;
fe251e2f 5691 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5692 if (r)
5693 return r;
5694 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5695 if (r)
5696 return r;
fe251e2f 5697
d78d6f39 5698 evergreen_uvd_resume(rdev);
f2ba57b5 5699
2898c348
CK		 5700 r = radeon_ib_pool_init(rdev);
5701 if (r) {
5702 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5703 return r;
2898c348 5704 }
b15ba512 5705
bfc1f97d 5706 r = radeon_audio_init(rdev);
69d2ae57
RM		 5707 if (r) {
5708 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5709 return r;
5710 }
5711
bcc1c2a1
AD		 5712 return 0;
5713}
5714
5715int evergreen_resume(struct radeon_device *rdev)
5716{
5717 int r;
5718
86f5c9ed
AD		 5719 /* reset the asic, the gfx blocks are often in a bad state
5720 * after the driver is unloaded or after a resume
5721 */
5722 if (radeon_asic_reset(rdev))
5723 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5724 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5725 * posting will perform necessary task to bring back GPU into good
5726 * shape.
5727 */
5728 /* post card */
5729 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5730
d4788db3
AD		 5731 /* init golden registers */
5732 evergreen_init_golden_registers(rdev);
5733
bc6a6295
AD		 5734 if (rdev->pm.pm_method == PM_METHOD_DPM)
5735 radeon_pm_resume(rdev);
6c7bccea 5736
b15ba512 5737 rdev->accel_working = true;
bcc1c2a1
AD		 5738 r = evergreen_startup(rdev);
5739 if (r) {
755d819e 5740 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5741 rdev->accel_working = false;
bcc1c2a1
AD		 5742 return r;
5743 }
fe251e2f 5744
bcc1c2a1
AD		 5745 return r;
5746
5747}
5748
5749int evergreen_suspend(struct radeon_device *rdev)
5750{
6c7bccea 5751 radeon_pm_suspend(rdev);
7991d665 5752 radeon_audio_fini(rdev);
d78d6f39
JG		 5753 if (rdev->has_uvd) {
5754 uvd_v1_0_fini(rdev);
5755 radeon_uvd_suspend(rdev);
5756 }
bcc1c2a1 5757 r700_cp_stop(rdev);
233d1ad5 5758 r600_dma_stop(rdev);
45f9a39b 5759 evergreen_irq_suspend(rdev);
724c80e1 5760 radeon_wb_disable(rdev);
bcc1c2a1 5761 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5762
5763 return 0;
5764}
5765
bcc1c2a1
AD		 5766/* Plan is to move initialization in that function and use
5767 * helper function so that radeon_device_init pretty much
5768 * do nothing more than calling asic specific function. This
5769 * should also allow to remove a bunch of callback function
5770 * like vram_info.
5771 */
5772int evergreen_init(struct radeon_device *rdev)
5773{
5774 int r;
5775
bcc1c2a1
AD		 5776 /* Read BIOS */
5777 if (!radeon_get_bios(rdev)) {
5778 if (ASIC_IS_AVIVO(rdev))
5779 return -EINVAL;
5780 }
5781 /* Must be an ATOMBIOS */
5782 if (!rdev->is_atom_bios) {
755d819e 5783 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5784 return -EINVAL;
5785 }
5786 r = radeon_atombios_init(rdev);
5787 if (r)
5788 return r;
86f5c9ed
AD		 5789 /* reset the asic, the gfx blocks are often in a bad state
5790 * after the driver is unloaded or after a resume
5791 */
5792 if (radeon_asic_reset(rdev))
5793 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5794 /* Post card if necessary */
fd909c37 5795 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5796 if (!rdev->bios) {
5797 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5798 return -EINVAL;
5799 }
5800 DRM_INFO("GPU not posted. posting now...\n");
5801 atom_asic_init(rdev->mode_info.atom_context);
5802 }
d4788db3
AD		 5803 /* init golden registers */
5804 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5805 /* Initialize scratch registers */
5806 r600_scratch_init(rdev);
5807 /* Initialize surface registers */
5808 radeon_surface_init(rdev);
5809 /* Initialize clocks */
5810 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5811 /* Fence driver */
5812 r = radeon_fence_driver_init(rdev);
5813 if (r)
5814 return r;
d594e46a
JG		 5815 /* initialize AGP */
5816 if (rdev->flags & RADEON_IS_AGP) {
5817 r = radeon_agp_init(rdev);
5818 if (r)
5819 radeon_agp_disable(rdev);
5820 }
5821 /* initialize memory controller */
bcc1c2a1
AD		 5822 r = evergreen_mc_init(rdev);
5823 if (r)
5824 return r;
5825 /* Memory manager */
5826 r = radeon_bo_init(rdev);
5827 if (r)
5828 return r;
45f9a39b 5829
01ac8794
AD		 5830 if (ASIC_IS_DCE5(rdev)) {
5831 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5832 r = ni_init_microcode(rdev);
5833 if (r) {
5834 DRM_ERROR("Failed to load firmware!\n");
5835 return r;
5836 }
5837 }
5838 } else {
5839 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5840 r = r600_init_microcode(rdev);
5841 if (r) {
5842 DRM_ERROR("Failed to load firmware!\n");
5843 return r;
5844 }
5845 }
5846 }
5847
6c7bccea
AD		 5848 /* Initialize power management */
5849 radeon_pm_init(rdev);
5850
e32eb50d
CK		 5851 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5852 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5853
233d1ad5
AD		 5854 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5855 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5856
d78d6f39 5857 evergreen_uvd_init(rdev);
f2ba57b5 5858
bcc1c2a1
AD		 5859 rdev->ih.ring_obj = NULL;
5860 r600_ih_ring_init(rdev, 64 * 1024);
5861
5862 r = r600_pcie_gart_init(rdev);
5863 if (r)
5864 return r;
0fcdb61e 5865
148a03bc 5866 rdev->accel_working = true;
bcc1c2a1
AD		 5867 r = evergreen_startup(rdev);
5868 if (r) {
fe251e2f
AD		 5869 dev_err(rdev->dev, "disabling GPU acceleration\n");
5870 r700_cp_fini(rdev);
233d1ad5 5871 r600_dma_fini(rdev);
fe251e2f 5872 r600_irq_fini(rdev);
2948f5e6
AD		 5873 if (rdev->flags & RADEON_IS_IGP)
5874 sumo_rlc_fini(rdev);
724c80e1 5875 radeon_wb_fini(rdev);
2898c348 5876 radeon_ib_pool_fini(rdev);
fe251e2f 5877 radeon_irq_kms_fini(rdev);
0fcdb61e 5878 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5879 rdev->accel_working = false;
5880 }
77e00f2e
AD		 5881
5882 /* Don't start up if the MC ucode is missing on BTC parts.
5883 * The default clocks and voltages before the MC ucode
5884 * is loaded are not suffient for advanced operations.
5885 */
5886 if (ASIC_IS_DCE5(rdev)) {
5887 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5888 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5889 return -EINVAL;
5890 }
5891 }
5892
bcc1c2a1
AD		 5893 return 0;
5894}
5895
5896void evergreen_fini(struct radeon_device *rdev)
5897{
6c7bccea 5898 radeon_pm_fini(rdev);
7991d665 5899 radeon_audio_fini(rdev);
45f9a39b 5900 r700_cp_fini(rdev);
233d1ad5 5901 r600_dma_fini(rdev);
bcc1c2a1 5902 r600_irq_fini(rdev);
2948f5e6
AD		 5903 if (rdev->flags & RADEON_IS_IGP)
5904 sumo_rlc_fini(rdev);
724c80e1 5905 radeon_wb_fini(rdev);
2898c348 5906 radeon_ib_pool_fini(rdev);
bcc1c2a1 5907 radeon_irq_kms_fini(rdev);
e409b128 5908 uvd_v1_0_fini(rdev);
f2ba57b5 5909 radeon_uvd_fini(rdev);
d9654413 5910 evergreen_pcie_gart_fini(rdev);
16cdf04d 5911 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5912 radeon_gem_fini(rdev);
5913 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5914 radeon_agp_fini(rdev);
5915 radeon_bo_fini(rdev);
5916 radeon_atombios_fini(rdev);
5917 kfree(rdev->bios);
5918 rdev->bios = NULL;
bcc1c2a1 5919}
9e46a48d 5920
b07759bf 5921void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5922{
7e0e4196 5923 u32 link_width_cntl, speed_cntl;
9e46a48d 5924
d42dd579
AD		 5925 if (radeon_pcie_gen2 == 0)
5926 return;
5927
9e46a48d
AD		 5928 if (rdev->flags & RADEON_IS_IGP)
5929 return;
5930
5931 if (!(rdev->flags & RADEON_IS_PCIE))
5932 return;
5933
5934 /* x2 cards have a special sequence */
5935 if (ASIC_IS_X2(rdev))
5936 return;
5937
7e0e4196
KSS		 5938 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5939 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5940 return;
5941
492d2b61 5942 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5943 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5944 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5945 return;
5946 }
5947
197bbb3d
DA		 5948 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5949
9e46a48d
AD		 5950 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5951 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5952
492d2b61 5953 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5954 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5955 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5956
492d2b61 5957 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5958 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5959 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5960
492d2b61 5961 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5962 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5963 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5964
492d2b61 5965 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5966 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5967 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5968
492d2b61 5969 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5970 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5971 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5972
5973 } else {
492d2b61 5974 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5975 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5976 if (1)
5977 link_width_cntl |= LC_UPCONFIGURE_DIS;
5978 else
5979 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5980 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5981 }
5982}
f52382d7
AD		 5983
5984void evergreen_program_aspm(struct radeon_device *rdev)
5985{
5986 u32 data, orig;
5987 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5988 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5989 /* fusion_platform = true
5990 * if the system is a fusion system
5991 * (APU or DGPU in a fusion system).
5992 * todo: check if the system is a fusion platform.
5993 */
5994 bool fusion_platform = false;
5995
1294d4a3
AD		 5996 if (radeon_aspm == 0)
5997 return;
5998
f52382d7
AD		 5999 if (!(rdev->flags & RADEON_IS_PCIE))
6000 return;
6001
6002 switch (rdev->family) {
6003 case CHIP_CYPRESS:
6004 case CHIP_HEMLOCK:
6005 case CHIP_JUNIPER:
6006 case CHIP_REDWOOD:
6007 case CHIP_CEDAR:
6008 case CHIP_SUMO:
6009 case CHIP_SUMO2:
6010 case CHIP_PALM:
6011 case CHIP_ARUBA:
6012 disable_l0s = true;
6013 break;
6014 default:
6015 disable_l0s = false;
6016 break;
6017 }
6018
6019 if (rdev->flags & RADEON_IS_IGP)
6020 fusion_platform = true; /* XXX also dGPUs in a fusion system */
6021
6022 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
6023 if (fusion_platform)
6024 data &= ~MULTI_PIF;
6025 else
6026 data |= MULTI_PIF;
6027 if (data != orig)
6028 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
6029
6030 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
6031 if (fusion_platform)
6032 data &= ~MULTI_PIF;
6033 else
6034 data |= MULTI_PIF;
6035 if (data != orig)
6036 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
6037
6038 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6039 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6040 if (!disable_l0s) {
6041 if (rdev->family >= CHIP_BARTS)
6042 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
6043 else
6044 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
6045 }
6046
6047 if (!disable_l1) {
6048 if (rdev->family >= CHIP_BARTS)
6049 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
6050 else
6051 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
6052
6053 if (!disable_plloff_in_l1) {
6054 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6055 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6056 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6057 if (data != orig)
6058 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6059
6060 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6061 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6062 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6063 if (data != orig)
6064 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6065
6066 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6067 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6068 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6069 if (data != orig)
6070 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6071
6072 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6073 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6074 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6075 if (data != orig)
6076 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6077
6078 if (rdev->family >= CHIP_BARTS) {
6079 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6080 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6081 data |= PLL_RAMP_UP_TIME_0(4);
6082 if (data != orig)
6083 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6084
6085 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6086 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6087 data |= PLL_RAMP_UP_TIME_1(4);
6088 if (data != orig)
6089 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6090
6091 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6092 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6093 data |= PLL_RAMP_UP_TIME_0(4);
6094 if (data != orig)
6095 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6096
6097 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6098 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6099 data |= PLL_RAMP_UP_TIME_1(4);
6100 if (data != orig)
6101 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6102 }
6103
6104 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6105 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6106 data |= LC_DYN_LANES_PWR_STATE(3);
6107 if (data != orig)
6108 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6109
6110 if (rdev->family >= CHIP_BARTS) {
6111 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6112 data &= ~LS2_EXIT_TIME_MASK;
6113 data |= LS2_EXIT_TIME(1);
6114 if (data != orig)
6115 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6116
6117 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6118 data &= ~LS2_EXIT_TIME_MASK;
6119 data |= LS2_EXIT_TIME(1);
6120 if (data != orig)
6121 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6122 }
6123 }
6124 }
6125
6126 /* evergreen parts only */
6127 if (rdev->family < CHIP_BARTS)
6128 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
6129
6130 if (pcie_lc_cntl != pcie_lc_cntl_old)
6131 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
6132}
Linux kernel - Deliverables
This page took 0.75899 seconds and 5 git commands to generate.