projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/ttm: fix kref count mess in ttm_bo_move_to_lru_tail
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
5a0e3ad6 25#include <linux/slab.h>
760285e7 26#include <drm/drmP.h>
bcc1c2a1 27#include "radeon.h"
e6990375 28#include "radeon_asic.h"
bfc1f97d 29#include "radeon_audio.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
9e5acbc2
DV		 38/*
39 * Indirect registers accessor
40 */
41u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
42{
43 unsigned long flags;
44 u32 r;
45
46 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
47 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
48 r = RREG32(EVERGREEN_CG_IND_DATA);
49 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
50 return r;
51}
52
53void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
54{
55 unsigned long flags;
56
57 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
58 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
59 WREG32(EVERGREEN_CG_IND_DATA, (v));
60 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
61}
62
63u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
64{
65 unsigned long flags;
66 u32 r;
67
68 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
69 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
70 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
71 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
72 return r;
73}
74
75void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
76{
77 unsigned long flags;
78
79 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
80 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
81 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
82 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
83}
84
85u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
86{
87 unsigned long flags;
88 u32 r;
89
90 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
91 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
92 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
93 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
94 return r;
95}
96
97void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
98{
99 unsigned long flags;
100
101 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
102 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
103 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
104 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
105}
106
4a15903d
AD		 107static const u32 crtc_offsets[6] =
108{
109 EVERGREEN_CRTC0_REGISTER_OFFSET,
110 EVERGREEN_CRTC1_REGISTER_OFFSET,
111 EVERGREEN_CRTC2_REGISTER_OFFSET,
112 EVERGREEN_CRTC3_REGISTER_OFFSET,
113 EVERGREEN_CRTC4_REGISTER_OFFSET,
114 EVERGREEN_CRTC5_REGISTER_OFFSET
115};
116
2948f5e6
AD		 117#include "clearstate_evergreen.h"
118
1fd11777 119static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 120{
121 0x98fc,
122 0x9830,
123 0x9834,
124 0x9838,
125 0x9870,
126 0x9874,
127 0x8a14,
128 0x8b24,
129 0x8bcc,
130 0x8b10,
131 0x8d00,
132 0x8d04,
133 0x8c00,
134 0x8c04,
135 0x8c08,
136 0x8c0c,
137 0x8d8c,
138 0x8c20,
139 0x8c24,
140 0x8c28,
141 0x8c18,
142 0x8c1c,
143 0x8cf0,
144 0x8e2c,
145 0x8e38,
146 0x8c30,
147 0x9508,
148 0x9688,
149 0x9608,
150 0x960c,
151 0x9610,
152 0x9614,
153 0x88c4,
154 0x88d4,
155 0xa008,
156 0x900c,
157 0x9100,
158 0x913c,
159 0x98f8,
160 0x98f4,
161 0x9b7c,
162 0x3f8c,
163 0x8950,
164 0x8954,
165 0x8a18,
166 0x8b28,
167 0x9144,
168 0x9148,
169 0x914c,
170 0x3f90,
171 0x3f94,
172 0x915c,
173 0x9160,
174 0x9178,
175 0x917c,
176 0x9180,
177 0x918c,
178 0x9190,
179 0x9194,
180 0x9198,
181 0x919c,
182 0x91a8,
183 0x91ac,
184 0x91b0,
185 0x91b4,
186 0x91b8,
187 0x91c4,
188 0x91c8,
189 0x91cc,
190 0x91d0,
191 0x91d4,
192 0x91e0,
193 0x91e4,
194 0x91ec,
195 0x91f0,
196 0x91f4,
197 0x9200,
198 0x9204,
199 0x929c,
200 0x9150,
201 0x802c,
202};
2948f5e6 203
bcc1c2a1
AD		 204static void evergreen_gpu_init(struct radeon_device *rdev);
205void evergreen_fini(struct radeon_device *rdev);
b07759bf 206void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 207void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 208extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
209 int ring, u32 cp_int_cntl);
54e2e49c
AD		 210extern void cayman_vm_decode_fault(struct radeon_device *rdev,
211 u32 status, u32 addr);
22c775ce 212void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 213
59a82d0e
AD		 214extern u32 si_get_csb_size(struct radeon_device *rdev);
215extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 216extern u32 cik_get_csb_size(struct radeon_device *rdev);
217extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
b5470b03 218extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
bcc1c2a1 219
d4788db3
AD		 220static const u32 evergreen_golden_registers[] =
221{
222 0x3f90, 0xffff0000, 0xff000000,
223 0x9148, 0xffff0000, 0xff000000,
224 0x3f94, 0xffff0000, 0xff000000,
225 0x914c, 0xffff0000, 0xff000000,
226 0x9b7c, 0xffffffff, 0x00000000,
227 0x8a14, 0xffffffff, 0x00000007,
228 0x8b10, 0xffffffff, 0x00000000,
229 0x960c, 0xffffffff, 0x54763210,
230 0x88c4, 0xffffffff, 0x000000c2,
231 0x88d4, 0xffffffff, 0x00000010,
232 0x8974, 0xffffffff, 0x00000000,
233 0xc78, 0x00000080, 0x00000080,
234 0x5eb4, 0xffffffff, 0x00000002,
235 0x5e78, 0xffffffff, 0x001000f0,
236 0x6104, 0x01000300, 0x00000000,
237 0x5bc0, 0x00300000, 0x00000000,
238 0x7030, 0xffffffff, 0x00000011,
239 0x7c30, 0xffffffff, 0x00000011,
240 0x10830, 0xffffffff, 0x00000011,
241 0x11430, 0xffffffff, 0x00000011,
242 0x12030, 0xffffffff, 0x00000011,
243 0x12c30, 0xffffffff, 0x00000011,
244 0xd02c, 0xffffffff, 0x08421000,
245 0x240c, 0xffffffff, 0x00000380,
246 0x8b24, 0xffffffff, 0x00ff0fff,
247 0x28a4c, 0x06000000, 0x06000000,
248 0x10c, 0x00000001, 0x00000001,
249 0x8d00, 0xffffffff, 0x100e4848,
250 0x8d04, 0xffffffff, 0x00164745,
251 0x8c00, 0xffffffff, 0xe4000003,
252 0x8c04, 0xffffffff, 0x40600060,
253 0x8c08, 0xffffffff, 0x001c001c,
254 0x8cf0, 0xffffffff, 0x08e00620,
255 0x8c20, 0xffffffff, 0x00800080,
256 0x8c24, 0xffffffff, 0x00800080,
257 0x8c18, 0xffffffff, 0x20202078,
258 0x8c1c, 0xffffffff, 0x00001010,
259 0x28350, 0xffffffff, 0x00000000,
260 0xa008, 0xffffffff, 0x00010000,
6abafb78 261 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 262 0x9508, 0xffffffff, 0x00000002,
263 0x913c, 0x0000000f, 0x0000000a
264};
265
266static const u32 evergreen_golden_registers2[] =
267{
268 0x2f4c, 0xffffffff, 0x00000000,
269 0x54f4, 0xffffffff, 0x00000000,
270 0x54f0, 0xffffffff, 0x00000000,
271 0x5498, 0xffffffff, 0x00000000,
272 0x549c, 0xffffffff, 0x00000000,
273 0x5494, 0xffffffff, 0x00000000,
274 0x53cc, 0xffffffff, 0x00000000,
275 0x53c8, 0xffffffff, 0x00000000,
276 0x53c4, 0xffffffff, 0x00000000,
277 0x53c0, 0xffffffff, 0x00000000,
278 0x53bc, 0xffffffff, 0x00000000,
279 0x53b8, 0xffffffff, 0x00000000,
280 0x53b4, 0xffffffff, 0x00000000,
281 0x53b0, 0xffffffff, 0x00000000
282};
283
284static const u32 cypress_mgcg_init[] =
285{
286 0x802c, 0xffffffff, 0xc0000000,
287 0x5448, 0xffffffff, 0x00000100,
288 0x55e4, 0xffffffff, 0x00000100,
289 0x160c, 0xffffffff, 0x00000100,
290 0x5644, 0xffffffff, 0x00000100,
291 0xc164, 0xffffffff, 0x00000100,
292 0x8a18, 0xffffffff, 0x00000100,
293 0x897c, 0xffffffff, 0x06000100,
294 0x8b28, 0xffffffff, 0x00000100,
295 0x9144, 0xffffffff, 0x00000100,
296 0x9a60, 0xffffffff, 0x00000100,
297 0x9868, 0xffffffff, 0x00000100,
298 0x8d58, 0xffffffff, 0x00000100,
299 0x9510, 0xffffffff, 0x00000100,
300 0x949c, 0xffffffff, 0x00000100,
301 0x9654, 0xffffffff, 0x00000100,
302 0x9030, 0xffffffff, 0x00000100,
303 0x9034, 0xffffffff, 0x00000100,
304 0x9038, 0xffffffff, 0x00000100,
305 0x903c, 0xffffffff, 0x00000100,
306 0x9040, 0xffffffff, 0x00000100,
307 0xa200, 0xffffffff, 0x00000100,
308 0xa204, 0xffffffff, 0x00000100,
309 0xa208, 0xffffffff, 0x00000100,
310 0xa20c, 0xffffffff, 0x00000100,
311 0x971c, 0xffffffff, 0x00000100,
312 0x977c, 0xffffffff, 0x00000100,
313 0x3f80, 0xffffffff, 0x00000100,
314 0xa210, 0xffffffff, 0x00000100,
315 0xa214, 0xffffffff, 0x00000100,
316 0x4d8, 0xffffffff, 0x00000100,
317 0x9784, 0xffffffff, 0x00000100,
318 0x9698, 0xffffffff, 0x00000100,
319 0x4d4, 0xffffffff, 0x00000200,
320 0x30cc, 0xffffffff, 0x00000100,
321 0xd0c0, 0xffffffff, 0xff000100,
322 0x802c, 0xffffffff, 0x40000000,
323 0x915c, 0xffffffff, 0x00010000,
324 0x9160, 0xffffffff, 0x00030002,
325 0x9178, 0xffffffff, 0x00070000,
326 0x917c, 0xffffffff, 0x00030002,
327 0x9180, 0xffffffff, 0x00050004,
328 0x918c, 0xffffffff, 0x00010006,
329 0x9190, 0xffffffff, 0x00090008,
330 0x9194, 0xffffffff, 0x00070000,
331 0x9198, 0xffffffff, 0x00030002,
332 0x919c, 0xffffffff, 0x00050004,
333 0x91a8, 0xffffffff, 0x00010006,
334 0x91ac, 0xffffffff, 0x00090008,
335 0x91b0, 0xffffffff, 0x00070000,
336 0x91b4, 0xffffffff, 0x00030002,
337 0x91b8, 0xffffffff, 0x00050004,
338 0x91c4, 0xffffffff, 0x00010006,
339 0x91c8, 0xffffffff, 0x00090008,
340 0x91cc, 0xffffffff, 0x00070000,
341 0x91d0, 0xffffffff, 0x00030002,
342 0x91d4, 0xffffffff, 0x00050004,
343 0x91e0, 0xffffffff, 0x00010006,
344 0x91e4, 0xffffffff, 0x00090008,
345 0x91e8, 0xffffffff, 0x00000000,
346 0x91ec, 0xffffffff, 0x00070000,
347 0x91f0, 0xffffffff, 0x00030002,
348 0x91f4, 0xffffffff, 0x00050004,
349 0x9200, 0xffffffff, 0x00010006,
350 0x9204, 0xffffffff, 0x00090008,
351 0x9208, 0xffffffff, 0x00070000,
352 0x920c, 0xffffffff, 0x00030002,
353 0x9210, 0xffffffff, 0x00050004,
354 0x921c, 0xffffffff, 0x00010006,
355 0x9220, 0xffffffff, 0x00090008,
356 0x9224, 0xffffffff, 0x00070000,
357 0x9228, 0xffffffff, 0x00030002,
358 0x922c, 0xffffffff, 0x00050004,
359 0x9238, 0xffffffff, 0x00010006,
360 0x923c, 0xffffffff, 0x00090008,
361 0x9240, 0xffffffff, 0x00070000,
362 0x9244, 0xffffffff, 0x00030002,
363 0x9248, 0xffffffff, 0x00050004,
364 0x9254, 0xffffffff, 0x00010006,
365 0x9258, 0xffffffff, 0x00090008,
366 0x925c, 0xffffffff, 0x00070000,
367 0x9260, 0xffffffff, 0x00030002,
368 0x9264, 0xffffffff, 0x00050004,
369 0x9270, 0xffffffff, 0x00010006,
370 0x9274, 0xffffffff, 0x00090008,
371 0x9278, 0xffffffff, 0x00070000,
372 0x927c, 0xffffffff, 0x00030002,
373 0x9280, 0xffffffff, 0x00050004,
374 0x928c, 0xffffffff, 0x00010006,
375 0x9290, 0xffffffff, 0x00090008,
376 0x9294, 0xffffffff, 0x00000000,
377 0x929c, 0xffffffff, 0x00000001,
378 0x802c, 0xffffffff, 0x40010000,
379 0x915c, 0xffffffff, 0x00010000,
380 0x9160, 0xffffffff, 0x00030002,
381 0x9178, 0xffffffff, 0x00070000,
382 0x917c, 0xffffffff, 0x00030002,
383 0x9180, 0xffffffff, 0x00050004,
384 0x918c, 0xffffffff, 0x00010006,
385 0x9190, 0xffffffff, 0x00090008,
386 0x9194, 0xffffffff, 0x00070000,
387 0x9198, 0xffffffff, 0x00030002,
388 0x919c, 0xffffffff, 0x00050004,
389 0x91a8, 0xffffffff, 0x00010006,
390 0x91ac, 0xffffffff, 0x00090008,
391 0x91b0, 0xffffffff, 0x00070000,
392 0x91b4, 0xffffffff, 0x00030002,
393 0x91b8, 0xffffffff, 0x00050004,
394 0x91c4, 0xffffffff, 0x00010006,
395 0x91c8, 0xffffffff, 0x00090008,
396 0x91cc, 0xffffffff, 0x00070000,
397 0x91d0, 0xffffffff, 0x00030002,
398 0x91d4, 0xffffffff, 0x00050004,
399 0x91e0, 0xffffffff, 0x00010006,
400 0x91e4, 0xffffffff, 0x00090008,
401 0x91e8, 0xffffffff, 0x00000000,
402 0x91ec, 0xffffffff, 0x00070000,
403 0x91f0, 0xffffffff, 0x00030002,
404 0x91f4, 0xffffffff, 0x00050004,
405 0x9200, 0xffffffff, 0x00010006,
406 0x9204, 0xffffffff, 0x00090008,
407 0x9208, 0xffffffff, 0x00070000,
408 0x920c, 0xffffffff, 0x00030002,
409 0x9210, 0xffffffff, 0x00050004,
410 0x921c, 0xffffffff, 0x00010006,
411 0x9220, 0xffffffff, 0x00090008,
412 0x9224, 0xffffffff, 0x00070000,
413 0x9228, 0xffffffff, 0x00030002,
414 0x922c, 0xffffffff, 0x00050004,
415 0x9238, 0xffffffff, 0x00010006,
416 0x923c, 0xffffffff, 0x00090008,
417 0x9240, 0xffffffff, 0x00070000,
418 0x9244, 0xffffffff, 0x00030002,
419 0x9248, 0xffffffff, 0x00050004,
420 0x9254, 0xffffffff, 0x00010006,
421 0x9258, 0xffffffff, 0x00090008,
422 0x925c, 0xffffffff, 0x00070000,
423 0x9260, 0xffffffff, 0x00030002,
424 0x9264, 0xffffffff, 0x00050004,
425 0x9270, 0xffffffff, 0x00010006,
426 0x9274, 0xffffffff, 0x00090008,
427 0x9278, 0xffffffff, 0x00070000,
428 0x927c, 0xffffffff, 0x00030002,
429 0x9280, 0xffffffff, 0x00050004,
430 0x928c, 0xffffffff, 0x00010006,
431 0x9290, 0xffffffff, 0x00090008,
432 0x9294, 0xffffffff, 0x00000000,
433 0x929c, 0xffffffff, 0x00000001,
434 0x802c, 0xffffffff, 0xc0000000
435};
436
437static const u32 redwood_mgcg_init[] =
438{
439 0x802c, 0xffffffff, 0xc0000000,
440 0x5448, 0xffffffff, 0x00000100,
441 0x55e4, 0xffffffff, 0x00000100,
442 0x160c, 0xffffffff, 0x00000100,
443 0x5644, 0xffffffff, 0x00000100,
444 0xc164, 0xffffffff, 0x00000100,
445 0x8a18, 0xffffffff, 0x00000100,
446 0x897c, 0xffffffff, 0x06000100,
447 0x8b28, 0xffffffff, 0x00000100,
448 0x9144, 0xffffffff, 0x00000100,
449 0x9a60, 0xffffffff, 0x00000100,
450 0x9868, 0xffffffff, 0x00000100,
451 0x8d58, 0xffffffff, 0x00000100,
452 0x9510, 0xffffffff, 0x00000100,
453 0x949c, 0xffffffff, 0x00000100,
454 0x9654, 0xffffffff, 0x00000100,
455 0x9030, 0xffffffff, 0x00000100,
456 0x9034, 0xffffffff, 0x00000100,
457 0x9038, 0xffffffff, 0x00000100,
458 0x903c, 0xffffffff, 0x00000100,
459 0x9040, 0xffffffff, 0x00000100,
460 0xa200, 0xffffffff, 0x00000100,
461 0xa204, 0xffffffff, 0x00000100,
462 0xa208, 0xffffffff, 0x00000100,
463 0xa20c, 0xffffffff, 0x00000100,
464 0x971c, 0xffffffff, 0x00000100,
465 0x977c, 0xffffffff, 0x00000100,
466 0x3f80, 0xffffffff, 0x00000100,
467 0xa210, 0xffffffff, 0x00000100,
468 0xa214, 0xffffffff, 0x00000100,
469 0x4d8, 0xffffffff, 0x00000100,
470 0x9784, 0xffffffff, 0x00000100,
471 0x9698, 0xffffffff, 0x00000100,
472 0x4d4, 0xffffffff, 0x00000200,
473 0x30cc, 0xffffffff, 0x00000100,
474 0xd0c0, 0xffffffff, 0xff000100,
475 0x802c, 0xffffffff, 0x40000000,
476 0x915c, 0xffffffff, 0x00010000,
477 0x9160, 0xffffffff, 0x00030002,
478 0x9178, 0xffffffff, 0x00070000,
479 0x917c, 0xffffffff, 0x00030002,
480 0x9180, 0xffffffff, 0x00050004,
481 0x918c, 0xffffffff, 0x00010006,
482 0x9190, 0xffffffff, 0x00090008,
483 0x9194, 0xffffffff, 0x00070000,
484 0x9198, 0xffffffff, 0x00030002,
485 0x919c, 0xffffffff, 0x00050004,
486 0x91a8, 0xffffffff, 0x00010006,
487 0x91ac, 0xffffffff, 0x00090008,
488 0x91b0, 0xffffffff, 0x00070000,
489 0x91b4, 0xffffffff, 0x00030002,
490 0x91b8, 0xffffffff, 0x00050004,
491 0x91c4, 0xffffffff, 0x00010006,
492 0x91c8, 0xffffffff, 0x00090008,
493 0x91cc, 0xffffffff, 0x00070000,
494 0x91d0, 0xffffffff, 0x00030002,
495 0x91d4, 0xffffffff, 0x00050004,
496 0x91e0, 0xffffffff, 0x00010006,
497 0x91e4, 0xffffffff, 0x00090008,
498 0x91e8, 0xffffffff, 0x00000000,
499 0x91ec, 0xffffffff, 0x00070000,
500 0x91f0, 0xffffffff, 0x00030002,
501 0x91f4, 0xffffffff, 0x00050004,
502 0x9200, 0xffffffff, 0x00010006,
503 0x9204, 0xffffffff, 0x00090008,
504 0x9294, 0xffffffff, 0x00000000,
505 0x929c, 0xffffffff, 0x00000001,
506 0x802c, 0xffffffff, 0xc0000000
507};
508
509static const u32 cedar_golden_registers[] =
510{
511 0x3f90, 0xffff0000, 0xff000000,
512 0x9148, 0xffff0000, 0xff000000,
513 0x3f94, 0xffff0000, 0xff000000,
514 0x914c, 0xffff0000, 0xff000000,
515 0x9b7c, 0xffffffff, 0x00000000,
516 0x8a14, 0xffffffff, 0x00000007,
517 0x8b10, 0xffffffff, 0x00000000,
518 0x960c, 0xffffffff, 0x54763210,
519 0x88c4, 0xffffffff, 0x000000c2,
520 0x88d4, 0xffffffff, 0x00000000,
521 0x8974, 0xffffffff, 0x00000000,
522 0xc78, 0x00000080, 0x00000080,
523 0x5eb4, 0xffffffff, 0x00000002,
524 0x5e78, 0xffffffff, 0x001000f0,
525 0x6104, 0x01000300, 0x00000000,
526 0x5bc0, 0x00300000, 0x00000000,
527 0x7030, 0xffffffff, 0x00000011,
528 0x7c30, 0xffffffff, 0x00000011,
529 0x10830, 0xffffffff, 0x00000011,
530 0x11430, 0xffffffff, 0x00000011,
531 0xd02c, 0xffffffff, 0x08421000,
532 0x240c, 0xffffffff, 0x00000380,
533 0x8b24, 0xffffffff, 0x00ff0fff,
534 0x28a4c, 0x06000000, 0x06000000,
535 0x10c, 0x00000001, 0x00000001,
536 0x8d00, 0xffffffff, 0x100e4848,
537 0x8d04, 0xffffffff, 0x00164745,
538 0x8c00, 0xffffffff, 0xe4000003,
539 0x8c04, 0xffffffff, 0x40600060,
540 0x8c08, 0xffffffff, 0x001c001c,
541 0x8cf0, 0xffffffff, 0x08e00410,
542 0x8c20, 0xffffffff, 0x00800080,
543 0x8c24, 0xffffffff, 0x00800080,
544 0x8c18, 0xffffffff, 0x20202078,
545 0x8c1c, 0xffffffff, 0x00001010,
546 0x28350, 0xffffffff, 0x00000000,
547 0xa008, 0xffffffff, 0x00010000,
6abafb78 548 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 549 0x9508, 0xffffffff, 0x00000002
550};
551
552static const u32 cedar_mgcg_init[] =
553{
554 0x802c, 0xffffffff, 0xc0000000,
555 0x5448, 0xffffffff, 0x00000100,
556 0x55e4, 0xffffffff, 0x00000100,
557 0x160c, 0xffffffff, 0x00000100,
558 0x5644, 0xffffffff, 0x00000100,
559 0xc164, 0xffffffff, 0x00000100,
560 0x8a18, 0xffffffff, 0x00000100,
561 0x897c, 0xffffffff, 0x06000100,
562 0x8b28, 0xffffffff, 0x00000100,
563 0x9144, 0xffffffff, 0x00000100,
564 0x9a60, 0xffffffff, 0x00000100,
565 0x9868, 0xffffffff, 0x00000100,
566 0x8d58, 0xffffffff, 0x00000100,
567 0x9510, 0xffffffff, 0x00000100,
568 0x949c, 0xffffffff, 0x00000100,
569 0x9654, 0xffffffff, 0x00000100,
570 0x9030, 0xffffffff, 0x00000100,
571 0x9034, 0xffffffff, 0x00000100,
572 0x9038, 0xffffffff, 0x00000100,
573 0x903c, 0xffffffff, 0x00000100,
574 0x9040, 0xffffffff, 0x00000100,
575 0xa200, 0xffffffff, 0x00000100,
576 0xa204, 0xffffffff, 0x00000100,
577 0xa208, 0xffffffff, 0x00000100,
578 0xa20c, 0xffffffff, 0x00000100,
579 0x971c, 0xffffffff, 0x00000100,
580 0x977c, 0xffffffff, 0x00000100,
581 0x3f80, 0xffffffff, 0x00000100,
582 0xa210, 0xffffffff, 0x00000100,
583 0xa214, 0xffffffff, 0x00000100,
584 0x4d8, 0xffffffff, 0x00000100,
585 0x9784, 0xffffffff, 0x00000100,
586 0x9698, 0xffffffff, 0x00000100,
587 0x4d4, 0xffffffff, 0x00000200,
588 0x30cc, 0xffffffff, 0x00000100,
589 0xd0c0, 0xffffffff, 0xff000100,
590 0x802c, 0xffffffff, 0x40000000,
591 0x915c, 0xffffffff, 0x00010000,
592 0x9178, 0xffffffff, 0x00050000,
593 0x917c, 0xffffffff, 0x00030002,
594 0x918c, 0xffffffff, 0x00010004,
595 0x9190, 0xffffffff, 0x00070006,
596 0x9194, 0xffffffff, 0x00050000,
597 0x9198, 0xffffffff, 0x00030002,
598 0x91a8, 0xffffffff, 0x00010004,
599 0x91ac, 0xffffffff, 0x00070006,
600 0x91e8, 0xffffffff, 0x00000000,
601 0x9294, 0xffffffff, 0x00000000,
602 0x929c, 0xffffffff, 0x00000001,
603 0x802c, 0xffffffff, 0xc0000000
604};
605
606static const u32 juniper_mgcg_init[] =
607{
608 0x802c, 0xffffffff, 0xc0000000,
609 0x5448, 0xffffffff, 0x00000100,
610 0x55e4, 0xffffffff, 0x00000100,
611 0x160c, 0xffffffff, 0x00000100,
612 0x5644, 0xffffffff, 0x00000100,
613 0xc164, 0xffffffff, 0x00000100,
614 0x8a18, 0xffffffff, 0x00000100,
615 0x897c, 0xffffffff, 0x06000100,
616 0x8b28, 0xffffffff, 0x00000100,
617 0x9144, 0xffffffff, 0x00000100,
618 0x9a60, 0xffffffff, 0x00000100,
619 0x9868, 0xffffffff, 0x00000100,
620 0x8d58, 0xffffffff, 0x00000100,
621 0x9510, 0xffffffff, 0x00000100,
622 0x949c, 0xffffffff, 0x00000100,
623 0x9654, 0xffffffff, 0x00000100,
624 0x9030, 0xffffffff, 0x00000100,
625 0x9034, 0xffffffff, 0x00000100,
626 0x9038, 0xffffffff, 0x00000100,
627 0x903c, 0xffffffff, 0x00000100,
628 0x9040, 0xffffffff, 0x00000100,
629 0xa200, 0xffffffff, 0x00000100,
630 0xa204, 0xffffffff, 0x00000100,
631 0xa208, 0xffffffff, 0x00000100,
632 0xa20c, 0xffffffff, 0x00000100,
633 0x971c, 0xffffffff, 0x00000100,
634 0xd0c0, 0xffffffff, 0xff000100,
635 0x802c, 0xffffffff, 0x40000000,
636 0x915c, 0xffffffff, 0x00010000,
637 0x9160, 0xffffffff, 0x00030002,
638 0x9178, 0xffffffff, 0x00070000,
639 0x917c, 0xffffffff, 0x00030002,
640 0x9180, 0xffffffff, 0x00050004,
641 0x918c, 0xffffffff, 0x00010006,
642 0x9190, 0xffffffff, 0x00090008,
643 0x9194, 0xffffffff, 0x00070000,
644 0x9198, 0xffffffff, 0x00030002,
645 0x919c, 0xffffffff, 0x00050004,
646 0x91a8, 0xffffffff, 0x00010006,
647 0x91ac, 0xffffffff, 0x00090008,
648 0x91b0, 0xffffffff, 0x00070000,
649 0x91b4, 0xffffffff, 0x00030002,
650 0x91b8, 0xffffffff, 0x00050004,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91c8, 0xffffffff, 0x00090008,
653 0x91cc, 0xffffffff, 0x00070000,
654 0x91d0, 0xffffffff, 0x00030002,
655 0x91d4, 0xffffffff, 0x00050004,
656 0x91e0, 0xffffffff, 0x00010006,
657 0x91e4, 0xffffffff, 0x00090008,
658 0x91e8, 0xffffffff, 0x00000000,
659 0x91ec, 0xffffffff, 0x00070000,
660 0x91f0, 0xffffffff, 0x00030002,
661 0x91f4, 0xffffffff, 0x00050004,
662 0x9200, 0xffffffff, 0x00010006,
663 0x9204, 0xffffffff, 0x00090008,
664 0x9208, 0xffffffff, 0x00070000,
665 0x920c, 0xffffffff, 0x00030002,
666 0x9210, 0xffffffff, 0x00050004,
667 0x921c, 0xffffffff, 0x00010006,
668 0x9220, 0xffffffff, 0x00090008,
669 0x9224, 0xffffffff, 0x00070000,
670 0x9228, 0xffffffff, 0x00030002,
671 0x922c, 0xffffffff, 0x00050004,
672 0x9238, 0xffffffff, 0x00010006,
673 0x923c, 0xffffffff, 0x00090008,
674 0x9240, 0xffffffff, 0x00070000,
675 0x9244, 0xffffffff, 0x00030002,
676 0x9248, 0xffffffff, 0x00050004,
677 0x9254, 0xffffffff, 0x00010006,
678 0x9258, 0xffffffff, 0x00090008,
679 0x925c, 0xffffffff, 0x00070000,
680 0x9260, 0xffffffff, 0x00030002,
681 0x9264, 0xffffffff, 0x00050004,
682 0x9270, 0xffffffff, 0x00010006,
683 0x9274, 0xffffffff, 0x00090008,
684 0x9278, 0xffffffff, 0x00070000,
685 0x927c, 0xffffffff, 0x00030002,
686 0x9280, 0xffffffff, 0x00050004,
687 0x928c, 0xffffffff, 0x00010006,
688 0x9290, 0xffffffff, 0x00090008,
689 0x9294, 0xffffffff, 0x00000000,
690 0x929c, 0xffffffff, 0x00000001,
691 0x802c, 0xffffffff, 0xc0000000,
692 0x977c, 0xffffffff, 0x00000100,
693 0x3f80, 0xffffffff, 0x00000100,
694 0xa210, 0xffffffff, 0x00000100,
695 0xa214, 0xffffffff, 0x00000100,
696 0x4d8, 0xffffffff, 0x00000100,
697 0x9784, 0xffffffff, 0x00000100,
698 0x9698, 0xffffffff, 0x00000100,
699 0x4d4, 0xffffffff, 0x00000200,
700 0x30cc, 0xffffffff, 0x00000100,
701 0x802c, 0xffffffff, 0xc0000000
702};
703
704static const u32 supersumo_golden_registers[] =
705{
706 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 707 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 708 0x7030, 0xffffffff, 0x00000011,
709 0x7c30, 0xffffffff, 0x00000011,
710 0x6104, 0x01000300, 0x00000000,
711 0x5bc0, 0x00300000, 0x00000000,
712 0x8c04, 0xffffffff, 0x40600060,
713 0x8c08, 0xffffffff, 0x001c001c,
714 0x8c20, 0xffffffff, 0x00800080,
715 0x8c24, 0xffffffff, 0x00800080,
716 0x8c18, 0xffffffff, 0x20202078,
717 0x8c1c, 0xffffffff, 0x00001010,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x91c4, 0xffffffff, 0x00010006,
721 0x91e0, 0xffffffff, 0x00010006,
722 0x9200, 0xffffffff, 0x00010006,
723 0x9150, 0xffffffff, 0x6e944040,
724 0x917c, 0xffffffff, 0x00030002,
725 0x9180, 0xffffffff, 0x00050004,
726 0x9198, 0xffffffff, 0x00030002,
727 0x919c, 0xffffffff, 0x00050004,
728 0x91b4, 0xffffffff, 0x00030002,
729 0x91b8, 0xffffffff, 0x00050004,
730 0x91d0, 0xffffffff, 0x00030002,
731 0x91d4, 0xffffffff, 0x00050004,
732 0x91f0, 0xffffffff, 0x00030002,
733 0x91f4, 0xffffffff, 0x00050004,
734 0x915c, 0xffffffff, 0x00010000,
735 0x9160, 0xffffffff, 0x00030002,
736 0x3f90, 0xffff0000, 0xff000000,
737 0x9178, 0xffffffff, 0x00070000,
738 0x9194, 0xffffffff, 0x00070000,
739 0x91b0, 0xffffffff, 0x00070000,
740 0x91cc, 0xffffffff, 0x00070000,
741 0x91ec, 0xffffffff, 0x00070000,
742 0x9148, 0xffff0000, 0xff000000,
743 0x9190, 0xffffffff, 0x00090008,
744 0x91ac, 0xffffffff, 0x00090008,
745 0x91c8, 0xffffffff, 0x00090008,
746 0x91e4, 0xffffffff, 0x00090008,
747 0x9204, 0xffffffff, 0x00090008,
748 0x3f94, 0xffff0000, 0xff000000,
749 0x914c, 0xffff0000, 0xff000000,
750 0x929c, 0xffffffff, 0x00000001,
751 0x8a18, 0xffffffff, 0x00000100,
752 0x8b28, 0xffffffff, 0x00000100,
753 0x9144, 0xffffffff, 0x00000100,
754 0x5644, 0xffffffff, 0x00000100,
755 0x9b7c, 0xffffffff, 0x00000000,
756 0x8030, 0xffffffff, 0x0000100a,
757 0x8a14, 0xffffffff, 0x00000007,
758 0x8b24, 0xffffffff, 0x00ff0fff,
759 0x8b10, 0xffffffff, 0x00000000,
760 0x28a4c, 0x06000000, 0x06000000,
761 0x4d8, 0xffffffff, 0x00000100,
762 0x913c, 0xffff000f, 0x0100000a,
763 0x960c, 0xffffffff, 0x54763210,
764 0x88c4, 0xffffffff, 0x000000c2,
765 0x88d4, 0xffffffff, 0x00000010,
766 0x8974, 0xffffffff, 0x00000000,
767 0xc78, 0x00000080, 0x00000080,
768 0x5e78, 0xffffffff, 0x001000f0,
769 0xd02c, 0xffffffff, 0x08421000,
770 0xa008, 0xffffffff, 0x00010000,
771 0x8d00, 0xffffffff, 0x100e4848,
772 0x8d04, 0xffffffff, 0x00164745,
773 0x8c00, 0xffffffff, 0xe4000003,
774 0x8cf0, 0x1fffffff, 0x08e00620,
775 0x28350, 0xffffffff, 0x00000000,
776 0x9508, 0xffffffff, 0x00000002
777};
778
779static const u32 sumo_golden_registers[] =
780{
781 0x900c, 0x00ffffff, 0x0017071f,
782 0x8c18, 0xffffffff, 0x10101060,
783 0x8c1c, 0xffffffff, 0x00001010,
784 0x8c30, 0x0000000f, 0x00000005,
785 0x9688, 0x0000000f, 0x00000007
786};
787
788static const u32 wrestler_golden_registers[] =
789{
790 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 791 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 792 0x7030, 0xffffffff, 0x00000011,
793 0x7c30, 0xffffffff, 0x00000011,
794 0x6104, 0x01000300, 0x00000000,
795 0x5bc0, 0x00300000, 0x00000000,
796 0x918c, 0xffffffff, 0x00010006,
797 0x91a8, 0xffffffff, 0x00010006,
798 0x9150, 0xffffffff, 0x6e944040,
799 0x917c, 0xffffffff, 0x00030002,
800 0x9198, 0xffffffff, 0x00030002,
801 0x915c, 0xffffffff, 0x00010000,
802 0x3f90, 0xffff0000, 0xff000000,
803 0x9178, 0xffffffff, 0x00070000,
804 0x9194, 0xffffffff, 0x00070000,
805 0x9148, 0xffff0000, 0xff000000,
806 0x9190, 0xffffffff, 0x00090008,
807 0x91ac, 0xffffffff, 0x00090008,
808 0x3f94, 0xffff0000, 0xff000000,
809 0x914c, 0xffff0000, 0xff000000,
810 0x929c, 0xffffffff, 0x00000001,
811 0x8a18, 0xffffffff, 0x00000100,
812 0x8b28, 0xffffffff, 0x00000100,
813 0x9144, 0xffffffff, 0x00000100,
814 0x9b7c, 0xffffffff, 0x00000000,
815 0x8030, 0xffffffff, 0x0000100a,
816 0x8a14, 0xffffffff, 0x00000001,
817 0x8b24, 0xffffffff, 0x00ff0fff,
818 0x8b10, 0xffffffff, 0x00000000,
819 0x28a4c, 0x06000000, 0x06000000,
820 0x4d8, 0xffffffff, 0x00000100,
821 0x913c, 0xffff000f, 0x0100000a,
822 0x960c, 0xffffffff, 0x54763210,
823 0x88c4, 0xffffffff, 0x000000c2,
824 0x88d4, 0xffffffff, 0x00000010,
825 0x8974, 0xffffffff, 0x00000000,
826 0xc78, 0x00000080, 0x00000080,
827 0x5e78, 0xffffffff, 0x001000f0,
828 0xd02c, 0xffffffff, 0x08421000,
829 0xa008, 0xffffffff, 0x00010000,
830 0x8d00, 0xffffffff, 0x100e4848,
831 0x8d04, 0xffffffff, 0x00164745,
832 0x8c00, 0xffffffff, 0xe4000003,
833 0x8cf0, 0x1fffffff, 0x08e00410,
834 0x28350, 0xffffffff, 0x00000000,
835 0x9508, 0xffffffff, 0x00000002,
836 0x900c, 0xffffffff, 0x0017071f,
837 0x8c18, 0xffffffff, 0x10101060,
838 0x8c1c, 0xffffffff, 0x00001010
839};
840
841static const u32 barts_golden_registers[] =
842{
843 0x5eb4, 0xffffffff, 0x00000002,
844 0x5e78, 0x8f311ff1, 0x001000f0,
845 0x3f90, 0xffff0000, 0xff000000,
846 0x9148, 0xffff0000, 0xff000000,
847 0x3f94, 0xffff0000, 0xff000000,
848 0x914c, 0xffff0000, 0xff000000,
849 0xc78, 0x00000080, 0x00000080,
850 0xbd4, 0x70073777, 0x00010001,
851 0xd02c, 0xbfffff1f, 0x08421000,
852 0xd0b8, 0x03773777, 0x02011003,
853 0x5bc0, 0x00200000, 0x50100000,
854 0x98f8, 0x33773777, 0x02011003,
855 0x98fc, 0xffffffff, 0x76543210,
856 0x7030, 0x31000311, 0x00000011,
857 0x2f48, 0x00000007, 0x02011003,
858 0x6b28, 0x00000010, 0x00000012,
859 0x7728, 0x00000010, 0x00000012,
860 0x10328, 0x00000010, 0x00000012,
861 0x10f28, 0x00000010, 0x00000012,
862 0x11b28, 0x00000010, 0x00000012,
863 0x12728, 0x00000010, 0x00000012,
864 0x240c, 0x000007ff, 0x00000380,
865 0x8a14, 0xf000001f, 0x00000007,
866 0x8b24, 0x3fff3fff, 0x00ff0fff,
867 0x8b10, 0x0000ff0f, 0x00000000,
868 0x28a4c, 0x07ffffff, 0x06000000,
869 0x10c, 0x00000001, 0x00010003,
870 0xa02c, 0xffffffff, 0x0000009b,
871 0x913c, 0x0000000f, 0x0100000a,
872 0x8d00, 0xffff7f7f, 0x100e4848,
873 0x8d04, 0x00ffffff, 0x00164745,
874 0x8c00, 0xfffc0003, 0xe4000003,
875 0x8c04, 0xf8ff00ff, 0x40600060,
876 0x8c08, 0x00ff00ff, 0x001c001c,
877 0x8cf0, 0x1fff1fff, 0x08e00620,
878 0x8c20, 0x0fff0fff, 0x00800080,
879 0x8c24, 0x0fff0fff, 0x00800080,
880 0x8c18, 0xffffffff, 0x20202078,
881 0x8c1c, 0x0000ffff, 0x00001010,
882 0x28350, 0x00000f01, 0x00000000,
883 0x9508, 0x3700001f, 0x00000002,
884 0x960c, 0xffffffff, 0x54763210,
885 0x88c4, 0x001f3ae3, 0x000000c2,
886 0x88d4, 0x0000001f, 0x00000010,
887 0x8974, 0xffffffff, 0x00000000
888};
889
890static const u32 turks_golden_registers[] =
891{
892 0x5eb4, 0xffffffff, 0x00000002,
893 0x5e78, 0x8f311ff1, 0x001000f0,
894 0x8c8, 0x00003000, 0x00001070,
895 0x8cc, 0x000fffff, 0x00040035,
896 0x3f90, 0xffff0000, 0xfff00000,
897 0x9148, 0xffff0000, 0xfff00000,
898 0x3f94, 0xffff0000, 0xfff00000,
899 0x914c, 0xffff0000, 0xfff00000,
900 0xc78, 0x00000080, 0x00000080,
901 0xbd4, 0x00073007, 0x00010002,
902 0xd02c, 0xbfffff1f, 0x08421000,
903 0xd0b8, 0x03773777, 0x02010002,
904 0x5bc0, 0x00200000, 0x50100000,
905 0x98f8, 0x33773777, 0x00010002,
906 0x98fc, 0xffffffff, 0x33221100,
907 0x7030, 0x31000311, 0x00000011,
908 0x2f48, 0x33773777, 0x00010002,
909 0x6b28, 0x00000010, 0x00000012,
910 0x7728, 0x00000010, 0x00000012,
911 0x10328, 0x00000010, 0x00000012,
912 0x10f28, 0x00000010, 0x00000012,
913 0x11b28, 0x00000010, 0x00000012,
914 0x12728, 0x00000010, 0x00000012,
915 0x240c, 0x000007ff, 0x00000380,
916 0x8a14, 0xf000001f, 0x00000007,
917 0x8b24, 0x3fff3fff, 0x00ff0fff,
918 0x8b10, 0x0000ff0f, 0x00000000,
919 0x28a4c, 0x07ffffff, 0x06000000,
920 0x10c, 0x00000001, 0x00010003,
921 0xa02c, 0xffffffff, 0x0000009b,
922 0x913c, 0x0000000f, 0x0100000a,
923 0x8d00, 0xffff7f7f, 0x100e4848,
924 0x8d04, 0x00ffffff, 0x00164745,
925 0x8c00, 0xfffc0003, 0xe4000003,
926 0x8c04, 0xf8ff00ff, 0x40600060,
927 0x8c08, 0x00ff00ff, 0x001c001c,
928 0x8cf0, 0x1fff1fff, 0x08e00410,
929 0x8c20, 0x0fff0fff, 0x00800080,
930 0x8c24, 0x0fff0fff, 0x00800080,
931 0x8c18, 0xffffffff, 0x20202078,
932 0x8c1c, 0x0000ffff, 0x00001010,
933 0x28350, 0x00000f01, 0x00000000,
934 0x9508, 0x3700001f, 0x00000002,
935 0x960c, 0xffffffff, 0x54763210,
936 0x88c4, 0x001f3ae3, 0x000000c2,
937 0x88d4, 0x0000001f, 0x00000010,
938 0x8974, 0xffffffff, 0x00000000
939};
940
941static const u32 caicos_golden_registers[] =
942{
943 0x5eb4, 0xffffffff, 0x00000002,
944 0x5e78, 0x8f311ff1, 0x001000f0,
945 0x8c8, 0x00003420, 0x00001450,
946 0x8cc, 0x000fffff, 0x00040035,
947 0x3f90, 0xffff0000, 0xfffc0000,
948 0x9148, 0xffff0000, 0xfffc0000,
949 0x3f94, 0xffff0000, 0xfffc0000,
950 0x914c, 0xffff0000, 0xfffc0000,
951 0xc78, 0x00000080, 0x00000080,
952 0xbd4, 0x00073007, 0x00010001,
953 0xd02c, 0xbfffff1f, 0x08421000,
954 0xd0b8, 0x03773777, 0x02010001,
955 0x5bc0, 0x00200000, 0x50100000,
956 0x98f8, 0x33773777, 0x02010001,
957 0x98fc, 0xffffffff, 0x33221100,
958 0x7030, 0x31000311, 0x00000011,
959 0x2f48, 0x33773777, 0x02010001,
960 0x6b28, 0x00000010, 0x00000012,
961 0x7728, 0x00000010, 0x00000012,
962 0x10328, 0x00000010, 0x00000012,
963 0x10f28, 0x00000010, 0x00000012,
964 0x11b28, 0x00000010, 0x00000012,
965 0x12728, 0x00000010, 0x00000012,
966 0x240c, 0x000007ff, 0x00000380,
967 0x8a14, 0xf000001f, 0x00000001,
968 0x8b24, 0x3fff3fff, 0x00ff0fff,
969 0x8b10, 0x0000ff0f, 0x00000000,
970 0x28a4c, 0x07ffffff, 0x06000000,
971 0x10c, 0x00000001, 0x00010003,
972 0xa02c, 0xffffffff, 0x0000009b,
973 0x913c, 0x0000000f, 0x0100000a,
974 0x8d00, 0xffff7f7f, 0x100e4848,
975 0x8d04, 0x00ffffff, 0x00164745,
976 0x8c00, 0xfffc0003, 0xe4000003,
977 0x8c04, 0xf8ff00ff, 0x40600060,
978 0x8c08, 0x00ff00ff, 0x001c001c,
979 0x8cf0, 0x1fff1fff, 0x08e00410,
980 0x8c20, 0x0fff0fff, 0x00800080,
981 0x8c24, 0x0fff0fff, 0x00800080,
982 0x8c18, 0xffffffff, 0x20202078,
983 0x8c1c, 0x0000ffff, 0x00001010,
984 0x28350, 0x00000f01, 0x00000000,
985 0x9508, 0x3700001f, 0x00000002,
986 0x960c, 0xffffffff, 0x54763210,
987 0x88c4, 0x001f3ae3, 0x000000c2,
988 0x88d4, 0x0000001f, 0x00000010,
989 0x8974, 0xffffffff, 0x00000000
990};
991
992static void evergreen_init_golden_registers(struct radeon_device *rdev)
993{
994 switch (rdev->family) {
995 case CHIP_CYPRESS:
996 case CHIP_HEMLOCK:
997 radeon_program_register_sequence(rdev,
998 evergreen_golden_registers,
999 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1000 radeon_program_register_sequence(rdev,
1001 evergreen_golden_registers2,
1002 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1003 radeon_program_register_sequence(rdev,
1004 cypress_mgcg_init,
1005 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1006 break;
1007 case CHIP_JUNIPER:
1008 radeon_program_register_sequence(rdev,
1009 evergreen_golden_registers,
1010 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1011 radeon_program_register_sequence(rdev,
1012 evergreen_golden_registers2,
1013 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1014 radeon_program_register_sequence(rdev,
1015 juniper_mgcg_init,
1016 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1017 break;
1018 case CHIP_REDWOOD:
1019 radeon_program_register_sequence(rdev,
1020 evergreen_golden_registers,
1021 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1022 radeon_program_register_sequence(rdev,
1023 evergreen_golden_registers2,
1024 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1025 radeon_program_register_sequence(rdev,
1026 redwood_mgcg_init,
1027 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1028 break;
1029 case CHIP_CEDAR:
1030 radeon_program_register_sequence(rdev,
1031 cedar_golden_registers,
1032 (const u32)ARRAY_SIZE(cedar_golden_registers));
1033 radeon_program_register_sequence(rdev,
1034 evergreen_golden_registers2,
1035 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1036 radeon_program_register_sequence(rdev,
1037 cedar_mgcg_init,
1038 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1039 break;
1040 case CHIP_PALM:
1041 radeon_program_register_sequence(rdev,
1042 wrestler_golden_registers,
1043 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1044 break;
1045 case CHIP_SUMO:
1046 radeon_program_register_sequence(rdev,
1047 supersumo_golden_registers,
1048 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1049 break;
1050 case CHIP_SUMO2:
1051 radeon_program_register_sequence(rdev,
1052 supersumo_golden_registers,
1053 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054 radeon_program_register_sequence(rdev,
1055 sumo_golden_registers,
1056 (const u32)ARRAY_SIZE(sumo_golden_registers));
1057 break;
1058 case CHIP_BARTS:
1059 radeon_program_register_sequence(rdev,
1060 barts_golden_registers,
1061 (const u32)ARRAY_SIZE(barts_golden_registers));
1062 break;
1063 case CHIP_TURKS:
1064 radeon_program_register_sequence(rdev,
1065 turks_golden_registers,
1066 (const u32)ARRAY_SIZE(turks_golden_registers));
1067 break;
1068 case CHIP_CAICOS:
1069 radeon_program_register_sequence(rdev,
1070 caicos_golden_registers,
1071 (const u32)ARRAY_SIZE(caicos_golden_registers));
1072 break;
1073 default:
1074 break;
1075 }
1076}
1077
ff609975
AD		 1078/**
1079 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1080 *
1081 * @rdev: radeon_device pointer
1082 * @reg: register offset in bytes
1083 * @val: register value
1084 *
1085 * Returns 0 for success or -EINVAL for an invalid register
1086 *
1087 */
1088int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1089 u32 reg, u32 *val)
1090{
1091 switch (reg) {
1092 case GRBM_STATUS:
1093 case GRBM_STATUS_SE0:
1094 case GRBM_STATUS_SE1:
1095 case SRBM_STATUS:
1096 case SRBM_STATUS2:
1097 case DMA_STATUS_REG:
1098 case UVD_STATUS:
1099 *val = RREG32(reg);
1100 return 0;
1101 default:
1102 return -EINVAL;
1103 }
1104}
1105
285484e2
JG		 1106void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1107 unsigned *bankh, unsigned *mtaspect,
1108 unsigned *tile_split)
1109{
1110 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1111 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1112 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1113 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1114 switch (*bankw) {
1115 default:
1116 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1117 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1118 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1119 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1120 }
1121 switch (*bankh) {
1122 default:
1123 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1124 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1125 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1126 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1127 }
1128 switch (*mtaspect) {
1129 default:
1130 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1131 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1132 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1133 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1134 }
1135}
1136
23d33ba3
AD		 1137static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1138 u32 cntl_reg, u32 status_reg)
1139{
1140 int r, i;
1141 struct atom_clock_dividers dividers;
1142
3cf8bb1a 1143 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
23d33ba3
AD		 1144 clock, false, &dividers);
1145 if (r)
1146 return r;
1147
1148 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1149
1150 for (i = 0; i < 100; i++) {
1151 if (RREG32(status_reg) & DCLK_STATUS)
1152 break;
1153 mdelay(10);
1154 }
1155 if (i == 100)
1156 return -ETIMEDOUT;
1157
1158 return 0;
1159}
1160
1161int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1162{
1163 int r = 0;
1164 u32 cg_scratch = RREG32(CG_SCRATCH1);
1165
1166 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1167 if (r)
1168 goto done;
1169 cg_scratch &= 0xffff0000;
1170 cg_scratch |= vclk / 100; /* Mhz */
1171
1172 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1173 if (r)
1174 goto done;
1175 cg_scratch &= 0x0000ffff;
1176 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1177
1178done:
1179 WREG32(CG_SCRATCH1, cg_scratch);
1180
1181 return r;
1182}
1183
a8b4925c
AD		 1184int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1185{
1186 /* start off with something large */
facd112d 1187 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1188 int r;
1189
4ed10835
CK		 1190 /* bypass vclk and dclk with bclk */
1191 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1192 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1193 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1194
1195 /* put PLL in bypass mode */
1196 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1197
1198 if (!vclk || !dclk) {
1199 /* keep the Bypass mode, put PLL to sleep */
1200 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1201 return 0;
1202 }
1203
facd112d
CK		 1204 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1205 16384, 0x03FFFFFF, 0, 128, 5,
1206 &fb_div, &vclk_div, &dclk_div);
1207 if (r)
1208 return r;
a8b4925c
AD		 1209
1210 /* set VCO_MODE to 1 */
1211 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1212
1213 /* toggle UPLL_SLEEP to 1 then back to 0 */
1214 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1215 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1216
1217 /* deassert UPLL_RESET */
1218 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1219
1220 mdelay(1);
1221
facd112d 1222 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1223 if (r)
1224 return r;
1225
1226 /* assert UPLL_RESET again */
1227 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1228
1229 /* disable spread spectrum. */
1230 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1231
1232 /* set feedback divider */
facd112d 1233 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1234
1235 /* set ref divider to 0 */
1236 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1237
facd112d 1238 if (fb_div < 307200)
a8b4925c
AD		 1239 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1240 else
1241 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1242
1243 /* set PDIV_A and PDIV_B */
1244 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1245 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1246 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1247
1248 /* give the PLL some time to settle */
1249 mdelay(15);
1250
1251 /* deassert PLL_RESET */
1252 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1253
1254 mdelay(15);
1255
1256 /* switch from bypass mode to normal mode */
1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1258
facd112d 1259 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1260 if (r)
1261 return r;
1262
1263 /* switch VCLK and DCLK selection */
1264 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1265 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1266 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1267
1268 mdelay(100);
1269
1270 return 0;
1271}
1272
d054ac16
AD		 1273void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1274{
c11592fe
YW		 1275 int readrq;
1276 u16 v;
d054ac16 1277
c11592fe
YW		 1278 readrq = pcie_get_readrq(rdev->pdev);
1279 v = ffs(readrq) - 8;
d054ac16
AD		 1280 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1281 * to avoid hangs or perfomance issues
1282 */
c11592fe
YW		 1283 if ((v == 0) || (v == 6) || (v == 7))
1284 pcie_set_readrq(rdev->pdev, 512);
d054ac16
AD		 1285}
1286
134b480f
AD		 1287void dce4_program_fmt(struct drm_encoder *encoder)
1288{
1289 struct drm_device *dev = encoder->dev;
1290 struct radeon_device *rdev = dev->dev_private;
1291 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1292 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1293 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1294 int bpc = 0;
1295 u32 tmp = 0;
6214bb74 1296 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1297
6214bb74
AD		 1298 if (connector) {
1299 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1300 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1301 dither = radeon_connector->dither;
1302 }
134b480f
AD		 1303
1304 /* LVDS/eDP FMT is set up by atom */
1305 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1306 return;
1307
1308 /* not needed for analog */
1309 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1310 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1311 return;
1312
1313 if (bpc == 0)
1314 return;
1315
1316 switch (bpc) {
1317 case 6:
6214bb74 1318 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1319 /* XXX sort out optimal dither settings */
1320 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1321 FMT_SPATIAL_DITHER_EN);
1322 else
1323 tmp |= FMT_TRUNCATE_EN;
1324 break;
1325 case 8:
6214bb74 1326 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1327 /* XXX sort out optimal dither settings */
1328 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329 FMT_RGB_RANDOM_ENABLE |
1330 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1331 else
1332 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1333 break;
1334 case 10:
1335 default:
1336 /* not needed */
1337 break;
1338 }
1339
1340 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1341}
1342
10257a6d
AD		 1343static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1344{
1345 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1346 return true;
1347 else
1348 return false;
1349}
1350
1351static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1352{
1353 u32 pos1, pos2;
1354
1355 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1356 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1357
1358 if (pos1 != pos2)
1359 return true;
1360 else
1361 return false;
1362}
1363
377edc8b
AD		 1364/**
1365 * dce4_wait_for_vblank - vblank wait asic callback.
1366 *
1367 * @rdev: radeon_device pointer
1368 * @crtc: crtc to wait for vblank on
1369 *
1370 * Wait for vblank on the requested crtc (evergreen+).
1371 */
3ae19b75
AD		 1372void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1373{
10257a6d 1374 unsigned i = 0;
3ae19b75 1375
4a15903d
AD		 1376 if (crtc >= rdev->num_crtc)
1377 return;
1378
10257a6d
AD		 1379 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1380 return;
1381
1382 /* depending on when we hit vblank, we may be close to active; if so,
1383 * wait for another frame.
1384 */
1385 while (dce4_is_in_vblank(rdev, crtc)) {
1386 if (i++ % 100 == 0) {
1387 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1388 break;
3ae19b75 1389 }
10257a6d
AD		 1390 }
1391
1392 while (!dce4_is_in_vblank(rdev, crtc)) {
1393 if (i++ % 100 == 0) {
1394 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1395 break;
3ae19b75
AD		 1396 }
1397 }
1398}
1399
377edc8b
AD		 1400/**
1401 * evergreen_page_flip - pageflip callback.
1402 *
1403 * @rdev: radeon_device pointer
1404 * @crtc_id: crtc to cleanup pageflip on
1405 * @crtc_base: new address of the crtc (GPU MC address)
1406 *
cbd3e249
AD		 1407 * Triggers the actual pageflip by updating the primary
1408 * surface base address (evergreen+).
377edc8b 1409 */
157fa14d 1410void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
6f34be50
AD		 1411{
1412 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1413
1414 /* update the scanout addresses */
6f34be50
AD		 1415 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1416 upper_32_bits(crtc_base));
1417 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1418 (u32)crtc_base);
cbd3e249
AD		 1419 /* post the write */
1420 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
157fa14d
CK		 1421}
1422
1423/**
1424 * evergreen_page_flip_pending - check if page flip is still pending
1425 *
1426 * @rdev: radeon_device pointer
1427 * @crtc_id: crtc to check
1428 *
1429 * Returns the current update pending status.
1430 */
1431bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1432{
1433 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1434
1435 /* Return current update_pending status: */
157fa14d
CK		 1436 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1437 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
6f34be50
AD		 1438}
1439
21a8122a 1440/* get temperature in millidegrees */
20d391d7 1441int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1442{
1c88d74f
AD		 1443 u32 temp, toffset;
1444 int actual_temp = 0;
67b3f823
AD		 1445
1446 if (rdev->family == CHIP_JUNIPER) {
1447 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1448 TOFFSET_SHIFT;
1449 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1450 TS0_ADC_DOUT_SHIFT;
1451
1452 if (toffset & 0x100)
1453 actual_temp = temp / 2 - (0x200 - toffset);
1454 else
1455 actual_temp = temp / 2 + toffset;
1456
1457 actual_temp = actual_temp * 1000;
1458
1459 } else {
1460 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1461 ASIC_T_SHIFT;
1462
1463 if (temp & 0x400)
1464 actual_temp = -256;
1465 else if (temp & 0x200)
1466 actual_temp = 255;
1467 else if (temp & 0x100) {
1468 actual_temp = temp & 0x1ff;
1469 actual_temp |= ~0x1ff;
1470 } else
1471 actual_temp = temp & 0xff;
1472
1473 actual_temp = (actual_temp * 1000) / 2;
1474 }
21a8122a 1475
67b3f823 1476 return actual_temp;
21a8122a
AD		 1477}
1478
20d391d7 1479int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1480{
1481 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1482 int actual_temp = temp - 49;
e33df25f
AD		 1483
1484 return actual_temp * 1000;
1485}
1486
377edc8b
AD		 1487/**
1488 * sumo_pm_init_profile - Initialize power profiles callback.
1489 *
1490 * @rdev: radeon_device pointer
1491 *
1492 * Initialize the power states used in profile mode
1493 * (sumo, trinity, SI).
1494 * Used for profile mode only.
1495 */
a4c9e2ee
AD		 1496void sumo_pm_init_profile(struct radeon_device *rdev)
1497{
1498 int idx;
1499
1500 /* default */
1501 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1502 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1503 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1505
1506 /* low,mid sh/mh */
1507 if (rdev->flags & RADEON_IS_MOBILITY)
1508 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1509 else
1510 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1511
1512 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1513 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1514 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1515 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1516
1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1519 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1520 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1521
1522 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1523 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1524 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1526
1527 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1530 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1531
1532 /* high sh/mh */
1533 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1534 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1537 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1538 rdev->pm.power_state[idx].num_clock_modes - 1;
1539
1540 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1544 rdev->pm.power_state[idx].num_clock_modes - 1;
1545}
1546
27810fb2
AD		 1547/**
1548 * btc_pm_init_profile - Initialize power profiles callback.
1549 *
1550 * @rdev: radeon_device pointer
1551 *
1552 * Initialize the power states used in profile mode
1553 * (BTC, cayman).
1554 * Used for profile mode only.
1555 */
1556void btc_pm_init_profile(struct radeon_device *rdev)
1557{
1558 int idx;
1559
1560 /* default */
1561 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1562 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1563 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1564 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1565 /* starting with BTC, there is one state that is used for both
1566 * MH and SH. Difference is that we always use the high clock index for
1567 * mclk.
1568 */
1569 if (rdev->flags & RADEON_IS_MOBILITY)
1570 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1571 else
1572 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1573 /* low sh */
1574 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1575 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1576 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1577 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1578 /* mid sh */
1579 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1580 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1581 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1582 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1583 /* high sh */
1584 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1585 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1586 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1587 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1588 /* low mh */
1589 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1590 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1591 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1592 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1593 /* mid mh */
1594 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1595 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1596 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1597 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1598 /* high mh */
1599 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1600 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1601 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1602 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1603}
1604
377edc8b
AD		 1605/**
1606 * evergreen_pm_misc - set additional pm hw parameters callback.
1607 *
1608 * @rdev: radeon_device pointer
1609 *
1610 * Set non-clock parameters associated with a power state
1611 * (voltage, etc.) (evergreen+).
1612 */
49e02b73
AD		 1613void evergreen_pm_misc(struct radeon_device *rdev)
1614{
a081a9d6
RM		 1615 int req_ps_idx = rdev->pm.requested_power_state_index;
1616 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1617 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1618 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1619
2feea49a 1620 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1621 /* 0xff0x are flags rather then an actual voltage */
1622 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1623 return;
2feea49a 1624 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1625 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1626 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1627 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1628 }
7ae764b1
AD		 1629
1630 /* starting with BTC, there is one state that is used for both
1631 * MH and SH. Difference is that we always use the high clock index for
1632 * mclk and vddci.
1633 */
1634 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1635 (rdev->family >= CHIP_BARTS) &&
1636 rdev->pm.active_crtc_count &&
1637 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1638 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1639 voltage = &rdev->pm.power_state[req_ps_idx].
1640 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1641
c6cf7777
AD		 1642 /* 0xff0x are flags rather then an actual voltage */
1643 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1644 return;
2feea49a
AD		 1645 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1646 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1647 rdev->pm.current_vddci = voltage->vddci;
1648 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1649 }
1650 }
49e02b73
AD		 1651}
1652
377edc8b
AD		 1653/**
1654 * evergreen_pm_prepare - pre-power state change callback.
1655 *
1656 * @rdev: radeon_device pointer
1657 *
1658 * Prepare for a power state change (evergreen+).
1659 */
49e02b73
AD		 1660void evergreen_pm_prepare(struct radeon_device *rdev)
1661{
1662 struct drm_device *ddev = rdev->ddev;
1663 struct drm_crtc *crtc;
1664 struct radeon_crtc *radeon_crtc;
1665 u32 tmp;
1666
1667 /* disable any active CRTCs */
1668 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1669 radeon_crtc = to_radeon_crtc(crtc);
1670 if (radeon_crtc->enabled) {
1671 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1672 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1673 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1674 }
1675 }
1676}
1677
377edc8b
AD		 1678/**
1679 * evergreen_pm_finish - post-power state change callback.
1680 *
1681 * @rdev: radeon_device pointer
1682 *
1683 * Clean up after a power state change (evergreen+).
1684 */
49e02b73
AD		 1685void evergreen_pm_finish(struct radeon_device *rdev)
1686{
1687 struct drm_device *ddev = rdev->ddev;
1688 struct drm_crtc *crtc;
1689 struct radeon_crtc *radeon_crtc;
1690 u32 tmp;
1691
1692 /* enable any active CRTCs */
1693 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1694 radeon_crtc = to_radeon_crtc(crtc);
1695 if (radeon_crtc->enabled) {
1696 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1697 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1698 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1699 }
1700 }
1701}
1702
377edc8b
AD		 1703/**
1704 * evergreen_hpd_sense - hpd sense callback.
1705 *
1706 * @rdev: radeon_device pointer
1707 * @hpd: hpd (hotplug detect) pin
1708 *
1709 * Checks if a digital monitor is connected (evergreen+).
1710 * Returns true if connected, false if not connected.
1711 */
bcc1c2a1
AD		 1712bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1713{
1714 bool connected = false;
0ca2ab52
AD		 1715
1716 switch (hpd) {
1717 case RADEON_HPD_1:
1718 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1719 connected = true;
1720 break;
1721 case RADEON_HPD_2:
1722 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1723 connected = true;
1724 break;
1725 case RADEON_HPD_3:
1726 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1727 connected = true;
1728 break;
1729 case RADEON_HPD_4:
1730 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1731 connected = true;
1732 break;
1733 case RADEON_HPD_5:
1734 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1735 connected = true;
1736 break;
1737 case RADEON_HPD_6:
1738 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1739 connected = true;
ebc54ffe 1740 break;
0ca2ab52
AD		 1741 default:
1742 break;
1743 }
1744
bcc1c2a1
AD		 1745 return connected;
1746}
1747
377edc8b
AD		 1748/**
1749 * evergreen_hpd_set_polarity - hpd set polarity callback.
1750 *
1751 * @rdev: radeon_device pointer
1752 * @hpd: hpd (hotplug detect) pin
1753 *
1754 * Set the polarity of the hpd pin (evergreen+).
1755 */
bcc1c2a1
AD		 1756void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1757 enum radeon_hpd_id hpd)
1758{
0ca2ab52
AD		 1759 u32 tmp;
1760 bool connected = evergreen_hpd_sense(rdev, hpd);
1761
1762 switch (hpd) {
1763 case RADEON_HPD_1:
1764 tmp = RREG32(DC_HPD1_INT_CONTROL);
1765 if (connected)
1766 tmp &= ~DC_HPDx_INT_POLARITY;
1767 else
1768 tmp |= DC_HPDx_INT_POLARITY;
1769 WREG32(DC_HPD1_INT_CONTROL, tmp);
1770 break;
1771 case RADEON_HPD_2:
1772 tmp = RREG32(DC_HPD2_INT_CONTROL);
1773 if (connected)
1774 tmp &= ~DC_HPDx_INT_POLARITY;
1775 else
1776 tmp |= DC_HPDx_INT_POLARITY;
1777 WREG32(DC_HPD2_INT_CONTROL, tmp);
1778 break;
1779 case RADEON_HPD_3:
1780 tmp = RREG32(DC_HPD3_INT_CONTROL);
1781 if (connected)
1782 tmp &= ~DC_HPDx_INT_POLARITY;
1783 else
1784 tmp |= DC_HPDx_INT_POLARITY;
1785 WREG32(DC_HPD3_INT_CONTROL, tmp);
1786 break;
1787 case RADEON_HPD_4:
1788 tmp = RREG32(DC_HPD4_INT_CONTROL);
1789 if (connected)
1790 tmp &= ~DC_HPDx_INT_POLARITY;
1791 else
1792 tmp |= DC_HPDx_INT_POLARITY;
1793 WREG32(DC_HPD4_INT_CONTROL, tmp);
1794 break;
1795 case RADEON_HPD_5:
1796 tmp = RREG32(DC_HPD5_INT_CONTROL);
1797 if (connected)
1798 tmp &= ~DC_HPDx_INT_POLARITY;
1799 else
1800 tmp |= DC_HPDx_INT_POLARITY;
1801 WREG32(DC_HPD5_INT_CONTROL, tmp);
1802 break;
1803 case RADEON_HPD_6:
1804 tmp = RREG32(DC_HPD6_INT_CONTROL);
1805 if (connected)
1806 tmp &= ~DC_HPDx_INT_POLARITY;
1807 else
1808 tmp |= DC_HPDx_INT_POLARITY;
1809 WREG32(DC_HPD6_INT_CONTROL, tmp);
1810 break;
1811 default:
1812 break;
1813 }
bcc1c2a1
AD		 1814}
1815
377edc8b
AD		 1816/**
1817 * evergreen_hpd_init - hpd setup callback.
1818 *
1819 * @rdev: radeon_device pointer
1820 *
1821 * Setup the hpd pins used by the card (evergreen+).
1822 * Enable the pin, set the polarity, and enable the hpd interrupts.
1823 */
bcc1c2a1
AD		 1824void evergreen_hpd_init(struct radeon_device *rdev)
1825{
0ca2ab52
AD		 1826 struct drm_device *dev = rdev->ddev;
1827 struct drm_connector *connector;
fb98257a 1828 unsigned enabled = 0;
0ca2ab52
AD		 1829 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1830 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1831
0ca2ab52
AD		 1832 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1833 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1834
1835 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1836 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1837 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1838 * aux dp channel on imac and help (but not completely fix)
1839 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1840 * also avoid interrupt storms during dpms.
1841 */
1842 continue;
1843 }
0ca2ab52
AD		 1844 switch (radeon_connector->hpd.hpd) {
1845 case RADEON_HPD_1:
1846 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1847 break;
1848 case RADEON_HPD_2:
1849 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1850 break;
1851 case RADEON_HPD_3:
1852 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1853 break;
1854 case RADEON_HPD_4:
1855 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1856 break;
1857 case RADEON_HPD_5:
1858 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1859 break;
1860 case RADEON_HPD_6:
1861 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1862 break;
1863 default:
1864 break;
1865 }
64912e99 1866 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1867 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1868 }
fb98257a 1869 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1870}
1871
377edc8b
AD		 1872/**
1873 * evergreen_hpd_fini - hpd tear down callback.
1874 *
1875 * @rdev: radeon_device pointer
1876 *
1877 * Tear down the hpd pins used by the card (evergreen+).
1878 * Disable the hpd interrupts.
1879 */
0ca2ab52 1880void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1881{
0ca2ab52
AD		 1882 struct drm_device *dev = rdev->ddev;
1883 struct drm_connector *connector;
fb98257a 1884 unsigned disabled = 0;
0ca2ab52
AD		 1885
1886 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1887 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1888 switch (radeon_connector->hpd.hpd) {
1889 case RADEON_HPD_1:
1890 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1891 break;
1892 case RADEON_HPD_2:
1893 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1894 break;
1895 case RADEON_HPD_3:
1896 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1897 break;
1898 case RADEON_HPD_4:
1899 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1900 break;
1901 case RADEON_HPD_5:
1902 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1903 break;
1904 case RADEON_HPD_6:
1905 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1906 break;
1907 default:
1908 break;
1909 }
fb98257a 1910 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1911 }
fb98257a 1912 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1913}
1914
f9d9c362
AD		 1915/* watermark setup */
1916
1917static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1918 struct radeon_crtc *radeon_crtc,
1919 struct drm_display_mode *mode,
1920 struct drm_display_mode *other_mode)
1921{
0b31e023
AD		 1922 u32 tmp, buffer_alloc, i;
1923 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1924 /*
1925 * Line Buffer Setup
1926 * There are 3 line buffers, each one shared by 2 display controllers.
1927 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1928 * the display controllers. The paritioning is done via one of four
1929 * preset allocations specified in bits 2:0:
1930 * first display controller
1931 * 0 - first half of lb (3840 * 2)
1932 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1933 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1934 * 3 - first 1/4 of lb (1920 * 2)
1935 * second display controller
1936 * 4 - second half of lb (3840 * 2)
1937 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1938 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1939 * 7 - last 1/4 of lb (1920 * 2)
1940 */
12dfc843
AD		 1941 /* this can get tricky if we have two large displays on a paired group
1942 * of crtcs. Ideally for multiple large displays we'd assign them to
1943 * non-linked crtcs for maximum line buffer allocation.
1944 */
1945 if (radeon_crtc->base.enabled && mode) {
0b31e023 1946 if (other_mode) {
f9d9c362 1947 tmp = 0; /* 1/2 */
0b31e023
AD		 1948 buffer_alloc = 1;
1949 } else {
12dfc843 1950 tmp = 2; /* whole */
0b31e023
AD		 1951 buffer_alloc = 2;
1952 }
1953 } else {
12dfc843 1954 tmp = 0;
0b31e023
AD		 1955 buffer_alloc = 0;
1956 }
f9d9c362
AD		 1957
1958 /* second controller of the pair uses second half of the lb */
1959 if (radeon_crtc->crtc_id % 2)
1960 tmp += 4;
1961 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1962
0b31e023
AD		 1963 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1964 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1965 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1966 for (i = 0; i < rdev->usec_timeout; i++) {
1967 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1968 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1969 break;
1970 udelay(1);
1971 }
1972 }
1973
12dfc843
AD		 1974 if (radeon_crtc->base.enabled && mode) {
1975 switch (tmp) {
1976 case 0:
1977 case 4:
1978 default:
1979 if (ASIC_IS_DCE5(rdev))
1980 return 4096 * 2;
1981 else
1982 return 3840 * 2;
1983 case 1:
1984 case 5:
1985 if (ASIC_IS_DCE5(rdev))
1986 return 6144 * 2;
1987 else
1988 return 5760 * 2;
1989 case 2:
1990 case 6:
1991 if (ASIC_IS_DCE5(rdev))
1992 return 8192 * 2;
1993 else
1994 return 7680 * 2;
1995 case 3:
1996 case 7:
1997 if (ASIC_IS_DCE5(rdev))
1998 return 2048 * 2;
1999 else
2000 return 1920 * 2;
2001 }
f9d9c362 2002 }
12dfc843
AD		 2003
2004 /* controller not enabled, so no lb used */
2005 return 0;
f9d9c362
AD		 2006}
2007
ca7db22b 2008u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 2009{
2010 u32 tmp = RREG32(MC_SHARED_CHMAP);
2011
2012 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2013 case 0:
2014 default:
2015 return 1;
2016 case 1:
2017 return 2;
2018 case 2:
2019 return 4;
2020 case 3:
2021 return 8;
2022 }
2023}
2024
2025struct evergreen_wm_params {
2026 u32 dram_channels; /* number of dram channels */
2027 u32 yclk; /* bandwidth per dram data pin in kHz */
2028 u32 sclk; /* engine clock in kHz */
2029 u32 disp_clk; /* display clock in kHz */
2030 u32 src_width; /* viewport width */
2031 u32 active_time; /* active display time in ns */
2032 u32 blank_time; /* blank time in ns */
2033 bool interlaced; /* mode is interlaced */
2034 fixed20_12 vsc; /* vertical scale ratio */
2035 u32 num_heads; /* number of active crtcs */
2036 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2037 u32 lb_size; /* line buffer allocated to pipe */
2038 u32 vtaps; /* vertical scaler taps */
2039};
2040
2041static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
2042{
2043 /* Calculate DRAM Bandwidth and the part allocated to display. */
2044 fixed20_12 dram_efficiency; /* 0.7 */
2045 fixed20_12 yclk, dram_channels, bandwidth;
2046 fixed20_12 a;
2047
2048 a.full = dfixed_const(1000);
2049 yclk.full = dfixed_const(wm->yclk);
2050 yclk.full = dfixed_div(yclk, a);
2051 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2052 a.full = dfixed_const(10);
2053 dram_efficiency.full = dfixed_const(7);
2054 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2055 bandwidth.full = dfixed_mul(dram_channels, yclk);
2056 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2057
2058 return dfixed_trunc(bandwidth);
2059}
2060
2061static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2062{
2063 /* Calculate DRAM Bandwidth and the part allocated to display. */
2064 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2065 fixed20_12 yclk, dram_channels, bandwidth;
2066 fixed20_12 a;
2067
2068 a.full = dfixed_const(1000);
2069 yclk.full = dfixed_const(wm->yclk);
2070 yclk.full = dfixed_div(yclk, a);
2071 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2072 a.full = dfixed_const(10);
2073 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2074 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2075 bandwidth.full = dfixed_mul(dram_channels, yclk);
2076 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2077
2078 return dfixed_trunc(bandwidth);
2079}
2080
2081static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2082{
2083 /* Calculate the display Data return Bandwidth */
2084 fixed20_12 return_efficiency; /* 0.8 */
2085 fixed20_12 sclk, bandwidth;
2086 fixed20_12 a;
2087
2088 a.full = dfixed_const(1000);
2089 sclk.full = dfixed_const(wm->sclk);
2090 sclk.full = dfixed_div(sclk, a);
2091 a.full = dfixed_const(10);
2092 return_efficiency.full = dfixed_const(8);
2093 return_efficiency.full = dfixed_div(return_efficiency, a);
2094 a.full = dfixed_const(32);
2095 bandwidth.full = dfixed_mul(a, sclk);
2096 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2097
2098 return dfixed_trunc(bandwidth);
2099}
2100
2101static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2102{
2103 /* Calculate the DMIF Request Bandwidth */
2104 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2105 fixed20_12 disp_clk, bandwidth;
2106 fixed20_12 a;
2107
2108 a.full = dfixed_const(1000);
2109 disp_clk.full = dfixed_const(wm->disp_clk);
2110 disp_clk.full = dfixed_div(disp_clk, a);
2111 a.full = dfixed_const(10);
2112 disp_clk_request_efficiency.full = dfixed_const(8);
2113 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2114 a.full = dfixed_const(32);
2115 bandwidth.full = dfixed_mul(a, disp_clk);
2116 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2117
2118 return dfixed_trunc(bandwidth);
2119}
2120
2121static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2122{
2123 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2124 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2125 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2126 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2127
2128 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2129}
2130
2131static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2132{
2133 /* Calculate the display mode Average Bandwidth
2134 * DisplayMode should contain the source and destination dimensions,
2135 * timing, etc.
2136 */
2137 fixed20_12 bpp;
2138 fixed20_12 line_time;
2139 fixed20_12 src_width;
2140 fixed20_12 bandwidth;
2141 fixed20_12 a;
2142
2143 a.full = dfixed_const(1000);
2144 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2145 line_time.full = dfixed_div(line_time, a);
2146 bpp.full = dfixed_const(wm->bytes_per_pixel);
2147 src_width.full = dfixed_const(wm->src_width);
2148 bandwidth.full = dfixed_mul(src_width, bpp);
2149 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2150 bandwidth.full = dfixed_div(bandwidth, line_time);
2151
2152 return dfixed_trunc(bandwidth);
2153}
2154
2155static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2156{
2157 /* First calcualte the latency in ns */
2158 u32 mc_latency = 2000; /* 2000 ns. */
2159 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2160 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2161 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2162 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2163 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2164 (wm->num_heads * cursor_line_pair_return_time);
2165 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2166 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2167 fixed20_12 a, b, c;
2168
2169 if (wm->num_heads == 0)
2170 return 0;
2171
2172 a.full = dfixed_const(2);
2173 b.full = dfixed_const(1);
2174 if ((wm->vsc.full > a.full) ||
2175 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2176 (wm->vtaps >= 5) ||
2177 ((wm->vsc.full >= a.full) && wm->interlaced))
2178 max_src_lines_per_dst_line = 4;
2179 else
2180 max_src_lines_per_dst_line = 2;
2181
2182 a.full = dfixed_const(available_bandwidth);
2183 b.full = dfixed_const(wm->num_heads);
2184 a.full = dfixed_div(a, b);
2185
2186 b.full = dfixed_const(1000);
2187 c.full = dfixed_const(wm->disp_clk);
2188 b.full = dfixed_div(c, b);
2189 c.full = dfixed_const(wm->bytes_per_pixel);
2190 b.full = dfixed_mul(b, c);
2191
2192 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2193
2194 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2195 b.full = dfixed_const(1000);
2196 c.full = dfixed_const(lb_fill_bw);
2197 b.full = dfixed_div(c, b);
2198 a.full = dfixed_div(a, b);
2199 line_fill_time = dfixed_trunc(a);
2200
2201 if (line_fill_time < wm->active_time)
2202 return latency;
2203 else
2204 return latency + (line_fill_time - wm->active_time);
2205
2206}
2207
2208static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2209{
2210 if (evergreen_average_bandwidth(wm) <=
2211 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2212 return true;
2213 else
2214 return false;
2215};
2216
2217static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2218{
2219 if (evergreen_average_bandwidth(wm) <=
2220 (evergreen_available_bandwidth(wm) / wm->num_heads))
2221 return true;
2222 else
2223 return false;
2224};
2225
2226static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2227{
2228 u32 lb_partitions = wm->lb_size / wm->src_width;
2229 u32 line_time = wm->active_time + wm->blank_time;
2230 u32 latency_tolerant_lines;
2231 u32 latency_hiding;
2232 fixed20_12 a;
2233
2234 a.full = dfixed_const(1);
2235 if (wm->vsc.full > a.full)
2236 latency_tolerant_lines = 1;
2237 else {
2238 if (lb_partitions <= (wm->vtaps + 1))
2239 latency_tolerant_lines = 1;
2240 else
2241 latency_tolerant_lines = 2;
2242 }
2243
2244 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2245
2246 if (evergreen_latency_watermark(wm) <= latency_hiding)
2247 return true;
2248 else
2249 return false;
2250}
2251
2252static void evergreen_program_watermarks(struct radeon_device *rdev,
2253 struct radeon_crtc *radeon_crtc,
2254 u32 lb_size, u32 num_heads)
2255{
2256 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2257 struct evergreen_wm_params wm_low, wm_high;
2258 u32 dram_channels;
f9d9c362
AD		 2259 u32 pixel_period;
2260 u32 line_time = 0;
2261 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2262 u32 priority_a_mark = 0, priority_b_mark = 0;
2263 u32 priority_a_cnt = PRIORITY_OFF;
2264 u32 priority_b_cnt = PRIORITY_OFF;
2265 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2266 u32 tmp, arb_control3;
2267 fixed20_12 a, b, c;
2268
2269 if (radeon_crtc->base.enabled && num_heads && mode) {
2270 pixel_period = 1000000 / (u32)mode->clock;
2271 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2272 priority_a_cnt = 0;
2273 priority_b_cnt = 0;
cf0cfdd7
AD		 2274 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275
2276 /* watermark for high clocks */
2277 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2278 wm_high.yclk =
2279 radeon_dpm_get_mclk(rdev, false) * 10;
2280 wm_high.sclk =
2281 radeon_dpm_get_sclk(rdev, false) * 10;
2282 } else {
2283 wm_high.yclk = rdev->pm.current_mclk * 10;
2284 wm_high.sclk = rdev->pm.current_sclk * 10;
2285 }
f9d9c362 2286
cf0cfdd7
AD		 2287 wm_high.disp_clk = mode->clock;
2288 wm_high.src_width = mode->crtc_hdisplay;
2289 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2290 wm_high.blank_time = line_time - wm_high.active_time;
2291 wm_high.interlaced = false;
f9d9c362 2292 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2293 wm_high.interlaced = true;
2294 wm_high.vsc = radeon_crtc->vsc;
2295 wm_high.vtaps = 1;
f9d9c362 2296 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2297 wm_high.vtaps = 2;
2298 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2299 wm_high.lb_size = lb_size;
2300 wm_high.dram_channels = dram_channels;
2301 wm_high.num_heads = num_heads;
2302
2303 /* watermark for low clocks */
2304 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2305 wm_low.yclk =
2306 radeon_dpm_get_mclk(rdev, true) * 10;
2307 wm_low.sclk =
2308 radeon_dpm_get_sclk(rdev, true) * 10;
2309 } else {
2310 wm_low.yclk = rdev->pm.current_mclk * 10;
2311 wm_low.sclk = rdev->pm.current_sclk * 10;
2312 }
2313
2314 wm_low.disp_clk = mode->clock;
2315 wm_low.src_width = mode->crtc_hdisplay;
2316 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2317 wm_low.blank_time = line_time - wm_low.active_time;
2318 wm_low.interlaced = false;
2319 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2320 wm_low.interlaced = true;
2321 wm_low.vsc = radeon_crtc->vsc;
2322 wm_low.vtaps = 1;
2323 if (radeon_crtc->rmx_type != RMX_OFF)
2324 wm_low.vtaps = 2;
2325 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2326 wm_low.lb_size = lb_size;
2327 wm_low.dram_channels = dram_channels;
2328 wm_low.num_heads = num_heads;
f9d9c362
AD		 2329
2330 /* set for high clocks */
cf0cfdd7 2331 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2332 /* set for low clocks */
cf0cfdd7 2333 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2334
2335 /* possibly force display priority to high */
2336 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2337 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2338 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2339 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2340 (rdev->disp_priority == 2)) {
cf0cfdd7 2341 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2342 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2343 }
2344 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2345 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2346 !evergreen_check_latency_hiding(&wm_low) ||
2347 (rdev->disp_priority == 2)) {
2348 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2349 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2350 }
2351
2352 a.full = dfixed_const(1000);
2353 b.full = dfixed_const(mode->clock);
2354 b.full = dfixed_div(b, a);
2355 c.full = dfixed_const(latency_watermark_a);
2356 c.full = dfixed_mul(c, b);
2357 c.full = dfixed_mul(c, radeon_crtc->hsc);
2358 c.full = dfixed_div(c, a);
2359 a.full = dfixed_const(16);
2360 c.full = dfixed_div(c, a);
2361 priority_a_mark = dfixed_trunc(c);
2362 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2363
2364 a.full = dfixed_const(1000);
2365 b.full = dfixed_const(mode->clock);
2366 b.full = dfixed_div(b, a);
2367 c.full = dfixed_const(latency_watermark_b);
2368 c.full = dfixed_mul(c, b);
2369 c.full = dfixed_mul(c, radeon_crtc->hsc);
2370 c.full = dfixed_div(c, a);
2371 a.full = dfixed_const(16);
2372 c.full = dfixed_div(c, a);
2373 priority_b_mark = dfixed_trunc(c);
2374 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
5b5561b3
MK		 2375
2376 /* Save number of lines the linebuffer leads before the scanout */
2377 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
f9d9c362
AD		 2378 }
2379
2380 /* select wm A */
2381 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2382 tmp = arb_control3;
2383 tmp &= ~LATENCY_WATERMARK_MASK(3);
2384 tmp |= LATENCY_WATERMARK_MASK(1);
2385 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2386 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2387 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2388 LATENCY_HIGH_WATERMARK(line_time)));
2389 /* select wm B */
2390 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2391 tmp &= ~LATENCY_WATERMARK_MASK(3);
2392 tmp |= LATENCY_WATERMARK_MASK(2);
2393 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2394 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2395 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2396 LATENCY_HIGH_WATERMARK(line_time)));
2397 /* restore original selection */
2398 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2399
2400 /* write the priority marks */
2401 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2402 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2403
7178d2a6
AD		 2404 /* save values for DPM */
2405 radeon_crtc->line_time = line_time;
2406 radeon_crtc->wm_high = latency_watermark_a;
2407 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2408}
2409
377edc8b
AD		 2410/**
2411 * evergreen_bandwidth_update - update display watermarks callback.
2412 *
2413 * @rdev: radeon_device pointer
2414 *
2415 * Update the display watermarks based on the requested mode(s)
2416 * (evergreen+).
2417 */
0ca2ab52 2418void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2419{
f9d9c362
AD		 2420 struct drm_display_mode *mode0 = NULL;
2421 struct drm_display_mode *mode1 = NULL;
2422 u32 num_heads = 0, lb_size;
2423 int i;
2424
8efe82ca
AD		 2425 if (!rdev->mode_info.mode_config_initialized)
2426 return;
2427
f9d9c362
AD		 2428 radeon_update_display_priority(rdev);
2429
2430 for (i = 0; i < rdev->num_crtc; i++) {
2431 if (rdev->mode_info.crtcs[i]->base.enabled)
2432 num_heads++;
2433 }
2434 for (i = 0; i < rdev->num_crtc; i += 2) {
2435 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2436 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2437 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2438 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2439 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2440 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2441 }
bcc1c2a1
AD		 2442}
2443
377edc8b
AD		 2444/**
2445 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2446 *
2447 * @rdev: radeon_device pointer
2448 *
2449 * Wait for the MC (memory controller) to be idle.
2450 * (evergreen+).
2451 * Returns 0 if the MC is idle, -1 if not.
2452 */
b9952a8a 2453int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2454{
2455 unsigned i;
2456 u32 tmp;
2457
2458 for (i = 0; i < rdev->usec_timeout; i++) {
2459 /* read MC_STATUS */
2460 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2461 if (!tmp)
2462 return 0;
2463 udelay(1);
2464 }
2465 return -1;
2466}
2467
2468/*
2469 * GART
2470 */
0fcdb61e
AD		 2471void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2472{
2473 unsigned i;
2474 u32 tmp;
2475
6f2f48a9
AD		 2476 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2477
0fcdb61e
AD		 2478 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2479 for (i = 0; i < rdev->usec_timeout; i++) {
2480 /* read MC_STATUS */
2481 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2482 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2483 if (tmp == 2) {
2484 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2485 return;
2486 }
2487 if (tmp) {
2488 return;
2489 }
2490 udelay(1);
2491 }
2492}
2493
1109ca09 2494static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2495{
2496 u32 tmp;
0fcdb61e 2497 int r;
bcc1c2a1 2498
c9a1be96 2499 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2500 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2501 return -EINVAL;
2502 }
2503 r = radeon_gart_table_vram_pin(rdev);
2504 if (r)
2505 return r;
2506 /* Setup L2 cache */
2507 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2508 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2509 EFFECTIVE_L2_QUEUE_SIZE(7));
2510 WREG32(VM_L2_CNTL2, 0);
2511 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2512 /* Setup TLB control */
2513 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2514 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2515 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2516 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2517 if (rdev->flags & RADEON_IS_IGP) {
2518 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2519 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2520 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2521 } else {
2522 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2523 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2524 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2525 if ((rdev->family == CHIP_JUNIPER) ||
2526 (rdev->family == CHIP_CYPRESS) ||
2527 (rdev->family == CHIP_HEMLOCK) ||
2528 (rdev->family == CHIP_BARTS))
2529 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2530 }
bcc1c2a1
AD		 2531 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2532 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2533 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2534 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2535 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 2536 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
bcc1c2a1
AD		 2537 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2538 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2539 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2540 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2541 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2542 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2543
0fcdb61e 2544 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2545 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2546 (unsigned)(rdev->mc.gtt_size >> 20),
2547 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2548 rdev->gart.ready = true;
2549 return 0;
2550}
2551
1109ca09 2552static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2553{
2554 u32 tmp;
bcc1c2a1
AD		 2555
2556 /* Disable all tables */
0fcdb61e
AD		 2557 WREG32(VM_CONTEXT0_CNTL, 0);
2558 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2559
2560 /* Setup L2 cache */
2561 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2562 EFFECTIVE_L2_QUEUE_SIZE(7));
2563 WREG32(VM_L2_CNTL2, 0);
2564 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2565 /* Setup TLB control */
2566 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2567 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2568 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2569 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2570 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2571 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2572 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2573 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2574 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2575}
2576
1109ca09 2577static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2578{
2579 evergreen_pcie_gart_disable(rdev);
2580 radeon_gart_table_vram_free(rdev);
2581 radeon_gart_fini(rdev);
2582}
2583
2584
1109ca09 2585static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2586{
2587 u32 tmp;
bcc1c2a1
AD		 2588
2589 /* Setup L2 cache */
2590 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2591 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2592 EFFECTIVE_L2_QUEUE_SIZE(7));
2593 WREG32(VM_L2_CNTL2, 0);
2594 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2595 /* Setup TLB control */
2596 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2597 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2598 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2599 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2600 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2601 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2602 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2603 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2604 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2605 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2606 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2607 WREG32(VM_CONTEXT0_CNTL, 0);
2608 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2609}
2610
b9952a8a 2611void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2612{
62444b74
AD		 2613 u32 crtc_enabled, tmp, frame_count, blackout;
2614 int i, j;
2615
5153550a
AD		 2616 if (!ASIC_IS_NODCE(rdev)) {
2617 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2618 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2619
5153550a
AD		 2620 /* disable VGA render */
2621 WREG32(VGA_RENDER_CONTROL, 0);
2622 }
62444b74
AD		 2623 /* blank the display controllers */
2624 for (i = 0; i < rdev->num_crtc; i++) {
2625 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2626 if (crtc_enabled) {
2627 save->crtc_enabled[i] = true;
2628 if (ASIC_IS_DCE6(rdev)) {
2629 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2630 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2631 radeon_wait_for_vblank(rdev, i);
abf1457b 2632 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2633 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2634 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
f0d7bfb9 2635 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2636 }
2637 } else {
2638 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2639 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2640 radeon_wait_for_vblank(rdev, i);
abf1457b 2641 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2642 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2643 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2644 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2645 }
2646 }
2647 /* wait for the next frame */
2648 frame_count = radeon_get_vblank_counter(rdev, i);
2649 for (j = 0; j < rdev->usec_timeout; j++) {
2650 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2651 break;
2652 udelay(1);
2653 }
abf1457b
AD		 2654
2655 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2656 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2657 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2658 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2659 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2660 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2661 save->crtc_enabled[i] = false;
2662 /* ***** */
804cc4a0
AD		 2663 } else {
2664 save->crtc_enabled[i] = false;
62444b74 2665 }
18007401 2666 }
bcc1c2a1 2667
62444b74
AD		 2668 radeon_mc_wait_for_idle(rdev);
2669
2670 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2671 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2672 /* Block CPU access */
2673 WREG32(BIF_FB_EN, 0);
2674 /* blackout the MC */
2675 blackout &= ~BLACKOUT_MODE_MASK;
2676 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2677 }
ed39fadd
AD		 2678 /* wait for the MC to settle */
2679 udelay(100);
968c0166
AD		 2680
2681 /* lock double buffered regs */
2682 for (i = 0; i < rdev->num_crtc; i++) {
2683 if (save->crtc_enabled[i]) {
2684 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2685 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2686 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2687 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2688 }
2689 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2690 if (!(tmp & 1)) {
2691 tmp |= 1;
2692 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2693 }
2694 }
2695 }
bcc1c2a1
AD		 2696}
2697
b9952a8a 2698void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2699{
62444b74
AD		 2700 u32 tmp, frame_count;
2701 int i, j;
18007401 2702
62444b74
AD		 2703 /* update crtc base addresses */
2704 for (i = 0; i < rdev->num_crtc; i++) {
2705 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2706 upper_32_bits(rdev->mc.vram_start));
62444b74 2707 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2708 upper_32_bits(rdev->mc.vram_start));
62444b74 2709 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2710 (u32)rdev->mc.vram_start);
62444b74 2711 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2712 (u32)rdev->mc.vram_start);
2713 }
5153550a
AD		 2714
2715 if (!ASIC_IS_NODCE(rdev)) {
2716 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2717 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2718 }
62444b74 2719
968c0166
AD		 2720 /* unlock regs and wait for update */
2721 for (i = 0; i < rdev->num_crtc; i++) {
2722 if (save->crtc_enabled[i]) {
2723 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
f53f81b2
MK		 2724 if ((tmp & 0x7) != 3) {
2725 tmp &= ~0x7;
2726 tmp |= 0x3;
968c0166
AD		 2727 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2728 }
2729 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2730 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2731 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2732 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2733 }
2734 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2735 if (tmp & 1) {
2736 tmp &= ~1;
2737 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2738 }
2739 for (j = 0; j < rdev->usec_timeout; j++) {
2740 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2741 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2742 break;
2743 udelay(1);
2744 }
2745 }
2746 }
2747
62444b74
AD		 2748 /* unblackout the MC */
2749 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2750 tmp &= ~BLACKOUT_MODE_MASK;
2751 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2752 /* allow CPU access */
2753 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2754
2755 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2756 if (save->crtc_enabled[i]) {
62444b74
AD		 2757 if (ASIC_IS_DCE6(rdev)) {
2758 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
3157c589 2759 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2760 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2761 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2762 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2763 } else {
2764 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2765 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2766 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2767 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2768 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2769 }
2770 /* wait for the next frame */
2771 frame_count = radeon_get_vblank_counter(rdev, i);
2772 for (j = 0; j < rdev->usec_timeout; j++) {
2773 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2774 break;
2775 udelay(1);
2776 }
2777 }
2778 }
5153550a
AD		 2779 if (!ASIC_IS_NODCE(rdev)) {
2780 /* Unlock vga access */
2781 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2782 mdelay(1);
2783 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2784 }
bcc1c2a1
AD		 2785}
2786
755d819e 2787void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2788{
2789 struct evergreen_mc_save save;
2790 u32 tmp;
2791 int i, j;
2792
2793 /* Initialize HDP */
2794 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2795 WREG32((0x2c14 + j), 0x00000000);
2796 WREG32((0x2c18 + j), 0x00000000);
2797 WREG32((0x2c1c + j), 0x00000000);
2798 WREG32((0x2c20 + j), 0x00000000);
2799 WREG32((0x2c24 + j), 0x00000000);
2800 }
2801 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2802
2803 evergreen_mc_stop(rdev, &save);
2804 if (evergreen_mc_wait_for_idle(rdev)) {
2805 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2806 }
2807 /* Lockout access through VGA aperture*/
2808 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2809 /* Update configuration */
2810 if (rdev->flags & RADEON_IS_AGP) {
2811 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2812 /* VRAM before AGP */
2813 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2814 rdev->mc.vram_start >> 12);
2815 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2816 rdev->mc.gtt_end >> 12);
2817 } else {
2818 /* VRAM after AGP */
2819 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2820 rdev->mc.gtt_start >> 12);
2821 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2822 rdev->mc.vram_end >> 12);
2823 }
2824 } else {
2825 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2826 rdev->mc.vram_start >> 12);
2827 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2828 rdev->mc.vram_end >> 12);
2829 }
3b9832f6 2830 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2831 /* llano/ontario only */
2832 if ((rdev->family == CHIP_PALM) ||
2833 (rdev->family == CHIP_SUMO) ||
2834 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2835 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2836 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2837 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2838 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2839 }
bcc1c2a1
AD		 2840 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2841 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2842 WREG32(MC_VM_FB_LOCATION, tmp);
2843 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2844 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2845 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2846 if (rdev->flags & RADEON_IS_AGP) {
2847 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2848 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2849 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2850 } else {
2851 WREG32(MC_VM_AGP_BASE, 0);
2852 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2853 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2854 }
2855 if (evergreen_mc_wait_for_idle(rdev)) {
2856 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2857 }
2858 evergreen_mc_resume(rdev, &save);
2859 /* we need to own VRAM, so turn off the VGA renderer here
2860 * to stop it overwriting our objects */
2861 rv515_vga_render_disable(rdev);
2862}
2863
bcc1c2a1
AD		 2864/*
2865 * CP.
2866 */
12920591
AD		 2867void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2868{
876dc9f3 2869 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2870 u32 next_rptr;
7b1f2485 2871
12920591 2872 /* set to DX10/11 mode */
e32eb50d
CK		 2873 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2874 radeon_ring_write(ring, 1);
45df6803
CK		 2875
2876 if (ring->rptr_save_reg) {
89d35807 2877 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2878 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2879 radeon_ring_write(ring, ((ring->rptr_save_reg -
2880 PACKET3_SET_CONFIG_REG_START) >> 2));
2881 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2882 } else if (rdev->wb.enabled) {
2883 next_rptr = ring->wptr + 5 + 4;
2884 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2885 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2886 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2887 radeon_ring_write(ring, next_rptr);
2888 radeon_ring_write(ring, 0);
45df6803
CK		 2889 }
2890
e32eb50d
CK		 2891 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2892 radeon_ring_write(ring,
0f234f5f
AD		 2893#ifdef __BIG_ENDIAN
2894 (2 << 0) |
2895#endif
2896 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2897 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2898 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2899}
2900
bcc1c2a1
AD		 2901
2902static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2903{
fe251e2f
AD		 2904 const __be32 *fw_data;
2905 int i;
2906
2907 if (!rdev->me_fw || !rdev->pfp_fw)
2908 return -EINVAL;
bcc1c2a1 2909
fe251e2f 2910 r700_cp_stop(rdev);
0f234f5f
AD		 2911 WREG32(CP_RB_CNTL,
2912#ifdef __BIG_ENDIAN
2913 BUF_SWAP_32BIT |
2914#endif
2915 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2916
2917 fw_data = (const __be32 *)rdev->pfp_fw->data;
2918 WREG32(CP_PFP_UCODE_ADDR, 0);
2919 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2920 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2921 WREG32(CP_PFP_UCODE_ADDR, 0);
2922
2923 fw_data = (const __be32 *)rdev->me_fw->data;
2924 WREG32(CP_ME_RAM_WADDR, 0);
2925 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2926 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2927
2928 WREG32(CP_PFP_UCODE_ADDR, 0);
2929 WREG32(CP_ME_RAM_WADDR, 0);
2930 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2931 return 0;
2932}
2933
7e7b41d2
AD		 2934static int evergreen_cp_start(struct radeon_device *rdev)
2935{
e32eb50d 2936 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2937 int r, i;
7e7b41d2
AD		 2938 uint32_t cp_me;
2939
e32eb50d 2940 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2941 if (r) {
2942 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2943 return r;
2944 }
e32eb50d
CK		 2945 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2946 radeon_ring_write(ring, 0x1);
2947 radeon_ring_write(ring, 0x0);
2948 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2949 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2950 radeon_ring_write(ring, 0);
2951 radeon_ring_write(ring, 0);
1538a9e0 2952 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2953
2954 cp_me = 0xff;
2955 WREG32(CP_ME_CNTL, cp_me);
2956
e32eb50d 2957 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2958 if (r) {
2959 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2960 return r;
2961 }
2281a378
AD		 2962
2963 /* setup clear context state */
e32eb50d
CK		 2964 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2965 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2966
2967 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2968 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2969
e32eb50d
CK		 2970 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2971 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2972
2973 /* set clear context state */
e32eb50d
CK		 2974 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2975 radeon_ring_write(ring, 0);
2281a378
AD		 2976
2977 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2978 radeon_ring_write(ring, 0xc0026f00);
2979 radeon_ring_write(ring, 0x00000000);
2980 radeon_ring_write(ring, 0x00000000);
2981 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2982
2983 /* Clear consts */
e32eb50d
CK		 2984 radeon_ring_write(ring, 0xc0036f00);
2985 radeon_ring_write(ring, 0x00000bc4);
2986 radeon_ring_write(ring, 0xffffffff);
2987 radeon_ring_write(ring, 0xffffffff);
2988 radeon_ring_write(ring, 0xffffffff);
2281a378 2989
e32eb50d
CK		 2990 radeon_ring_write(ring, 0xc0026900);
2991 radeon_ring_write(ring, 0x00000316);
2992 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2993 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2994
1538a9e0 2995 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2996
2997 return 0;
2998}
2999
1109ca09 3000static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 3001{
e32eb50d 3002 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 3003 u32 tmp;
3004 u32 rb_bufsz;
3005 int r;
3006
3007 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3008 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3009 SOFT_RESET_PA |
3010 SOFT_RESET_SH |
3011 SOFT_RESET_VGT |
a49a50da 3012 SOFT_RESET_SPI |
fe251e2f
AD		 3013 SOFT_RESET_SX));
3014 RREG32(GRBM_SOFT_RESET);
3015 mdelay(15);
3016 WREG32(GRBM_SOFT_RESET, 0);
3017 RREG32(GRBM_SOFT_RESET);
3018
3019 /* Set ring buffer size */
b72a8925
DV		 3020 rb_bufsz = order_base_2(ring->ring_size / 8);
3021 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 3022#ifdef __BIG_ENDIAN
3023 tmp |= BUF_SWAP_32BIT;
32fcdbf4 3024#endif
fe251e2f 3025 WREG32(CP_RB_CNTL, tmp);
15d3332f 3026 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 3027 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 3028
3029 /* Set the write pointer delay */
3030 WREG32(CP_RB_WPTR_DELAY, 0);
3031
3032 /* Initialize the ring buffer's read and write pointers */
3033 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3034 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 3035 ring->wptr = 0;
3036 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 3037
48fc7f7e 3038 /* set the wb address whether it's enabled or not */
0f234f5f 3039 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 3040 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 3041 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3042 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3043
3044 if (rdev->wb.enabled)
3045 WREG32(SCRATCH_UMSK, 0xff);
3046 else {
3047 tmp |= RB_NO_UPDATE;
3048 WREG32(SCRATCH_UMSK, 0);
3049 }
3050
fe251e2f
AD		 3051 mdelay(1);
3052 WREG32(CP_RB_CNTL, tmp);
3053
e32eb50d 3054 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 3055 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3056
7e7b41d2 3057 evergreen_cp_start(rdev);
e32eb50d 3058 ring->ready = true;
f712812e 3059 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 3060 if (r) {
e32eb50d 3061 ring->ready = false;
fe251e2f
AD		 3062 return r;
3063 }
3064 return 0;
3065}
bcc1c2a1
AD		 3066
3067/*
3068 * Core functions
3069 */
bcc1c2a1
AD		 3070static void evergreen_gpu_init(struct radeon_device *rdev)
3071{
416a2bd2 3072 u32 gb_addr_config;
32fcdbf4 3073 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 3074 u32 sx_debug_1;
3075 u32 smx_dc_ctl0;
3076 u32 sq_config;
3077 u32 sq_lds_resource_mgmt;
3078 u32 sq_gpr_resource_mgmt_1;
3079 u32 sq_gpr_resource_mgmt_2;
3080 u32 sq_gpr_resource_mgmt_3;
3081 u32 sq_thread_resource_mgmt;
3082 u32 sq_thread_resource_mgmt_2;
3083 u32 sq_stack_resource_mgmt_1;
3084 u32 sq_stack_resource_mgmt_2;
3085 u32 sq_stack_resource_mgmt_3;
3086 u32 vgt_cache_invalidation;
f25a5c63 3087 u32 hdp_host_path_cntl, tmp;
416a2bd2 3088 u32 disabled_rb_mask;
072c44bf 3089 int i, j, ps_thread_count;
32fcdbf4
AD		 3090
3091 switch (rdev->family) {
3092 case CHIP_CYPRESS:
3093 case CHIP_HEMLOCK:
3094 rdev->config.evergreen.num_ses = 2;
3095 rdev->config.evergreen.max_pipes = 4;
3096 rdev->config.evergreen.max_tile_pipes = 8;
3097 rdev->config.evergreen.max_simds = 10;
3098 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3099 rdev->config.evergreen.max_gprs = 256;
3100 rdev->config.evergreen.max_threads = 248;
3101 rdev->config.evergreen.max_gs_threads = 32;
3102 rdev->config.evergreen.max_stack_entries = 512;
3103 rdev->config.evergreen.sx_num_of_sets = 4;
3104 rdev->config.evergreen.sx_max_export_size = 256;
3105 rdev->config.evergreen.sx_max_export_pos_size = 64;
3106 rdev->config.evergreen.sx_max_export_smx_size = 192;
3107 rdev->config.evergreen.max_hw_contexts = 8;
3108 rdev->config.evergreen.sq_num_cf_insts = 2;
3109
3110 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3111 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3112 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3113 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3114 break;
3115 case CHIP_JUNIPER:
3116 rdev->config.evergreen.num_ses = 1;
3117 rdev->config.evergreen.max_pipes = 4;
3118 rdev->config.evergreen.max_tile_pipes = 4;
3119 rdev->config.evergreen.max_simds = 10;
3120 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3121 rdev->config.evergreen.max_gprs = 256;
3122 rdev->config.evergreen.max_threads = 248;
3123 rdev->config.evergreen.max_gs_threads = 32;
3124 rdev->config.evergreen.max_stack_entries = 512;
3125 rdev->config.evergreen.sx_num_of_sets = 4;
3126 rdev->config.evergreen.sx_max_export_size = 256;
3127 rdev->config.evergreen.sx_max_export_pos_size = 64;
3128 rdev->config.evergreen.sx_max_export_smx_size = 192;
3129 rdev->config.evergreen.max_hw_contexts = 8;
3130 rdev->config.evergreen.sq_num_cf_insts = 2;
3131
3132 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3133 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3134 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3135 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3136 break;
3137 case CHIP_REDWOOD:
3138 rdev->config.evergreen.num_ses = 1;
3139 rdev->config.evergreen.max_pipes = 4;
3140 rdev->config.evergreen.max_tile_pipes = 4;
3141 rdev->config.evergreen.max_simds = 5;
3142 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3143 rdev->config.evergreen.max_gprs = 256;
3144 rdev->config.evergreen.max_threads = 248;
3145 rdev->config.evergreen.max_gs_threads = 32;
3146 rdev->config.evergreen.max_stack_entries = 256;
3147 rdev->config.evergreen.sx_num_of_sets = 4;
3148 rdev->config.evergreen.sx_max_export_size = 256;
3149 rdev->config.evergreen.sx_max_export_pos_size = 64;
3150 rdev->config.evergreen.sx_max_export_smx_size = 192;
3151 rdev->config.evergreen.max_hw_contexts = 8;
3152 rdev->config.evergreen.sq_num_cf_insts = 2;
3153
3154 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3155 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3156 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3157 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3158 break;
3159 case CHIP_CEDAR:
3160 default:
3161 rdev->config.evergreen.num_ses = 1;
3162 rdev->config.evergreen.max_pipes = 2;
3163 rdev->config.evergreen.max_tile_pipes = 2;
3164 rdev->config.evergreen.max_simds = 2;
3165 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3166 rdev->config.evergreen.max_gprs = 256;
3167 rdev->config.evergreen.max_threads = 192;
3168 rdev->config.evergreen.max_gs_threads = 16;
3169 rdev->config.evergreen.max_stack_entries = 256;
3170 rdev->config.evergreen.sx_num_of_sets = 4;
3171 rdev->config.evergreen.sx_max_export_size = 128;
3172 rdev->config.evergreen.sx_max_export_pos_size = 32;
3173 rdev->config.evergreen.sx_max_export_smx_size = 96;
3174 rdev->config.evergreen.max_hw_contexts = 4;
3175 rdev->config.evergreen.sq_num_cf_insts = 1;
3176
d5e455e4
AD		 3177 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3178 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3179 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3180 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3181 break;
3182 case CHIP_PALM:
3183 rdev->config.evergreen.num_ses = 1;
3184 rdev->config.evergreen.max_pipes = 2;
3185 rdev->config.evergreen.max_tile_pipes = 2;
3186 rdev->config.evergreen.max_simds = 2;
3187 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3188 rdev->config.evergreen.max_gprs = 256;
3189 rdev->config.evergreen.max_threads = 192;
3190 rdev->config.evergreen.max_gs_threads = 16;
3191 rdev->config.evergreen.max_stack_entries = 256;
3192 rdev->config.evergreen.sx_num_of_sets = 4;
3193 rdev->config.evergreen.sx_max_export_size = 128;
3194 rdev->config.evergreen.sx_max_export_pos_size = 32;
3195 rdev->config.evergreen.sx_max_export_smx_size = 96;
3196 rdev->config.evergreen.max_hw_contexts = 4;
3197 rdev->config.evergreen.sq_num_cf_insts = 1;
3198
d5c5a72f
AD		 3199 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3200 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3201 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3202 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3203 break;
3204 case CHIP_SUMO:
3205 rdev->config.evergreen.num_ses = 1;
3206 rdev->config.evergreen.max_pipes = 4;
bd25f078 3207 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3208 if (rdev->pdev->device == 0x9648)
3209 rdev->config.evergreen.max_simds = 3;
3210 else if ((rdev->pdev->device == 0x9647) ||
3211 (rdev->pdev->device == 0x964a))
3212 rdev->config.evergreen.max_simds = 4;
3213 else
3214 rdev->config.evergreen.max_simds = 5;
3215 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3216 rdev->config.evergreen.max_gprs = 256;
3217 rdev->config.evergreen.max_threads = 248;
3218 rdev->config.evergreen.max_gs_threads = 32;
3219 rdev->config.evergreen.max_stack_entries = 256;
3220 rdev->config.evergreen.sx_num_of_sets = 4;
3221 rdev->config.evergreen.sx_max_export_size = 256;
3222 rdev->config.evergreen.sx_max_export_pos_size = 64;
3223 rdev->config.evergreen.sx_max_export_smx_size = 192;
3224 rdev->config.evergreen.max_hw_contexts = 8;
3225 rdev->config.evergreen.sq_num_cf_insts = 2;
3226
3227 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3228 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3229 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3230 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3231 break;
3232 case CHIP_SUMO2:
3233 rdev->config.evergreen.num_ses = 1;
3234 rdev->config.evergreen.max_pipes = 4;
3235 rdev->config.evergreen.max_tile_pipes = 4;
3236 rdev->config.evergreen.max_simds = 2;
3237 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3238 rdev->config.evergreen.max_gprs = 256;
3239 rdev->config.evergreen.max_threads = 248;
3240 rdev->config.evergreen.max_gs_threads = 32;
3241 rdev->config.evergreen.max_stack_entries = 512;
3242 rdev->config.evergreen.sx_num_of_sets = 4;
3243 rdev->config.evergreen.sx_max_export_size = 256;
3244 rdev->config.evergreen.sx_max_export_pos_size = 64;
3245 rdev->config.evergreen.sx_max_export_smx_size = 192;
50b8f5ae 3246 rdev->config.evergreen.max_hw_contexts = 4;
d5c5a72f
AD		 3247 rdev->config.evergreen.sq_num_cf_insts = 2;
3248
adb68fa2
AD		 3249 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3250 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3251 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3252 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3253 break;
3254 case CHIP_BARTS:
3255 rdev->config.evergreen.num_ses = 2;
3256 rdev->config.evergreen.max_pipes = 4;
3257 rdev->config.evergreen.max_tile_pipes = 8;
3258 rdev->config.evergreen.max_simds = 7;
3259 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3260 rdev->config.evergreen.max_gprs = 256;
3261 rdev->config.evergreen.max_threads = 248;
3262 rdev->config.evergreen.max_gs_threads = 32;
3263 rdev->config.evergreen.max_stack_entries = 512;
3264 rdev->config.evergreen.sx_num_of_sets = 4;
3265 rdev->config.evergreen.sx_max_export_size = 256;
3266 rdev->config.evergreen.sx_max_export_pos_size = 64;
3267 rdev->config.evergreen.sx_max_export_smx_size = 192;
3268 rdev->config.evergreen.max_hw_contexts = 8;
3269 rdev->config.evergreen.sq_num_cf_insts = 2;
3270
3271 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3272 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3273 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3274 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3275 break;
3276 case CHIP_TURKS:
3277 rdev->config.evergreen.num_ses = 1;
3278 rdev->config.evergreen.max_pipes = 4;
3279 rdev->config.evergreen.max_tile_pipes = 4;
3280 rdev->config.evergreen.max_simds = 6;
3281 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3282 rdev->config.evergreen.max_gprs = 256;
3283 rdev->config.evergreen.max_threads = 248;
3284 rdev->config.evergreen.max_gs_threads = 32;
3285 rdev->config.evergreen.max_stack_entries = 256;
3286 rdev->config.evergreen.sx_num_of_sets = 4;
3287 rdev->config.evergreen.sx_max_export_size = 256;
3288 rdev->config.evergreen.sx_max_export_pos_size = 64;
3289 rdev->config.evergreen.sx_max_export_smx_size = 192;
3290 rdev->config.evergreen.max_hw_contexts = 8;
3291 rdev->config.evergreen.sq_num_cf_insts = 2;
3292
3293 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3294 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3295 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3296 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3297 break;
3298 case CHIP_CAICOS:
3299 rdev->config.evergreen.num_ses = 1;
bd25f078 3300 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3301 rdev->config.evergreen.max_tile_pipes = 2;
3302 rdev->config.evergreen.max_simds = 2;
3303 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3304 rdev->config.evergreen.max_gprs = 256;
3305 rdev->config.evergreen.max_threads = 192;
3306 rdev->config.evergreen.max_gs_threads = 16;
3307 rdev->config.evergreen.max_stack_entries = 256;
3308 rdev->config.evergreen.sx_num_of_sets = 4;
3309 rdev->config.evergreen.sx_max_export_size = 128;
3310 rdev->config.evergreen.sx_max_export_pos_size = 32;
3311 rdev->config.evergreen.sx_max_export_smx_size = 96;
3312 rdev->config.evergreen.max_hw_contexts = 4;
3313 rdev->config.evergreen.sq_num_cf_insts = 1;
3314
32fcdbf4
AD		 3315 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3316 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3317 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3318 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3319 break;
3320 }
3321
3322 /* Initialize HDP */
3323 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3324 WREG32((0x2c14 + j), 0x00000000);
3325 WREG32((0x2c18 + j), 0x00000000);
3326 WREG32((0x2c1c + j), 0x00000000);
3327 WREG32((0x2c20 + j), 0x00000000);
3328 WREG32((0x2c24 + j), 0x00000000);
3329 }
3330
3331 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
acc1522a
CK		 3332 WREG32(SRBM_INT_CNTL, 0x1);
3333 WREG32(SRBM_INT_ACK, 0x1);
32fcdbf4 3334
d054ac16
AD		 3335 evergreen_fix_pci_max_read_req_size(rdev);
3336
32fcdbf4 3337 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3338 if ((rdev->family == CHIP_PALM) ||
3339 (rdev->family == CHIP_SUMO) ||
3340 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3341 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3342 else
3343 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3344
1aa52bd3
AD		 3345 /* setup tiling info dword. gb_addr_config is not adequate since it does
3346 * not have bank info, so create a custom tiling dword.
3347 * bits 3:0 num_pipes
3348 * bits 7:4 num_banks
3349 * bits 11:8 group_size
3350 * bits 15:12 row_size
3351 */
3352 rdev->config.evergreen.tile_config = 0;
3353 switch (rdev->config.evergreen.max_tile_pipes) {
3354 case 1:
3355 default:
3356 rdev->config.evergreen.tile_config |= (0 << 0);
3357 break;
3358 case 2:
3359 rdev->config.evergreen.tile_config |= (1 << 0);
3360 break;
3361 case 4:
3362 rdev->config.evergreen.tile_config |= (2 << 0);
3363 break;
3364 case 8:
3365 rdev->config.evergreen.tile_config |= (3 << 0);
3366 break;
3367 }
d698a34d 3368 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3369 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3370 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3371 else {
c8d15edc
AD		 3372 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3373 case 0: /* four banks */
29d65406 3374 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3375 break;
3376 case 1: /* eight banks */
3377 rdev->config.evergreen.tile_config |= 1 << 4;
3378 break;
3379 case 2: /* sixteen banks */
3380 default:
3381 rdev->config.evergreen.tile_config |= 2 << 4;
3382 break;
3383 }
29d65406 3384 }
416a2bd2 3385 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3386 rdev->config.evergreen.tile_config |=
3387 ((gb_addr_config & 0x30000000) >> 28) << 12;
3388
416a2bd2
AD		 3389 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3390 u32 efuse_straps_4;
3391 u32 efuse_straps_3;
32fcdbf4 3392
ff82bbc4
AD		 3393 efuse_straps_4 = RREG32_RCU(0x204);
3394 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3395 tmp = (((efuse_straps_4 & 0xf) << 4) |
3396 ((efuse_straps_3 & 0xf0000000) >> 28));
3397 } else {
3398 tmp = 0;
3399 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3400 u32 rb_disable_bitmap;
3401
3402 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3403 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3404 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3405 tmp <<= 4;
3406 tmp |= rb_disable_bitmap;
32fcdbf4 3407 }
416a2bd2
AD		 3408 }
3409 /* enabled rb are just the one not disabled :) */
3410 disabled_rb_mask = tmp;
cedb655a
AD		 3411 tmp = 0;
3412 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3413 tmp |= (1 << i);
3414 /* if all the backends are disabled, fix it up here */
3415 if ((disabled_rb_mask & tmp) == tmp) {
3416 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3417 disabled_rb_mask &= ~(1 << i);
3418 }
32fcdbf4 3419
65fcf668
AD		 3420 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3421 u32 simd_disable_bitmap;
3422
3423 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3424 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3425 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3426 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3427 tmp <<= 16;
3428 tmp |= simd_disable_bitmap;
3429 }
3430 rdev->config.evergreen.active_simds = hweight32(~tmp);
3431
416a2bd2
AD		 3432 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3433 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3434
416a2bd2
AD		 3435 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3436 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3437 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3438 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3439 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3440 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3441 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3442
f7eb9730
AD		 3443 if ((rdev->config.evergreen.max_backends == 1) &&
3444 (rdev->flags & RADEON_IS_IGP)) {
3445 if ((disabled_rb_mask & 3) == 1) {
3446 /* RB0 disabled, RB1 enabled */
3447 tmp = 0x11111111;
3448 } else {
3449 /* RB1 disabled, RB0 enabled */
3450 tmp = 0x00000000;
3451 }
3452 } else {
3453 tmp = gb_addr_config & NUM_PIPES_MASK;
3454 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3455 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3456 }
416a2bd2 3457 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3458
3459 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3460 WREG32(CGTS_TCC_DISABLE, 0);
3461 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3462 WREG32(CGTS_USER_TCC_DISABLE, 0);
3463
3464 /* set HW defaults for 3D engine */
3465 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3466 ROQ_IB2_START(0x2b)));
3467
3468 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3469
3470 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3471 SYNC_GRADIENT |
3472 SYNC_WALKER |
3473 SYNC_ALIGNER));
3474
3475 sx_debug_1 = RREG32(SX_DEBUG_1);
3476 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3477 WREG32(SX_DEBUG_1, sx_debug_1);
3478
3479
3480 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3481 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3482 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3483 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3484
b866d133
AD		 3485 if (rdev->family <= CHIP_SUMO2)
3486 WREG32(SMX_SAR_CTL0, 0x00010000);
3487
32fcdbf4
AD		 3488 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3489 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3490 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3491
3492 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3493 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3494 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3495
3496 WREG32(VGT_NUM_INSTANCES, 1);
3497 WREG32(SPI_CONFIG_CNTL, 0);
3498 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3499 WREG32(CP_PERFMON_CNTL, 0);
3500
3501 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3502 FETCH_FIFO_HIWATER(0x4) |
3503 DONE_FIFO_HIWATER(0xe0) |
3504 ALU_UPDATE_FIFO_HIWATER(0x8)));
3505
3506 sq_config = RREG32(SQ_CONFIG);
3507 sq_config &= ~(PS_PRIO(3) |
3508 VS_PRIO(3) |
3509 GS_PRIO(3) |
3510 ES_PRIO(3));
3511 sq_config |= (VC_ENABLE |
3512 EXPORT_SRC_C |
3513 PS_PRIO(0) |
3514 VS_PRIO(1) |
3515 GS_PRIO(2) |
3516 ES_PRIO(3));
3517
d5e455e4
AD		 3518 switch (rdev->family) {
3519 case CHIP_CEDAR:
3520 case CHIP_PALM:
d5c5a72f
AD		 3521 case CHIP_SUMO:
3522 case CHIP_SUMO2:
adb68fa2 3523 case CHIP_CAICOS:
32fcdbf4
AD		 3524 /* no vertex cache */
3525 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3526 break;
3527 default:
3528 break;
3529 }
32fcdbf4
AD		 3530
3531 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3532
3533 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3534 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3535 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3536 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3537 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3538 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3539 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3540
d5e455e4
AD		 3541 switch (rdev->family) {
3542 case CHIP_CEDAR:
3543 case CHIP_PALM:
d5c5a72f
AD		 3544 case CHIP_SUMO:
3545 case CHIP_SUMO2:
32fcdbf4 3546 ps_thread_count = 96;
d5e455e4
AD		 3547 break;
3548 default:
32fcdbf4 3549 ps_thread_count = 128;
d5e455e4
AD		 3550 break;
3551 }
32fcdbf4
AD		 3552
3553 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3554 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3555 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3556 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3557 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3558 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3559
3560 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3561 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3562 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3563 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3564 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3565 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3566
3567 WREG32(SQ_CONFIG, sq_config);
3568 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3569 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3570 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3571 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3572 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3573 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3574 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3575 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3576 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3577 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3578
3579 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3580 FORCE_EOV_MAX_REZ_CNT(255)));
3581
d5e455e4
AD		 3582 switch (rdev->family) {
3583 case CHIP_CEDAR:
3584 case CHIP_PALM:
d5c5a72f
AD		 3585 case CHIP_SUMO:
3586 case CHIP_SUMO2:
adb68fa2 3587 case CHIP_CAICOS:
32fcdbf4 3588 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3589 break;
3590 default:
32fcdbf4 3591 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3592 break;
3593 }
32fcdbf4
AD		 3594 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3595 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3596
3597 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3598 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3599 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3600
60a4a3e0
AD		 3601 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3602 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3603
32fcdbf4
AD		 3604 WREG32(CB_PERF_CTR0_SEL_0, 0);
3605 WREG32(CB_PERF_CTR0_SEL_1, 0);
3606 WREG32(CB_PERF_CTR1_SEL_0, 0);
3607 WREG32(CB_PERF_CTR1_SEL_1, 0);
3608 WREG32(CB_PERF_CTR2_SEL_0, 0);
3609 WREG32(CB_PERF_CTR2_SEL_1, 0);
3610 WREG32(CB_PERF_CTR3_SEL_0, 0);
3611 WREG32(CB_PERF_CTR3_SEL_1, 0);
3612
60a4a3e0
AD		 3613 /* clear render buffer base addresses */
3614 WREG32(CB_COLOR0_BASE, 0);
3615 WREG32(CB_COLOR1_BASE, 0);
3616 WREG32(CB_COLOR2_BASE, 0);
3617 WREG32(CB_COLOR3_BASE, 0);
3618 WREG32(CB_COLOR4_BASE, 0);
3619 WREG32(CB_COLOR5_BASE, 0);
3620 WREG32(CB_COLOR6_BASE, 0);
3621 WREG32(CB_COLOR7_BASE, 0);
3622 WREG32(CB_COLOR8_BASE, 0);
3623 WREG32(CB_COLOR9_BASE, 0);
3624 WREG32(CB_COLOR10_BASE, 0);
3625 WREG32(CB_COLOR11_BASE, 0);
3626
3627 /* set the shader const cache sizes to 0 */
3628 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3629 WREG32(i, 0);
3630 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3631 WREG32(i, 0);
3632
f25a5c63
AD		 3633 tmp = RREG32(HDP_MISC_CNTL);
3634 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3635 WREG32(HDP_MISC_CNTL, tmp);
3636
32fcdbf4
AD		 3637 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3638 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3639
3640 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3641
3642 udelay(50);
3643
bcc1c2a1
AD		 3644}
3645
3646int evergreen_mc_init(struct radeon_device *rdev)
3647{
bcc1c2a1
AD		 3648 u32 tmp;
3649 int chansize, numchan;
bcc1c2a1
AD		 3650
3651 /* Get VRAM informations */
3652 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3653 if ((rdev->family == CHIP_PALM) ||
3654 (rdev->family == CHIP_SUMO) ||
3655 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3656 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3657 else
3658 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3659 if (tmp & CHANSIZE_OVERRIDE) {
3660 chansize = 16;
3661 } else if (tmp & CHANSIZE_MASK) {
3662 chansize = 64;
3663 } else {
3664 chansize = 32;
3665 }
3666 tmp = RREG32(MC_SHARED_CHMAP);
3667 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3668 case 0:
3669 default:
3670 numchan = 1;
3671 break;
3672 case 1:
3673 numchan = 2;
3674 break;
3675 case 2:
3676 numchan = 4;
3677 break;
3678 case 3:
3679 numchan = 8;
3680 break;
3681 }
3682 rdev->mc.vram_width = numchan * chansize;
3683 /* Could aper size report 0 ? */
01d73a69
JC		 3684 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3685 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3686 /* Setup GPU memory space */
05b3ef69
AD		 3687 if ((rdev->family == CHIP_PALM) ||
3688 (rdev->family == CHIP_SUMO) ||
3689 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3690 /* size in bytes on fusion */
3691 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3692 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3693 } else {
05b3ef69 3694 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3695 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3696 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3697 }
51e5fcd3 3698 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3699 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3700 radeon_update_bandwidth_info(rdev);
3701
bcc1c2a1
AD		 3702 return 0;
3703}
d594e46a 3704
187e3593 3705void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3706{
64c56e8c 3707 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3708 RREG32(GRBM_STATUS));
64c56e8c 3709 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3710 RREG32(GRBM_STATUS_SE0));
64c56e8c 3711 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3712 RREG32(GRBM_STATUS_SE1));
64c56e8c 3713 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3714 RREG32(SRBM_STATUS));
a65a4369
AD		 3715 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3716 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3717 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3718 RREG32(CP_STALLED_STAT1));
3719 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3720 RREG32(CP_STALLED_STAT2));
3721 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3722 RREG32(CP_BUSY_STAT));
3723 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3724 RREG32(CP_STAT));
eaaa6983
JG		 3725 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3726 RREG32(DMA_STATUS_REG));
168757ea
AD		 3727 if (rdev->family >= CHIP_CAYMAN) {
3728 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3729 RREG32(DMA_STATUS_REG + 0x800));
3730 }
0ecebb9e
AD		 3731}
3732
168757ea 3733bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3734{
a65a4369
AD		 3735 u32 crtc_hung = 0;
3736 u32 crtc_status[6];
3737 u32 i, j, tmp;
3738
3739 for (i = 0; i < rdev->num_crtc; i++) {
3740 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3741 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3742 crtc_hung |= (1 << i);
3743 }
3744 }
3745
3746 for (j = 0; j < 10; j++) {
3747 for (i = 0; i < rdev->num_crtc; i++) {
3748 if (crtc_hung & (1 << i)) {
3749 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3750 if (tmp != crtc_status[i])
3751 crtc_hung &= ~(1 << i);
3752 }
3753 }
3754 if (crtc_hung == 0)
3755 return false;
3756 udelay(100);
3757 }
3758
3759 return true;
3760}
3761
2483b4ea 3762u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3763{
3764 u32 reset_mask = 0;
b7630473 3765 u32 tmp;
0ecebb9e 3766
a65a4369
AD		 3767 /* GRBM_STATUS */
3768 tmp = RREG32(GRBM_STATUS);
3769 if (tmp & (PA_BUSY | SC_BUSY |
3770 SH_BUSY | SX_BUSY |
3771 TA_BUSY | VGT_BUSY |
3772 DB_BUSY | CB_BUSY |
3773 SPI_BUSY | VGT_BUSY_NO_DMA))
3774 reset_mask |= RADEON_RESET_GFX;
3775
3776 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3777 CP_BUSY | CP_COHERENCY_BUSY))
3778 reset_mask |= RADEON_RESET_CP;
3779
3780 if (tmp & GRBM_EE_BUSY)
3781 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3782
a65a4369
AD		 3783 /* DMA_STATUS_REG */
3784 tmp = RREG32(DMA_STATUS_REG);
3785 if (!(tmp & DMA_IDLE))
3786 reset_mask |= RADEON_RESET_DMA;
3787
3788 /* SRBM_STATUS2 */
3789 tmp = RREG32(SRBM_STATUS2);
3790 if (tmp & DMA_BUSY)
3791 reset_mask |= RADEON_RESET_DMA;
3792
3793 /* SRBM_STATUS */
3794 tmp = RREG32(SRBM_STATUS);
3795 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3796 reset_mask |= RADEON_RESET_RLC;
3797
3798 if (tmp & IH_BUSY)
3799 reset_mask |= RADEON_RESET_IH;
3800
3801 if (tmp & SEM_BUSY)
3802 reset_mask |= RADEON_RESET_SEM;
3803
3804 if (tmp & GRBM_RQ_PENDING)
3805 reset_mask |= RADEON_RESET_GRBM;
3806
3807 if (tmp & VMC_BUSY)
3808 reset_mask |= RADEON_RESET_VMC;
3809
3810 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3811 MCC_BUSY | MCD_BUSY))
3812 reset_mask |= RADEON_RESET_MC;
3813
3814 if (evergreen_is_display_hung(rdev))
3815 reset_mask |= RADEON_RESET_DISPLAY;
3816
3817 /* VM_L2_STATUS */
3818 tmp = RREG32(VM_L2_STATUS);
3819 if (tmp & L2_BUSY)
3820 reset_mask |= RADEON_RESET_VMC;
3821
d808fc88
AD		 3822 /* Skip MC reset as it's mostly likely not hung, just busy */
3823 if (reset_mask & RADEON_RESET_MC) {
3824 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3825 reset_mask &= ~RADEON_RESET_MC;
3826 }
3827
a65a4369
AD		 3828 return reset_mask;
3829}
3830
3831static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3832{
3833 struct evergreen_mc_save save;
3834 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3835 u32 tmp;
19fc42ed 3836
0ecebb9e 3837 if (reset_mask == 0)
a65a4369 3838 return;
0ecebb9e
AD		 3839
3840 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3841
b7630473
AD		 3842 evergreen_print_gpu_status_regs(rdev);
3843
b7630473
AD		 3844 /* Disable CP parsing/prefetching */
3845 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3846
3847 if (reset_mask & RADEON_RESET_DMA) {
3848 /* Disable DMA */
3849 tmp = RREG32(DMA_RB_CNTL);
3850 tmp &= ~DMA_RB_ENABLE;
3851 WREG32(DMA_RB_CNTL, tmp);
3852 }
3853
b21b6e7a
AD		 3854 udelay(50);
3855
3856 evergreen_mc_stop(rdev, &save);
3857 if (evergreen_mc_wait_for_idle(rdev)) {
3858 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3859 }
3860
b7630473
AD		 3861 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3862 grbm_soft_reset |= SOFT_RESET_DB |
3863 SOFT_RESET_CB |
3864 SOFT_RESET_PA |
3865 SOFT_RESET_SC |
3866 SOFT_RESET_SPI |
3867 SOFT_RESET_SX |
3868 SOFT_RESET_SH |
3869 SOFT_RESET_TC |
3870 SOFT_RESET_TA |
3871 SOFT_RESET_VC |
3872 SOFT_RESET_VGT;
3873 }
3874
3875 if (reset_mask & RADEON_RESET_CP) {
3876 grbm_soft_reset |= SOFT_RESET_CP |
3877 SOFT_RESET_VGT;
3878
3879 srbm_soft_reset |= SOFT_RESET_GRBM;
3880 }
0ecebb9e
AD		 3881
3882 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3883 srbm_soft_reset |= SOFT_RESET_DMA;
3884
a65a4369
AD		 3885 if (reset_mask & RADEON_RESET_DISPLAY)
3886 srbm_soft_reset |= SOFT_RESET_DC;
3887
3888 if (reset_mask & RADEON_RESET_RLC)
3889 srbm_soft_reset |= SOFT_RESET_RLC;
3890
3891 if (reset_mask & RADEON_RESET_SEM)
3892 srbm_soft_reset |= SOFT_RESET_SEM;
3893
3894 if (reset_mask & RADEON_RESET_IH)
3895 srbm_soft_reset |= SOFT_RESET_IH;
3896
3897 if (reset_mask & RADEON_RESET_GRBM)
3898 srbm_soft_reset |= SOFT_RESET_GRBM;
3899
3900 if (reset_mask & RADEON_RESET_VMC)
3901 srbm_soft_reset |= SOFT_RESET_VMC;
3902
24178ec4
AD		 3903 if (!(rdev->flags & RADEON_IS_IGP)) {
3904 if (reset_mask & RADEON_RESET_MC)
3905 srbm_soft_reset |= SOFT_RESET_MC;
3906 }
a65a4369 3907
b7630473
AD		 3908 if (grbm_soft_reset) {
3909 tmp = RREG32(GRBM_SOFT_RESET);
3910 tmp |= grbm_soft_reset;
3911 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3912 WREG32(GRBM_SOFT_RESET, tmp);
3913 tmp = RREG32(GRBM_SOFT_RESET);
3914
3915 udelay(50);
3916
3917 tmp &= ~grbm_soft_reset;
3918 WREG32(GRBM_SOFT_RESET, tmp);
3919 tmp = RREG32(GRBM_SOFT_RESET);
3920 }
3921
3922 if (srbm_soft_reset) {
3923 tmp = RREG32(SRBM_SOFT_RESET);
3924 tmp |= srbm_soft_reset;
3925 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3926 WREG32(SRBM_SOFT_RESET, tmp);
3927 tmp = RREG32(SRBM_SOFT_RESET);
3928
3929 udelay(50);
3930
3931 tmp &= ~srbm_soft_reset;
3932 WREG32(SRBM_SOFT_RESET, tmp);
3933 tmp = RREG32(SRBM_SOFT_RESET);
3934 }
0ecebb9e
AD		 3935
3936 /* Wait a little for things to settle down */
3937 udelay(50);
3938
747943ea 3939 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3940 udelay(50);
3941
b7630473 3942 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3943}
3944
b5470b03
AD		 3945void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3946{
3947 struct evergreen_mc_save save;
3948 u32 tmp, i;
3949
3950 dev_info(rdev->dev, "GPU pci config reset\n");
3951
3952 /* disable dpm? */
3953
3954 /* Disable CP parsing/prefetching */
3955 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3956 udelay(50);
3957 /* Disable DMA */
3958 tmp = RREG32(DMA_RB_CNTL);
3959 tmp &= ~DMA_RB_ENABLE;
3960 WREG32(DMA_RB_CNTL, tmp);
3961 /* XXX other engines? */
3962
3963 /* halt the rlc */
3964 r600_rlc_stop(rdev);
3965
3966 udelay(50);
3967
3968 /* set mclk/sclk to bypass */
3969 rv770_set_clk_bypass_mode(rdev);
3970 /* disable BM */
3971 pci_clear_master(rdev->pdev);
3972 /* disable mem access */
3973 evergreen_mc_stop(rdev, &save);
3974 if (evergreen_mc_wait_for_idle(rdev)) {
3975 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3976 }
3977 /* reset */
3978 radeon_pci_config_reset(rdev);
3979 /* wait for asic to come out of reset */
3980 for (i = 0; i < rdev->usec_timeout; i++) {
3981 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3982 break;
3983 udelay(1);
3984 }
3985}
3986
a2d07b74 3987int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3988{
a65a4369
AD		 3989 u32 reset_mask;
3990
3991 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3992
3993 if (reset_mask)
3994 r600_set_bios_scratch_engine_hung(rdev, true);
3995
b5470b03 3996 /* try soft reset */
a65a4369
AD		 3997 evergreen_gpu_soft_reset(rdev, reset_mask);
3998
3999 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4000
b5470b03
AD		 4001 /* try pci config reset */
4002 if (reset_mask && radeon_hard_reset)
4003 evergreen_gpu_pci_config_reset(rdev);
4004
4005 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4006
a65a4369
AD		 4007 if (!reset_mask)
4008 r600_set_bios_scratch_engine_hung(rdev, false);
4009
4010 return 0;
747943ea
AD		 4011}
4012
123bc183
AD		 4013/**
4014 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4015 *
4016 * @rdev: radeon_device pointer
4017 * @ring: radeon_ring structure holding ring information
4018 *
4019 * Check if the GFX engine is locked up.
4020 * Returns true if the engine appears to be locked up, false if not.
4021 */
4022bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4023{
4024 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4025
4026 if (!(reset_mask & (RADEON_RESET_GFX |
4027 RADEON_RESET_COMPUTE |
4028 RADEON_RESET_CP))) {
ff212f25 4029 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 4030 return false;
4031 }
123bc183
AD		 4032 return radeon_ring_test_lockup(rdev, ring);
4033}
4034
2948f5e6
AD		 4035/*
4036 * RLC
4037 */
4038#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4039#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4040
4041void sumo_rlc_fini(struct radeon_device *rdev)
4042{
4043 int r;
4044
4045 /* save restore block */
4046 if (rdev->rlc.save_restore_obj) {
4047 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4048 if (unlikely(r != 0))
4049 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4050 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4051 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4052
4053 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4054 rdev->rlc.save_restore_obj = NULL;
4055 }
4056
4057 /* clear state block */
4058 if (rdev->rlc.clear_state_obj) {
4059 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4060 if (unlikely(r != 0))
4061 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4062 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4063 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4064
4065 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4066 rdev->rlc.clear_state_obj = NULL;
4067 }
22c775ce
AD		 4068
4069 /* clear state block */
4070 if (rdev->rlc.cp_table_obj) {
4071 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4072 if (unlikely(r != 0))
4073 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4074 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4075 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4076
4077 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4078 rdev->rlc.cp_table_obj = NULL;
4079 }
2948f5e6
AD		 4080}
4081
22c775ce
AD		 4082#define CP_ME_TABLE_SIZE 96
4083
2948f5e6
AD		 4084int sumo_rlc_init(struct radeon_device *rdev)
4085{
1fd11777 4086 const u32 *src_ptr;
2948f5e6
AD		 4087 volatile u32 *dst_ptr;
4088 u32 dws, data, i, j, k, reg_num;
59a82d0e 4089 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 4090 u64 reg_list_mc_addr;
1fd11777 4091 const struct cs_section_def *cs_data;
2948f5e6
AD		 4092 int r;
4093
4094 src_ptr = rdev->rlc.reg_list;
4095 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 4096 if (rdev->family >= CHIP_BONAIRE) {
4097 dws += (5 * 16) + 48 + 48 + 64;
4098 }
2948f5e6
AD		 4099 cs_data = rdev->rlc.cs_data;
4100
10b7ca7e
AD		 4101 if (src_ptr) {
4102 /* save restore block */
4103 if (rdev->rlc.save_restore_obj == NULL) {
4104 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4105 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4106 NULL, &rdev->rlc.save_restore_obj);
10b7ca7e
AD		 4107 if (r) {
4108 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4109 return r;
4110 }
4111 }
4112
4113 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4114 if (unlikely(r != 0)) {
4115 sumo_rlc_fini(rdev);
4116 return r;
4117 }
4118 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4119 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4120 if (r) {
10b7ca7e
AD		 4121 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4122 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4123 sumo_rlc_fini(rdev);
2948f5e6
AD		 4124 return r;
4125 }
2948f5e6 4126
10b7ca7e
AD		 4127 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4128 if (r) {
4129 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4130 sumo_rlc_fini(rdev);
4131 return r;
4132 }
4133 /* write the sr buffer */
4134 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4135 if (rdev->family >= CHIP_TAHITI) {
4136 /* SI */
59a82d0e 4137 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6ba81e53 4138 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
1fd11777
AD		 4139 } else {
4140 /* ON/LN/TN */
4141 /* format:
4142 * dw0: (reg2 << 16) | reg1
4143 * dw1: reg1 save space
4144 * dw2: reg2 save space
4145 */
4146 for (i = 0; i < dws; i++) {
4147 data = src_ptr[i] >> 2;
4148 i++;
4149 if (i < dws)
4150 data |= (src_ptr[i] >> 2) << 16;
4151 j = (((i - 1) * 3) / 2);
6ba81e53 4152 dst_ptr[j] = cpu_to_le32(data);
1fd11777
AD		 4153 }
4154 j = ((i * 3) / 2);
6ba81e53 4155 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
10b7ca7e 4156 }
10b7ca7e 4157 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4158 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4159 }
2948f5e6 4160
10b7ca7e
AD		 4161 if (cs_data) {
4162 /* clear state block */
a0f38609
AD		 4163 if (rdev->family >= CHIP_BONAIRE) {
4164 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4165 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4166 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4167 dws = rdev->rlc.clear_state_size + (256 / 4);
4168 } else {
4169 reg_list_num = 0;
4170 dws = 0;
4171 for (i = 0; cs_data[i].section != NULL; i++) {
4172 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4173 reg_list_num++;
4174 dws += cs_data[i].section[j].reg_count;
4175 }
10b7ca7e 4176 }
59a82d0e
AD		 4177 reg_list_blk_index = (3 * reg_list_num + 2);
4178 dws += reg_list_blk_index;
4179 rdev->rlc.clear_state_size = dws;
2948f5e6 4180 }
2948f5e6 4181
10b7ca7e 4182 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4183 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4184 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4185 NULL, &rdev->rlc.clear_state_obj);
10b7ca7e
AD		 4186 if (r) {
4187 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4188 sumo_rlc_fini(rdev);
4189 return r;
4190 }
4191 }
4192 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4193 if (unlikely(r != 0)) {
4194 sumo_rlc_fini(rdev);
4195 return r;
4196 }
4197 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4198 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4199 if (r) {
10b7ca7e
AD		 4200 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4201 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4202 sumo_rlc_fini(rdev);
4203 return r;
4204 }
2948f5e6 4205
10b7ca7e
AD		 4206 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4207 if (r) {
4208 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4209 sumo_rlc_fini(rdev);
4210 return r;
4211 }
4212 /* set up the cs buffer */
4213 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4214 if (rdev->family >= CHIP_BONAIRE) {
4215 cik_get_csb_buffer(rdev, dst_ptr);
4216 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e 4217 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
6ba81e53
AD		 4218 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4219 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4220 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
59a82d0e
AD		 4221 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4222 } else {
4223 reg_list_hdr_blk_index = 0;
4224 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4225 data = upper_32_bits(reg_list_mc_addr);
6ba81e53 4226 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
2948f5e6 4227 reg_list_hdr_blk_index++;
59a82d0e
AD		 4228 for (i = 0; cs_data[i].section != NULL; i++) {
4229 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4230 reg_num = cs_data[i].section[j].reg_count;
4231 data = reg_list_mc_addr & 0xffffffff;
6ba81e53 4232 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4233 reg_list_hdr_blk_index++;
4234
4235 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
6ba81e53 4236 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4237 reg_list_hdr_blk_index++;
4238
4239 data = 0x08000000 | (reg_num * 4);
6ba81e53 4240 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4241 reg_list_hdr_blk_index++;
4242
4243 for (k = 0; k < reg_num; k++) {
4244 data = cs_data[i].section[j].extent[k];
6ba81e53 4245 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
59a82d0e
AD		 4246 }
4247 reg_list_mc_addr += reg_num * 4;
4248 reg_list_blk_index += reg_num;
10b7ca7e 4249 }
2948f5e6 4250 }
6ba81e53 4251 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
2948f5e6 4252 }
10b7ca7e
AD		 4253 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4254 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4255 }
2948f5e6 4256
22c775ce
AD		 4257 if (rdev->rlc.cp_table_size) {
4258 if (rdev->rlc.cp_table_obj == NULL) {
02376d82
MD		 4259 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4260 PAGE_SIZE, true,
4261 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4262 NULL, &rdev->rlc.cp_table_obj);
22c775ce
AD		 4263 if (r) {
4264 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4265 sumo_rlc_fini(rdev);
4266 return r;
4267 }
4268 }
4269
4270 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4271 if (unlikely(r != 0)) {
4272 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4273 sumo_rlc_fini(rdev);
4274 return r;
4275 }
4276 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4277 &rdev->rlc.cp_table_gpu_addr);
4278 if (r) {
4279 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4280 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4281 sumo_rlc_fini(rdev);
4282 return r;
4283 }
4284 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4285 if (r) {
4286 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4287 sumo_rlc_fini(rdev);
4288 return r;
4289 }
4290
4291 cik_init_cp_pg_table(rdev);
4292
4293 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4294 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4295
4296 }
2948f5e6
AD		 4297
4298 return 0;
4299}
4300
4301static void evergreen_rlc_start(struct radeon_device *rdev)
4302{
8ba10463
AD		 4303 u32 mask = RLC_ENABLE;
4304
4305 if (rdev->flags & RADEON_IS_IGP) {
4306 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4307 }
4308
4309 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4310}
4311
4312int evergreen_rlc_resume(struct radeon_device *rdev)
4313{
4314 u32 i;
4315 const __be32 *fw_data;
4316
4317 if (!rdev->rlc_fw)
4318 return -EINVAL;
4319
4320 r600_rlc_stop(rdev);
4321
4322 WREG32(RLC_HB_CNTL, 0);
4323
4324 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4325 if (rdev->family == CHIP_ARUBA) {
4326 u32 always_on_bitmap =
4327 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4328 /* find out the number of active simds */
4329 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4330 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4331 tmp = hweight32(~tmp);
4332 if (tmp == rdev->config.cayman.max_simds_per_se) {
4333 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4334 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4335 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4336 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4337 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4338 }
4339 } else {
4340 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4341 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4342 }
2948f5e6
AD		 4343 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4344 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4345 } else {
4346 WREG32(RLC_HB_BASE, 0);
4347 WREG32(RLC_HB_RPTR, 0);
4348 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4349 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4350 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4351 }
2948f5e6
AD		 4352 WREG32(RLC_MC_CNTL, 0);
4353 WREG32(RLC_UCODE_CNTL, 0);
4354
4355 fw_data = (const __be32 *)rdev->rlc_fw->data;
4356 if (rdev->family >= CHIP_ARUBA) {
4357 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4358 WREG32(RLC_UCODE_ADDR, i);
4359 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4360 }
4361 } else if (rdev->family >= CHIP_CAYMAN) {
4362 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4363 WREG32(RLC_UCODE_ADDR, i);
4364 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4365 }
4366 } else {
4367 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4368 WREG32(RLC_UCODE_ADDR, i);
4369 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4370 }
4371 }
4372 WREG32(RLC_UCODE_ADDR, 0);
4373
4374 evergreen_rlc_start(rdev);
4375
4376 return 0;
4377}
4378
45f9a39b
AD		 4379/* Interrupts */
4380
4381u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4382{
46437057 4383 if (crtc >= rdev->num_crtc)
45f9a39b 4384 return 0;
46437057
AD		 4385 else
4386 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4387}
4388
4389void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4390{
4391 u32 tmp;
4392
1b37078b
AD		 4393 if (rdev->family >= CHIP_CAYMAN) {
4394 cayman_cp_int_cntl_setup(rdev, 0,
4395 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4396 cayman_cp_int_cntl_setup(rdev, 1, 0);
4397 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4398 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4399 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4400 } else
4401 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4402 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4403 WREG32(DMA_CNTL, tmp);
45f9a39b 4404 WREG32(GRBM_INT_CNTL, 0);
acc1522a 4405 WREG32(SRBM_INT_CNTL, 0);
45f9a39b
AD		 4406 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4407 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4408 if (rdev->num_crtc >= 4) {
18007401
AD		 4409 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4410 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4411 }
4412 if (rdev->num_crtc >= 6) {
18007401
AD		 4413 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4414 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4415 }
45f9a39b
AD		 4416
4417 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4418 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4419 if (rdev->num_crtc >= 4) {
18007401
AD		 4420 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4421 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4422 }
4423 if (rdev->num_crtc >= 6) {
18007401
AD		 4424 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4425 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4426 }
45f9a39b 4427
e9a321c6
AD		 4428 /* only one DAC on DCE5 */
4429 if (!ASIC_IS_DCE5(rdev))
05b3ef69 4430 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4431 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4432
4433 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4434 WREG32(DC_HPD1_INT_CONTROL, tmp);
4435 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4436 WREG32(DC_HPD2_INT_CONTROL, tmp);
4437 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4438 WREG32(DC_HPD3_INT_CONTROL, tmp);
4439 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4440 WREG32(DC_HPD4_INT_CONTROL, tmp);
4441 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4442 WREG32(DC_HPD5_INT_CONTROL, tmp);
4443 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4444 WREG32(DC_HPD6_INT_CONTROL, tmp);
4445
4446}
4447
4448int evergreen_irq_set(struct radeon_device *rdev)
4449{
4450 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4451 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4452 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4453 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4454 u32 grbm_int_cntl = 0;
f122c610 4455 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4456 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4457 u32 thermal_int = 0;
45f9a39b
AD		 4458
4459 if (!rdev->irq.installed) {
fce7d61b 4460 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4461 return -EINVAL;
4462 }
4463 /* don't enable anything if the ih is disabled */
4464 if (!rdev->ih.enabled) {
4465 r600_disable_interrupts(rdev);
4466 /* force the active interrupt state to all disabled */
4467 evergreen_disable_interrupt_state(rdev);
4468 return 0;
4469 }
4470
2bc67b4d
DA		 4471 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4472 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4473 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4474 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4475 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4476 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
d70229f7
AD		 4477 if (rdev->family == CHIP_ARUBA)
4478 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4479 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4480 else
4481 thermal_int = RREG32(CG_THERMAL_INT) &
4482 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4483
f122c610
AD		 4484 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4485 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4486 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4487 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4488 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4489 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4490
233d1ad5
AD		 4491 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4492
1b37078b
AD		 4493 if (rdev->family >= CHIP_CAYMAN) {
4494 /* enable CP interrupts on all rings */
736fc37f 4495 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4496 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4497 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4498 }
736fc37f 4499 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4500 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4501 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4502 }
736fc37f 4503 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4504 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4505 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4506 }
4507 } else {
736fc37f 4508 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4509 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4510 cp_int_cntl |= RB_INT_ENABLE;
4511 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4512 }
45f9a39b 4513 }
1b37078b 4514
233d1ad5
AD		 4515 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4516 DRM_DEBUG("r600_irq_set: sw int dma\n");
4517 dma_cntl |= TRAP_ENABLE;
4518 }
4519
f60cbd11
AD		 4520 if (rdev->family >= CHIP_CAYMAN) {
4521 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4522 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4523 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4524 dma_cntl1 |= TRAP_ENABLE;
4525 }
4526 }
4527
dc50ba7f
AD		 4528 if (rdev->irq.dpm_thermal) {
4529 DRM_DEBUG("dpm thermal\n");
4530 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4531 }
4532
6f34be50 4533 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4534 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4535 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4536 crtc1 |= VBLANK_INT_MASK;
4537 }
6f34be50 4538 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4539 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4540 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4541 crtc2 |= VBLANK_INT_MASK;
4542 }
6f34be50 4543 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4544 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4545 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4546 crtc3 |= VBLANK_INT_MASK;
4547 }
6f34be50 4548 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4549 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4550 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4551 crtc4 |= VBLANK_INT_MASK;
4552 }
6f34be50 4553 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4554 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4555 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4556 crtc5 |= VBLANK_INT_MASK;
4557 }
6f34be50 4558 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4559 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4560 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4561 crtc6 |= VBLANK_INT_MASK;
4562 }
4563 if (rdev->irq.hpd[0]) {
4564 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2bc67b4d 4565 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4566 }
4567 if (rdev->irq.hpd[1]) {
4568 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2bc67b4d 4569 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4570 }
4571 if (rdev->irq.hpd[2]) {
4572 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2bc67b4d 4573 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4574 }
4575 if (rdev->irq.hpd[3]) {
4576 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2bc67b4d 4577 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4578 }
4579 if (rdev->irq.hpd[4]) {
4580 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2bc67b4d 4581 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b
AD		 4582 }
4583 if (rdev->irq.hpd[5]) {
4584 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2bc67b4d 4585 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
45f9a39b 4586 }
f122c610
AD		 4587 if (rdev->irq.afmt[0]) {
4588 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4589 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4590 }
4591 if (rdev->irq.afmt[1]) {
4592 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4593 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4594 }
4595 if (rdev->irq.afmt[2]) {
4596 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4597 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4598 }
4599 if (rdev->irq.afmt[3]) {
4600 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4601 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4602 }
4603 if (rdev->irq.afmt[4]) {
4604 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4605 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4606 }
4607 if (rdev->irq.afmt[5]) {
4608 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4609 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4610 }
45f9a39b 4611
1b37078b
AD		 4612 if (rdev->family >= CHIP_CAYMAN) {
4613 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4614 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4615 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4616 } else
4617 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4618
4619 WREG32(DMA_CNTL, dma_cntl);
4620
f60cbd11
AD		 4621 if (rdev->family >= CHIP_CAYMAN)
4622 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4623
2031f77c 4624 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4625
4626 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4627 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4628 if (rdev->num_crtc >= 4) {
18007401
AD		 4629 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4630 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4631 }
4632 if (rdev->num_crtc >= 6) {
18007401
AD		 4633 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4634 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4635 }
45f9a39b 4636
f5d636d2
CK		 4637 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4638 GRPH_PFLIP_INT_MASK);
4639 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4640 GRPH_PFLIP_INT_MASK);
b7eff394 4641 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 4642 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4643 GRPH_PFLIP_INT_MASK);
4644 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4645 GRPH_PFLIP_INT_MASK);
b7eff394
AD		 4646 }
4647 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 4648 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4649 GRPH_PFLIP_INT_MASK);
4650 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4651 GRPH_PFLIP_INT_MASK);
b7eff394 4652 }
6f34be50 4653
45f9a39b
AD		 4654 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4655 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4656 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4657 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4658 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4659 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4660 if (rdev->family == CHIP_ARUBA)
4661 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4662 else
4663 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4664
f122c610
AD		 4665 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4666 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4667 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4668 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4669 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4670 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4671
c320bb5f
AD		 4672 /* posting read */
4673 RREG32(SRBM_STATUS);
4674
bcc1c2a1
AD		 4675 return 0;
4676}
4677
cbdd4501 4678static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4679{
4680 u32 tmp;
4681
6f34be50
AD		 4682 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4683 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4684 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4685 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4686 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4687 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4688 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4689 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4690 if (rdev->num_crtc >= 4) {
4691 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4692 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4693 }
4694 if (rdev->num_crtc >= 6) {
4695 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4696 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4697 }
6f34be50 4698
f122c610
AD		 4699 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4700 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4701 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4702 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4703 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4704 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4705
6f34be50
AD		 4706 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4707 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4708 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4709 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4710 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4711 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4712 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4713 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4714 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4715 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4716 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4717 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4718
b7eff394
AD		 4719 if (rdev->num_crtc >= 4) {
4720 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4721 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4722 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4723 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4724 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4725 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4726 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4727 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4728 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4729 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4730 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4731 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4732 }
4733
4734 if (rdev->num_crtc >= 6) {
4735 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4736 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4737 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4738 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4739 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4740 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4741 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4742 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4743 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4744 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4745 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4746 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4747 }
45f9a39b 4748
6f34be50 4749 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4750 tmp = RREG32(DC_HPD1_INT_CONTROL);
4751 tmp |= DC_HPDx_INT_ACK;
4752 WREG32(DC_HPD1_INT_CONTROL, tmp);
4753 }
6f34be50 4754 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4755 tmp = RREG32(DC_HPD2_INT_CONTROL);
4756 tmp |= DC_HPDx_INT_ACK;
4757 WREG32(DC_HPD2_INT_CONTROL, tmp);
4758 }
6f34be50 4759 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4760 tmp = RREG32(DC_HPD3_INT_CONTROL);
4761 tmp |= DC_HPDx_INT_ACK;
4762 WREG32(DC_HPD3_INT_CONTROL, tmp);
4763 }
6f34be50 4764 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4765 tmp = RREG32(DC_HPD4_INT_CONTROL);
4766 tmp |= DC_HPDx_INT_ACK;
4767 WREG32(DC_HPD4_INT_CONTROL, tmp);
4768 }
6f34be50 4769 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4770 tmp = RREG32(DC_HPD5_INT_CONTROL);
4771 tmp |= DC_HPDx_INT_ACK;
4772 WREG32(DC_HPD5_INT_CONTROL, tmp);
4773 }
6f34be50 4774 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4775 tmp = RREG32(DC_HPD5_INT_CONTROL);
4776 tmp |= DC_HPDx_INT_ACK;
4777 WREG32(DC_HPD6_INT_CONTROL, tmp);
4778 }
2bc67b4d
DA		 4779
4780 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4781 tmp = RREG32(DC_HPD1_INT_CONTROL);
4782 tmp |= DC_HPDx_RX_INT_ACK;
4783 WREG32(DC_HPD1_INT_CONTROL, tmp);
4784 }
4785 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4786 tmp = RREG32(DC_HPD2_INT_CONTROL);
4787 tmp |= DC_HPDx_RX_INT_ACK;
4788 WREG32(DC_HPD2_INT_CONTROL, tmp);
4789 }
4790 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4791 tmp = RREG32(DC_HPD3_INT_CONTROL);
4792 tmp |= DC_HPDx_RX_INT_ACK;
4793 WREG32(DC_HPD3_INT_CONTROL, tmp);
4794 }
4795 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4796 tmp = RREG32(DC_HPD4_INT_CONTROL);
4797 tmp |= DC_HPDx_RX_INT_ACK;
4798 WREG32(DC_HPD4_INT_CONTROL, tmp);
4799 }
4800 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4801 tmp = RREG32(DC_HPD5_INT_CONTROL);
4802 tmp |= DC_HPDx_RX_INT_ACK;
4803 WREG32(DC_HPD5_INT_CONTROL, tmp);
4804 }
4805 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4806 tmp = RREG32(DC_HPD5_INT_CONTROL);
4807 tmp |= DC_HPDx_RX_INT_ACK;
4808 WREG32(DC_HPD6_INT_CONTROL, tmp);
4809 }
4810
f122c610
AD		 4811 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4812 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4813 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4814 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4815 }
4816 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4817 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4818 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4819 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4820 }
4821 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4822 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4823 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4824 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4825 }
4826 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4827 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4828 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4829 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4830 }
4831 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4832 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4833 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4834 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4835 }
4836 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4837 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4838 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4839 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4840 }
45f9a39b
AD		 4841}
4842
1109ca09 4843static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4844{
45f9a39b
AD		 4845 r600_disable_interrupts(rdev);
4846 /* Wait and acknowledge irq */
4847 mdelay(1);
6f34be50 4848 evergreen_irq_ack(rdev);
45f9a39b
AD		 4849 evergreen_disable_interrupt_state(rdev);
4850}
4851
755d819e 4852void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4853{
4854 evergreen_irq_disable(rdev);
4855 r600_rlc_stop(rdev);
4856}
4857
cbdd4501 4858static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4859{
4860 u32 wptr, tmp;
4861
724c80e1 4862 if (rdev->wb.enabled)
204ae24d 4863 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4864 else
4865 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4866
4867 if (wptr & RB_OVERFLOW) {
11bab0ae 4868 wptr &= ~RB_OVERFLOW;
45f9a39b
AD		 4869 /* When a ring buffer overflow happen start parsing interrupt
4870 * from the last not overwritten vector (wptr + 16). Hopefully
4871 * this should allow us to catchup.
4872 */
6cc2fda2
MD		 4873 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4874 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
45f9a39b
AD		 4875 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4876 tmp = RREG32(IH_RB_CNTL);
4877 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4878 WREG32(IH_RB_CNTL, tmp);
4879 }
4880 return (wptr & rdev->ih.ptr_mask);
4881}
4882
4883int evergreen_irq_process(struct radeon_device *rdev)
4884{
682f1a54
DA		 4885 u32 wptr;
4886 u32 rptr;
45f9a39b
AD		 4887 u32 src_id, src_data;
4888 u32 ring_index;
45f9a39b 4889 bool queue_hotplug = false;
f122c610 4890 bool queue_hdmi = false;
2bc67b4d 4891 bool queue_dp = false;
dc50ba7f 4892 bool queue_thermal = false;
54e2e49c 4893 u32 status, addr;
45f9a39b 4894
682f1a54 4895 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4896 return IRQ_NONE;
4897
682f1a54 4898 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4899
4900restart_ih:
4901 /* is somebody else already processing irqs? */
4902 if (atomic_xchg(&rdev->ih.lock, 1))
4903 return IRQ_NONE;
4904
682f1a54 4905 rptr = rdev->ih.rptr;
07f18f0b 4906 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4907
964f6645
BH		 4908 /* Order reading of wptr vs. reading of IH ring data */
4909 rmb();
4910
45f9a39b 4911 /* display interrupts */
6f34be50 4912 evergreen_irq_ack(rdev);
45f9a39b 4913
45f9a39b
AD		 4914 while (rptr != wptr) {
4915 /* wptr/rptr are in bytes! */
4916 ring_index = rptr / 4;
0f234f5f
AD		 4917 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4918 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4919
4920 switch (src_id) {
4921 case 1: /* D1 vblank/vline */
4922 switch (src_data) {
4923 case 0: /* D1 vblank */
07f18f0b
MK		 4924 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
4925 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
4926
4927 if (rdev->irq.crtc_vblank_int[0]) {
4928 drm_handle_vblank(rdev->ddev, 0);
4929 rdev->pm.vblank_sync = true;
4930 wake_up(&rdev->irq.vblank_queue);
45f9a39b 4931 }
07f18f0b
MK		 4932 if (atomic_read(&rdev->irq.pflip[0]))
4933 radeon_crtc_handle_vblank(rdev, 0);
4934 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4935 DRM_DEBUG("IH: D1 vblank\n");
4936
45f9a39b
AD		 4937 break;
4938 case 1: /* D1 vline */
07f18f0b
MK		 4939 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
4940 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
4941
4942 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4943 DRM_DEBUG("IH: D1 vline\n");
4944
45f9a39b
AD		 4945 break;
4946 default:
4947 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4948 break;
4949 }
4950 break;
4951 case 2: /* D2 vblank/vline */
4952 switch (src_data) {
4953 case 0: /* D2 vblank */
07f18f0b
MK		 4954 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
4955 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
4956
4957 if (rdev->irq.crtc_vblank_int[1]) {
4958 drm_handle_vblank(rdev->ddev, 1);
4959 rdev->pm.vblank_sync = true;
4960 wake_up(&rdev->irq.vblank_queue);
45f9a39b 4961 }
07f18f0b
MK		 4962 if (atomic_read(&rdev->irq.pflip[1]))
4963 radeon_crtc_handle_vblank(rdev, 1);
4964 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4965 DRM_DEBUG("IH: D2 vblank\n");
4966
45f9a39b
AD		 4967 break;
4968 case 1: /* D2 vline */
07f18f0b
MK		 4969 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
4970 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
4971
4972 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4973 DRM_DEBUG("IH: D2 vline\n");
4974
45f9a39b
AD		 4975 break;
4976 default:
4977 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4978 break;
4979 }
4980 break;
4981 case 3: /* D3 vblank/vline */
4982 switch (src_data) {
4983 case 0: /* D3 vblank */
07f18f0b
MK		 4984 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
4985 DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
4986
4987 if (rdev->irq.crtc_vblank_int[2]) {
4988 drm_handle_vblank(rdev->ddev, 2);
4989 rdev->pm.vblank_sync = true;
4990 wake_up(&rdev->irq.vblank_queue);
45f9a39b 4991 }
07f18f0b
MK		 4992 if (atomic_read(&rdev->irq.pflip[2]))
4993 radeon_crtc_handle_vblank(rdev, 2);
4994 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4995 DRM_DEBUG("IH: D3 vblank\n");
4996
45f9a39b
AD		 4997 break;
4998 case 1: /* D3 vline */
07f18f0b
MK		 4999 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
5000 DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
5001
5002 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5003 DRM_DEBUG("IH: D3 vline\n");
5004
45f9a39b
AD		 5005 break;
5006 default:
5007 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5008 break;
5009 }
5010 break;
5011 case 4: /* D4 vblank/vline */
5012 switch (src_data) {
5013 case 0: /* D4 vblank */
07f18f0b
MK		 5014 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5015 DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5016
5017 if (rdev->irq.crtc_vblank_int[3]) {
5018 drm_handle_vblank(rdev->ddev, 3);
5019 rdev->pm.vblank_sync = true;
5020 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5021 }
07f18f0b
MK		 5022 if (atomic_read(&rdev->irq.pflip[3]))
5023 radeon_crtc_handle_vblank(rdev, 3);
5024 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5025 DRM_DEBUG("IH: D4 vblank\n");
5026
45f9a39b
AD		 5027 break;
5028 case 1: /* D4 vline */
07f18f0b
MK		 5029 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5030 DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5031
5032 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5033 DRM_DEBUG("IH: D4 vline\n");
5034
45f9a39b
AD		 5035 break;
5036 default:
5037 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5038 break;
5039 }
5040 break;
5041 case 5: /* D5 vblank/vline */
5042 switch (src_data) {
5043 case 0: /* D5 vblank */
07f18f0b
MK		 5044 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5045 DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5046
5047 if (rdev->irq.crtc_vblank_int[4]) {
5048 drm_handle_vblank(rdev->ddev, 4);
5049 rdev->pm.vblank_sync = true;
5050 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5051 }
07f18f0b
MK		 5052 if (atomic_read(&rdev->irq.pflip[4]))
5053 radeon_crtc_handle_vblank(rdev, 4);
5054 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5055 DRM_DEBUG("IH: D5 vblank\n");
5056
45f9a39b
AD		 5057 break;
5058 case 1: /* D5 vline */
07f18f0b
MK		 5059 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5060 DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5061
5062 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5063 DRM_DEBUG("IH: D5 vline\n");
5064
45f9a39b
AD		 5065 break;
5066 default:
5067 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5068 break;
5069 }
5070 break;
5071 case 6: /* D6 vblank/vline */
5072 switch (src_data) {
5073 case 0: /* D6 vblank */
07f18f0b
MK		 5074 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5075 DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5076
5077 if (rdev->irq.crtc_vblank_int[5]) {
5078 drm_handle_vblank(rdev->ddev, 5);
5079 rdev->pm.vblank_sync = true;
5080 wake_up(&rdev->irq.vblank_queue);
45f9a39b 5081 }
07f18f0b
MK		 5082 if (atomic_read(&rdev->irq.pflip[5]))
5083 radeon_crtc_handle_vblank(rdev, 5);
5084 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5085 DRM_DEBUG("IH: D6 vblank\n");
5086
45f9a39b
AD		 5087 break;
5088 case 1: /* D6 vline */
07f18f0b
MK		 5089 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5090 DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5091
5092 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5093 DRM_DEBUG("IH: D6 vline\n");
5094
45f9a39b
AD		 5095 break;
5096 default:
5097 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5098 break;
5099 }
5100 break;
f5d636d2
CK		 5101 case 8: /* D1 page flip */
5102 case 10: /* D2 page flip */
5103 case 12: /* D3 page flip */
5104 case 14: /* D4 page flip */
5105 case 16: /* D5 page flip */
5106 case 18: /* D6 page flip */
5107 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 5108 if (radeon_use_pflipirq > 0)
5109 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 5110 break;
45f9a39b
AD		 5111 case 42: /* HPD hotplug */
5112 switch (src_data) {
5113 case 0:
07f18f0b
MK		 5114 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5115 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5116
5117 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5118 queue_hotplug = true;
5119 DRM_DEBUG("IH: HPD1\n");
45f9a39b
AD		 5120 break;
5121 case 1:
07f18f0b
MK		 5122 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5123 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5124
5125 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5126 queue_hotplug = true;
5127 DRM_DEBUG("IH: HPD2\n");
45f9a39b
AD		 5128 break;
5129 case 2:
07f18f0b
MK		 5130 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5131 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5132
5133 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5134 queue_hotplug = true;
5135 DRM_DEBUG("IH: HPD3\n");
45f9a39b
AD		 5136 break;
5137 case 3:
07f18f0b
MK		 5138 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5139 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5140
5141 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5142 queue_hotplug = true;
5143 DRM_DEBUG("IH: HPD4\n");
45f9a39b
AD		 5144 break;
5145 case 4:
07f18f0b
MK		 5146 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5147 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5148
5149 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5150 queue_hotplug = true;
5151 DRM_DEBUG("IH: HPD5\n");
45f9a39b
AD		 5152 break;
5153 case 5:
07f18f0b
MK		 5154 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5155 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5156
5157 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5158 queue_hotplug = true;
5159 DRM_DEBUG("IH: HPD6\n");
45f9a39b 5160 break;
2bc67b4d 5161 case 6:
07f18f0b
MK		 5162 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5163 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5164
5165 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5166 queue_dp = true;
5167 DRM_DEBUG("IH: HPD_RX 1\n");
2bc67b4d
DA		 5168 break;
5169 case 7:
07f18f0b
MK		 5170 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5171 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5172
5173 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5174 queue_dp = true;
5175 DRM_DEBUG("IH: HPD_RX 2\n");
2bc67b4d
DA		 5176 break;
5177 case 8:
07f18f0b
MK		 5178 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5179 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5180
5181 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5182 queue_dp = true;
5183 DRM_DEBUG("IH: HPD_RX 3\n");
2bc67b4d
DA		 5184 break;
5185 case 9:
07f18f0b
MK		 5186 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5187 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5188
5189 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5190 queue_dp = true;
5191 DRM_DEBUG("IH: HPD_RX 4\n");
2bc67b4d
DA		 5192 break;
5193 case 10:
07f18f0b
MK		 5194 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5195 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5196
5197 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5198 queue_dp = true;
5199 DRM_DEBUG("IH: HPD_RX 5\n");
2bc67b4d
DA		 5200 break;
5201 case 11:
07f18f0b
MK		 5202 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5203 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5204
5205 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5206 queue_dp = true;
5207 DRM_DEBUG("IH: HPD_RX 6\n");
2bc67b4d 5208 break;
45f9a39b
AD		 5209 default:
5210 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5211 break;
5212 }
5213 break;
f122c610
AD		 5214 case 44: /* hdmi */
5215 switch (src_data) {
5216 case 0:
07f18f0b
MK		 5217 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5218 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5219
5220 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5221 queue_hdmi = true;
5222 DRM_DEBUG("IH: HDMI0\n");
f122c610
AD		 5223 break;
5224 case 1:
07f18f0b
MK		 5225 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5226 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5227
5228 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5229 queue_hdmi = true;
5230 DRM_DEBUG("IH: HDMI1\n");
f122c610
AD		 5231 break;
5232 case 2:
07f18f0b
MK		 5233 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5234 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5235
5236 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5237 queue_hdmi = true;
5238 DRM_DEBUG("IH: HDMI2\n");
f122c610
AD		 5239 break;
5240 case 3:
07f18f0b
MK		 5241 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5242 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5243
5244 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5245 queue_hdmi = true;
5246 DRM_DEBUG("IH: HDMI3\n");
f122c610
AD		 5247 break;
5248 case 4:
07f18f0b
MK		 5249 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5250 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5251
5252 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5253 queue_hdmi = true;
5254 DRM_DEBUG("IH: HDMI4\n");
f122c610
AD		 5255 break;
5256 case 5:
07f18f0b
MK		 5257 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5258 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5259
5260 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5261 queue_hdmi = true;
5262 DRM_DEBUG("IH: HDMI5\n");
f122c610
AD		 5263 break;
5264 default:
5265 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5266 break;
5267 }
acc1522a
CK		 5268 case 96:
5269 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5270 WREG32(SRBM_INT_ACK, 0x1);
5271 break;
f2ba57b5
CK		 5272 case 124: /* UVD */
5273 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5274 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5275 break;
ae133a11
CK		 5276 case 146:
5277 case 147:
54e2e49c
AD		 5278 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5279 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 5280 /* reset addr and status */
5281 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5282 if (addr == 0x0 && status == 0x0)
5283 break;
ae133a11
CK		 5284 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5285 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5286 addr);
ae133a11 5287 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5288 status);
5289 cayman_vm_decode_fault(rdev, status, addr);
ae133a11 5290 break;
45f9a39b
AD		 5291 case 176: /* CP_INT in ring buffer */
5292 case 177: /* CP_INT in IB1 */
5293 case 178: /* CP_INT in IB2 */
5294 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5295 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5296 break;
5297 case 181: /* CP EOP event */
5298 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5299 if (rdev->family >= CHIP_CAYMAN) {
5300 switch (src_data) {
5301 case 0:
5302 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5303 break;
5304 case 1:
5305 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5306 break;
5307 case 2:
5308 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5309 break;
5310 }
5311 } else
5312 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5313 break;
233d1ad5
AD		 5314 case 224: /* DMA trap event */
5315 DRM_DEBUG("IH: DMA trap\n");
5316 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5317 break;
dc50ba7f
AD		 5318 case 230: /* thermal low to high */
5319 DRM_DEBUG("IH: thermal low to high\n");
5320 rdev->pm.dpm.thermal.high_to_low = false;
5321 queue_thermal = true;
5322 break;
5323 case 231: /* thermal high to low */
5324 DRM_DEBUG("IH: thermal high to low\n");
5325 rdev->pm.dpm.thermal.high_to_low = true;
5326 queue_thermal = true;
5327 break;
2031f77c 5328 case 233: /* GUI IDLE */
303c805c 5329 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5330 break;
f60cbd11
AD		 5331 case 244: /* DMA trap event */
5332 if (rdev->family >= CHIP_CAYMAN) {
5333 DRM_DEBUG("IH: DMA1 trap\n");
5334 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5335 }
5336 break;
45f9a39b
AD		 5337 default:
5338 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5339 break;
5340 }
5341
5342 /* wptr/rptr are in bytes! */
5343 rptr += 16;
5344 rptr &= rdev->ih.ptr_mask;
f55e03b9 5345 WREG32(IH_RB_RPTR, rptr);
45f9a39b 5346 }
2bc67b4d
DA		 5347 if (queue_dp)
5348 schedule_work(&rdev->dp_work);
45f9a39b 5349 if (queue_hotplug)
cb5d4166 5350 schedule_delayed_work(&rdev->hotplug_work, 0);
f122c610
AD		 5351 if (queue_hdmi)
5352 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5353 if (queue_thermal && rdev->pm.dpm_enabled)
5354 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b 5355 rdev->ih.rptr = rptr;
c20dc369
CK		 5356 atomic_set(&rdev->ih.lock, 0);
5357
5358 /* make sure wptr hasn't changed while processing */
5359 wptr = evergreen_get_ih_wptr(rdev);
5360 if (wptr != rptr)
5361 goto restart_ih;
5362
45f9a39b
AD		 5363 return IRQ_HANDLED;
5364}
5365
bcc1c2a1
AD		 5366static int evergreen_startup(struct radeon_device *rdev)
5367{
f2ba57b5 5368 struct radeon_ring *ring;
bcc1c2a1
AD		 5369 int r;
5370
9e46a48d 5371 /* enable pcie gen2 link */
cd54033a 5372 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5373 /* enable aspm */
5374 evergreen_program_aspm(rdev);
9e46a48d 5375
e5903d39
AD		 5376 /* scratch needs to be initialized before MC */
5377 r = r600_vram_scratch_init(rdev);
5378 if (r)
5379 return r;
5380
6fab3feb
AD		 5381 evergreen_mc_program(rdev);
5382
6c7bccea 5383 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
755d819e 5384 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5385 if (r) {
0af62b01 5386 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5387 return r;
5388 }
5389 }
fe251e2f 5390
bcc1c2a1 5391 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5392 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5393 } else {
5394 r = evergreen_pcie_gart_enable(rdev);
5395 if (r)
5396 return r;
5397 }
bcc1c2a1 5398 evergreen_gpu_init(rdev);
bcc1c2a1 5399
2948f5e6
AD		 5400 /* allocate rlc buffers */
5401 if (rdev->flags & RADEON_IS_IGP) {
5402 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5403 rdev->rlc.reg_list_size =
5404 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5405 rdev->rlc.cs_data = evergreen_cs_data;
5406 r = sumo_rlc_init(rdev);
5407 if (r) {
5408 DRM_ERROR("Failed to init rlc BOs!\n");
5409 return r;
5410 }
5411 }
5412
724c80e1
AD		 5413 /* allocate wb buffer */
5414 r = radeon_wb_init(rdev);
5415 if (r)
5416 return r;
5417
30eb77f4
JG		 5418 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5419 if (r) {
5420 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5421 return r;
5422 }
5423
233d1ad5
AD		 5424 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5425 if (r) {
5426 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5427 return r;
5428 }
5429
e409b128 5430 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5431 if (!r) {
5432 r = radeon_fence_driver_start_ring(rdev,
5433 R600_RING_TYPE_UVD_INDEX);
5434 if (r)
5435 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5436 }
5437
5438 if (r)
5439 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5440
bcc1c2a1 5441 /* Enable IRQ */
e49f3959
AH		 5442 if (!rdev->irq.installed) {
5443 r = radeon_irq_kms_init(rdev);
5444 if (r)
5445 return r;
5446 }
5447
bcc1c2a1
AD		 5448 r = r600_irq_init(rdev);
5449 if (r) {
5450 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5451 radeon_irq_kms_fini(rdev);
5452 return r;
5453 }
45f9a39b 5454 evergreen_irq_set(rdev);
bcc1c2a1 5455
f2ba57b5 5456 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5457 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 5458 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5459 if (r)
5460 return r;
233d1ad5
AD		 5461
5462 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5463 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 5464 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5465 if (r)
5466 return r;
5467
bcc1c2a1
AD		 5468 r = evergreen_cp_load_microcode(rdev);
5469 if (r)
5470 return r;
fe251e2f 5471 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5472 if (r)
5473 return r;
5474 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5475 if (r)
5476 return r;
fe251e2f 5477
f2ba57b5
CK		 5478 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5479 if (ring->ring_size) {
02c9f7fa 5480 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 5481 RADEON_CP_PACKET2);
f2ba57b5 5482 if (!r)
e409b128 5483 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5484
5485 if (r)
5486 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5487 }
5488
2898c348
CK		 5489 r = radeon_ib_pool_init(rdev);
5490 if (r) {
5491 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5492 return r;
2898c348 5493 }
b15ba512 5494
bfc1f97d 5495 r = radeon_audio_init(rdev);
69d2ae57
RM		 5496 if (r) {
5497 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5498 return r;
5499 }
5500
bcc1c2a1
AD		 5501 return 0;
5502}
5503
5504int evergreen_resume(struct radeon_device *rdev)
5505{
5506 int r;
5507
86f5c9ed
AD		 5508 /* reset the asic, the gfx blocks are often in a bad state
5509 * after the driver is unloaded or after a resume
5510 */
5511 if (radeon_asic_reset(rdev))
5512 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5513 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5514 * posting will perform necessary task to bring back GPU into good
5515 * shape.
5516 */
5517 /* post card */
5518 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5519
d4788db3
AD		 5520 /* init golden registers */
5521 evergreen_init_golden_registers(rdev);
5522
bc6a6295
AD		 5523 if (rdev->pm.pm_method == PM_METHOD_DPM)
5524 radeon_pm_resume(rdev);
6c7bccea 5525
b15ba512 5526 rdev->accel_working = true;
bcc1c2a1
AD		 5527 r = evergreen_startup(rdev);
5528 if (r) {
755d819e 5529 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5530 rdev->accel_working = false;
bcc1c2a1
AD		 5531 return r;
5532 }
fe251e2f 5533
bcc1c2a1
AD		 5534 return r;
5535
5536}
5537
5538int evergreen_suspend(struct radeon_device *rdev)
5539{
6c7bccea 5540 radeon_pm_suspend(rdev);
7991d665 5541 radeon_audio_fini(rdev);
e409b128 5542 uvd_v1_0_fini(rdev);
f2ba57b5 5543 radeon_uvd_suspend(rdev);
bcc1c2a1 5544 r700_cp_stop(rdev);
233d1ad5 5545 r600_dma_stop(rdev);
45f9a39b 5546 evergreen_irq_suspend(rdev);
724c80e1 5547 radeon_wb_disable(rdev);
bcc1c2a1 5548 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5549
5550 return 0;
5551}
5552
bcc1c2a1
AD		 5553/* Plan is to move initialization in that function and use
5554 * helper function so that radeon_device_init pretty much
5555 * do nothing more than calling asic specific function. This
5556 * should also allow to remove a bunch of callback function
5557 * like vram_info.
5558 */
5559int evergreen_init(struct radeon_device *rdev)
5560{
5561 int r;
5562
bcc1c2a1
AD		 5563 /* Read BIOS */
5564 if (!radeon_get_bios(rdev)) {
5565 if (ASIC_IS_AVIVO(rdev))
5566 return -EINVAL;
5567 }
5568 /* Must be an ATOMBIOS */
5569 if (!rdev->is_atom_bios) {
755d819e 5570 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5571 return -EINVAL;
5572 }
5573 r = radeon_atombios_init(rdev);
5574 if (r)
5575 return r;
86f5c9ed
AD		 5576 /* reset the asic, the gfx blocks are often in a bad state
5577 * after the driver is unloaded or after a resume
5578 */
5579 if (radeon_asic_reset(rdev))
5580 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5581 /* Post card if necessary */
fd909c37 5582 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5583 if (!rdev->bios) {
5584 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5585 return -EINVAL;
5586 }
5587 DRM_INFO("GPU not posted. posting now...\n");
5588 atom_asic_init(rdev->mode_info.atom_context);
5589 }
d4788db3
AD		 5590 /* init golden registers */
5591 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5592 /* Initialize scratch registers */
5593 r600_scratch_init(rdev);
5594 /* Initialize surface registers */
5595 radeon_surface_init(rdev);
5596 /* Initialize clocks */
5597 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5598 /* Fence driver */
5599 r = radeon_fence_driver_init(rdev);
5600 if (r)
5601 return r;
d594e46a
JG		 5602 /* initialize AGP */
5603 if (rdev->flags & RADEON_IS_AGP) {
5604 r = radeon_agp_init(rdev);
5605 if (r)
5606 radeon_agp_disable(rdev);
5607 }
5608 /* initialize memory controller */
bcc1c2a1
AD		 5609 r = evergreen_mc_init(rdev);
5610 if (r)
5611 return r;
5612 /* Memory manager */
5613 r = radeon_bo_init(rdev);
5614 if (r)
5615 return r;
45f9a39b 5616
01ac8794
AD		 5617 if (ASIC_IS_DCE5(rdev)) {
5618 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5619 r = ni_init_microcode(rdev);
5620 if (r) {
5621 DRM_ERROR("Failed to load firmware!\n");
5622 return r;
5623 }
5624 }
5625 } else {
5626 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5627 r = r600_init_microcode(rdev);
5628 if (r) {
5629 DRM_ERROR("Failed to load firmware!\n");
5630 return r;
5631 }
5632 }
5633 }
5634
6c7bccea
AD		 5635 /* Initialize power management */
5636 radeon_pm_init(rdev);
5637
e32eb50d
CK		 5638 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5639 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5640
233d1ad5
AD		 5641 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5642 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5643
f2ba57b5
CK		 5644 r = radeon_uvd_init(rdev);
5645 if (!r) {
5646 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5647 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5648 4096);
5649 }
5650
bcc1c2a1
AD		 5651 rdev->ih.ring_obj = NULL;
5652 r600_ih_ring_init(rdev, 64 * 1024);
5653
5654 r = r600_pcie_gart_init(rdev);
5655 if (r)
5656 return r;
0fcdb61e 5657
148a03bc 5658 rdev->accel_working = true;
bcc1c2a1
AD		 5659 r = evergreen_startup(rdev);
5660 if (r) {
fe251e2f
AD		 5661 dev_err(rdev->dev, "disabling GPU acceleration\n");
5662 r700_cp_fini(rdev);
233d1ad5 5663 r600_dma_fini(rdev);
fe251e2f 5664 r600_irq_fini(rdev);
2948f5e6
AD		 5665 if (rdev->flags & RADEON_IS_IGP)
5666 sumo_rlc_fini(rdev);
724c80e1 5667 radeon_wb_fini(rdev);
2898c348 5668 radeon_ib_pool_fini(rdev);
fe251e2f 5669 radeon_irq_kms_fini(rdev);
0fcdb61e 5670 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5671 rdev->accel_working = false;
5672 }
77e00f2e
AD		 5673
5674 /* Don't start up if the MC ucode is missing on BTC parts.
5675 * The default clocks and voltages before the MC ucode
5676 * is loaded are not suffient for advanced operations.
5677 */
5678 if (ASIC_IS_DCE5(rdev)) {
5679 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5680 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5681 return -EINVAL;
5682 }
5683 }
5684
bcc1c2a1
AD		 5685 return 0;
5686}
5687
5688void evergreen_fini(struct radeon_device *rdev)
5689{
6c7bccea 5690 radeon_pm_fini(rdev);
7991d665 5691 radeon_audio_fini(rdev);
45f9a39b 5692 r700_cp_fini(rdev);
233d1ad5 5693 r600_dma_fini(rdev);
bcc1c2a1 5694 r600_irq_fini(rdev);
2948f5e6
AD		 5695 if (rdev->flags & RADEON_IS_IGP)
5696 sumo_rlc_fini(rdev);
724c80e1 5697 radeon_wb_fini(rdev);
2898c348 5698 radeon_ib_pool_fini(rdev);
bcc1c2a1 5699 radeon_irq_kms_fini(rdev);
e409b128 5700 uvd_v1_0_fini(rdev);
f2ba57b5 5701 radeon_uvd_fini(rdev);
d9654413 5702 evergreen_pcie_gart_fini(rdev);
16cdf04d 5703 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5704 radeon_gem_fini(rdev);
5705 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5706 radeon_agp_fini(rdev);
5707 radeon_bo_fini(rdev);
5708 radeon_atombios_fini(rdev);
5709 kfree(rdev->bios);
5710 rdev->bios = NULL;
bcc1c2a1 5711}
9e46a48d 5712
b07759bf 5713void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5714{
7e0e4196 5715 u32 link_width_cntl, speed_cntl;
9e46a48d 5716
d42dd579
AD		 5717 if (radeon_pcie_gen2 == 0)
5718 return;
5719
9e46a48d
AD		 5720 if (rdev->flags & RADEON_IS_IGP)
5721 return;
5722
5723 if (!(rdev->flags & RADEON_IS_PCIE))
5724 return;
5725
5726 /* x2 cards have a special sequence */
5727 if (ASIC_IS_X2(rdev))
5728 return;
5729
7e0e4196
KSS		 5730 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5731 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5732 return;
5733
492d2b61 5734 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5735 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5736 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5737 return;
5738 }
5739
197bbb3d
DA		 5740 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5741
9e46a48d
AD		 5742 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5743 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5744
492d2b61 5745 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5746 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5747 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5748
492d2b61 5749 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5750 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5751 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5752
492d2b61 5753 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5754 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5755 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5756
492d2b61 5757 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5758 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5759 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5760
492d2b61 5761 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5762 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5763 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5764
5765 } else {
492d2b61 5766 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5767 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5768 if (1)
5769 link_width_cntl |= LC_UPCONFIGURE_DIS;
5770 else
5771 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5772 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5773 }
5774}
f52382d7
AD		 5775
5776void evergreen_program_aspm(struct radeon_device *rdev)
5777{
5778 u32 data, orig;
5779 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5780 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5781 /* fusion_platform = true
5782 * if the system is a fusion system
5783 * (APU or DGPU in a fusion system).
5784 * todo: check if the system is a fusion platform.
5785 */
5786 bool fusion_platform = false;
5787
1294d4a3
AD		 5788 if (radeon_aspm == 0)
5789 return;
5790
f52382d7
AD		 5791 if (!(rdev->flags & RADEON_IS_PCIE))
5792 return;
5793
5794 switch (rdev->family) {
5795 case CHIP_CYPRESS:
5796 case CHIP_HEMLOCK:
5797 case CHIP_JUNIPER:
5798 case CHIP_REDWOOD:
5799 case CHIP_CEDAR:
5800 case CHIP_SUMO:
5801 case CHIP_SUMO2:
5802 case CHIP_PALM:
5803 case CHIP_ARUBA:
5804 disable_l0s = true;
5805 break;
5806 default:
5807 disable_l0s = false;
5808 break;
5809 }
5810
5811 if (rdev->flags & RADEON_IS_IGP)
5812 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5813
5814 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5815 if (fusion_platform)
5816 data &= ~MULTI_PIF;
5817 else
5818 data |= MULTI_PIF;
5819 if (data != orig)
5820 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5821
5822 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5823 if (fusion_platform)
5824 data &= ~MULTI_PIF;
5825 else
5826 data |= MULTI_PIF;
5827 if (data != orig)
5828 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5829
5830 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5831 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5832 if (!disable_l0s) {
5833 if (rdev->family >= CHIP_BARTS)
5834 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5835 else
5836 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5837 }
5838
5839 if (!disable_l1) {
5840 if (rdev->family >= CHIP_BARTS)
5841 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5842 else
5843 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5844
5845 if (!disable_plloff_in_l1) {
5846 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5847 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5848 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5849 if (data != orig)
5850 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5851
5852 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5853 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5854 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5855 if (data != orig)
5856 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5857
5858 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5859 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5860 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5861 if (data != orig)
5862 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5863
5864 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5865 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5866 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5867 if (data != orig)
5868 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5869
5870 if (rdev->family >= CHIP_BARTS) {
5871 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5872 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5873 data |= PLL_RAMP_UP_TIME_0(4);
5874 if (data != orig)
5875 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5876
5877 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5878 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5879 data |= PLL_RAMP_UP_TIME_1(4);
5880 if (data != orig)
5881 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5882
5883 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5884 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5885 data |= PLL_RAMP_UP_TIME_0(4);
5886 if (data != orig)
5887 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5888
5889 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5890 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5891 data |= PLL_RAMP_UP_TIME_1(4);
5892 if (data != orig)
5893 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5894 }
5895
5896 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5897 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5898 data |= LC_DYN_LANES_PWR_STATE(3);
5899 if (data != orig)
5900 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5901
5902 if (rdev->family >= CHIP_BARTS) {
5903 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5904 data &= ~LS2_EXIT_TIME_MASK;
5905 data |= LS2_EXIT_TIME(1);
5906 if (data != orig)
5907 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5908
5909 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5910 data &= ~LS2_EXIT_TIME_MASK;
5911 data |= LS2_EXIT_TIME(1);
5912 if (data != orig)
5913 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5914 }
5915 }
5916 }
5917
5918 /* evergreen parts only */
5919 if (rdev->family < CHIP_BARTS)
5920 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5921
5922 if (pcie_lc_cntl != pcie_lc_cntl_old)
5923 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5924}
Linux kernel - Deliverables
This page took 0.750336 seconds and 5 git commands to generate.