projects / deliverable / linux.git / blob
? search:


0acde1949c18d4bcc52790754ab41f3ae74ffd6f
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <drm/drmP.h>
27 #include "radeon.h"
28 #include "radeon_asic.h"
29 #include "radeon_audio.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
32 #include "atom.h"
33 #include "avivod.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
37
38 /*
39 * Indirect registers accessor
40 */
41 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
42 {
43 unsigned long flags;
44 u32 r;
45
46 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
47 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
48 r = RREG32(EVERGREEN_CG_IND_DATA);
49 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
50 return r;
51 }
52
53 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
54 {
55 unsigned long flags;
56
57 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
58 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
59 WREG32(EVERGREEN_CG_IND_DATA, (v));
60 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
61 }
62
63 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
64 {
65 unsigned long flags;
66 u32 r;
67
68 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
69 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
70 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
71 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
72 return r;
73 }
74
75 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
76 {
77 unsigned long flags;
78
79 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
80 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
81 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
82 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
83 }
84
85 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
86 {
87 unsigned long flags;
88 u32 r;
89
90 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
91 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
92 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
93 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
94 return r;
95 }
96
97 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
98 {
99 unsigned long flags;
100
101 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
102 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
103 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
104 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
105 }
106
107 static const u32 crtc_offsets[6] =
108 {
109 EVERGREEN_CRTC0_REGISTER_OFFSET,
110 EVERGREEN_CRTC1_REGISTER_OFFSET,
111 EVERGREEN_CRTC2_REGISTER_OFFSET,
112 EVERGREEN_CRTC3_REGISTER_OFFSET,
113 EVERGREEN_CRTC4_REGISTER_OFFSET,
114 EVERGREEN_CRTC5_REGISTER_OFFSET
115 };
116
117 #include "clearstate_evergreen.h"
118
119 static const u32 sumo_rlc_save_restore_register_list[] =
120 {
121 0x98fc,
122 0x9830,
123 0x9834,
124 0x9838,
125 0x9870,
126 0x9874,
127 0x8a14,
128 0x8b24,
129 0x8bcc,
130 0x8b10,
131 0x8d00,
132 0x8d04,
133 0x8c00,
134 0x8c04,
135 0x8c08,
136 0x8c0c,
137 0x8d8c,
138 0x8c20,
139 0x8c24,
140 0x8c28,
141 0x8c18,
142 0x8c1c,
143 0x8cf0,
144 0x8e2c,
145 0x8e38,
146 0x8c30,
147 0x9508,
148 0x9688,
149 0x9608,
150 0x960c,
151 0x9610,
152 0x9614,
153 0x88c4,
154 0x88d4,
155 0xa008,
156 0x900c,
157 0x9100,
158 0x913c,
159 0x98f8,
160 0x98f4,
161 0x9b7c,
162 0x3f8c,
163 0x8950,
164 0x8954,
165 0x8a18,
166 0x8b28,
167 0x9144,
168 0x9148,
169 0x914c,
170 0x3f90,
171 0x3f94,
172 0x915c,
173 0x9160,
174 0x9178,
175 0x917c,
176 0x9180,
177 0x918c,
178 0x9190,
179 0x9194,
180 0x9198,
181 0x919c,
182 0x91a8,
183 0x91ac,
184 0x91b0,
185 0x91b4,
186 0x91b8,
187 0x91c4,
188 0x91c8,
189 0x91cc,
190 0x91d0,
191 0x91d4,
192 0x91e0,
193 0x91e4,
194 0x91ec,
195 0x91f0,
196 0x91f4,
197 0x9200,
198 0x9204,
199 0x929c,
200 0x9150,
201 0x802c,
202 };
203
204 static void evergreen_gpu_init(struct radeon_device *rdev);
205 void evergreen_fini(struct radeon_device *rdev);
206 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
207 void evergreen_program_aspm(struct radeon_device *rdev);
208 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
209 int ring, u32 cp_int_cntl);
210 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
211 u32 status, u32 addr);
212 void cik_init_cp_pg_table(struct radeon_device *rdev);
213
214 extern u32 si_get_csb_size(struct radeon_device *rdev);
215 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
216 extern u32 cik_get_csb_size(struct radeon_device *rdev);
217 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
218 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
219
220 static const u32 evergreen_golden_registers[] =
221 {
222 0x3f90, 0xffff0000, 0xff000000,
223 0x9148, 0xffff0000, 0xff000000,
224 0x3f94, 0xffff0000, 0xff000000,
225 0x914c, 0xffff0000, 0xff000000,
226 0x9b7c, 0xffffffff, 0x00000000,
227 0x8a14, 0xffffffff, 0x00000007,
228 0x8b10, 0xffffffff, 0x00000000,
229 0x960c, 0xffffffff, 0x54763210,
230 0x88c4, 0xffffffff, 0x000000c2,
231 0x88d4, 0xffffffff, 0x00000010,
232 0x8974, 0xffffffff, 0x00000000,
233 0xc78, 0x00000080, 0x00000080,
234 0x5eb4, 0xffffffff, 0x00000002,
235 0x5e78, 0xffffffff, 0x001000f0,
236 0x6104, 0x01000300, 0x00000000,
237 0x5bc0, 0x00300000, 0x00000000,
238 0x7030, 0xffffffff, 0x00000011,
239 0x7c30, 0xffffffff, 0x00000011,
240 0x10830, 0xffffffff, 0x00000011,
241 0x11430, 0xffffffff, 0x00000011,
242 0x12030, 0xffffffff, 0x00000011,
243 0x12c30, 0xffffffff, 0x00000011,
244 0xd02c, 0xffffffff, 0x08421000,
245 0x240c, 0xffffffff, 0x00000380,
246 0x8b24, 0xffffffff, 0x00ff0fff,
247 0x28a4c, 0x06000000, 0x06000000,
248 0x10c, 0x00000001, 0x00000001,
249 0x8d00, 0xffffffff, 0x100e4848,
250 0x8d04, 0xffffffff, 0x00164745,
251 0x8c00, 0xffffffff, 0xe4000003,
252 0x8c04, 0xffffffff, 0x40600060,
253 0x8c08, 0xffffffff, 0x001c001c,
254 0x8cf0, 0xffffffff, 0x08e00620,
255 0x8c20, 0xffffffff, 0x00800080,
256 0x8c24, 0xffffffff, 0x00800080,
257 0x8c18, 0xffffffff, 0x20202078,
258 0x8c1c, 0xffffffff, 0x00001010,
259 0x28350, 0xffffffff, 0x00000000,
260 0xa008, 0xffffffff, 0x00010000,
261 0x5c4, 0xffffffff, 0x00000001,
262 0x9508, 0xffffffff, 0x00000002,
263 0x913c, 0x0000000f, 0x0000000a
264 };
265
266 static const u32 evergreen_golden_registers2[] =
267 {
268 0x2f4c, 0xffffffff, 0x00000000,
269 0x54f4, 0xffffffff, 0x00000000,
270 0x54f0, 0xffffffff, 0x00000000,
271 0x5498, 0xffffffff, 0x00000000,
272 0x549c, 0xffffffff, 0x00000000,
273 0x5494, 0xffffffff, 0x00000000,
274 0x53cc, 0xffffffff, 0x00000000,
275 0x53c8, 0xffffffff, 0x00000000,
276 0x53c4, 0xffffffff, 0x00000000,
277 0x53c0, 0xffffffff, 0x00000000,
278 0x53bc, 0xffffffff, 0x00000000,
279 0x53b8, 0xffffffff, 0x00000000,
280 0x53b4, 0xffffffff, 0x00000000,
281 0x53b0, 0xffffffff, 0x00000000
282 };
283
284 static const u32 cypress_mgcg_init[] =
285 {
286 0x802c, 0xffffffff, 0xc0000000,
287 0x5448, 0xffffffff, 0x00000100,
288 0x55e4, 0xffffffff, 0x00000100,
289 0x160c, 0xffffffff, 0x00000100,
290 0x5644, 0xffffffff, 0x00000100,
291 0xc164, 0xffffffff, 0x00000100,
292 0x8a18, 0xffffffff, 0x00000100,
293 0x897c, 0xffffffff, 0x06000100,
294 0x8b28, 0xffffffff, 0x00000100,
295 0x9144, 0xffffffff, 0x00000100,
296 0x9a60, 0xffffffff, 0x00000100,
297 0x9868, 0xffffffff, 0x00000100,
298 0x8d58, 0xffffffff, 0x00000100,
299 0x9510, 0xffffffff, 0x00000100,
300 0x949c, 0xffffffff, 0x00000100,
301 0x9654, 0xffffffff, 0x00000100,
302 0x9030, 0xffffffff, 0x00000100,
303 0x9034, 0xffffffff, 0x00000100,
304 0x9038, 0xffffffff, 0x00000100,
305 0x903c, 0xffffffff, 0x00000100,
306 0x9040, 0xffffffff, 0x00000100,
307 0xa200, 0xffffffff, 0x00000100,
308 0xa204, 0xffffffff, 0x00000100,
309 0xa208, 0xffffffff, 0x00000100,
310 0xa20c, 0xffffffff, 0x00000100,
311 0x971c, 0xffffffff, 0x00000100,
312 0x977c, 0xffffffff, 0x00000100,
313 0x3f80, 0xffffffff, 0x00000100,
314 0xa210, 0xffffffff, 0x00000100,
315 0xa214, 0xffffffff, 0x00000100,
316 0x4d8, 0xffffffff, 0x00000100,
317 0x9784, 0xffffffff, 0x00000100,
318 0x9698, 0xffffffff, 0x00000100,
319 0x4d4, 0xffffffff, 0x00000200,
320 0x30cc, 0xffffffff, 0x00000100,
321 0xd0c0, 0xffffffff, 0xff000100,
322 0x802c, 0xffffffff, 0x40000000,
323 0x915c, 0xffffffff, 0x00010000,
324 0x9160, 0xffffffff, 0x00030002,
325 0x9178, 0xffffffff, 0x00070000,
326 0x917c, 0xffffffff, 0x00030002,
327 0x9180, 0xffffffff, 0x00050004,
328 0x918c, 0xffffffff, 0x00010006,
329 0x9190, 0xffffffff, 0x00090008,
330 0x9194, 0xffffffff, 0x00070000,
331 0x9198, 0xffffffff, 0x00030002,
332 0x919c, 0xffffffff, 0x00050004,
333 0x91a8, 0xffffffff, 0x00010006,
334 0x91ac, 0xffffffff, 0x00090008,
335 0x91b0, 0xffffffff, 0x00070000,
336 0x91b4, 0xffffffff, 0x00030002,
337 0x91b8, 0xffffffff, 0x00050004,
338 0x91c4, 0xffffffff, 0x00010006,
339 0x91c8, 0xffffffff, 0x00090008,
340 0x91cc, 0xffffffff, 0x00070000,
341 0x91d0, 0xffffffff, 0x00030002,
342 0x91d4, 0xffffffff, 0x00050004,
343 0x91e0, 0xffffffff, 0x00010006,
344 0x91e4, 0xffffffff, 0x00090008,
345 0x91e8, 0xffffffff, 0x00000000,
346 0x91ec, 0xffffffff, 0x00070000,
347 0x91f0, 0xffffffff, 0x00030002,
348 0x91f4, 0xffffffff, 0x00050004,
349 0x9200, 0xffffffff, 0x00010006,
350 0x9204, 0xffffffff, 0x00090008,
351 0x9208, 0xffffffff, 0x00070000,
352 0x920c, 0xffffffff, 0x00030002,
353 0x9210, 0xffffffff, 0x00050004,
354 0x921c, 0xffffffff, 0x00010006,
355 0x9220, 0xffffffff, 0x00090008,
356 0x9224, 0xffffffff, 0x00070000,
357 0x9228, 0xffffffff, 0x00030002,
358 0x922c, 0xffffffff, 0x00050004,
359 0x9238, 0xffffffff, 0x00010006,
360 0x923c, 0xffffffff, 0x00090008,
361 0x9240, 0xffffffff, 0x00070000,
362 0x9244, 0xffffffff, 0x00030002,
363 0x9248, 0xffffffff, 0x00050004,
364 0x9254, 0xffffffff, 0x00010006,
365 0x9258, 0xffffffff, 0x00090008,
366 0x925c, 0xffffffff, 0x00070000,
367 0x9260, 0xffffffff, 0x00030002,
368 0x9264, 0xffffffff, 0x00050004,
369 0x9270, 0xffffffff, 0x00010006,
370 0x9274, 0xffffffff, 0x00090008,
371 0x9278, 0xffffffff, 0x00070000,
372 0x927c, 0xffffffff, 0x00030002,
373 0x9280, 0xffffffff, 0x00050004,
374 0x928c, 0xffffffff, 0x00010006,
375 0x9290, 0xffffffff, 0x00090008,
376 0x9294, 0xffffffff, 0x00000000,
377 0x929c, 0xffffffff, 0x00000001,
378 0x802c, 0xffffffff, 0x40010000,
379 0x915c, 0xffffffff, 0x00010000,
380 0x9160, 0xffffffff, 0x00030002,
381 0x9178, 0xffffffff, 0x00070000,
382 0x917c, 0xffffffff, 0x00030002,
383 0x9180, 0xffffffff, 0x00050004,
384 0x918c, 0xffffffff, 0x00010006,
385 0x9190, 0xffffffff, 0x00090008,
386 0x9194, 0xffffffff, 0x00070000,
387 0x9198, 0xffffffff, 0x00030002,
388 0x919c, 0xffffffff, 0x00050004,
389 0x91a8, 0xffffffff, 0x00010006,
390 0x91ac, 0xffffffff, 0x00090008,
391 0x91b0, 0xffffffff, 0x00070000,
392 0x91b4, 0xffffffff, 0x00030002,
393 0x91b8, 0xffffffff, 0x00050004,
394 0x91c4, 0xffffffff, 0x00010006,
395 0x91c8, 0xffffffff, 0x00090008,
396 0x91cc, 0xffffffff, 0x00070000,
397 0x91d0, 0xffffffff, 0x00030002,
398 0x91d4, 0xffffffff, 0x00050004,
399 0x91e0, 0xffffffff, 0x00010006,
400 0x91e4, 0xffffffff, 0x00090008,
401 0x91e8, 0xffffffff, 0x00000000,
402 0x91ec, 0xffffffff, 0x00070000,
403 0x91f0, 0xffffffff, 0x00030002,
404 0x91f4, 0xffffffff, 0x00050004,
405 0x9200, 0xffffffff, 0x00010006,
406 0x9204, 0xffffffff, 0x00090008,
407 0x9208, 0xffffffff, 0x00070000,
408 0x920c, 0xffffffff, 0x00030002,
409 0x9210, 0xffffffff, 0x00050004,
410 0x921c, 0xffffffff, 0x00010006,
411 0x9220, 0xffffffff, 0x00090008,
412 0x9224, 0xffffffff, 0x00070000,
413 0x9228, 0xffffffff, 0x00030002,
414 0x922c, 0xffffffff, 0x00050004,
415 0x9238, 0xffffffff, 0x00010006,
416 0x923c, 0xffffffff, 0x00090008,
417 0x9240, 0xffffffff, 0x00070000,
418 0x9244, 0xffffffff, 0x00030002,
419 0x9248, 0xffffffff, 0x00050004,
420 0x9254, 0xffffffff, 0x00010006,
421 0x9258, 0xffffffff, 0x00090008,
422 0x925c, 0xffffffff, 0x00070000,
423 0x9260, 0xffffffff, 0x00030002,
424 0x9264, 0xffffffff, 0x00050004,
425 0x9270, 0xffffffff, 0x00010006,
426 0x9274, 0xffffffff, 0x00090008,
427 0x9278, 0xffffffff, 0x00070000,
428 0x927c, 0xffffffff, 0x00030002,
429 0x9280, 0xffffffff, 0x00050004,
430 0x928c, 0xffffffff, 0x00010006,
431 0x9290, 0xffffffff, 0x00090008,
432 0x9294, 0xffffffff, 0x00000000,
433 0x929c, 0xffffffff, 0x00000001,
434 0x802c, 0xffffffff, 0xc0000000
435 };
436
437 static const u32 redwood_mgcg_init[] =
438 {
439 0x802c, 0xffffffff, 0xc0000000,
440 0x5448, 0xffffffff, 0x00000100,
441 0x55e4, 0xffffffff, 0x00000100,
442 0x160c, 0xffffffff, 0x00000100,
443 0x5644, 0xffffffff, 0x00000100,
444 0xc164, 0xffffffff, 0x00000100,
445 0x8a18, 0xffffffff, 0x00000100,
446 0x897c, 0xffffffff, 0x06000100,
447 0x8b28, 0xffffffff, 0x00000100,
448 0x9144, 0xffffffff, 0x00000100,
449 0x9a60, 0xffffffff, 0x00000100,
450 0x9868, 0xffffffff, 0x00000100,
451 0x8d58, 0xffffffff, 0x00000100,
452 0x9510, 0xffffffff, 0x00000100,
453 0x949c, 0xffffffff, 0x00000100,
454 0x9654, 0xffffffff, 0x00000100,
455 0x9030, 0xffffffff, 0x00000100,
456 0x9034, 0xffffffff, 0x00000100,
457 0x9038, 0xffffffff, 0x00000100,
458 0x903c, 0xffffffff, 0x00000100,
459 0x9040, 0xffffffff, 0x00000100,
460 0xa200, 0xffffffff, 0x00000100,
461 0xa204, 0xffffffff, 0x00000100,
462 0xa208, 0xffffffff, 0x00000100,
463 0xa20c, 0xffffffff, 0x00000100,
464 0x971c, 0xffffffff, 0x00000100,
465 0x977c, 0xffffffff, 0x00000100,
466 0x3f80, 0xffffffff, 0x00000100,
467 0xa210, 0xffffffff, 0x00000100,
468 0xa214, 0xffffffff, 0x00000100,
469 0x4d8, 0xffffffff, 0x00000100,
470 0x9784, 0xffffffff, 0x00000100,
471 0x9698, 0xffffffff, 0x00000100,
472 0x4d4, 0xffffffff, 0x00000200,
473 0x30cc, 0xffffffff, 0x00000100,
474 0xd0c0, 0xffffffff, 0xff000100,
475 0x802c, 0xffffffff, 0x40000000,
476 0x915c, 0xffffffff, 0x00010000,
477 0x9160, 0xffffffff, 0x00030002,
478 0x9178, 0xffffffff, 0x00070000,
479 0x917c, 0xffffffff, 0x00030002,
480 0x9180, 0xffffffff, 0x00050004,
481 0x918c, 0xffffffff, 0x00010006,
482 0x9190, 0xffffffff, 0x00090008,
483 0x9194, 0xffffffff, 0x00070000,
484 0x9198, 0xffffffff, 0x00030002,
485 0x919c, 0xffffffff, 0x00050004,
486 0x91a8, 0xffffffff, 0x00010006,
487 0x91ac, 0xffffffff, 0x00090008,
488 0x91b0, 0xffffffff, 0x00070000,
489 0x91b4, 0xffffffff, 0x00030002,
490 0x91b8, 0xffffffff, 0x00050004,
491 0x91c4, 0xffffffff, 0x00010006,
492 0x91c8, 0xffffffff, 0x00090008,
493 0x91cc, 0xffffffff, 0x00070000,
494 0x91d0, 0xffffffff, 0x00030002,
495 0x91d4, 0xffffffff, 0x00050004,
496 0x91e0, 0xffffffff, 0x00010006,
497 0x91e4, 0xffffffff, 0x00090008,
498 0x91e8, 0xffffffff, 0x00000000,
499 0x91ec, 0xffffffff, 0x00070000,
500 0x91f0, 0xffffffff, 0x00030002,
501 0x91f4, 0xffffffff, 0x00050004,
502 0x9200, 0xffffffff, 0x00010006,
503 0x9204, 0xffffffff, 0x00090008,
504 0x9294, 0xffffffff, 0x00000000,
505 0x929c, 0xffffffff, 0x00000001,
506 0x802c, 0xffffffff, 0xc0000000
507 };
508
509 static const u32 cedar_golden_registers[] =
510 {
511 0x3f90, 0xffff0000, 0xff000000,
512 0x9148, 0xffff0000, 0xff000000,
513 0x3f94, 0xffff0000, 0xff000000,
514 0x914c, 0xffff0000, 0xff000000,
515 0x9b7c, 0xffffffff, 0x00000000,
516 0x8a14, 0xffffffff, 0x00000007,
517 0x8b10, 0xffffffff, 0x00000000,
518 0x960c, 0xffffffff, 0x54763210,
519 0x88c4, 0xffffffff, 0x000000c2,
520 0x88d4, 0xffffffff, 0x00000000,
521 0x8974, 0xffffffff, 0x00000000,
522 0xc78, 0x00000080, 0x00000080,
523 0x5eb4, 0xffffffff, 0x00000002,
524 0x5e78, 0xffffffff, 0x001000f0,
525 0x6104, 0x01000300, 0x00000000,
526 0x5bc0, 0x00300000, 0x00000000,
527 0x7030, 0xffffffff, 0x00000011,
528 0x7c30, 0xffffffff, 0x00000011,
529 0x10830, 0xffffffff, 0x00000011,
530 0x11430, 0xffffffff, 0x00000011,
531 0xd02c, 0xffffffff, 0x08421000,
532 0x240c, 0xffffffff, 0x00000380,
533 0x8b24, 0xffffffff, 0x00ff0fff,
534 0x28a4c, 0x06000000, 0x06000000,
535 0x10c, 0x00000001, 0x00000001,
536 0x8d00, 0xffffffff, 0x100e4848,
537 0x8d04, 0xffffffff, 0x00164745,
538 0x8c00, 0xffffffff, 0xe4000003,
539 0x8c04, 0xffffffff, 0x40600060,
540 0x8c08, 0xffffffff, 0x001c001c,
541 0x8cf0, 0xffffffff, 0x08e00410,
542 0x8c20, 0xffffffff, 0x00800080,
543 0x8c24, 0xffffffff, 0x00800080,
544 0x8c18, 0xffffffff, 0x20202078,
545 0x8c1c, 0xffffffff, 0x00001010,
546 0x28350, 0xffffffff, 0x00000000,
547 0xa008, 0xffffffff, 0x00010000,
548 0x5c4, 0xffffffff, 0x00000001,
549 0x9508, 0xffffffff, 0x00000002
550 };
551
552 static const u32 cedar_mgcg_init[] =
553 {
554 0x802c, 0xffffffff, 0xc0000000,
555 0x5448, 0xffffffff, 0x00000100,
556 0x55e4, 0xffffffff, 0x00000100,
557 0x160c, 0xffffffff, 0x00000100,
558 0x5644, 0xffffffff, 0x00000100,
559 0xc164, 0xffffffff, 0x00000100,
560 0x8a18, 0xffffffff, 0x00000100,
561 0x897c, 0xffffffff, 0x06000100,
562 0x8b28, 0xffffffff, 0x00000100,
563 0x9144, 0xffffffff, 0x00000100,
564 0x9a60, 0xffffffff, 0x00000100,
565 0x9868, 0xffffffff, 0x00000100,
566 0x8d58, 0xffffffff, 0x00000100,
567 0x9510, 0xffffffff, 0x00000100,
568 0x949c, 0xffffffff, 0x00000100,
569 0x9654, 0xffffffff, 0x00000100,
570 0x9030, 0xffffffff, 0x00000100,
571 0x9034, 0xffffffff, 0x00000100,
572 0x9038, 0xffffffff, 0x00000100,
573 0x903c, 0xffffffff, 0x00000100,
574 0x9040, 0xffffffff, 0x00000100,
575 0xa200, 0xffffffff, 0x00000100,
576 0xa204, 0xffffffff, 0x00000100,
577 0xa208, 0xffffffff, 0x00000100,
578 0xa20c, 0xffffffff, 0x00000100,
579 0x971c, 0xffffffff, 0x00000100,
580 0x977c, 0xffffffff, 0x00000100,
581 0x3f80, 0xffffffff, 0x00000100,
582 0xa210, 0xffffffff, 0x00000100,
583 0xa214, 0xffffffff, 0x00000100,
584 0x4d8, 0xffffffff, 0x00000100,
585 0x9784, 0xffffffff, 0x00000100,
586 0x9698, 0xffffffff, 0x00000100,
587 0x4d4, 0xffffffff, 0x00000200,
588 0x30cc, 0xffffffff, 0x00000100,
589 0xd0c0, 0xffffffff, 0xff000100,
590 0x802c, 0xffffffff, 0x40000000,
591 0x915c, 0xffffffff, 0x00010000,
592 0x9178, 0xffffffff, 0x00050000,
593 0x917c, 0xffffffff, 0x00030002,
594 0x918c, 0xffffffff, 0x00010004,
595 0x9190, 0xffffffff, 0x00070006,
596 0x9194, 0xffffffff, 0x00050000,
597 0x9198, 0xffffffff, 0x00030002,
598 0x91a8, 0xffffffff, 0x00010004,
599 0x91ac, 0xffffffff, 0x00070006,
600 0x91e8, 0xffffffff, 0x00000000,
601 0x9294, 0xffffffff, 0x00000000,
602 0x929c, 0xffffffff, 0x00000001,
603 0x802c, 0xffffffff, 0xc0000000
604 };
605
606 static const u32 juniper_mgcg_init[] =
607 {
608 0x802c, 0xffffffff, 0xc0000000,
609 0x5448, 0xffffffff, 0x00000100,
610 0x55e4, 0xffffffff, 0x00000100,
611 0x160c, 0xffffffff, 0x00000100,
612 0x5644, 0xffffffff, 0x00000100,
613 0xc164, 0xffffffff, 0x00000100,
614 0x8a18, 0xffffffff, 0x00000100,
615 0x897c, 0xffffffff, 0x06000100,
616 0x8b28, 0xffffffff, 0x00000100,
617 0x9144, 0xffffffff, 0x00000100,
618 0x9a60, 0xffffffff, 0x00000100,
619 0x9868, 0xffffffff, 0x00000100,
620 0x8d58, 0xffffffff, 0x00000100,
621 0x9510, 0xffffffff, 0x00000100,
622 0x949c, 0xffffffff, 0x00000100,
623 0x9654, 0xffffffff, 0x00000100,
624 0x9030, 0xffffffff, 0x00000100,
625 0x9034, 0xffffffff, 0x00000100,
626 0x9038, 0xffffffff, 0x00000100,
627 0x903c, 0xffffffff, 0x00000100,
628 0x9040, 0xffffffff, 0x00000100,
629 0xa200, 0xffffffff, 0x00000100,
630 0xa204, 0xffffffff, 0x00000100,
631 0xa208, 0xffffffff, 0x00000100,
632 0xa20c, 0xffffffff, 0x00000100,
633 0x971c, 0xffffffff, 0x00000100,
634 0xd0c0, 0xffffffff, 0xff000100,
635 0x802c, 0xffffffff, 0x40000000,
636 0x915c, 0xffffffff, 0x00010000,
637 0x9160, 0xffffffff, 0x00030002,
638 0x9178, 0xffffffff, 0x00070000,
639 0x917c, 0xffffffff, 0x00030002,
640 0x9180, 0xffffffff, 0x00050004,
641 0x918c, 0xffffffff, 0x00010006,
642 0x9190, 0xffffffff, 0x00090008,
643 0x9194, 0xffffffff, 0x00070000,
644 0x9198, 0xffffffff, 0x00030002,
645 0x919c, 0xffffffff, 0x00050004,
646 0x91a8, 0xffffffff, 0x00010006,
647 0x91ac, 0xffffffff, 0x00090008,
648 0x91b0, 0xffffffff, 0x00070000,
649 0x91b4, 0xffffffff, 0x00030002,
650 0x91b8, 0xffffffff, 0x00050004,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91c8, 0xffffffff, 0x00090008,
653 0x91cc, 0xffffffff, 0x00070000,
654 0x91d0, 0xffffffff, 0x00030002,
655 0x91d4, 0xffffffff, 0x00050004,
656 0x91e0, 0xffffffff, 0x00010006,
657 0x91e4, 0xffffffff, 0x00090008,
658 0x91e8, 0xffffffff, 0x00000000,
659 0x91ec, 0xffffffff, 0x00070000,
660 0x91f0, 0xffffffff, 0x00030002,
661 0x91f4, 0xffffffff, 0x00050004,
662 0x9200, 0xffffffff, 0x00010006,
663 0x9204, 0xffffffff, 0x00090008,
664 0x9208, 0xffffffff, 0x00070000,
665 0x920c, 0xffffffff, 0x00030002,
666 0x9210, 0xffffffff, 0x00050004,
667 0x921c, 0xffffffff, 0x00010006,
668 0x9220, 0xffffffff, 0x00090008,
669 0x9224, 0xffffffff, 0x00070000,
670 0x9228, 0xffffffff, 0x00030002,
671 0x922c, 0xffffffff, 0x00050004,
672 0x9238, 0xffffffff, 0x00010006,
673 0x923c, 0xffffffff, 0x00090008,
674 0x9240, 0xffffffff, 0x00070000,
675 0x9244, 0xffffffff, 0x00030002,
676 0x9248, 0xffffffff, 0x00050004,
677 0x9254, 0xffffffff, 0x00010006,
678 0x9258, 0xffffffff, 0x00090008,
679 0x925c, 0xffffffff, 0x00070000,
680 0x9260, 0xffffffff, 0x00030002,
681 0x9264, 0xffffffff, 0x00050004,
682 0x9270, 0xffffffff, 0x00010006,
683 0x9274, 0xffffffff, 0x00090008,
684 0x9278, 0xffffffff, 0x00070000,
685 0x927c, 0xffffffff, 0x00030002,
686 0x9280, 0xffffffff, 0x00050004,
687 0x928c, 0xffffffff, 0x00010006,
688 0x9290, 0xffffffff, 0x00090008,
689 0x9294, 0xffffffff, 0x00000000,
690 0x929c, 0xffffffff, 0x00000001,
691 0x802c, 0xffffffff, 0xc0000000,
692 0x977c, 0xffffffff, 0x00000100,
693 0x3f80, 0xffffffff, 0x00000100,
694 0xa210, 0xffffffff, 0x00000100,
695 0xa214, 0xffffffff, 0x00000100,
696 0x4d8, 0xffffffff, 0x00000100,
697 0x9784, 0xffffffff, 0x00000100,
698 0x9698, 0xffffffff, 0x00000100,
699 0x4d4, 0xffffffff, 0x00000200,
700 0x30cc, 0xffffffff, 0x00000100,
701 0x802c, 0xffffffff, 0xc0000000
702 };
703
704 static const u32 supersumo_golden_registers[] =
705 {
706 0x5eb4, 0xffffffff, 0x00000002,
707 0x5c4, 0xffffffff, 0x00000001,
708 0x7030, 0xffffffff, 0x00000011,
709 0x7c30, 0xffffffff, 0x00000011,
710 0x6104, 0x01000300, 0x00000000,
711 0x5bc0, 0x00300000, 0x00000000,
712 0x8c04, 0xffffffff, 0x40600060,
713 0x8c08, 0xffffffff, 0x001c001c,
714 0x8c20, 0xffffffff, 0x00800080,
715 0x8c24, 0xffffffff, 0x00800080,
716 0x8c18, 0xffffffff, 0x20202078,
717 0x8c1c, 0xffffffff, 0x00001010,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x91c4, 0xffffffff, 0x00010006,
721 0x91e0, 0xffffffff, 0x00010006,
722 0x9200, 0xffffffff, 0x00010006,
723 0x9150, 0xffffffff, 0x6e944040,
724 0x917c, 0xffffffff, 0x00030002,
725 0x9180, 0xffffffff, 0x00050004,
726 0x9198, 0xffffffff, 0x00030002,
727 0x919c, 0xffffffff, 0x00050004,
728 0x91b4, 0xffffffff, 0x00030002,
729 0x91b8, 0xffffffff, 0x00050004,
730 0x91d0, 0xffffffff, 0x00030002,
731 0x91d4, 0xffffffff, 0x00050004,
732 0x91f0, 0xffffffff, 0x00030002,
733 0x91f4, 0xffffffff, 0x00050004,
734 0x915c, 0xffffffff, 0x00010000,
735 0x9160, 0xffffffff, 0x00030002,
736 0x3f90, 0xffff0000, 0xff000000,
737 0x9178, 0xffffffff, 0x00070000,
738 0x9194, 0xffffffff, 0x00070000,
739 0x91b0, 0xffffffff, 0x00070000,
740 0x91cc, 0xffffffff, 0x00070000,
741 0x91ec, 0xffffffff, 0x00070000,
742 0x9148, 0xffff0000, 0xff000000,
743 0x9190, 0xffffffff, 0x00090008,
744 0x91ac, 0xffffffff, 0x00090008,
745 0x91c8, 0xffffffff, 0x00090008,
746 0x91e4, 0xffffffff, 0x00090008,
747 0x9204, 0xffffffff, 0x00090008,
748 0x3f94, 0xffff0000, 0xff000000,
749 0x914c, 0xffff0000, 0xff000000,
750 0x929c, 0xffffffff, 0x00000001,
751 0x8a18, 0xffffffff, 0x00000100,
752 0x8b28, 0xffffffff, 0x00000100,
753 0x9144, 0xffffffff, 0x00000100,
754 0x5644, 0xffffffff, 0x00000100,
755 0x9b7c, 0xffffffff, 0x00000000,
756 0x8030, 0xffffffff, 0x0000100a,
757 0x8a14, 0xffffffff, 0x00000007,
758 0x8b24, 0xffffffff, 0x00ff0fff,
759 0x8b10, 0xffffffff, 0x00000000,
760 0x28a4c, 0x06000000, 0x06000000,
761 0x4d8, 0xffffffff, 0x00000100,
762 0x913c, 0xffff000f, 0x0100000a,
763 0x960c, 0xffffffff, 0x54763210,
764 0x88c4, 0xffffffff, 0x000000c2,
765 0x88d4, 0xffffffff, 0x00000010,
766 0x8974, 0xffffffff, 0x00000000,
767 0xc78, 0x00000080, 0x00000080,
768 0x5e78, 0xffffffff, 0x001000f0,
769 0xd02c, 0xffffffff, 0x08421000,
770 0xa008, 0xffffffff, 0x00010000,
771 0x8d00, 0xffffffff, 0x100e4848,
772 0x8d04, 0xffffffff, 0x00164745,
773 0x8c00, 0xffffffff, 0xe4000003,
774 0x8cf0, 0x1fffffff, 0x08e00620,
775 0x28350, 0xffffffff, 0x00000000,
776 0x9508, 0xffffffff, 0x00000002
777 };
778
779 static const u32 sumo_golden_registers[] =
780 {
781 0x900c, 0x00ffffff, 0x0017071f,
782 0x8c18, 0xffffffff, 0x10101060,
783 0x8c1c, 0xffffffff, 0x00001010,
784 0x8c30, 0x0000000f, 0x00000005,
785 0x9688, 0x0000000f, 0x00000007
786 };
787
788 static const u32 wrestler_golden_registers[] =
789 {
790 0x5eb4, 0xffffffff, 0x00000002,
791 0x5c4, 0xffffffff, 0x00000001,
792 0x7030, 0xffffffff, 0x00000011,
793 0x7c30, 0xffffffff, 0x00000011,
794 0x6104, 0x01000300, 0x00000000,
795 0x5bc0, 0x00300000, 0x00000000,
796 0x918c, 0xffffffff, 0x00010006,
797 0x91a8, 0xffffffff, 0x00010006,
798 0x9150, 0xffffffff, 0x6e944040,
799 0x917c, 0xffffffff, 0x00030002,
800 0x9198, 0xffffffff, 0x00030002,
801 0x915c, 0xffffffff, 0x00010000,
802 0x3f90, 0xffff0000, 0xff000000,
803 0x9178, 0xffffffff, 0x00070000,
804 0x9194, 0xffffffff, 0x00070000,
805 0x9148, 0xffff0000, 0xff000000,
806 0x9190, 0xffffffff, 0x00090008,
807 0x91ac, 0xffffffff, 0x00090008,
808 0x3f94, 0xffff0000, 0xff000000,
809 0x914c, 0xffff0000, 0xff000000,
810 0x929c, 0xffffffff, 0x00000001,
811 0x8a18, 0xffffffff, 0x00000100,
812 0x8b28, 0xffffffff, 0x00000100,
813 0x9144, 0xffffffff, 0x00000100,
814 0x9b7c, 0xffffffff, 0x00000000,
815 0x8030, 0xffffffff, 0x0000100a,
816 0x8a14, 0xffffffff, 0x00000001,
817 0x8b24, 0xffffffff, 0x00ff0fff,
818 0x8b10, 0xffffffff, 0x00000000,
819 0x28a4c, 0x06000000, 0x06000000,
820 0x4d8, 0xffffffff, 0x00000100,
821 0x913c, 0xffff000f, 0x0100000a,
822 0x960c, 0xffffffff, 0x54763210,
823 0x88c4, 0xffffffff, 0x000000c2,
824 0x88d4, 0xffffffff, 0x00000010,
825 0x8974, 0xffffffff, 0x00000000,
826 0xc78, 0x00000080, 0x00000080,
827 0x5e78, 0xffffffff, 0x001000f0,
828 0xd02c, 0xffffffff, 0x08421000,
829 0xa008, 0xffffffff, 0x00010000,
830 0x8d00, 0xffffffff, 0x100e4848,
831 0x8d04, 0xffffffff, 0x00164745,
832 0x8c00, 0xffffffff, 0xe4000003,
833 0x8cf0, 0x1fffffff, 0x08e00410,
834 0x28350, 0xffffffff, 0x00000000,
835 0x9508, 0xffffffff, 0x00000002,
836 0x900c, 0xffffffff, 0x0017071f,
837 0x8c18, 0xffffffff, 0x10101060,
838 0x8c1c, 0xffffffff, 0x00001010
839 };
840
841 static const u32 barts_golden_registers[] =
842 {
843 0x5eb4, 0xffffffff, 0x00000002,
844 0x5e78, 0x8f311ff1, 0x001000f0,
845 0x3f90, 0xffff0000, 0xff000000,
846 0x9148, 0xffff0000, 0xff000000,
847 0x3f94, 0xffff0000, 0xff000000,
848 0x914c, 0xffff0000, 0xff000000,
849 0xc78, 0x00000080, 0x00000080,
850 0xbd4, 0x70073777, 0x00010001,
851 0xd02c, 0xbfffff1f, 0x08421000,
852 0xd0b8, 0x03773777, 0x02011003,
853 0x5bc0, 0x00200000, 0x50100000,
854 0x98f8, 0x33773777, 0x02011003,
855 0x98fc, 0xffffffff, 0x76543210,
856 0x7030, 0x31000311, 0x00000011,
857 0x2f48, 0x00000007, 0x02011003,
858 0x6b28, 0x00000010, 0x00000012,
859 0x7728, 0x00000010, 0x00000012,
860 0x10328, 0x00000010, 0x00000012,
861 0x10f28, 0x00000010, 0x00000012,
862 0x11b28, 0x00000010, 0x00000012,
863 0x12728, 0x00000010, 0x00000012,
864 0x240c, 0x000007ff, 0x00000380,
865 0x8a14, 0xf000001f, 0x00000007,
866 0x8b24, 0x3fff3fff, 0x00ff0fff,
867 0x8b10, 0x0000ff0f, 0x00000000,
868 0x28a4c, 0x07ffffff, 0x06000000,
869 0x10c, 0x00000001, 0x00010003,
870 0xa02c, 0xffffffff, 0x0000009b,
871 0x913c, 0x0000000f, 0x0100000a,
872 0x8d00, 0xffff7f7f, 0x100e4848,
873 0x8d04, 0x00ffffff, 0x00164745,
874 0x8c00, 0xfffc0003, 0xe4000003,
875 0x8c04, 0xf8ff00ff, 0x40600060,
876 0x8c08, 0x00ff00ff, 0x001c001c,
877 0x8cf0, 0x1fff1fff, 0x08e00620,
878 0x8c20, 0x0fff0fff, 0x00800080,
879 0x8c24, 0x0fff0fff, 0x00800080,
880 0x8c18, 0xffffffff, 0x20202078,
881 0x8c1c, 0x0000ffff, 0x00001010,
882 0x28350, 0x00000f01, 0x00000000,
883 0x9508, 0x3700001f, 0x00000002,
884 0x960c, 0xffffffff, 0x54763210,
885 0x88c4, 0x001f3ae3, 0x000000c2,
886 0x88d4, 0x0000001f, 0x00000010,
887 0x8974, 0xffffffff, 0x00000000
888 };
889
890 static const u32 turks_golden_registers[] =
891 {
892 0x5eb4, 0xffffffff, 0x00000002,
893 0x5e78, 0x8f311ff1, 0x001000f0,
894 0x8c8, 0x00003000, 0x00001070,
895 0x8cc, 0x000fffff, 0x00040035,
896 0x3f90, 0xffff0000, 0xfff00000,
897 0x9148, 0xffff0000, 0xfff00000,
898 0x3f94, 0xffff0000, 0xfff00000,
899 0x914c, 0xffff0000, 0xfff00000,
900 0xc78, 0x00000080, 0x00000080,
901 0xbd4, 0x00073007, 0x00010002,
902 0xd02c, 0xbfffff1f, 0x08421000,
903 0xd0b8, 0x03773777, 0x02010002,
904 0x5bc0, 0x00200000, 0x50100000,
905 0x98f8, 0x33773777, 0x00010002,
906 0x98fc, 0xffffffff, 0x33221100,
907 0x7030, 0x31000311, 0x00000011,
908 0x2f48, 0x33773777, 0x00010002,
909 0x6b28, 0x00000010, 0x00000012,
910 0x7728, 0x00000010, 0x00000012,
911 0x10328, 0x00000010, 0x00000012,
912 0x10f28, 0x00000010, 0x00000012,
913 0x11b28, 0x00000010, 0x00000012,
914 0x12728, 0x00000010, 0x00000012,
915 0x240c, 0x000007ff, 0x00000380,
916 0x8a14, 0xf000001f, 0x00000007,
917 0x8b24, 0x3fff3fff, 0x00ff0fff,
918 0x8b10, 0x0000ff0f, 0x00000000,
919 0x28a4c, 0x07ffffff, 0x06000000,
920 0x10c, 0x00000001, 0x00010003,
921 0xa02c, 0xffffffff, 0x0000009b,
922 0x913c, 0x0000000f, 0x0100000a,
923 0x8d00, 0xffff7f7f, 0x100e4848,
924 0x8d04, 0x00ffffff, 0x00164745,
925 0x8c00, 0xfffc0003, 0xe4000003,
926 0x8c04, 0xf8ff00ff, 0x40600060,
927 0x8c08, 0x00ff00ff, 0x001c001c,
928 0x8cf0, 0x1fff1fff, 0x08e00410,
929 0x8c20, 0x0fff0fff, 0x00800080,
930 0x8c24, 0x0fff0fff, 0x00800080,
931 0x8c18, 0xffffffff, 0x20202078,
932 0x8c1c, 0x0000ffff, 0x00001010,
933 0x28350, 0x00000f01, 0x00000000,
934 0x9508, 0x3700001f, 0x00000002,
935 0x960c, 0xffffffff, 0x54763210,
936 0x88c4, 0x001f3ae3, 0x000000c2,
937 0x88d4, 0x0000001f, 0x00000010,
938 0x8974, 0xffffffff, 0x00000000
939 };
940
941 static const u32 caicos_golden_registers[] =
942 {
943 0x5eb4, 0xffffffff, 0x00000002,
944 0x5e78, 0x8f311ff1, 0x001000f0,
945 0x8c8, 0x00003420, 0x00001450,
946 0x8cc, 0x000fffff, 0x00040035,
947 0x3f90, 0xffff0000, 0xfffc0000,
948 0x9148, 0xffff0000, 0xfffc0000,
949 0x3f94, 0xffff0000, 0xfffc0000,
950 0x914c, 0xffff0000, 0xfffc0000,
951 0xc78, 0x00000080, 0x00000080,
952 0xbd4, 0x00073007, 0x00010001,
953 0xd02c, 0xbfffff1f, 0x08421000,
954 0xd0b8, 0x03773777, 0x02010001,
955 0x5bc0, 0x00200000, 0x50100000,
956 0x98f8, 0x33773777, 0x02010001,
957 0x98fc, 0xffffffff, 0x33221100,
958 0x7030, 0x31000311, 0x00000011,
959 0x2f48, 0x33773777, 0x02010001,
960 0x6b28, 0x00000010, 0x00000012,
961 0x7728, 0x00000010, 0x00000012,
962 0x10328, 0x00000010, 0x00000012,
963 0x10f28, 0x00000010, 0x00000012,
964 0x11b28, 0x00000010, 0x00000012,
965 0x12728, 0x00000010, 0x00000012,
966 0x240c, 0x000007ff, 0x00000380,
967 0x8a14, 0xf000001f, 0x00000001,
968 0x8b24, 0x3fff3fff, 0x00ff0fff,
969 0x8b10, 0x0000ff0f, 0x00000000,
970 0x28a4c, 0x07ffffff, 0x06000000,
971 0x10c, 0x00000001, 0x00010003,
972 0xa02c, 0xffffffff, 0x0000009b,
973 0x913c, 0x0000000f, 0x0100000a,
974 0x8d00, 0xffff7f7f, 0x100e4848,
975 0x8d04, 0x00ffffff, 0x00164745,
976 0x8c00, 0xfffc0003, 0xe4000003,
977 0x8c04, 0xf8ff00ff, 0x40600060,
978 0x8c08, 0x00ff00ff, 0x001c001c,
979 0x8cf0, 0x1fff1fff, 0x08e00410,
980 0x8c20, 0x0fff0fff, 0x00800080,
981 0x8c24, 0x0fff0fff, 0x00800080,
982 0x8c18, 0xffffffff, 0x20202078,
983 0x8c1c, 0x0000ffff, 0x00001010,
984 0x28350, 0x00000f01, 0x00000000,
985 0x9508, 0x3700001f, 0x00000002,
986 0x960c, 0xffffffff, 0x54763210,
987 0x88c4, 0x001f3ae3, 0x000000c2,
988 0x88d4, 0x0000001f, 0x00000010,
989 0x8974, 0xffffffff, 0x00000000
990 };
991
992 static void evergreen_init_golden_registers(struct radeon_device *rdev)
993 {
994 switch (rdev->family) {
995 case CHIP_CYPRESS:
996 case CHIP_HEMLOCK:
997 radeon_program_register_sequence(rdev,
998 evergreen_golden_registers,
999 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1000 radeon_program_register_sequence(rdev,
1001 evergreen_golden_registers2,
1002 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1003 radeon_program_register_sequence(rdev,
1004 cypress_mgcg_init,
1005 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1006 break;
1007 case CHIP_JUNIPER:
1008 radeon_program_register_sequence(rdev,
1009 evergreen_golden_registers,
1010 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1011 radeon_program_register_sequence(rdev,
1012 evergreen_golden_registers2,
1013 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1014 radeon_program_register_sequence(rdev,
1015 juniper_mgcg_init,
1016 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1017 break;
1018 case CHIP_REDWOOD:
1019 radeon_program_register_sequence(rdev,
1020 evergreen_golden_registers,
1021 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1022 radeon_program_register_sequence(rdev,
1023 evergreen_golden_registers2,
1024 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1025 radeon_program_register_sequence(rdev,
1026 redwood_mgcg_init,
1027 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1028 break;
1029 case CHIP_CEDAR:
1030 radeon_program_register_sequence(rdev,
1031 cedar_golden_registers,
1032 (const u32)ARRAY_SIZE(cedar_golden_registers));
1033 radeon_program_register_sequence(rdev,
1034 evergreen_golden_registers2,
1035 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1036 radeon_program_register_sequence(rdev,
1037 cedar_mgcg_init,
1038 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1039 break;
1040 case CHIP_PALM:
1041 radeon_program_register_sequence(rdev,
1042 wrestler_golden_registers,
1043 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1044 break;
1045 case CHIP_SUMO:
1046 radeon_program_register_sequence(rdev,
1047 supersumo_golden_registers,
1048 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1049 break;
1050 case CHIP_SUMO2:
1051 radeon_program_register_sequence(rdev,
1052 supersumo_golden_registers,
1053 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054 radeon_program_register_sequence(rdev,
1055 sumo_golden_registers,
1056 (const u32)ARRAY_SIZE(sumo_golden_registers));
1057 break;
1058 case CHIP_BARTS:
1059 radeon_program_register_sequence(rdev,
1060 barts_golden_registers,
1061 (const u32)ARRAY_SIZE(barts_golden_registers));
1062 break;
1063 case CHIP_TURKS:
1064 radeon_program_register_sequence(rdev,
1065 turks_golden_registers,
1066 (const u32)ARRAY_SIZE(turks_golden_registers));
1067 break;
1068 case CHIP_CAICOS:
1069 radeon_program_register_sequence(rdev,
1070 caicos_golden_registers,
1071 (const u32)ARRAY_SIZE(caicos_golden_registers));
1072 break;
1073 default:
1074 break;
1075 }
1076 }
1077
1078 /**
1079 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1080 *
1081 * @rdev: radeon_device pointer
1082 * @reg: register offset in bytes
1083 * @val: register value
1084 *
1085 * Returns 0 for success or -EINVAL for an invalid register
1086 *
1087 */
1088 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1089 u32 reg, u32 *val)
1090 {
1091 switch (reg) {
1092 case GRBM_STATUS:
1093 case GRBM_STATUS_SE0:
1094 case GRBM_STATUS_SE1:
1095 case SRBM_STATUS:
1096 case SRBM_STATUS2:
1097 case DMA_STATUS_REG:
1098 case UVD_STATUS:
1099 *val = RREG32(reg);
1100 return 0;
1101 default:
1102 return -EINVAL;
1103 }
1104 }
1105
1106 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1107 unsigned *bankh, unsigned *mtaspect,
1108 unsigned *tile_split)
1109 {
1110 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1111 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1112 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1113 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1114 switch (*bankw) {
1115 default:
1116 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1117 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1118 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1119 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1120 }
1121 switch (*bankh) {
1122 default:
1123 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1124 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1125 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1126 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1127 }
1128 switch (*mtaspect) {
1129 default:
1130 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1131 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1132 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1133 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1134 }
1135 }
1136
1137 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1138 u32 cntl_reg, u32 status_reg)
1139 {
1140 int r, i;
1141 struct atom_clock_dividers dividers;
1142
1143 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1144 clock, false, &dividers);
1145 if (r)
1146 return r;
1147
1148 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1149
1150 for (i = 0; i < 100; i++) {
1151 if (RREG32(status_reg) & DCLK_STATUS)
1152 break;
1153 mdelay(10);
1154 }
1155 if (i == 100)
1156 return -ETIMEDOUT;
1157
1158 return 0;
1159 }
1160
1161 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1162 {
1163 int r = 0;
1164 u32 cg_scratch = RREG32(CG_SCRATCH1);
1165
1166 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1167 if (r)
1168 goto done;
1169 cg_scratch &= 0xffff0000;
1170 cg_scratch |= vclk / 100; /* Mhz */
1171
1172 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1173 if (r)
1174 goto done;
1175 cg_scratch &= 0x0000ffff;
1176 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1177
1178 done:
1179 WREG32(CG_SCRATCH1, cg_scratch);
1180
1181 return r;
1182 }
1183
1184 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1185 {
1186 /* start off with something large */
1187 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1188 int r;
1189
1190 /* bypass vclk and dclk with bclk */
1191 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1192 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1193 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1194
1195 /* put PLL in bypass mode */
1196 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1197
1198 if (!vclk || !dclk) {
1199 /* keep the Bypass mode, put PLL to sleep */
1200 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1201 return 0;
1202 }
1203
1204 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1205 16384, 0x03FFFFFF, 0, 128, 5,
1206 &fb_div, &vclk_div, &dclk_div);
1207 if (r)
1208 return r;
1209
1210 /* set VCO_MODE to 1 */
1211 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1212
1213 /* toggle UPLL_SLEEP to 1 then back to 0 */
1214 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1215 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1216
1217 /* deassert UPLL_RESET */
1218 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1219
1220 mdelay(1);
1221
1222 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1223 if (r)
1224 return r;
1225
1226 /* assert UPLL_RESET again */
1227 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1228
1229 /* disable spread spectrum. */
1230 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1231
1232 /* set feedback divider */
1233 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1234
1235 /* set ref divider to 0 */
1236 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1237
1238 if (fb_div < 307200)
1239 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1240 else
1241 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1242
1243 /* set PDIV_A and PDIV_B */
1244 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1245 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1246 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1247
1248 /* give the PLL some time to settle */
1249 mdelay(15);
1250
1251 /* deassert PLL_RESET */
1252 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1253
1254 mdelay(15);
1255
1256 /* switch from bypass mode to normal mode */
1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1258
1259 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1260 if (r)
1261 return r;
1262
1263 /* switch VCLK and DCLK selection */
1264 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1265 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1266 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1267
1268 mdelay(100);
1269
1270 return 0;
1271 }
1272
1273 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1274 {
1275 int readrq;
1276 u16 v;
1277
1278 readrq = pcie_get_readrq(rdev->pdev);
1279 v = ffs(readrq) - 8;
1280 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1281 * to avoid hangs or perfomance issues
1282 */
1283 if ((v == 0) || (v == 6) || (v == 7))
1284 pcie_set_readrq(rdev->pdev, 512);
1285 }
1286
1287 void dce4_program_fmt(struct drm_encoder *encoder)
1288 {
1289 struct drm_device *dev = encoder->dev;
1290 struct radeon_device *rdev = dev->dev_private;
1291 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1292 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1293 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1294 int bpc = 0;
1295 u32 tmp = 0;
1296 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1297
1298 if (connector) {
1299 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1300 bpc = radeon_get_monitor_bpc(connector);
1301 dither = radeon_connector->dither;
1302 }
1303
1304 /* LVDS/eDP FMT is set up by atom */
1305 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1306 return;
1307
1308 /* not needed for analog */
1309 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1310 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1311 return;
1312
1313 if (bpc == 0)
1314 return;
1315
1316 switch (bpc) {
1317 case 6:
1318 if (dither == RADEON_FMT_DITHER_ENABLE)
1319 /* XXX sort out optimal dither settings */
1320 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1321 FMT_SPATIAL_DITHER_EN);
1322 else
1323 tmp |= FMT_TRUNCATE_EN;
1324 break;
1325 case 8:
1326 if (dither == RADEON_FMT_DITHER_ENABLE)
1327 /* XXX sort out optimal dither settings */
1328 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329 FMT_RGB_RANDOM_ENABLE |
1330 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1331 else
1332 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1333 break;
1334 case 10:
1335 default:
1336 /* not needed */
1337 break;
1338 }
1339
1340 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1341 }
1342
1343 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1344 {
1345 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1346 return true;
1347 else
1348 return false;
1349 }
1350
1351 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1352 {
1353 u32 pos1, pos2;
1354
1355 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1356 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1357
1358 if (pos1 != pos2)
1359 return true;
1360 else
1361 return false;
1362 }
1363
1364 /**
1365 * dce4_wait_for_vblank - vblank wait asic callback.
1366 *
1367 * @rdev: radeon_device pointer
1368 * @crtc: crtc to wait for vblank on
1369 *
1370 * Wait for vblank on the requested crtc (evergreen+).
1371 */
1372 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1373 {
1374 unsigned i = 0;
1375
1376 if (crtc >= rdev->num_crtc)
1377 return;
1378
1379 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1380 return;
1381
1382 /* depending on when we hit vblank, we may be close to active; if so,
1383 * wait for another frame.
1384 */
1385 while (dce4_is_in_vblank(rdev, crtc)) {
1386 if (i++ % 100 == 0) {
1387 if (!dce4_is_counter_moving(rdev, crtc))
1388 break;
1389 }
1390 }
1391
1392 while (!dce4_is_in_vblank(rdev, crtc)) {
1393 if (i++ % 100 == 0) {
1394 if (!dce4_is_counter_moving(rdev, crtc))
1395 break;
1396 }
1397 }
1398 }
1399
1400 /**
1401 * evergreen_page_flip - pageflip callback.
1402 *
1403 * @rdev: radeon_device pointer
1404 * @crtc_id: crtc to cleanup pageflip on
1405 * @crtc_base: new address of the crtc (GPU MC address)
1406 *
1407 * Does the actual pageflip (evergreen+).
1408 * During vblank we take the crtc lock and wait for the update_pending
1409 * bit to go high, when it does, we release the lock, and allow the
1410 * double buffered update to take place.
1411 * Returns the current update pending status.
1412 */
1413 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1414 {
1415 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1416 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1417 int i;
1418
1419 /* Lock the graphics update lock */
1420 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1421 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1422
1423 /* update the scanout addresses */
1424 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1425 upper_32_bits(crtc_base));
1426 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1427 (u32)crtc_base);
1428
1429 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430 upper_32_bits(crtc_base));
1431 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1432 (u32)crtc_base);
1433
1434 /* Wait for update_pending to go high. */
1435 for (i = 0; i < rdev->usec_timeout; i++) {
1436 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1437 break;
1438 udelay(1);
1439 }
1440 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1441
1442 /* Unlock the lock, so double-buffering can take place inside vblank */
1443 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1444 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1445 }
1446
1447 /**
1448 * evergreen_page_flip_pending - check if page flip is still pending
1449 *
1450 * @rdev: radeon_device pointer
1451 * @crtc_id: crtc to check
1452 *
1453 * Returns the current update pending status.
1454 */
1455 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1456 {
1457 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1458
1459 /* Return current update_pending status: */
1460 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1461 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1462 }
1463
1464 /* get temperature in millidegrees */
1465 int evergreen_get_temp(struct radeon_device *rdev)
1466 {
1467 u32 temp, toffset;
1468 int actual_temp = 0;
1469
1470 if (rdev->family == CHIP_JUNIPER) {
1471 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1472 TOFFSET_SHIFT;
1473 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1474 TS0_ADC_DOUT_SHIFT;
1475
1476 if (toffset & 0x100)
1477 actual_temp = temp / 2 - (0x200 - toffset);
1478 else
1479 actual_temp = temp / 2 + toffset;
1480
1481 actual_temp = actual_temp * 1000;
1482
1483 } else {
1484 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1485 ASIC_T_SHIFT;
1486
1487 if (temp & 0x400)
1488 actual_temp = -256;
1489 else if (temp & 0x200)
1490 actual_temp = 255;
1491 else if (temp & 0x100) {
1492 actual_temp = temp & 0x1ff;
1493 actual_temp |= ~0x1ff;
1494 } else
1495 actual_temp = temp & 0xff;
1496
1497 actual_temp = (actual_temp * 1000) / 2;
1498 }
1499
1500 return actual_temp;
1501 }
1502
1503 int sumo_get_temp(struct radeon_device *rdev)
1504 {
1505 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1506 int actual_temp = temp - 49;
1507
1508 return actual_temp * 1000;
1509 }
1510
1511 /**
1512 * sumo_pm_init_profile - Initialize power profiles callback.
1513 *
1514 * @rdev: radeon_device pointer
1515 *
1516 * Initialize the power states used in profile mode
1517 * (sumo, trinity, SI).
1518 * Used for profile mode only.
1519 */
1520 void sumo_pm_init_profile(struct radeon_device *rdev)
1521 {
1522 int idx;
1523
1524 /* default */
1525 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1526 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1527 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1528 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1529
1530 /* low,mid sh/mh */
1531 if (rdev->flags & RADEON_IS_MOBILITY)
1532 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1533 else
1534 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1535
1536 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1537 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1538 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1539 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1540
1541 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1543 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1544 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1545
1546 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1547 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1548 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1549 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1550
1551 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1552 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1553 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1554 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1555
1556 /* high sh/mh */
1557 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1558 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1559 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1560 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1561 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1562 rdev->pm.power_state[idx].num_clock_modes - 1;
1563
1564 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1565 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1566 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1567 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1568 rdev->pm.power_state[idx].num_clock_modes - 1;
1569 }
1570
1571 /**
1572 * btc_pm_init_profile - Initialize power profiles callback.
1573 *
1574 * @rdev: radeon_device pointer
1575 *
1576 * Initialize the power states used in profile mode
1577 * (BTC, cayman).
1578 * Used for profile mode only.
1579 */
1580 void btc_pm_init_profile(struct radeon_device *rdev)
1581 {
1582 int idx;
1583
1584 /* default */
1585 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1586 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1587 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1588 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1589 /* starting with BTC, there is one state that is used for both
1590 * MH and SH. Difference is that we always use the high clock index for
1591 * mclk.
1592 */
1593 if (rdev->flags & RADEON_IS_MOBILITY)
1594 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1595 else
1596 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1597 /* low sh */
1598 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1599 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1600 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1601 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1602 /* mid sh */
1603 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1604 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1605 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1606 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1607 /* high sh */
1608 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1609 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1610 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1611 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1612 /* low mh */
1613 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1614 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1615 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1616 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1617 /* mid mh */
1618 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1619 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1620 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1621 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1622 /* high mh */
1623 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1624 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1625 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1626 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1627 }
1628
1629 /**
1630 * evergreen_pm_misc - set additional pm hw parameters callback.
1631 *
1632 * @rdev: radeon_device pointer
1633 *
1634 * Set non-clock parameters associated with a power state
1635 * (voltage, etc.) (evergreen+).
1636 */
1637 void evergreen_pm_misc(struct radeon_device *rdev)
1638 {
1639 int req_ps_idx = rdev->pm.requested_power_state_index;
1640 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1641 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1642 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1643
1644 if (voltage->type == VOLTAGE_SW) {
1645 /* 0xff0x are flags rather then an actual voltage */
1646 if ((voltage->voltage & 0xff00) == 0xff00)
1647 return;
1648 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1649 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1650 rdev->pm.current_vddc = voltage->voltage;
1651 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1652 }
1653
1654 /* starting with BTC, there is one state that is used for both
1655 * MH and SH. Difference is that we always use the high clock index for
1656 * mclk and vddci.
1657 */
1658 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1659 (rdev->family >= CHIP_BARTS) &&
1660 rdev->pm.active_crtc_count &&
1661 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1662 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1663 voltage = &rdev->pm.power_state[req_ps_idx].
1664 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1665
1666 /* 0xff0x are flags rather then an actual voltage */
1667 if ((voltage->vddci & 0xff00) == 0xff00)
1668 return;
1669 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1670 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1671 rdev->pm.current_vddci = voltage->vddci;
1672 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1673 }
1674 }
1675 }
1676
1677 /**
1678 * evergreen_pm_prepare - pre-power state change callback.
1679 *
1680 * @rdev: radeon_device pointer
1681 *
1682 * Prepare for a power state change (evergreen+).
1683 */
1684 void evergreen_pm_prepare(struct radeon_device *rdev)
1685 {
1686 struct drm_device *ddev = rdev->ddev;
1687 struct drm_crtc *crtc;
1688 struct radeon_crtc *radeon_crtc;
1689 u32 tmp;
1690
1691 /* disable any active CRTCs */
1692 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1693 radeon_crtc = to_radeon_crtc(crtc);
1694 if (radeon_crtc->enabled) {
1695 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1696 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1697 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1698 }
1699 }
1700 }
1701
1702 /**
1703 * evergreen_pm_finish - post-power state change callback.
1704 *
1705 * @rdev: radeon_device pointer
1706 *
1707 * Clean up after a power state change (evergreen+).
1708 */
1709 void evergreen_pm_finish(struct radeon_device *rdev)
1710 {
1711 struct drm_device *ddev = rdev->ddev;
1712 struct drm_crtc *crtc;
1713 struct radeon_crtc *radeon_crtc;
1714 u32 tmp;
1715
1716 /* enable any active CRTCs */
1717 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1718 radeon_crtc = to_radeon_crtc(crtc);
1719 if (radeon_crtc->enabled) {
1720 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1721 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1722 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1723 }
1724 }
1725 }
1726
1727 /**
1728 * evergreen_hpd_sense - hpd sense callback.
1729 *
1730 * @rdev: radeon_device pointer
1731 * @hpd: hpd (hotplug detect) pin
1732 *
1733 * Checks if a digital monitor is connected (evergreen+).
1734 * Returns true if connected, false if not connected.
1735 */
1736 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1737 {
1738 bool connected = false;
1739
1740 switch (hpd) {
1741 case RADEON_HPD_1:
1742 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1743 connected = true;
1744 break;
1745 case RADEON_HPD_2:
1746 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1747 connected = true;
1748 break;
1749 case RADEON_HPD_3:
1750 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1751 connected = true;
1752 break;
1753 case RADEON_HPD_4:
1754 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1755 connected = true;
1756 break;
1757 case RADEON_HPD_5:
1758 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1759 connected = true;
1760 break;
1761 case RADEON_HPD_6:
1762 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1763 connected = true;
1764 break;
1765 default:
1766 break;
1767 }
1768
1769 return connected;
1770 }
1771
1772 /**
1773 * evergreen_hpd_set_polarity - hpd set polarity callback.
1774 *
1775 * @rdev: radeon_device pointer
1776 * @hpd: hpd (hotplug detect) pin
1777 *
1778 * Set the polarity of the hpd pin (evergreen+).
1779 */
1780 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1781 enum radeon_hpd_id hpd)
1782 {
1783 u32 tmp;
1784 bool connected = evergreen_hpd_sense(rdev, hpd);
1785
1786 switch (hpd) {
1787 case RADEON_HPD_1:
1788 tmp = RREG32(DC_HPD1_INT_CONTROL);
1789 if (connected)
1790 tmp &= ~DC_HPDx_INT_POLARITY;
1791 else
1792 tmp |= DC_HPDx_INT_POLARITY;
1793 WREG32(DC_HPD1_INT_CONTROL, tmp);
1794 break;
1795 case RADEON_HPD_2:
1796 tmp = RREG32(DC_HPD2_INT_CONTROL);
1797 if (connected)
1798 tmp &= ~DC_HPDx_INT_POLARITY;
1799 else
1800 tmp |= DC_HPDx_INT_POLARITY;
1801 WREG32(DC_HPD2_INT_CONTROL, tmp);
1802 break;
1803 case RADEON_HPD_3:
1804 tmp = RREG32(DC_HPD3_INT_CONTROL);
1805 if (connected)
1806 tmp &= ~DC_HPDx_INT_POLARITY;
1807 else
1808 tmp |= DC_HPDx_INT_POLARITY;
1809 WREG32(DC_HPD3_INT_CONTROL, tmp);
1810 break;
1811 case RADEON_HPD_4:
1812 tmp = RREG32(DC_HPD4_INT_CONTROL);
1813 if (connected)
1814 tmp &= ~DC_HPDx_INT_POLARITY;
1815 else
1816 tmp |= DC_HPDx_INT_POLARITY;
1817 WREG32(DC_HPD4_INT_CONTROL, tmp);
1818 break;
1819 case RADEON_HPD_5:
1820 tmp = RREG32(DC_HPD5_INT_CONTROL);
1821 if (connected)
1822 tmp &= ~DC_HPDx_INT_POLARITY;
1823 else
1824 tmp |= DC_HPDx_INT_POLARITY;
1825 WREG32(DC_HPD5_INT_CONTROL, tmp);
1826 break;
1827 case RADEON_HPD_6:
1828 tmp = RREG32(DC_HPD6_INT_CONTROL);
1829 if (connected)
1830 tmp &= ~DC_HPDx_INT_POLARITY;
1831 else
1832 tmp |= DC_HPDx_INT_POLARITY;
1833 WREG32(DC_HPD6_INT_CONTROL, tmp);
1834 break;
1835 default:
1836 break;
1837 }
1838 }
1839
1840 /**
1841 * evergreen_hpd_init - hpd setup callback.
1842 *
1843 * @rdev: radeon_device pointer
1844 *
1845 * Setup the hpd pins used by the card (evergreen+).
1846 * Enable the pin, set the polarity, and enable the hpd interrupts.
1847 */
1848 void evergreen_hpd_init(struct radeon_device *rdev)
1849 {
1850 struct drm_device *dev = rdev->ddev;
1851 struct drm_connector *connector;
1852 unsigned enabled = 0;
1853 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1854 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1855
1856 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1857 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1858
1859 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1860 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1861 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1862 * aux dp channel on imac and help (but not completely fix)
1863 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1864 * also avoid interrupt storms during dpms.
1865 */
1866 continue;
1867 }
1868 switch (radeon_connector->hpd.hpd) {
1869 case RADEON_HPD_1:
1870 WREG32(DC_HPD1_CONTROL, tmp);
1871 break;
1872 case RADEON_HPD_2:
1873 WREG32(DC_HPD2_CONTROL, tmp);
1874 break;
1875 case RADEON_HPD_3:
1876 WREG32(DC_HPD3_CONTROL, tmp);
1877 break;
1878 case RADEON_HPD_4:
1879 WREG32(DC_HPD4_CONTROL, tmp);
1880 break;
1881 case RADEON_HPD_5:
1882 WREG32(DC_HPD5_CONTROL, tmp);
1883 break;
1884 case RADEON_HPD_6:
1885 WREG32(DC_HPD6_CONTROL, tmp);
1886 break;
1887 default:
1888 break;
1889 }
1890 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1891 enabled |= 1 << radeon_connector->hpd.hpd;
1892 }
1893 radeon_irq_kms_enable_hpd(rdev, enabled);
1894 }
1895
1896 /**
1897 * evergreen_hpd_fini - hpd tear down callback.
1898 *
1899 * @rdev: radeon_device pointer
1900 *
1901 * Tear down the hpd pins used by the card (evergreen+).
1902 * Disable the hpd interrupts.
1903 */
1904 void evergreen_hpd_fini(struct radeon_device *rdev)
1905 {
1906 struct drm_device *dev = rdev->ddev;
1907 struct drm_connector *connector;
1908 unsigned disabled = 0;
1909
1910 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1911 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1912 switch (radeon_connector->hpd.hpd) {
1913 case RADEON_HPD_1:
1914 WREG32(DC_HPD1_CONTROL, 0);
1915 break;
1916 case RADEON_HPD_2:
1917 WREG32(DC_HPD2_CONTROL, 0);
1918 break;
1919 case RADEON_HPD_3:
1920 WREG32(DC_HPD3_CONTROL, 0);
1921 break;
1922 case RADEON_HPD_4:
1923 WREG32(DC_HPD4_CONTROL, 0);
1924 break;
1925 case RADEON_HPD_5:
1926 WREG32(DC_HPD5_CONTROL, 0);
1927 break;
1928 case RADEON_HPD_6:
1929 WREG32(DC_HPD6_CONTROL, 0);
1930 break;
1931 default:
1932 break;
1933 }
1934 disabled |= 1 << radeon_connector->hpd.hpd;
1935 }
1936 radeon_irq_kms_disable_hpd(rdev, disabled);
1937 }
1938
1939 /* watermark setup */
1940
1941 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1942 struct radeon_crtc *radeon_crtc,
1943 struct drm_display_mode *mode,
1944 struct drm_display_mode *other_mode)
1945 {
1946 u32 tmp, buffer_alloc, i;
1947 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1948 /*
1949 * Line Buffer Setup
1950 * There are 3 line buffers, each one shared by 2 display controllers.
1951 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1952 * the display controllers. The paritioning is done via one of four
1953 * preset allocations specified in bits 2:0:
1954 * first display controller
1955 * 0 - first half of lb (3840 * 2)
1956 * 1 - first 3/4 of lb (5760 * 2)
1957 * 2 - whole lb (7680 * 2), other crtc must be disabled
1958 * 3 - first 1/4 of lb (1920 * 2)
1959 * second display controller
1960 * 4 - second half of lb (3840 * 2)
1961 * 5 - second 3/4 of lb (5760 * 2)
1962 * 6 - whole lb (7680 * 2), other crtc must be disabled
1963 * 7 - last 1/4 of lb (1920 * 2)
1964 */
1965 /* this can get tricky if we have two large displays on a paired group
1966 * of crtcs. Ideally for multiple large displays we'd assign them to
1967 * non-linked crtcs for maximum line buffer allocation.
1968 */
1969 if (radeon_crtc->base.enabled && mode) {
1970 if (other_mode) {
1971 tmp = 0; /* 1/2 */
1972 buffer_alloc = 1;
1973 } else {
1974 tmp = 2; /* whole */
1975 buffer_alloc = 2;
1976 }
1977 } else {
1978 tmp = 0;
1979 buffer_alloc = 0;
1980 }
1981
1982 /* second controller of the pair uses second half of the lb */
1983 if (radeon_crtc->crtc_id % 2)
1984 tmp += 4;
1985 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1986
1987 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1988 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1989 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1990 for (i = 0; i < rdev->usec_timeout; i++) {
1991 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1992 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1993 break;
1994 udelay(1);
1995 }
1996 }
1997
1998 if (radeon_crtc->base.enabled && mode) {
1999 switch (tmp) {
2000 case 0:
2001 case 4:
2002 default:
2003 if (ASIC_IS_DCE5(rdev))
2004 return 4096 * 2;
2005 else
2006 return 3840 * 2;
2007 case 1:
2008 case 5:
2009 if (ASIC_IS_DCE5(rdev))
2010 return 6144 * 2;
2011 else
2012 return 5760 * 2;
2013 case 2:
2014 case 6:
2015 if (ASIC_IS_DCE5(rdev))
2016 return 8192 * 2;
2017 else
2018 return 7680 * 2;
2019 case 3:
2020 case 7:
2021 if (ASIC_IS_DCE5(rdev))
2022 return 2048 * 2;
2023 else
2024 return 1920 * 2;
2025 }
2026 }
2027
2028 /* controller not enabled, so no lb used */
2029 return 0;
2030 }
2031
2032 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
2033 {
2034 u32 tmp = RREG32(MC_SHARED_CHMAP);
2035
2036 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2037 case 0:
2038 default:
2039 return 1;
2040 case 1:
2041 return 2;
2042 case 2:
2043 return 4;
2044 case 3:
2045 return 8;
2046 }
2047 }
2048
2049 struct evergreen_wm_params {
2050 u32 dram_channels; /* number of dram channels */
2051 u32 yclk; /* bandwidth per dram data pin in kHz */
2052 u32 sclk; /* engine clock in kHz */
2053 u32 disp_clk; /* display clock in kHz */
2054 u32 src_width; /* viewport width */
2055 u32 active_time; /* active display time in ns */
2056 u32 blank_time; /* blank time in ns */
2057 bool interlaced; /* mode is interlaced */
2058 fixed20_12 vsc; /* vertical scale ratio */
2059 u32 num_heads; /* number of active crtcs */
2060 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2061 u32 lb_size; /* line buffer allocated to pipe */
2062 u32 vtaps; /* vertical scaler taps */
2063 };
2064
2065 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
2066 {
2067 /* Calculate DRAM Bandwidth and the part allocated to display. */
2068 fixed20_12 dram_efficiency; /* 0.7 */
2069 fixed20_12 yclk, dram_channels, bandwidth;
2070 fixed20_12 a;
2071
2072 a.full = dfixed_const(1000);
2073 yclk.full = dfixed_const(wm->yclk);
2074 yclk.full = dfixed_div(yclk, a);
2075 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2076 a.full = dfixed_const(10);
2077 dram_efficiency.full = dfixed_const(7);
2078 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2079 bandwidth.full = dfixed_mul(dram_channels, yclk);
2080 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2081
2082 return dfixed_trunc(bandwidth);
2083 }
2084
2085 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2086 {
2087 /* Calculate DRAM Bandwidth and the part allocated to display. */
2088 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2089 fixed20_12 yclk, dram_channels, bandwidth;
2090 fixed20_12 a;
2091
2092 a.full = dfixed_const(1000);
2093 yclk.full = dfixed_const(wm->yclk);
2094 yclk.full = dfixed_div(yclk, a);
2095 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2096 a.full = dfixed_const(10);
2097 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2098 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2099 bandwidth.full = dfixed_mul(dram_channels, yclk);
2100 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2101
2102 return dfixed_trunc(bandwidth);
2103 }
2104
2105 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2106 {
2107 /* Calculate the display Data return Bandwidth */
2108 fixed20_12 return_efficiency; /* 0.8 */
2109 fixed20_12 sclk, bandwidth;
2110 fixed20_12 a;
2111
2112 a.full = dfixed_const(1000);
2113 sclk.full = dfixed_const(wm->sclk);
2114 sclk.full = dfixed_div(sclk, a);
2115 a.full = dfixed_const(10);
2116 return_efficiency.full = dfixed_const(8);
2117 return_efficiency.full = dfixed_div(return_efficiency, a);
2118 a.full = dfixed_const(32);
2119 bandwidth.full = dfixed_mul(a, sclk);
2120 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2121
2122 return dfixed_trunc(bandwidth);
2123 }
2124
2125 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2126 {
2127 /* Calculate the DMIF Request Bandwidth */
2128 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2129 fixed20_12 disp_clk, bandwidth;
2130 fixed20_12 a;
2131
2132 a.full = dfixed_const(1000);
2133 disp_clk.full = dfixed_const(wm->disp_clk);
2134 disp_clk.full = dfixed_div(disp_clk, a);
2135 a.full = dfixed_const(10);
2136 disp_clk_request_efficiency.full = dfixed_const(8);
2137 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2138 a.full = dfixed_const(32);
2139 bandwidth.full = dfixed_mul(a, disp_clk);
2140 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2141
2142 return dfixed_trunc(bandwidth);
2143 }
2144
2145 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2146 {
2147 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2148 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2149 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2150 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2151
2152 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2153 }
2154
2155 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2156 {
2157 /* Calculate the display mode Average Bandwidth
2158 * DisplayMode should contain the source and destination dimensions,
2159 * timing, etc.
2160 */
2161 fixed20_12 bpp;
2162 fixed20_12 line_time;
2163 fixed20_12 src_width;
2164 fixed20_12 bandwidth;
2165 fixed20_12 a;
2166
2167 a.full = dfixed_const(1000);
2168 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2169 line_time.full = dfixed_div(line_time, a);
2170 bpp.full = dfixed_const(wm->bytes_per_pixel);
2171 src_width.full = dfixed_const(wm->src_width);
2172 bandwidth.full = dfixed_mul(src_width, bpp);
2173 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2174 bandwidth.full = dfixed_div(bandwidth, line_time);
2175
2176 return dfixed_trunc(bandwidth);
2177 }
2178
2179 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2180 {
2181 /* First calcualte the latency in ns */
2182 u32 mc_latency = 2000; /* 2000 ns. */
2183 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2184 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2185 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2186 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2187 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2188 (wm->num_heads * cursor_line_pair_return_time);
2189 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2190 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2191 fixed20_12 a, b, c;
2192
2193 if (wm->num_heads == 0)
2194 return 0;
2195
2196 a.full = dfixed_const(2);
2197 b.full = dfixed_const(1);
2198 if ((wm->vsc.full > a.full) ||
2199 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2200 (wm->vtaps >= 5) ||
2201 ((wm->vsc.full >= a.full) && wm->interlaced))
2202 max_src_lines_per_dst_line = 4;
2203 else
2204 max_src_lines_per_dst_line = 2;
2205
2206 a.full = dfixed_const(available_bandwidth);
2207 b.full = dfixed_const(wm->num_heads);
2208 a.full = dfixed_div(a, b);
2209
2210 b.full = dfixed_const(1000);
2211 c.full = dfixed_const(wm->disp_clk);
2212 b.full = dfixed_div(c, b);
2213 c.full = dfixed_const(wm->bytes_per_pixel);
2214 b.full = dfixed_mul(b, c);
2215
2216 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2217
2218 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2219 b.full = dfixed_const(1000);
2220 c.full = dfixed_const(lb_fill_bw);
2221 b.full = dfixed_div(c, b);
2222 a.full = dfixed_div(a, b);
2223 line_fill_time = dfixed_trunc(a);
2224
2225 if (line_fill_time < wm->active_time)
2226 return latency;
2227 else
2228 return latency + (line_fill_time - wm->active_time);
2229
2230 }
2231
2232 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2233 {
2234 if (evergreen_average_bandwidth(wm) <=
2235 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2236 return true;
2237 else
2238 return false;
2239 };
2240
2241 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2242 {
2243 if (evergreen_average_bandwidth(wm) <=
2244 (evergreen_available_bandwidth(wm) / wm->num_heads))
2245 return true;
2246 else
2247 return false;
2248 };
2249
2250 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2251 {
2252 u32 lb_partitions = wm->lb_size / wm->src_width;
2253 u32 line_time = wm->active_time + wm->blank_time;
2254 u32 latency_tolerant_lines;
2255 u32 latency_hiding;
2256 fixed20_12 a;
2257
2258 a.full = dfixed_const(1);
2259 if (wm->vsc.full > a.full)
2260 latency_tolerant_lines = 1;
2261 else {
2262 if (lb_partitions <= (wm->vtaps + 1))
2263 latency_tolerant_lines = 1;
2264 else
2265 latency_tolerant_lines = 2;
2266 }
2267
2268 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2269
2270 if (evergreen_latency_watermark(wm) <= latency_hiding)
2271 return true;
2272 else
2273 return false;
2274 }
2275
2276 static void evergreen_program_watermarks(struct radeon_device *rdev,
2277 struct radeon_crtc *radeon_crtc,
2278 u32 lb_size, u32 num_heads)
2279 {
2280 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2281 struct evergreen_wm_params wm_low, wm_high;
2282 u32 dram_channels;
2283 u32 pixel_period;
2284 u32 line_time = 0;
2285 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2286 u32 priority_a_mark = 0, priority_b_mark = 0;
2287 u32 priority_a_cnt = PRIORITY_OFF;
2288 u32 priority_b_cnt = PRIORITY_OFF;
2289 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2290 u32 tmp, arb_control3;
2291 fixed20_12 a, b, c;
2292
2293 if (radeon_crtc->base.enabled && num_heads && mode) {
2294 pixel_period = 1000000 / (u32)mode->clock;
2295 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2296 priority_a_cnt = 0;
2297 priority_b_cnt = 0;
2298 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2299
2300 /* watermark for high clocks */
2301 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2302 wm_high.yclk =
2303 radeon_dpm_get_mclk(rdev, false) * 10;
2304 wm_high.sclk =
2305 radeon_dpm_get_sclk(rdev, false) * 10;
2306 } else {
2307 wm_high.yclk = rdev->pm.current_mclk * 10;
2308 wm_high.sclk = rdev->pm.current_sclk * 10;
2309 }
2310
2311 wm_high.disp_clk = mode->clock;
2312 wm_high.src_width = mode->crtc_hdisplay;
2313 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2314 wm_high.blank_time = line_time - wm_high.active_time;
2315 wm_high.interlaced = false;
2316 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2317 wm_high.interlaced = true;
2318 wm_high.vsc = radeon_crtc->vsc;
2319 wm_high.vtaps = 1;
2320 if (radeon_crtc->rmx_type != RMX_OFF)
2321 wm_high.vtaps = 2;
2322 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2323 wm_high.lb_size = lb_size;
2324 wm_high.dram_channels = dram_channels;
2325 wm_high.num_heads = num_heads;
2326
2327 /* watermark for low clocks */
2328 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2329 wm_low.yclk =
2330 radeon_dpm_get_mclk(rdev, true) * 10;
2331 wm_low.sclk =
2332 radeon_dpm_get_sclk(rdev, true) * 10;
2333 } else {
2334 wm_low.yclk = rdev->pm.current_mclk * 10;
2335 wm_low.sclk = rdev->pm.current_sclk * 10;
2336 }
2337
2338 wm_low.disp_clk = mode->clock;
2339 wm_low.src_width = mode->crtc_hdisplay;
2340 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2341 wm_low.blank_time = line_time - wm_low.active_time;
2342 wm_low.interlaced = false;
2343 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2344 wm_low.interlaced = true;
2345 wm_low.vsc = radeon_crtc->vsc;
2346 wm_low.vtaps = 1;
2347 if (radeon_crtc->rmx_type != RMX_OFF)
2348 wm_low.vtaps = 2;
2349 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2350 wm_low.lb_size = lb_size;
2351 wm_low.dram_channels = dram_channels;
2352 wm_low.num_heads = num_heads;
2353
2354 /* set for high clocks */
2355 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2356 /* set for low clocks */
2357 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2358
2359 /* possibly force display priority to high */
2360 /* should really do this at mode validation time... */
2361 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2362 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2363 !evergreen_check_latency_hiding(&wm_high) ||
2364 (rdev->disp_priority == 2)) {
2365 DRM_DEBUG_KMS("force priority a to high\n");
2366 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2367 }
2368 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2369 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2370 !evergreen_check_latency_hiding(&wm_low) ||
2371 (rdev->disp_priority == 2)) {
2372 DRM_DEBUG_KMS("force priority b to high\n");
2373 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2374 }
2375
2376 a.full = dfixed_const(1000);
2377 b.full = dfixed_const(mode->clock);
2378 b.full = dfixed_div(b, a);
2379 c.full = dfixed_const(latency_watermark_a);
2380 c.full = dfixed_mul(c, b);
2381 c.full = dfixed_mul(c, radeon_crtc->hsc);
2382 c.full = dfixed_div(c, a);
2383 a.full = dfixed_const(16);
2384 c.full = dfixed_div(c, a);
2385 priority_a_mark = dfixed_trunc(c);
2386 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2387
2388 a.full = dfixed_const(1000);
2389 b.full = dfixed_const(mode->clock);
2390 b.full = dfixed_div(b, a);
2391 c.full = dfixed_const(latency_watermark_b);
2392 c.full = dfixed_mul(c, b);
2393 c.full = dfixed_mul(c, radeon_crtc->hsc);
2394 c.full = dfixed_div(c, a);
2395 a.full = dfixed_const(16);
2396 c.full = dfixed_div(c, a);
2397 priority_b_mark = dfixed_trunc(c);
2398 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2399 }
2400
2401 /* select wm A */
2402 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2403 tmp = arb_control3;
2404 tmp &= ~LATENCY_WATERMARK_MASK(3);
2405 tmp |= LATENCY_WATERMARK_MASK(1);
2406 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2407 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2408 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2409 LATENCY_HIGH_WATERMARK(line_time)));
2410 /* select wm B */
2411 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2412 tmp &= ~LATENCY_WATERMARK_MASK(3);
2413 tmp |= LATENCY_WATERMARK_MASK(2);
2414 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2415 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2416 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2417 LATENCY_HIGH_WATERMARK(line_time)));
2418 /* restore original selection */
2419 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2420
2421 /* write the priority marks */
2422 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2423 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2424
2425 /* save values for DPM */
2426 radeon_crtc->line_time = line_time;
2427 radeon_crtc->wm_high = latency_watermark_a;
2428 radeon_crtc->wm_low = latency_watermark_b;
2429 }
2430
2431 /**
2432 * evergreen_bandwidth_update - update display watermarks callback.
2433 *
2434 * @rdev: radeon_device pointer
2435 *
2436 * Update the display watermarks based on the requested mode(s)
2437 * (evergreen+).
2438 */
2439 void evergreen_bandwidth_update(struct radeon_device *rdev)
2440 {
2441 struct drm_display_mode *mode0 = NULL;
2442 struct drm_display_mode *mode1 = NULL;
2443 u32 num_heads = 0, lb_size;
2444 int i;
2445
2446 if (!rdev->mode_info.mode_config_initialized)
2447 return;
2448
2449 radeon_update_display_priority(rdev);
2450
2451 for (i = 0; i < rdev->num_crtc; i++) {
2452 if (rdev->mode_info.crtcs[i]->base.enabled)
2453 num_heads++;
2454 }
2455 for (i = 0; i < rdev->num_crtc; i += 2) {
2456 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2457 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2458 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2459 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2460 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2461 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2462 }
2463 }
2464
2465 /**
2466 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2467 *
2468 * @rdev: radeon_device pointer
2469 *
2470 * Wait for the MC (memory controller) to be idle.
2471 * (evergreen+).
2472 * Returns 0 if the MC is idle, -1 if not.
2473 */
2474 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2475 {
2476 unsigned i;
2477 u32 tmp;
2478
2479 for (i = 0; i < rdev->usec_timeout; i++) {
2480 /* read MC_STATUS */
2481 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2482 if (!tmp)
2483 return 0;
2484 udelay(1);
2485 }
2486 return -1;
2487 }
2488
2489 /*
2490 * GART
2491 */
2492 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2493 {
2494 unsigned i;
2495 u32 tmp;
2496
2497 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2498
2499 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2500 for (i = 0; i < rdev->usec_timeout; i++) {
2501 /* read MC_STATUS */
2502 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2503 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2504 if (tmp == 2) {
2505 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2506 return;
2507 }
2508 if (tmp) {
2509 return;
2510 }
2511 udelay(1);
2512 }
2513 }
2514
2515 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2516 {
2517 u32 tmp;
2518 int r;
2519
2520 if (rdev->gart.robj == NULL) {
2521 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2522 return -EINVAL;
2523 }
2524 r = radeon_gart_table_vram_pin(rdev);
2525 if (r)
2526 return r;
2527 /* Setup L2 cache */
2528 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2529 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2530 EFFECTIVE_L2_QUEUE_SIZE(7));
2531 WREG32(VM_L2_CNTL2, 0);
2532 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2533 /* Setup TLB control */
2534 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2535 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2536 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2537 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2538 if (rdev->flags & RADEON_IS_IGP) {
2539 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2540 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2541 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2542 } else {
2543 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2544 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2545 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2546 if ((rdev->family == CHIP_JUNIPER) ||
2547 (rdev->family == CHIP_CYPRESS) ||
2548 (rdev->family == CHIP_HEMLOCK) ||
2549 (rdev->family == CHIP_BARTS))
2550 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2551 }
2552 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2553 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2554 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2555 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2556 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2557 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2558 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2559 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2560 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2561 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2562 (u32)(rdev->dummy_page.addr >> 12));
2563 WREG32(VM_CONTEXT1_CNTL, 0);
2564
2565 evergreen_pcie_gart_tlb_flush(rdev);
2566 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2567 (unsigned)(rdev->mc.gtt_size >> 20),
2568 (unsigned long long)rdev->gart.table_addr);
2569 rdev->gart.ready = true;
2570 return 0;
2571 }
2572
2573 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2574 {
2575 u32 tmp;
2576
2577 /* Disable all tables */
2578 WREG32(VM_CONTEXT0_CNTL, 0);
2579 WREG32(VM_CONTEXT1_CNTL, 0);
2580
2581 /* Setup L2 cache */
2582 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2583 EFFECTIVE_L2_QUEUE_SIZE(7));
2584 WREG32(VM_L2_CNTL2, 0);
2585 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2586 /* Setup TLB control */
2587 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2588 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2589 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2590 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2591 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2592 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2593 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2594 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2595 radeon_gart_table_vram_unpin(rdev);
2596 }
2597
2598 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2599 {
2600 evergreen_pcie_gart_disable(rdev);
2601 radeon_gart_table_vram_free(rdev);
2602 radeon_gart_fini(rdev);
2603 }
2604
2605
2606 static void evergreen_agp_enable(struct radeon_device *rdev)
2607 {
2608 u32 tmp;
2609
2610 /* Setup L2 cache */
2611 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2612 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2613 EFFECTIVE_L2_QUEUE_SIZE(7));
2614 WREG32(VM_L2_CNTL2, 0);
2615 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2616 /* Setup TLB control */
2617 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2618 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2619 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2620 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2621 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2622 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2623 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2624 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2625 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2626 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2627 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2628 WREG32(VM_CONTEXT0_CNTL, 0);
2629 WREG32(VM_CONTEXT1_CNTL, 0);
2630 }
2631
2632 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2633 {
2634 u32 crtc_enabled, tmp, frame_count, blackout;
2635 int i, j;
2636
2637 if (!ASIC_IS_NODCE(rdev)) {
2638 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2639 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2640
2641 /* disable VGA render */
2642 WREG32(VGA_RENDER_CONTROL, 0);
2643 }
2644 /* blank the display controllers */
2645 for (i = 0; i < rdev->num_crtc; i++) {
2646 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2647 if (crtc_enabled) {
2648 save->crtc_enabled[i] = true;
2649 if (ASIC_IS_DCE6(rdev)) {
2650 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2651 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2652 radeon_wait_for_vblank(rdev, i);
2653 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2654 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2655 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2656 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2657 }
2658 } else {
2659 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2660 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2661 radeon_wait_for_vblank(rdev, i);
2662 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2663 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2664 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2665 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2666 }
2667 }
2668 /* wait for the next frame */
2669 frame_count = radeon_get_vblank_counter(rdev, i);
2670 for (j = 0; j < rdev->usec_timeout; j++) {
2671 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2672 break;
2673 udelay(1);
2674 }
2675
2676 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2677 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2678 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2679 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2680 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2681 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2682 save->crtc_enabled[i] = false;
2683 /* ***** */
2684 } else {
2685 save->crtc_enabled[i] = false;
2686 }
2687 }
2688
2689 radeon_mc_wait_for_idle(rdev);
2690
2691 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2692 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2693 /* Block CPU access */
2694 WREG32(BIF_FB_EN, 0);
2695 /* blackout the MC */
2696 blackout &= ~BLACKOUT_MODE_MASK;
2697 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2698 }
2699 /* wait for the MC to settle */
2700 udelay(100);
2701
2702 /* lock double buffered regs */
2703 for (i = 0; i < rdev->num_crtc; i++) {
2704 if (save->crtc_enabled[i]) {
2705 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2706 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2707 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2708 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2709 }
2710 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2711 if (!(tmp & 1)) {
2712 tmp |= 1;
2713 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2714 }
2715 }
2716 }
2717 }
2718
2719 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2720 {
2721 u32 tmp, frame_count;
2722 int i, j;
2723
2724 /* update crtc base addresses */
2725 for (i = 0; i < rdev->num_crtc; i++) {
2726 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2727 upper_32_bits(rdev->mc.vram_start));
2728 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2729 upper_32_bits(rdev->mc.vram_start));
2730 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2731 (u32)rdev->mc.vram_start);
2732 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2733 (u32)rdev->mc.vram_start);
2734 }
2735
2736 if (!ASIC_IS_NODCE(rdev)) {
2737 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2738 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2739 }
2740
2741 /* unlock regs and wait for update */
2742 for (i = 0; i < rdev->num_crtc; i++) {
2743 if (save->crtc_enabled[i]) {
2744 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2745 if ((tmp & 0x7) != 3) {
2746 tmp &= ~0x7;
2747 tmp |= 0x3;
2748 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2749 }
2750 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2751 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2752 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2753 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2754 }
2755 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2756 if (tmp & 1) {
2757 tmp &= ~1;
2758 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2759 }
2760 for (j = 0; j < rdev->usec_timeout; j++) {
2761 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2762 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2763 break;
2764 udelay(1);
2765 }
2766 }
2767 }
2768
2769 /* unblackout the MC */
2770 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2771 tmp &= ~BLACKOUT_MODE_MASK;
2772 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2773 /* allow CPU access */
2774 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2775
2776 for (i = 0; i < rdev->num_crtc; i++) {
2777 if (save->crtc_enabled[i]) {
2778 if (ASIC_IS_DCE6(rdev)) {
2779 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2780 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2781 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2782 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2783 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2784 } else {
2785 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2786 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2787 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2788 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2789 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2790 }
2791 /* wait for the next frame */
2792 frame_count = radeon_get_vblank_counter(rdev, i);
2793 for (j = 0; j < rdev->usec_timeout; j++) {
2794 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2795 break;
2796 udelay(1);
2797 }
2798 }
2799 }
2800 if (!ASIC_IS_NODCE(rdev)) {
2801 /* Unlock vga access */
2802 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2803 mdelay(1);
2804 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2805 }
2806 }
2807
2808 void evergreen_mc_program(struct radeon_device *rdev)
2809 {
2810 struct evergreen_mc_save save;
2811 u32 tmp;
2812 int i, j;
2813
2814 /* Initialize HDP */
2815 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2816 WREG32((0x2c14 + j), 0x00000000);
2817 WREG32((0x2c18 + j), 0x00000000);
2818 WREG32((0x2c1c + j), 0x00000000);
2819 WREG32((0x2c20 + j), 0x00000000);
2820 WREG32((0x2c24 + j), 0x00000000);
2821 }
2822 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2823
2824 evergreen_mc_stop(rdev, &save);
2825 if (evergreen_mc_wait_for_idle(rdev)) {
2826 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2827 }
2828 /* Lockout access through VGA aperture*/
2829 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2830 /* Update configuration */
2831 if (rdev->flags & RADEON_IS_AGP) {
2832 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2833 /* VRAM before AGP */
2834 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2835 rdev->mc.vram_start >> 12);
2836 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2837 rdev->mc.gtt_end >> 12);
2838 } else {
2839 /* VRAM after AGP */
2840 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2841 rdev->mc.gtt_start >> 12);
2842 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2843 rdev->mc.vram_end >> 12);
2844 }
2845 } else {
2846 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2847 rdev->mc.vram_start >> 12);
2848 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2849 rdev->mc.vram_end >> 12);
2850 }
2851 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2852 /* llano/ontario only */
2853 if ((rdev->family == CHIP_PALM) ||
2854 (rdev->family == CHIP_SUMO) ||
2855 (rdev->family == CHIP_SUMO2)) {
2856 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2857 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2858 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2859 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2860 }
2861 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2862 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2863 WREG32(MC_VM_FB_LOCATION, tmp);
2864 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2865 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2866 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2867 if (rdev->flags & RADEON_IS_AGP) {
2868 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2869 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2870 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2871 } else {
2872 WREG32(MC_VM_AGP_BASE, 0);
2873 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2874 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2875 }
2876 if (evergreen_mc_wait_for_idle(rdev)) {
2877 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2878 }
2879 evergreen_mc_resume(rdev, &save);
2880 /* we need to own VRAM, so turn off the VGA renderer here
2881 * to stop it overwriting our objects */
2882 rv515_vga_render_disable(rdev);
2883 }
2884
2885 /*
2886 * CP.
2887 */
2888 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2889 {
2890 struct radeon_ring *ring = &rdev->ring[ib->ring];
2891 u32 next_rptr;
2892
2893 /* set to DX10/11 mode */
2894 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2895 radeon_ring_write(ring, 1);
2896
2897 if (ring->rptr_save_reg) {
2898 next_rptr = ring->wptr + 3 + 4;
2899 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2900 radeon_ring_write(ring, ((ring->rptr_save_reg -
2901 PACKET3_SET_CONFIG_REG_START) >> 2));
2902 radeon_ring_write(ring, next_rptr);
2903 } else if (rdev->wb.enabled) {
2904 next_rptr = ring->wptr + 5 + 4;
2905 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2906 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2907 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2908 radeon_ring_write(ring, next_rptr);
2909 radeon_ring_write(ring, 0);
2910 }
2911
2912 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2913 radeon_ring_write(ring,
2914 #ifdef __BIG_ENDIAN
2915 (2 << 0) |
2916 #endif
2917 (ib->gpu_addr & 0xFFFFFFFC));
2918 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2919 radeon_ring_write(ring, ib->length_dw);
2920 }
2921
2922
2923 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2924 {
2925 const __be32 *fw_data;
2926 int i;
2927
2928 if (!rdev->me_fw || !rdev->pfp_fw)
2929 return -EINVAL;
2930
2931 r700_cp_stop(rdev);
2932 WREG32(CP_RB_CNTL,
2933 #ifdef __BIG_ENDIAN
2934 BUF_SWAP_32BIT |
2935 #endif
2936 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2937
2938 fw_data = (const __be32 *)rdev->pfp_fw->data;
2939 WREG32(CP_PFP_UCODE_ADDR, 0);
2940 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2941 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2942 WREG32(CP_PFP_UCODE_ADDR, 0);
2943
2944 fw_data = (const __be32 *)rdev->me_fw->data;
2945 WREG32(CP_ME_RAM_WADDR, 0);
2946 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2947 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2948
2949 WREG32(CP_PFP_UCODE_ADDR, 0);
2950 WREG32(CP_ME_RAM_WADDR, 0);
2951 WREG32(CP_ME_RAM_RADDR, 0);
2952 return 0;
2953 }
2954
2955 static int evergreen_cp_start(struct radeon_device *rdev)
2956 {
2957 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2958 int r, i;
2959 uint32_t cp_me;
2960
2961 r = radeon_ring_lock(rdev, ring, 7);
2962 if (r) {
2963 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2964 return r;
2965 }
2966 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2967 radeon_ring_write(ring, 0x1);
2968 radeon_ring_write(ring, 0x0);
2969 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2970 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2971 radeon_ring_write(ring, 0);
2972 radeon_ring_write(ring, 0);
2973 radeon_ring_unlock_commit(rdev, ring, false);
2974
2975 cp_me = 0xff;
2976 WREG32(CP_ME_CNTL, cp_me);
2977
2978 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2979 if (r) {
2980 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2981 return r;
2982 }
2983
2984 /* setup clear context state */
2985 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2986 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2987
2988 for (i = 0; i < evergreen_default_size; i++)
2989 radeon_ring_write(ring, evergreen_default_state[i]);
2990
2991 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2992 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2993
2994 /* set clear context state */
2995 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2996 radeon_ring_write(ring, 0);
2997
2998 /* SQ_VTX_BASE_VTX_LOC */
2999 radeon_ring_write(ring, 0xc0026f00);
3000 radeon_ring_write(ring, 0x00000000);
3001 radeon_ring_write(ring, 0x00000000);
3002 radeon_ring_write(ring, 0x00000000);
3003
3004 /* Clear consts */
3005 radeon_ring_write(ring, 0xc0036f00);
3006 radeon_ring_write(ring, 0x00000bc4);
3007 radeon_ring_write(ring, 0xffffffff);
3008 radeon_ring_write(ring, 0xffffffff);
3009 radeon_ring_write(ring, 0xffffffff);
3010
3011 radeon_ring_write(ring, 0xc0026900);
3012 radeon_ring_write(ring, 0x00000316);
3013 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3014 radeon_ring_write(ring, 0x00000010); /* */
3015
3016 radeon_ring_unlock_commit(rdev, ring, false);
3017
3018 return 0;
3019 }
3020
3021 static int evergreen_cp_resume(struct radeon_device *rdev)
3022 {
3023 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3024 u32 tmp;
3025 u32 rb_bufsz;
3026 int r;
3027
3028 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3029 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3030 SOFT_RESET_PA |
3031 SOFT_RESET_SH |
3032 SOFT_RESET_VGT |
3033 SOFT_RESET_SPI |
3034 SOFT_RESET_SX));
3035 RREG32(GRBM_SOFT_RESET);
3036 mdelay(15);
3037 WREG32(GRBM_SOFT_RESET, 0);
3038 RREG32(GRBM_SOFT_RESET);
3039
3040 /* Set ring buffer size */
3041 rb_bufsz = order_base_2(ring->ring_size / 8);
3042 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3043 #ifdef __BIG_ENDIAN
3044 tmp |= BUF_SWAP_32BIT;
3045 #endif
3046 WREG32(CP_RB_CNTL, tmp);
3047 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3048 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3049
3050 /* Set the write pointer delay */
3051 WREG32(CP_RB_WPTR_DELAY, 0);
3052
3053 /* Initialize the ring buffer's read and write pointers */
3054 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3055 WREG32(CP_RB_RPTR_WR, 0);
3056 ring->wptr = 0;
3057 WREG32(CP_RB_WPTR, ring->wptr);
3058
3059 /* set the wb address whether it's enabled or not */
3060 WREG32(CP_RB_RPTR_ADDR,
3061 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3062 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3063 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3064
3065 if (rdev->wb.enabled)
3066 WREG32(SCRATCH_UMSK, 0xff);
3067 else {
3068 tmp |= RB_NO_UPDATE;
3069 WREG32(SCRATCH_UMSK, 0);
3070 }
3071
3072 mdelay(1);
3073 WREG32(CP_RB_CNTL, tmp);
3074
3075 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3076 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3077
3078 evergreen_cp_start(rdev);
3079 ring->ready = true;
3080 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3081 if (r) {
3082 ring->ready = false;
3083 return r;
3084 }
3085 return 0;
3086 }
3087
3088 /*
3089 * Core functions
3090 */
3091 static void evergreen_gpu_init(struct radeon_device *rdev)
3092 {
3093 u32 gb_addr_config;
3094 u32 mc_shared_chmap, mc_arb_ramcfg;
3095 u32 sx_debug_1;
3096 u32 smx_dc_ctl0;
3097 u32 sq_config;
3098 u32 sq_lds_resource_mgmt;
3099 u32 sq_gpr_resource_mgmt_1;
3100 u32 sq_gpr_resource_mgmt_2;
3101 u32 sq_gpr_resource_mgmt_3;
3102 u32 sq_thread_resource_mgmt;
3103 u32 sq_thread_resource_mgmt_2;
3104 u32 sq_stack_resource_mgmt_1;
3105 u32 sq_stack_resource_mgmt_2;
3106 u32 sq_stack_resource_mgmt_3;
3107 u32 vgt_cache_invalidation;
3108 u32 hdp_host_path_cntl, tmp;
3109 u32 disabled_rb_mask;
3110 int i, j, ps_thread_count;
3111
3112 switch (rdev->family) {
3113 case CHIP_CYPRESS:
3114 case CHIP_HEMLOCK:
3115 rdev->config.evergreen.num_ses = 2;
3116 rdev->config.evergreen.max_pipes = 4;
3117 rdev->config.evergreen.max_tile_pipes = 8;
3118 rdev->config.evergreen.max_simds = 10;
3119 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3120 rdev->config.evergreen.max_gprs = 256;
3121 rdev->config.evergreen.max_threads = 248;
3122 rdev->config.evergreen.max_gs_threads = 32;
3123 rdev->config.evergreen.max_stack_entries = 512;
3124 rdev->config.evergreen.sx_num_of_sets = 4;
3125 rdev->config.evergreen.sx_max_export_size = 256;
3126 rdev->config.evergreen.sx_max_export_pos_size = 64;
3127 rdev->config.evergreen.sx_max_export_smx_size = 192;
3128 rdev->config.evergreen.max_hw_contexts = 8;
3129 rdev->config.evergreen.sq_num_cf_insts = 2;
3130
3131 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3132 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3133 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3134 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3135 break;
3136 case CHIP_JUNIPER:
3137 rdev->config.evergreen.num_ses = 1;
3138 rdev->config.evergreen.max_pipes = 4;
3139 rdev->config.evergreen.max_tile_pipes = 4;
3140 rdev->config.evergreen.max_simds = 10;
3141 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3142 rdev->config.evergreen.max_gprs = 256;
3143 rdev->config.evergreen.max_threads = 248;
3144 rdev->config.evergreen.max_gs_threads = 32;
3145 rdev->config.evergreen.max_stack_entries = 512;
3146 rdev->config.evergreen.sx_num_of_sets = 4;
3147 rdev->config.evergreen.sx_max_export_size = 256;
3148 rdev->config.evergreen.sx_max_export_pos_size = 64;
3149 rdev->config.evergreen.sx_max_export_smx_size = 192;
3150 rdev->config.evergreen.max_hw_contexts = 8;
3151 rdev->config.evergreen.sq_num_cf_insts = 2;
3152
3153 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3154 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3155 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3156 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3157 break;
3158 case CHIP_REDWOOD:
3159 rdev->config.evergreen.num_ses = 1;
3160 rdev->config.evergreen.max_pipes = 4;
3161 rdev->config.evergreen.max_tile_pipes = 4;
3162 rdev->config.evergreen.max_simds = 5;
3163 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3164 rdev->config.evergreen.max_gprs = 256;
3165 rdev->config.evergreen.max_threads = 248;
3166 rdev->config.evergreen.max_gs_threads = 32;
3167 rdev->config.evergreen.max_stack_entries = 256;
3168 rdev->config.evergreen.sx_num_of_sets = 4;
3169 rdev->config.evergreen.sx_max_export_size = 256;
3170 rdev->config.evergreen.sx_max_export_pos_size = 64;
3171 rdev->config.evergreen.sx_max_export_smx_size = 192;
3172 rdev->config.evergreen.max_hw_contexts = 8;
3173 rdev->config.evergreen.sq_num_cf_insts = 2;
3174
3175 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3176 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3177 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3178 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3179 break;
3180 case CHIP_CEDAR:
3181 default:
3182 rdev->config.evergreen.num_ses = 1;
3183 rdev->config.evergreen.max_pipes = 2;
3184 rdev->config.evergreen.max_tile_pipes = 2;
3185 rdev->config.evergreen.max_simds = 2;
3186 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3187 rdev->config.evergreen.max_gprs = 256;
3188 rdev->config.evergreen.max_threads = 192;
3189 rdev->config.evergreen.max_gs_threads = 16;
3190 rdev->config.evergreen.max_stack_entries = 256;
3191 rdev->config.evergreen.sx_num_of_sets = 4;
3192 rdev->config.evergreen.sx_max_export_size = 128;
3193 rdev->config.evergreen.sx_max_export_pos_size = 32;
3194 rdev->config.evergreen.sx_max_export_smx_size = 96;
3195 rdev->config.evergreen.max_hw_contexts = 4;
3196 rdev->config.evergreen.sq_num_cf_insts = 1;
3197
3198 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3199 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3200 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3201 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3202 break;
3203 case CHIP_PALM:
3204 rdev->config.evergreen.num_ses = 1;
3205 rdev->config.evergreen.max_pipes = 2;
3206 rdev->config.evergreen.max_tile_pipes = 2;
3207 rdev->config.evergreen.max_simds = 2;
3208 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3209 rdev->config.evergreen.max_gprs = 256;
3210 rdev->config.evergreen.max_threads = 192;
3211 rdev->config.evergreen.max_gs_threads = 16;
3212 rdev->config.evergreen.max_stack_entries = 256;
3213 rdev->config.evergreen.sx_num_of_sets = 4;
3214 rdev->config.evergreen.sx_max_export_size = 128;
3215 rdev->config.evergreen.sx_max_export_pos_size = 32;
3216 rdev->config.evergreen.sx_max_export_smx_size = 96;
3217 rdev->config.evergreen.max_hw_contexts = 4;
3218 rdev->config.evergreen.sq_num_cf_insts = 1;
3219
3220 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3221 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3222 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3223 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3224 break;
3225 case CHIP_SUMO:
3226 rdev->config.evergreen.num_ses = 1;
3227 rdev->config.evergreen.max_pipes = 4;
3228 rdev->config.evergreen.max_tile_pipes = 4;
3229 if (rdev->pdev->device == 0x9648)
3230 rdev->config.evergreen.max_simds = 3;
3231 else if ((rdev->pdev->device == 0x9647) ||
3232 (rdev->pdev->device == 0x964a))
3233 rdev->config.evergreen.max_simds = 4;
3234 else
3235 rdev->config.evergreen.max_simds = 5;
3236 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3237 rdev->config.evergreen.max_gprs = 256;
3238 rdev->config.evergreen.max_threads = 248;
3239 rdev->config.evergreen.max_gs_threads = 32;
3240 rdev->config.evergreen.max_stack_entries = 256;
3241 rdev->config.evergreen.sx_num_of_sets = 4;
3242 rdev->config.evergreen.sx_max_export_size = 256;
3243 rdev->config.evergreen.sx_max_export_pos_size = 64;
3244 rdev->config.evergreen.sx_max_export_smx_size = 192;
3245 rdev->config.evergreen.max_hw_contexts = 8;
3246 rdev->config.evergreen.sq_num_cf_insts = 2;
3247
3248 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3249 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3250 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3251 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3252 break;
3253 case CHIP_SUMO2:
3254 rdev->config.evergreen.num_ses = 1;
3255 rdev->config.evergreen.max_pipes = 4;
3256 rdev->config.evergreen.max_tile_pipes = 4;
3257 rdev->config.evergreen.max_simds = 2;
3258 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3259 rdev->config.evergreen.max_gprs = 256;
3260 rdev->config.evergreen.max_threads = 248;
3261 rdev->config.evergreen.max_gs_threads = 32;
3262 rdev->config.evergreen.max_stack_entries = 512;
3263 rdev->config.evergreen.sx_num_of_sets = 4;
3264 rdev->config.evergreen.sx_max_export_size = 256;
3265 rdev->config.evergreen.sx_max_export_pos_size = 64;
3266 rdev->config.evergreen.sx_max_export_smx_size = 192;
3267 rdev->config.evergreen.max_hw_contexts = 4;
3268 rdev->config.evergreen.sq_num_cf_insts = 2;
3269
3270 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3271 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3272 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3273 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3274 break;
3275 case CHIP_BARTS:
3276 rdev->config.evergreen.num_ses = 2;
3277 rdev->config.evergreen.max_pipes = 4;
3278 rdev->config.evergreen.max_tile_pipes = 8;
3279 rdev->config.evergreen.max_simds = 7;
3280 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3281 rdev->config.evergreen.max_gprs = 256;
3282 rdev->config.evergreen.max_threads = 248;
3283 rdev->config.evergreen.max_gs_threads = 32;
3284 rdev->config.evergreen.max_stack_entries = 512;
3285 rdev->config.evergreen.sx_num_of_sets = 4;
3286 rdev->config.evergreen.sx_max_export_size = 256;
3287 rdev->config.evergreen.sx_max_export_pos_size = 64;
3288 rdev->config.evergreen.sx_max_export_smx_size = 192;
3289 rdev->config.evergreen.max_hw_contexts = 8;
3290 rdev->config.evergreen.sq_num_cf_insts = 2;
3291
3292 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3293 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3294 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3295 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3296 break;
3297 case CHIP_TURKS:
3298 rdev->config.evergreen.num_ses = 1;
3299 rdev->config.evergreen.max_pipes = 4;
3300 rdev->config.evergreen.max_tile_pipes = 4;
3301 rdev->config.evergreen.max_simds = 6;
3302 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3303 rdev->config.evergreen.max_gprs = 256;
3304 rdev->config.evergreen.max_threads = 248;
3305 rdev->config.evergreen.max_gs_threads = 32;
3306 rdev->config.evergreen.max_stack_entries = 256;
3307 rdev->config.evergreen.sx_num_of_sets = 4;
3308 rdev->config.evergreen.sx_max_export_size = 256;
3309 rdev->config.evergreen.sx_max_export_pos_size = 64;
3310 rdev->config.evergreen.sx_max_export_smx_size = 192;
3311 rdev->config.evergreen.max_hw_contexts = 8;
3312 rdev->config.evergreen.sq_num_cf_insts = 2;
3313
3314 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3315 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3316 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3317 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3318 break;
3319 case CHIP_CAICOS:
3320 rdev->config.evergreen.num_ses = 1;
3321 rdev->config.evergreen.max_pipes = 2;
3322 rdev->config.evergreen.max_tile_pipes = 2;
3323 rdev->config.evergreen.max_simds = 2;
3324 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3325 rdev->config.evergreen.max_gprs = 256;
3326 rdev->config.evergreen.max_threads = 192;
3327 rdev->config.evergreen.max_gs_threads = 16;
3328 rdev->config.evergreen.max_stack_entries = 256;
3329 rdev->config.evergreen.sx_num_of_sets = 4;
3330 rdev->config.evergreen.sx_max_export_size = 128;
3331 rdev->config.evergreen.sx_max_export_pos_size = 32;
3332 rdev->config.evergreen.sx_max_export_smx_size = 96;
3333 rdev->config.evergreen.max_hw_contexts = 4;
3334 rdev->config.evergreen.sq_num_cf_insts = 1;
3335
3336 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3337 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3338 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3339 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3340 break;
3341 }
3342
3343 /* Initialize HDP */
3344 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3345 WREG32((0x2c14 + j), 0x00000000);
3346 WREG32((0x2c18 + j), 0x00000000);
3347 WREG32((0x2c1c + j), 0x00000000);
3348 WREG32((0x2c20 + j), 0x00000000);
3349 WREG32((0x2c24 + j), 0x00000000);
3350 }
3351
3352 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3353 WREG32(SRBM_INT_CNTL, 0x1);
3354 WREG32(SRBM_INT_ACK, 0x1);
3355
3356 evergreen_fix_pci_max_read_req_size(rdev);
3357
3358 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3359 if ((rdev->family == CHIP_PALM) ||
3360 (rdev->family == CHIP_SUMO) ||
3361 (rdev->family == CHIP_SUMO2))
3362 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3363 else
3364 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3365
3366 /* setup tiling info dword. gb_addr_config is not adequate since it does
3367 * not have bank info, so create a custom tiling dword.
3368 * bits 3:0 num_pipes
3369 * bits 7:4 num_banks
3370 * bits 11:8 group_size
3371 * bits 15:12 row_size
3372 */
3373 rdev->config.evergreen.tile_config = 0;
3374 switch (rdev->config.evergreen.max_tile_pipes) {
3375 case 1:
3376 default:
3377 rdev->config.evergreen.tile_config |= (0 << 0);
3378 break;
3379 case 2:
3380 rdev->config.evergreen.tile_config |= (1 << 0);
3381 break;
3382 case 4:
3383 rdev->config.evergreen.tile_config |= (2 << 0);
3384 break;
3385 case 8:
3386 rdev->config.evergreen.tile_config |= (3 << 0);
3387 break;
3388 }
3389 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3390 if (rdev->flags & RADEON_IS_IGP)
3391 rdev->config.evergreen.tile_config |= 1 << 4;
3392 else {
3393 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3394 case 0: /* four banks */
3395 rdev->config.evergreen.tile_config |= 0 << 4;
3396 break;
3397 case 1: /* eight banks */
3398 rdev->config.evergreen.tile_config |= 1 << 4;
3399 break;
3400 case 2: /* sixteen banks */
3401 default:
3402 rdev->config.evergreen.tile_config |= 2 << 4;
3403 break;
3404 }
3405 }
3406 rdev->config.evergreen.tile_config |= 0 << 8;
3407 rdev->config.evergreen.tile_config |=
3408 ((gb_addr_config & 0x30000000) >> 28) << 12;
3409
3410 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3411 u32 efuse_straps_4;
3412 u32 efuse_straps_3;
3413
3414 efuse_straps_4 = RREG32_RCU(0x204);
3415 efuse_straps_3 = RREG32_RCU(0x203);
3416 tmp = (((efuse_straps_4 & 0xf) << 4) |
3417 ((efuse_straps_3 & 0xf0000000) >> 28));
3418 } else {
3419 tmp = 0;
3420 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3421 u32 rb_disable_bitmap;
3422
3423 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3424 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3425 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3426 tmp <<= 4;
3427 tmp |= rb_disable_bitmap;
3428 }
3429 }
3430 /* enabled rb are just the one not disabled :) */
3431 disabled_rb_mask = tmp;
3432 tmp = 0;
3433 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3434 tmp |= (1 << i);
3435 /* if all the backends are disabled, fix it up here */
3436 if ((disabled_rb_mask & tmp) == tmp) {
3437 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3438 disabled_rb_mask &= ~(1 << i);
3439 }
3440
3441 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3442 u32 simd_disable_bitmap;
3443
3444 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3445 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3446 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3447 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3448 tmp <<= 16;
3449 tmp |= simd_disable_bitmap;
3450 }
3451 rdev->config.evergreen.active_simds = hweight32(~tmp);
3452
3453 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3454 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3455
3456 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3457 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3458 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3459 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3460 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3461 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3462 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3463
3464 if ((rdev->config.evergreen.max_backends == 1) &&
3465 (rdev->flags & RADEON_IS_IGP)) {
3466 if ((disabled_rb_mask & 3) == 1) {
3467 /* RB0 disabled, RB1 enabled */
3468 tmp = 0x11111111;
3469 } else {
3470 /* RB1 disabled, RB0 enabled */
3471 tmp = 0x00000000;
3472 }
3473 } else {
3474 tmp = gb_addr_config & NUM_PIPES_MASK;
3475 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3476 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3477 }
3478 WREG32(GB_BACKEND_MAP, tmp);
3479
3480 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3481 WREG32(CGTS_TCC_DISABLE, 0);
3482 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3483 WREG32(CGTS_USER_TCC_DISABLE, 0);
3484
3485 /* set HW defaults for 3D engine */
3486 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3487 ROQ_IB2_START(0x2b)));
3488
3489 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3490
3491 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3492 SYNC_GRADIENT |
3493 SYNC_WALKER |
3494 SYNC_ALIGNER));
3495
3496 sx_debug_1 = RREG32(SX_DEBUG_1);
3497 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3498 WREG32(SX_DEBUG_1, sx_debug_1);
3499
3500
3501 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3502 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3503 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3504 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3505
3506 if (rdev->family <= CHIP_SUMO2)
3507 WREG32(SMX_SAR_CTL0, 0x00010000);
3508
3509 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3510 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3511 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3512
3513 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3514 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3515 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3516
3517 WREG32(VGT_NUM_INSTANCES, 1);
3518 WREG32(SPI_CONFIG_CNTL, 0);
3519 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3520 WREG32(CP_PERFMON_CNTL, 0);
3521
3522 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3523 FETCH_FIFO_HIWATER(0x4) |
3524 DONE_FIFO_HIWATER(0xe0) |
3525 ALU_UPDATE_FIFO_HIWATER(0x8)));
3526
3527 sq_config = RREG32(SQ_CONFIG);
3528 sq_config &= ~(PS_PRIO(3) |
3529 VS_PRIO(3) |
3530 GS_PRIO(3) |
3531 ES_PRIO(3));
3532 sq_config |= (VC_ENABLE |
3533 EXPORT_SRC_C |
3534 PS_PRIO(0) |
3535 VS_PRIO(1) |
3536 GS_PRIO(2) |
3537 ES_PRIO(3));
3538
3539 switch (rdev->family) {
3540 case CHIP_CEDAR:
3541 case CHIP_PALM:
3542 case CHIP_SUMO:
3543 case CHIP_SUMO2:
3544 case CHIP_CAICOS:
3545 /* no vertex cache */
3546 sq_config &= ~VC_ENABLE;
3547 break;
3548 default:
3549 break;
3550 }
3551
3552 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3553
3554 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3555 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3556 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3557 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3558 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3559 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3560 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3561
3562 switch (rdev->family) {
3563 case CHIP_CEDAR:
3564 case CHIP_PALM:
3565 case CHIP_SUMO:
3566 case CHIP_SUMO2:
3567 ps_thread_count = 96;
3568 break;
3569 default:
3570 ps_thread_count = 128;
3571 break;
3572 }
3573
3574 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3575 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3576 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3577 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3578 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3579 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3580
3581 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3582 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3583 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3584 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3585 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3586 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3587
3588 WREG32(SQ_CONFIG, sq_config);
3589 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3590 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3591 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3592 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3593 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3594 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3595 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3596 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3597 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3598 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3599
3600 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3601 FORCE_EOV_MAX_REZ_CNT(255)));
3602
3603 switch (rdev->family) {
3604 case CHIP_CEDAR:
3605 case CHIP_PALM:
3606 case CHIP_SUMO:
3607 case CHIP_SUMO2:
3608 case CHIP_CAICOS:
3609 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3610 break;
3611 default:
3612 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3613 break;
3614 }
3615 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3616 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3617
3618 WREG32(VGT_GS_VERTEX_REUSE, 16);
3619 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3620 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3621
3622 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3623 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3624
3625 WREG32(CB_PERF_CTR0_SEL_0, 0);
3626 WREG32(CB_PERF_CTR0_SEL_1, 0);
3627 WREG32(CB_PERF_CTR1_SEL_0, 0);
3628 WREG32(CB_PERF_CTR1_SEL_1, 0);
3629 WREG32(CB_PERF_CTR2_SEL_0, 0);
3630 WREG32(CB_PERF_CTR2_SEL_1, 0);
3631 WREG32(CB_PERF_CTR3_SEL_0, 0);
3632 WREG32(CB_PERF_CTR3_SEL_1, 0);
3633
3634 /* clear render buffer base addresses */
3635 WREG32(CB_COLOR0_BASE, 0);
3636 WREG32(CB_COLOR1_BASE, 0);
3637 WREG32(CB_COLOR2_BASE, 0);
3638 WREG32(CB_COLOR3_BASE, 0);
3639 WREG32(CB_COLOR4_BASE, 0);
3640 WREG32(CB_COLOR5_BASE, 0);
3641 WREG32(CB_COLOR6_BASE, 0);
3642 WREG32(CB_COLOR7_BASE, 0);
3643 WREG32(CB_COLOR8_BASE, 0);
3644 WREG32(CB_COLOR9_BASE, 0);
3645 WREG32(CB_COLOR10_BASE, 0);
3646 WREG32(CB_COLOR11_BASE, 0);
3647
3648 /* set the shader const cache sizes to 0 */
3649 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3650 WREG32(i, 0);
3651 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3652 WREG32(i, 0);
3653
3654 tmp = RREG32(HDP_MISC_CNTL);
3655 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3656 WREG32(HDP_MISC_CNTL, tmp);
3657
3658 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3659 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3660
3661 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3662
3663 udelay(50);
3664
3665 }
3666
3667 int evergreen_mc_init(struct radeon_device *rdev)
3668 {
3669 u32 tmp;
3670 int chansize, numchan;
3671
3672 /* Get VRAM informations */
3673 rdev->mc.vram_is_ddr = true;
3674 if ((rdev->family == CHIP_PALM) ||
3675 (rdev->family == CHIP_SUMO) ||
3676 (rdev->family == CHIP_SUMO2))
3677 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3678 else
3679 tmp = RREG32(MC_ARB_RAMCFG);
3680 if (tmp & CHANSIZE_OVERRIDE) {
3681 chansize = 16;
3682 } else if (tmp & CHANSIZE_MASK) {
3683 chansize = 64;
3684 } else {
3685 chansize = 32;
3686 }
3687 tmp = RREG32(MC_SHARED_CHMAP);
3688 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3689 case 0:
3690 default:
3691 numchan = 1;
3692 break;
3693 case 1:
3694 numchan = 2;
3695 break;
3696 case 2:
3697 numchan = 4;
3698 break;
3699 case 3:
3700 numchan = 8;
3701 break;
3702 }
3703 rdev->mc.vram_width = numchan * chansize;
3704 /* Could aper size report 0 ? */
3705 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3706 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3707 /* Setup GPU memory space */
3708 if ((rdev->family == CHIP_PALM) ||
3709 (rdev->family == CHIP_SUMO) ||
3710 (rdev->family == CHIP_SUMO2)) {
3711 /* size in bytes on fusion */
3712 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3713 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3714 } else {
3715 /* size in MB on evergreen/cayman/tn */
3716 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3717 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3718 }
3719 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3720 r700_vram_gtt_location(rdev, &rdev->mc);
3721 radeon_update_bandwidth_info(rdev);
3722
3723 return 0;
3724 }
3725
3726 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3727 {
3728 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3729 RREG32(GRBM_STATUS));
3730 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3731 RREG32(GRBM_STATUS_SE0));
3732 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3733 RREG32(GRBM_STATUS_SE1));
3734 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3735 RREG32(SRBM_STATUS));
3736 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3737 RREG32(SRBM_STATUS2));
3738 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3739 RREG32(CP_STALLED_STAT1));
3740 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3741 RREG32(CP_STALLED_STAT2));
3742 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3743 RREG32(CP_BUSY_STAT));
3744 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3745 RREG32(CP_STAT));
3746 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3747 RREG32(DMA_STATUS_REG));
3748 if (rdev->family >= CHIP_CAYMAN) {
3749 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3750 RREG32(DMA_STATUS_REG + 0x800));
3751 }
3752 }
3753
3754 bool evergreen_is_display_hung(struct radeon_device *rdev)
3755 {
3756 u32 crtc_hung = 0;
3757 u32 crtc_status[6];
3758 u32 i, j, tmp;
3759
3760 for (i = 0; i < rdev->num_crtc; i++) {
3761 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3762 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3763 crtc_hung |= (1 << i);
3764 }
3765 }
3766
3767 for (j = 0; j < 10; j++) {
3768 for (i = 0; i < rdev->num_crtc; i++) {
3769 if (crtc_hung & (1 << i)) {
3770 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3771 if (tmp != crtc_status[i])
3772 crtc_hung &= ~(1 << i);
3773 }
3774 }
3775 if (crtc_hung == 0)
3776 return false;
3777 udelay(100);
3778 }
3779
3780 return true;
3781 }
3782
3783 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3784 {
3785 u32 reset_mask = 0;
3786 u32 tmp;
3787
3788 /* GRBM_STATUS */
3789 tmp = RREG32(GRBM_STATUS);
3790 if (tmp & (PA_BUSY | SC_BUSY |
3791 SH_BUSY | SX_BUSY |
3792 TA_BUSY | VGT_BUSY |
3793 DB_BUSY | CB_BUSY |
3794 SPI_BUSY | VGT_BUSY_NO_DMA))
3795 reset_mask |= RADEON_RESET_GFX;
3796
3797 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3798 CP_BUSY | CP_COHERENCY_BUSY))
3799 reset_mask |= RADEON_RESET_CP;
3800
3801 if (tmp & GRBM_EE_BUSY)
3802 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3803
3804 /* DMA_STATUS_REG */
3805 tmp = RREG32(DMA_STATUS_REG);
3806 if (!(tmp & DMA_IDLE))
3807 reset_mask |= RADEON_RESET_DMA;
3808
3809 /* SRBM_STATUS2 */
3810 tmp = RREG32(SRBM_STATUS2);
3811 if (tmp & DMA_BUSY)
3812 reset_mask |= RADEON_RESET_DMA;
3813
3814 /* SRBM_STATUS */
3815 tmp = RREG32(SRBM_STATUS);
3816 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3817 reset_mask |= RADEON_RESET_RLC;
3818
3819 if (tmp & IH_BUSY)
3820 reset_mask |= RADEON_RESET_IH;
3821
3822 if (tmp & SEM_BUSY)
3823 reset_mask |= RADEON_RESET_SEM;
3824
3825 if (tmp & GRBM_RQ_PENDING)
3826 reset_mask |= RADEON_RESET_GRBM;
3827
3828 if (tmp & VMC_BUSY)
3829 reset_mask |= RADEON_RESET_VMC;
3830
3831 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3832 MCC_BUSY | MCD_BUSY))
3833 reset_mask |= RADEON_RESET_MC;
3834
3835 if (evergreen_is_display_hung(rdev))
3836 reset_mask |= RADEON_RESET_DISPLAY;
3837
3838 /* VM_L2_STATUS */
3839 tmp = RREG32(VM_L2_STATUS);
3840 if (tmp & L2_BUSY)
3841 reset_mask |= RADEON_RESET_VMC;
3842
3843 /* Skip MC reset as it's mostly likely not hung, just busy */
3844 if (reset_mask & RADEON_RESET_MC) {
3845 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3846 reset_mask &= ~RADEON_RESET_MC;
3847 }
3848
3849 return reset_mask;
3850 }
3851
3852 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3853 {
3854 struct evergreen_mc_save save;
3855 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3856 u32 tmp;
3857
3858 if (reset_mask == 0)
3859 return;
3860
3861 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3862
3863 evergreen_print_gpu_status_regs(rdev);
3864
3865 /* Disable CP parsing/prefetching */
3866 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3867
3868 if (reset_mask & RADEON_RESET_DMA) {
3869 /* Disable DMA */
3870 tmp = RREG32(DMA_RB_CNTL);
3871 tmp &= ~DMA_RB_ENABLE;
3872 WREG32(DMA_RB_CNTL, tmp);
3873 }
3874
3875 udelay(50);
3876
3877 evergreen_mc_stop(rdev, &save);
3878 if (evergreen_mc_wait_for_idle(rdev)) {
3879 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3880 }
3881
3882 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3883 grbm_soft_reset |= SOFT_RESET_DB |
3884 SOFT_RESET_CB |
3885 SOFT_RESET_PA |
3886 SOFT_RESET_SC |
3887 SOFT_RESET_SPI |
3888 SOFT_RESET_SX |
3889 SOFT_RESET_SH |
3890 SOFT_RESET_TC |
3891 SOFT_RESET_TA |
3892 SOFT_RESET_VC |
3893 SOFT_RESET_VGT;
3894 }
3895
3896 if (reset_mask & RADEON_RESET_CP) {
3897 grbm_soft_reset |= SOFT_RESET_CP |
3898 SOFT_RESET_VGT;
3899
3900 srbm_soft_reset |= SOFT_RESET_GRBM;
3901 }
3902
3903 if (reset_mask & RADEON_RESET_DMA)
3904 srbm_soft_reset |= SOFT_RESET_DMA;
3905
3906 if (reset_mask & RADEON_RESET_DISPLAY)
3907 srbm_soft_reset |= SOFT_RESET_DC;
3908
3909 if (reset_mask & RADEON_RESET_RLC)
3910 srbm_soft_reset |= SOFT_RESET_RLC;
3911
3912 if (reset_mask & RADEON_RESET_SEM)
3913 srbm_soft_reset |= SOFT_RESET_SEM;
3914
3915 if (reset_mask & RADEON_RESET_IH)
3916 srbm_soft_reset |= SOFT_RESET_IH;
3917
3918 if (reset_mask & RADEON_RESET_GRBM)
3919 srbm_soft_reset |= SOFT_RESET_GRBM;
3920
3921 if (reset_mask & RADEON_RESET_VMC)
3922 srbm_soft_reset |= SOFT_RESET_VMC;
3923
3924 if (!(rdev->flags & RADEON_IS_IGP)) {
3925 if (reset_mask & RADEON_RESET_MC)
3926 srbm_soft_reset |= SOFT_RESET_MC;
3927 }
3928
3929 if (grbm_soft_reset) {
3930 tmp = RREG32(GRBM_SOFT_RESET);
3931 tmp |= grbm_soft_reset;
3932 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3933 WREG32(GRBM_SOFT_RESET, tmp);
3934 tmp = RREG32(GRBM_SOFT_RESET);
3935
3936 udelay(50);
3937
3938 tmp &= ~grbm_soft_reset;
3939 WREG32(GRBM_SOFT_RESET, tmp);
3940 tmp = RREG32(GRBM_SOFT_RESET);
3941 }
3942
3943 if (srbm_soft_reset) {
3944 tmp = RREG32(SRBM_SOFT_RESET);
3945 tmp |= srbm_soft_reset;
3946 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3947 WREG32(SRBM_SOFT_RESET, tmp);
3948 tmp = RREG32(SRBM_SOFT_RESET);
3949
3950 udelay(50);
3951
3952 tmp &= ~srbm_soft_reset;
3953 WREG32(SRBM_SOFT_RESET, tmp);
3954 tmp = RREG32(SRBM_SOFT_RESET);
3955 }
3956
3957 /* Wait a little for things to settle down */
3958 udelay(50);
3959
3960 evergreen_mc_resume(rdev, &save);
3961 udelay(50);
3962
3963 evergreen_print_gpu_status_regs(rdev);
3964 }
3965
3966 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3967 {
3968 struct evergreen_mc_save save;
3969 u32 tmp, i;
3970
3971 dev_info(rdev->dev, "GPU pci config reset\n");
3972
3973 /* disable dpm? */
3974
3975 /* Disable CP parsing/prefetching */
3976 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3977 udelay(50);
3978 /* Disable DMA */
3979 tmp = RREG32(DMA_RB_CNTL);
3980 tmp &= ~DMA_RB_ENABLE;
3981 WREG32(DMA_RB_CNTL, tmp);
3982 /* XXX other engines? */
3983
3984 /* halt the rlc */
3985 r600_rlc_stop(rdev);
3986
3987 udelay(50);
3988
3989 /* set mclk/sclk to bypass */
3990 rv770_set_clk_bypass_mode(rdev);
3991 /* disable BM */
3992 pci_clear_master(rdev->pdev);
3993 /* disable mem access */
3994 evergreen_mc_stop(rdev, &save);
3995 if (evergreen_mc_wait_for_idle(rdev)) {
3996 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3997 }
3998 /* reset */
3999 radeon_pci_config_reset(rdev);
4000 /* wait for asic to come out of reset */
4001 for (i = 0; i < rdev->usec_timeout; i++) {
4002 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4003 break;
4004 udelay(1);
4005 }
4006 }
4007
4008 int evergreen_asic_reset(struct radeon_device *rdev)
4009 {
4010 u32 reset_mask;
4011
4012 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4013
4014 if (reset_mask)
4015 r600_set_bios_scratch_engine_hung(rdev, true);
4016
4017 /* try soft reset */
4018 evergreen_gpu_soft_reset(rdev, reset_mask);
4019
4020 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4021
4022 /* try pci config reset */
4023 if (reset_mask && radeon_hard_reset)
4024 evergreen_gpu_pci_config_reset(rdev);
4025
4026 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4027
4028 if (!reset_mask)
4029 r600_set_bios_scratch_engine_hung(rdev, false);
4030
4031 return 0;
4032 }
4033
4034 /**
4035 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4036 *
4037 * @rdev: radeon_device pointer
4038 * @ring: radeon_ring structure holding ring information
4039 *
4040 * Check if the GFX engine is locked up.
4041 * Returns true if the engine appears to be locked up, false if not.
4042 */
4043 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4044 {
4045 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4046
4047 if (!(reset_mask & (RADEON_RESET_GFX |
4048 RADEON_RESET_COMPUTE |
4049 RADEON_RESET_CP))) {
4050 radeon_ring_lockup_update(rdev, ring);
4051 return false;
4052 }
4053 return radeon_ring_test_lockup(rdev, ring);
4054 }
4055
4056 /*
4057 * RLC
4058 */
4059 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4060 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
4061
4062 void sumo_rlc_fini(struct radeon_device *rdev)
4063 {
4064 int r;
4065
4066 /* save restore block */
4067 if (rdev->rlc.save_restore_obj) {
4068 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4069 if (unlikely(r != 0))
4070 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4071 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4072 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4073
4074 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4075 rdev->rlc.save_restore_obj = NULL;
4076 }
4077
4078 /* clear state block */
4079 if (rdev->rlc.clear_state_obj) {
4080 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4081 if (unlikely(r != 0))
4082 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4083 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4084 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4085
4086 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4087 rdev->rlc.clear_state_obj = NULL;
4088 }
4089
4090 /* clear state block */
4091 if (rdev->rlc.cp_table_obj) {
4092 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4093 if (unlikely(r != 0))
4094 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4095 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4096 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4097
4098 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4099 rdev->rlc.cp_table_obj = NULL;
4100 }
4101 }
4102
4103 #define CP_ME_TABLE_SIZE 96
4104
4105 int sumo_rlc_init(struct radeon_device *rdev)
4106 {
4107 const u32 *src_ptr;
4108 volatile u32 *dst_ptr;
4109 u32 dws, data, i, j, k, reg_num;
4110 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4111 u64 reg_list_mc_addr;
4112 const struct cs_section_def *cs_data;
4113 int r;
4114
4115 src_ptr = rdev->rlc.reg_list;
4116 dws = rdev->rlc.reg_list_size;
4117 if (rdev->family >= CHIP_BONAIRE) {
4118 dws += (5 * 16) + 48 + 48 + 64;
4119 }
4120 cs_data = rdev->rlc.cs_data;
4121
4122 if (src_ptr) {
4123 /* save restore block */
4124 if (rdev->rlc.save_restore_obj == NULL) {
4125 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4126 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4127 NULL, &rdev->rlc.save_restore_obj);
4128 if (r) {
4129 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4130 return r;
4131 }
4132 }
4133
4134 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4135 if (unlikely(r != 0)) {
4136 sumo_rlc_fini(rdev);
4137 return r;
4138 }
4139 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4140 &rdev->rlc.save_restore_gpu_addr);
4141 if (r) {
4142 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4143 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4144 sumo_rlc_fini(rdev);
4145 return r;
4146 }
4147
4148 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4149 if (r) {
4150 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4151 sumo_rlc_fini(rdev);
4152 return r;
4153 }
4154 /* write the sr buffer */
4155 dst_ptr = rdev->rlc.sr_ptr;
4156 if (rdev->family >= CHIP_TAHITI) {
4157 /* SI */
4158 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4159 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4160 } else {
4161 /* ON/LN/TN */
4162 /* format:
4163 * dw0: (reg2 << 16) | reg1
4164 * dw1: reg1 save space
4165 * dw2: reg2 save space
4166 */
4167 for (i = 0; i < dws; i++) {
4168 data = src_ptr[i] >> 2;
4169 i++;
4170 if (i < dws)
4171 data |= (src_ptr[i] >> 2) << 16;
4172 j = (((i - 1) * 3) / 2);
4173 dst_ptr[j] = cpu_to_le32(data);
4174 }
4175 j = ((i * 3) / 2);
4176 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4177 }
4178 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4179 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4180 }
4181
4182 if (cs_data) {
4183 /* clear state block */
4184 if (rdev->family >= CHIP_BONAIRE) {
4185 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4186 } else if (rdev->family >= CHIP_TAHITI) {
4187 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4188 dws = rdev->rlc.clear_state_size + (256 / 4);
4189 } else {
4190 reg_list_num = 0;
4191 dws = 0;
4192 for (i = 0; cs_data[i].section != NULL; i++) {
4193 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4194 reg_list_num++;
4195 dws += cs_data[i].section[j].reg_count;
4196 }
4197 }
4198 reg_list_blk_index = (3 * reg_list_num + 2);
4199 dws += reg_list_blk_index;
4200 rdev->rlc.clear_state_size = dws;
4201 }
4202
4203 if (rdev->rlc.clear_state_obj == NULL) {
4204 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4205 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4206 NULL, &rdev->rlc.clear_state_obj);
4207 if (r) {
4208 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4209 sumo_rlc_fini(rdev);
4210 return r;
4211 }
4212 }
4213 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4214 if (unlikely(r != 0)) {
4215 sumo_rlc_fini(rdev);
4216 return r;
4217 }
4218 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4219 &rdev->rlc.clear_state_gpu_addr);
4220 if (r) {
4221 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4222 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4223 sumo_rlc_fini(rdev);
4224 return r;
4225 }
4226
4227 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4228 if (r) {
4229 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4230 sumo_rlc_fini(rdev);
4231 return r;
4232 }
4233 /* set up the cs buffer */
4234 dst_ptr = rdev->rlc.cs_ptr;
4235 if (rdev->family >= CHIP_BONAIRE) {
4236 cik_get_csb_buffer(rdev, dst_ptr);
4237 } else if (rdev->family >= CHIP_TAHITI) {
4238 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4239 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4240 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4241 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4242 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4243 } else {
4244 reg_list_hdr_blk_index = 0;
4245 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4246 data = upper_32_bits(reg_list_mc_addr);
4247 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4248 reg_list_hdr_blk_index++;
4249 for (i = 0; cs_data[i].section != NULL; i++) {
4250 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4251 reg_num = cs_data[i].section[j].reg_count;
4252 data = reg_list_mc_addr & 0xffffffff;
4253 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4254 reg_list_hdr_blk_index++;
4255
4256 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4257 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4258 reg_list_hdr_blk_index++;
4259
4260 data = 0x08000000 | (reg_num * 4);
4261 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4262 reg_list_hdr_blk_index++;
4263
4264 for (k = 0; k < reg_num; k++) {
4265 data = cs_data[i].section[j].extent[k];
4266 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4267 }
4268 reg_list_mc_addr += reg_num * 4;
4269 reg_list_blk_index += reg_num;
4270 }
4271 }
4272 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4273 }
4274 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4275 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4276 }
4277
4278 if (rdev->rlc.cp_table_size) {
4279 if (rdev->rlc.cp_table_obj == NULL) {
4280 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4281 PAGE_SIZE, true,
4282 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4283 NULL, &rdev->rlc.cp_table_obj);
4284 if (r) {
4285 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4286 sumo_rlc_fini(rdev);
4287 return r;
4288 }
4289 }
4290
4291 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4292 if (unlikely(r != 0)) {
4293 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4294 sumo_rlc_fini(rdev);
4295 return r;
4296 }
4297 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4298 &rdev->rlc.cp_table_gpu_addr);
4299 if (r) {
4300 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4301 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4302 sumo_rlc_fini(rdev);
4303 return r;
4304 }
4305 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4306 if (r) {
4307 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4308 sumo_rlc_fini(rdev);
4309 return r;
4310 }
4311
4312 cik_init_cp_pg_table(rdev);
4313
4314 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4315 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4316
4317 }
4318
4319 return 0;
4320 }
4321
4322 static void evergreen_rlc_start(struct radeon_device *rdev)
4323 {
4324 u32 mask = RLC_ENABLE;
4325
4326 if (rdev->flags & RADEON_IS_IGP) {
4327 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4328 }
4329
4330 WREG32(RLC_CNTL, mask);
4331 }
4332
4333 int evergreen_rlc_resume(struct radeon_device *rdev)
4334 {
4335 u32 i;
4336 const __be32 *fw_data;
4337
4338 if (!rdev->rlc_fw)
4339 return -EINVAL;
4340
4341 r600_rlc_stop(rdev);
4342
4343 WREG32(RLC_HB_CNTL, 0);
4344
4345 if (rdev->flags & RADEON_IS_IGP) {
4346 if (rdev->family == CHIP_ARUBA) {
4347 u32 always_on_bitmap =
4348 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4349 /* find out the number of active simds */
4350 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4351 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4352 tmp = hweight32(~tmp);
4353 if (tmp == rdev->config.cayman.max_simds_per_se) {
4354 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4355 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4356 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4357 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4358 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4359 }
4360 } else {
4361 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4362 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4363 }
4364 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4365 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4366 } else {
4367 WREG32(RLC_HB_BASE, 0);
4368 WREG32(RLC_HB_RPTR, 0);
4369 WREG32(RLC_HB_WPTR, 0);
4370 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4371 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4372 }
4373 WREG32(RLC_MC_CNTL, 0);
4374 WREG32(RLC_UCODE_CNTL, 0);
4375
4376 fw_data = (const __be32 *)rdev->rlc_fw->data;
4377 if (rdev->family >= CHIP_ARUBA) {
4378 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4379 WREG32(RLC_UCODE_ADDR, i);
4380 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4381 }
4382 } else if (rdev->family >= CHIP_CAYMAN) {
4383 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4384 WREG32(RLC_UCODE_ADDR, i);
4385 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4386 }
4387 } else {
4388 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4389 WREG32(RLC_UCODE_ADDR, i);
4390 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4391 }
4392 }
4393 WREG32(RLC_UCODE_ADDR, 0);
4394
4395 evergreen_rlc_start(rdev);
4396
4397 return 0;
4398 }
4399
4400 /* Interrupts */
4401
4402 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4403 {
4404 if (crtc >= rdev->num_crtc)
4405 return 0;
4406 else
4407 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4408 }
4409
4410 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4411 {
4412 u32 tmp;
4413
4414 if (rdev->family >= CHIP_CAYMAN) {
4415 cayman_cp_int_cntl_setup(rdev, 0,
4416 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4417 cayman_cp_int_cntl_setup(rdev, 1, 0);
4418 cayman_cp_int_cntl_setup(rdev, 2, 0);
4419 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4420 WREG32(CAYMAN_DMA1_CNTL, tmp);
4421 } else
4422 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4423 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4424 WREG32(DMA_CNTL, tmp);
4425 WREG32(GRBM_INT_CNTL, 0);
4426 WREG32(SRBM_INT_CNTL, 0);
4427 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4428 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4429 if (rdev->num_crtc >= 4) {
4430 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4431 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4432 }
4433 if (rdev->num_crtc >= 6) {
4434 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4435 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4436 }
4437
4438 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4439 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4440 if (rdev->num_crtc >= 4) {
4441 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4442 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4443 }
4444 if (rdev->num_crtc >= 6) {
4445 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4446 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4447 }
4448
4449 /* only one DAC on DCE5 */
4450 if (!ASIC_IS_DCE5(rdev))
4451 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4452 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4453
4454 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4455 WREG32(DC_HPD1_INT_CONTROL, tmp);
4456 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4457 WREG32(DC_HPD2_INT_CONTROL, tmp);
4458 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4459 WREG32(DC_HPD3_INT_CONTROL, tmp);
4460 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4461 WREG32(DC_HPD4_INT_CONTROL, tmp);
4462 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4463 WREG32(DC_HPD5_INT_CONTROL, tmp);
4464 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4465 WREG32(DC_HPD6_INT_CONTROL, tmp);
4466
4467 }
4468
4469 int evergreen_irq_set(struct radeon_device *rdev)
4470 {
4471 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4472 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4473 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4474 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4475 u32 grbm_int_cntl = 0;
4476 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4477 u32 dma_cntl, dma_cntl1 = 0;
4478 u32 thermal_int = 0;
4479
4480 if (!rdev->irq.installed) {
4481 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4482 return -EINVAL;
4483 }
4484 /* don't enable anything if the ih is disabled */
4485 if (!rdev->ih.enabled) {
4486 r600_disable_interrupts(rdev);
4487 /* force the active interrupt state to all disabled */
4488 evergreen_disable_interrupt_state(rdev);
4489 return 0;
4490 }
4491
4492 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4493 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4494 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4495 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4496 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4497 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4498 if (rdev->family == CHIP_ARUBA)
4499 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4500 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4501 else
4502 thermal_int = RREG32(CG_THERMAL_INT) &
4503 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4504
4505 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4506 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4507 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4508 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4509 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4510 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4511
4512 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4513
4514 if (rdev->family >= CHIP_CAYMAN) {
4515 /* enable CP interrupts on all rings */
4516 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4517 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4518 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4519 }
4520 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4521 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4522 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4523 }
4524 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4525 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4526 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4527 }
4528 } else {
4529 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4530 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4531 cp_int_cntl |= RB_INT_ENABLE;
4532 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4533 }
4534 }
4535
4536 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4537 DRM_DEBUG("r600_irq_set: sw int dma\n");
4538 dma_cntl |= TRAP_ENABLE;
4539 }
4540
4541 if (rdev->family >= CHIP_CAYMAN) {
4542 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4543 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4544 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4545 dma_cntl1 |= TRAP_ENABLE;
4546 }
4547 }
4548
4549 if (rdev->irq.dpm_thermal) {
4550 DRM_DEBUG("dpm thermal\n");
4551 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4552 }
4553
4554 if (rdev->irq.crtc_vblank_int[0] ||
4555 atomic_read(&rdev->irq.pflip[0])) {
4556 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4557 crtc1 |= VBLANK_INT_MASK;
4558 }
4559 if (rdev->irq.crtc_vblank_int[1] ||
4560 atomic_read(&rdev->irq.pflip[1])) {
4561 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4562 crtc2 |= VBLANK_INT_MASK;
4563 }
4564 if (rdev->irq.crtc_vblank_int[2] ||
4565 atomic_read(&rdev->irq.pflip[2])) {
4566 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4567 crtc3 |= VBLANK_INT_MASK;
4568 }
4569 if (rdev->irq.crtc_vblank_int[3] ||
4570 atomic_read(&rdev->irq.pflip[3])) {
4571 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4572 crtc4 |= VBLANK_INT_MASK;
4573 }
4574 if (rdev->irq.crtc_vblank_int[4] ||
4575 atomic_read(&rdev->irq.pflip[4])) {
4576 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4577 crtc5 |= VBLANK_INT_MASK;
4578 }
4579 if (rdev->irq.crtc_vblank_int[5] ||
4580 atomic_read(&rdev->irq.pflip[5])) {
4581 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4582 crtc6 |= VBLANK_INT_MASK;
4583 }
4584 if (rdev->irq.hpd[0]) {
4585 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4586 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4587 }
4588 if (rdev->irq.hpd[1]) {
4589 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4590 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4591 }
4592 if (rdev->irq.hpd[2]) {
4593 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4594 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4595 }
4596 if (rdev->irq.hpd[3]) {
4597 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4598 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4599 }
4600 if (rdev->irq.hpd[4]) {
4601 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4602 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4603 }
4604 if (rdev->irq.hpd[5]) {
4605 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4606 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4607 }
4608 if (rdev->irq.afmt[0]) {
4609 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4610 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4611 }
4612 if (rdev->irq.afmt[1]) {
4613 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4614 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4615 }
4616 if (rdev->irq.afmt[2]) {
4617 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4618 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4619 }
4620 if (rdev->irq.afmt[3]) {
4621 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4622 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4623 }
4624 if (rdev->irq.afmt[4]) {
4625 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4626 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4627 }
4628 if (rdev->irq.afmt[5]) {
4629 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4630 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4631 }
4632
4633 if (rdev->family >= CHIP_CAYMAN) {
4634 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4635 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4636 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4637 } else
4638 WREG32(CP_INT_CNTL, cp_int_cntl);
4639
4640 WREG32(DMA_CNTL, dma_cntl);
4641
4642 if (rdev->family >= CHIP_CAYMAN)
4643 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4644
4645 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4646
4647 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4648 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4649 if (rdev->num_crtc >= 4) {
4650 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4651 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4652 }
4653 if (rdev->num_crtc >= 6) {
4654 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4655 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4656 }
4657
4658 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4659 GRPH_PFLIP_INT_MASK);
4660 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4661 GRPH_PFLIP_INT_MASK);
4662 if (rdev->num_crtc >= 4) {
4663 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4664 GRPH_PFLIP_INT_MASK);
4665 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4666 GRPH_PFLIP_INT_MASK);
4667 }
4668 if (rdev->num_crtc >= 6) {
4669 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4670 GRPH_PFLIP_INT_MASK);
4671 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4672 GRPH_PFLIP_INT_MASK);
4673 }
4674
4675 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4676 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4677 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4678 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4679 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4680 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4681 if (rdev->family == CHIP_ARUBA)
4682 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4683 else
4684 WREG32(CG_THERMAL_INT, thermal_int);
4685
4686 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4687 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4688 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4689 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4690 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4691 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4692
4693 /* posting read */
4694 RREG32(SRBM_STATUS);
4695
4696 return 0;
4697 }
4698
4699 static void evergreen_irq_ack(struct radeon_device *rdev)
4700 {
4701 u32 tmp;
4702
4703 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4704 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4705 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4706 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4707 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4708 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4709 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4710 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4711 if (rdev->num_crtc >= 4) {
4712 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4713 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4714 }
4715 if (rdev->num_crtc >= 6) {
4716 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4717 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4718 }
4719
4720 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4721 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4722 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4723 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4724 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4725 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4726
4727 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4728 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4729 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4730 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4731 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4732 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4733 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4734 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4735 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4736 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4737 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4738 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4739
4740 if (rdev->num_crtc >= 4) {
4741 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4742 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4743 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4744 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4745 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4746 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4747 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4748 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4749 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4750 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4751 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4752 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4753 }
4754
4755 if (rdev->num_crtc >= 6) {
4756 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4757 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4758 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4759 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4760 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4761 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4762 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4763 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4764 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4765 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4766 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4767 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4768 }
4769
4770 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4771 tmp = RREG32(DC_HPD1_INT_CONTROL);
4772 tmp |= DC_HPDx_INT_ACK;
4773 WREG32(DC_HPD1_INT_CONTROL, tmp);
4774 }
4775 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4776 tmp = RREG32(DC_HPD2_INT_CONTROL);
4777 tmp |= DC_HPDx_INT_ACK;
4778 WREG32(DC_HPD2_INT_CONTROL, tmp);
4779 }
4780 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4781 tmp = RREG32(DC_HPD3_INT_CONTROL);
4782 tmp |= DC_HPDx_INT_ACK;
4783 WREG32(DC_HPD3_INT_CONTROL, tmp);
4784 }
4785 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4786 tmp = RREG32(DC_HPD4_INT_CONTROL);
4787 tmp |= DC_HPDx_INT_ACK;
4788 WREG32(DC_HPD4_INT_CONTROL, tmp);
4789 }
4790 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4791 tmp = RREG32(DC_HPD5_INT_CONTROL);
4792 tmp |= DC_HPDx_INT_ACK;
4793 WREG32(DC_HPD5_INT_CONTROL, tmp);
4794 }
4795 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4796 tmp = RREG32(DC_HPD5_INT_CONTROL);
4797 tmp |= DC_HPDx_INT_ACK;
4798 WREG32(DC_HPD6_INT_CONTROL, tmp);
4799 }
4800
4801 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4802 tmp = RREG32(DC_HPD1_INT_CONTROL);
4803 tmp |= DC_HPDx_RX_INT_ACK;
4804 WREG32(DC_HPD1_INT_CONTROL, tmp);
4805 }
4806 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4807 tmp = RREG32(DC_HPD2_INT_CONTROL);
4808 tmp |= DC_HPDx_RX_INT_ACK;
4809 WREG32(DC_HPD2_INT_CONTROL, tmp);
4810 }
4811 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4812 tmp = RREG32(DC_HPD3_INT_CONTROL);
4813 tmp |= DC_HPDx_RX_INT_ACK;
4814 WREG32(DC_HPD3_INT_CONTROL, tmp);
4815 }
4816 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4817 tmp = RREG32(DC_HPD4_INT_CONTROL);
4818 tmp |= DC_HPDx_RX_INT_ACK;
4819 WREG32(DC_HPD4_INT_CONTROL, tmp);
4820 }
4821 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4822 tmp = RREG32(DC_HPD5_INT_CONTROL);
4823 tmp |= DC_HPDx_RX_INT_ACK;
4824 WREG32(DC_HPD5_INT_CONTROL, tmp);
4825 }
4826 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4827 tmp = RREG32(DC_HPD5_INT_CONTROL);
4828 tmp |= DC_HPDx_RX_INT_ACK;
4829 WREG32(DC_HPD6_INT_CONTROL, tmp);
4830 }
4831
4832 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4833 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4834 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4835 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4836 }
4837 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4838 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4839 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4840 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4841 }
4842 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4843 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4844 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4845 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4846 }
4847 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4848 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4849 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4850 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4851 }
4852 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4853 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4854 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4855 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4856 }
4857 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4858 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4859 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4860 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4861 }
4862 }
4863
4864 static void evergreen_irq_disable(struct radeon_device *rdev)
4865 {
4866 r600_disable_interrupts(rdev);
4867 /* Wait and acknowledge irq */
4868 mdelay(1);
4869 evergreen_irq_ack(rdev);
4870 evergreen_disable_interrupt_state(rdev);
4871 }
4872
4873 void evergreen_irq_suspend(struct radeon_device *rdev)
4874 {
4875 evergreen_irq_disable(rdev);
4876 r600_rlc_stop(rdev);
4877 }
4878
4879 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4880 {
4881 u32 wptr, tmp;
4882
4883 if (rdev->wb.enabled)
4884 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4885 else
4886 wptr = RREG32(IH_RB_WPTR);
4887
4888 if (wptr & RB_OVERFLOW) {
4889 wptr &= ~RB_OVERFLOW;
4890 /* When a ring buffer overflow happen start parsing interrupt
4891 * from the last not overwritten vector (wptr + 16). Hopefully
4892 * this should allow us to catchup.
4893 */
4894 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4895 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4896 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4897 tmp = RREG32(IH_RB_CNTL);
4898 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4899 WREG32(IH_RB_CNTL, tmp);
4900 }
4901 return (wptr & rdev->ih.ptr_mask);
4902 }
4903
4904 int evergreen_irq_process(struct radeon_device *rdev)
4905 {
4906 u32 wptr;
4907 u32 rptr;
4908 u32 src_id, src_data;
4909 u32 ring_index;
4910 bool queue_hotplug = false;
4911 bool queue_hdmi = false;
4912 bool queue_dp = false;
4913 bool queue_thermal = false;
4914 u32 status, addr;
4915
4916 if (!rdev->ih.enabled || rdev->shutdown)
4917 return IRQ_NONE;
4918
4919 wptr = evergreen_get_ih_wptr(rdev);
4920
4921 restart_ih:
4922 /* is somebody else already processing irqs? */
4923 if (atomic_xchg(&rdev->ih.lock, 1))
4924 return IRQ_NONE;
4925
4926 rptr = rdev->ih.rptr;
4927 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4928
4929 /* Order reading of wptr vs. reading of IH ring data */
4930 rmb();
4931
4932 /* display interrupts */
4933 evergreen_irq_ack(rdev);
4934
4935 while (rptr != wptr) {
4936 /* wptr/rptr are in bytes! */
4937 ring_index = rptr / 4;
4938 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4939 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4940
4941 switch (src_id) {
4942 case 1: /* D1 vblank/vline */
4943 switch (src_data) {
4944 case 0: /* D1 vblank */
4945 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
4946 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
4947
4948 if (rdev->irq.crtc_vblank_int[0]) {
4949 drm_handle_vblank(rdev->ddev, 0);
4950 rdev->pm.vblank_sync = true;
4951 wake_up(&rdev->irq.vblank_queue);
4952 }
4953 if (atomic_read(&rdev->irq.pflip[0]))
4954 radeon_crtc_handle_vblank(rdev, 0);
4955 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4956 DRM_DEBUG("IH: D1 vblank\n");
4957
4958 break;
4959 case 1: /* D1 vline */
4960 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
4961 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
4962
4963 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4964 DRM_DEBUG("IH: D1 vline\n");
4965
4966 break;
4967 default:
4968 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4969 break;
4970 }
4971 break;
4972 case 2: /* D2 vblank/vline */
4973 switch (src_data) {
4974 case 0: /* D2 vblank */
4975 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
4976 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
4977
4978 if (rdev->irq.crtc_vblank_int[1]) {
4979 drm_handle_vblank(rdev->ddev, 1);
4980 rdev->pm.vblank_sync = true;
4981 wake_up(&rdev->irq.vblank_queue);
4982 }
4983 if (atomic_read(&rdev->irq.pflip[1]))
4984 radeon_crtc_handle_vblank(rdev, 1);
4985 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4986 DRM_DEBUG("IH: D2 vblank\n");
4987
4988 break;
4989 case 1: /* D2 vline */
4990 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
4991 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
4992
4993 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4994 DRM_DEBUG("IH: D2 vline\n");
4995
4996 break;
4997 default:
4998 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4999 break;
5000 }
5001 break;
5002 case 3: /* D3 vblank/vline */
5003 switch (src_data) {
5004 case 0: /* D3 vblank */
5005 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
5006 DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
5007
5008 if (rdev->irq.crtc_vblank_int[2]) {
5009 drm_handle_vblank(rdev->ddev, 2);
5010 rdev->pm.vblank_sync = true;
5011 wake_up(&rdev->irq.vblank_queue);
5012 }
5013 if (atomic_read(&rdev->irq.pflip[2]))
5014 radeon_crtc_handle_vblank(rdev, 2);
5015 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5016 DRM_DEBUG("IH: D3 vblank\n");
5017
5018 break;
5019 case 1: /* D3 vline */
5020 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
5021 DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
5022
5023 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5024 DRM_DEBUG("IH: D3 vline\n");
5025
5026 break;
5027 default:
5028 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5029 break;
5030 }
5031 break;
5032 case 4: /* D4 vblank/vline */
5033 switch (src_data) {
5034 case 0: /* D4 vblank */
5035 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5036 DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5037
5038 if (rdev->irq.crtc_vblank_int[3]) {
5039 drm_handle_vblank(rdev->ddev, 3);
5040 rdev->pm.vblank_sync = true;
5041 wake_up(&rdev->irq.vblank_queue);
5042 }
5043 if (atomic_read(&rdev->irq.pflip[3]))
5044 radeon_crtc_handle_vblank(rdev, 3);
5045 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5046 DRM_DEBUG("IH: D4 vblank\n");
5047
5048 break;
5049 case 1: /* D4 vline */
5050 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5051 DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5052
5053 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5054 DRM_DEBUG("IH: D4 vline\n");
5055
5056 break;
5057 default:
5058 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5059 break;
5060 }
5061 break;
5062 case 5: /* D5 vblank/vline */
5063 switch (src_data) {
5064 case 0: /* D5 vblank */
5065 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5066 DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5067
5068 if (rdev->irq.crtc_vblank_int[4]) {
5069 drm_handle_vblank(rdev->ddev, 4);
5070 rdev->pm.vblank_sync = true;
5071 wake_up(&rdev->irq.vblank_queue);
5072 }
5073 if (atomic_read(&rdev->irq.pflip[4]))
5074 radeon_crtc_handle_vblank(rdev, 4);
5075 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5076 DRM_DEBUG("IH: D5 vblank\n");
5077
5078 break;
5079 case 1: /* D5 vline */
5080 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5081 DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5082
5083 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5084 DRM_DEBUG("IH: D5 vline\n");
5085
5086 break;
5087 default:
5088 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5089 break;
5090 }
5091 break;
5092 case 6: /* D6 vblank/vline */
5093 switch (src_data) {
5094 case 0: /* D6 vblank */
5095 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5096 DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5097
5098 if (rdev->irq.crtc_vblank_int[5]) {
5099 drm_handle_vblank(rdev->ddev, 5);
5100 rdev->pm.vblank_sync = true;
5101 wake_up(&rdev->irq.vblank_queue);
5102 }
5103 if (atomic_read(&rdev->irq.pflip[5]))
5104 radeon_crtc_handle_vblank(rdev, 5);
5105 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5106 DRM_DEBUG("IH: D6 vblank\n");
5107
5108 break;
5109 case 1: /* D6 vline */
5110 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5111 DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5112
5113 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5114 DRM_DEBUG("IH: D6 vline\n");
5115
5116 break;
5117 default:
5118 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5119 break;
5120 }
5121 break;
5122 case 8: /* D1 page flip */
5123 case 10: /* D2 page flip */
5124 case 12: /* D3 page flip */
5125 case 14: /* D4 page flip */
5126 case 16: /* D5 page flip */
5127 case 18: /* D6 page flip */
5128 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
5129 if (radeon_use_pflipirq > 0)
5130 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
5131 break;
5132 case 42: /* HPD hotplug */
5133 switch (src_data) {
5134 case 0:
5135 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5136 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5137
5138 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5139 queue_hotplug = true;
5140 DRM_DEBUG("IH: HPD1\n");
5141 break;
5142 case 1:
5143 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5144 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5145
5146 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5147 queue_hotplug = true;
5148 DRM_DEBUG("IH: HPD2\n");
5149 break;
5150 case 2:
5151 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5152 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5153
5154 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5155 queue_hotplug = true;
5156 DRM_DEBUG("IH: HPD3\n");
5157 break;
5158 case 3:
5159 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5160 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5161
5162 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5163 queue_hotplug = true;
5164 DRM_DEBUG("IH: HPD4\n");
5165 break;
5166 case 4:
5167 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5168 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5169
5170 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5171 queue_hotplug = true;
5172 DRM_DEBUG("IH: HPD5\n");
5173 break;
5174 case 5:
5175 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5176 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5177
5178 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5179 queue_hotplug = true;
5180 DRM_DEBUG("IH: HPD6\n");
5181 break;
5182 case 6:
5183 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5184 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5185
5186 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5187 queue_dp = true;
5188 DRM_DEBUG("IH: HPD_RX 1\n");
5189 break;
5190 case 7:
5191 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5192 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5193
5194 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5195 queue_dp = true;
5196 DRM_DEBUG("IH: HPD_RX 2\n");
5197 break;
5198 case 8:
5199 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5200 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5201
5202 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5203 queue_dp = true;
5204 DRM_DEBUG("IH: HPD_RX 3\n");
5205 break;
5206 case 9:
5207 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5208 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5209
5210 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5211 queue_dp = true;
5212 DRM_DEBUG("IH: HPD_RX 4\n");
5213 break;
5214 case 10:
5215 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5216 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5217
5218 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5219 queue_dp = true;
5220 DRM_DEBUG("IH: HPD_RX 5\n");
5221 break;
5222 case 11:
5223 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5224 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5225
5226 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5227 queue_dp = true;
5228 DRM_DEBUG("IH: HPD_RX 6\n");
5229 break;
5230 default:
5231 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5232 break;
5233 }
5234 break;
5235 case 44: /* hdmi */
5236 switch (src_data) {
5237 case 0:
5238 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5239 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5240
5241 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5242 queue_hdmi = true;
5243 DRM_DEBUG("IH: HDMI0\n");
5244 break;
5245 case 1:
5246 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5247 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5248
5249 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5250 queue_hdmi = true;
5251 DRM_DEBUG("IH: HDMI1\n");
5252 break;
5253 case 2:
5254 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5255 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5256
5257 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5258 queue_hdmi = true;
5259 DRM_DEBUG("IH: HDMI2\n");
5260 break;
5261 case 3:
5262 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5263 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5264
5265 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5266 queue_hdmi = true;
5267 DRM_DEBUG("IH: HDMI3\n");
5268 break;
5269 case 4:
5270 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5271 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5272
5273 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5274 queue_hdmi = true;
5275 DRM_DEBUG("IH: HDMI4\n");
5276 break;
5277 case 5:
5278 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5279 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5280
5281 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5282 queue_hdmi = true;
5283 DRM_DEBUG("IH: HDMI5\n");
5284 break;
5285 default:
5286 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5287 break;
5288 }
5289 case 96:
5290 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5291 WREG32(SRBM_INT_ACK, 0x1);
5292 break;
5293 case 124: /* UVD */
5294 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5295 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5296 break;
5297 case 146:
5298 case 147:
5299 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5300 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5301 /* reset addr and status */
5302 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5303 if (addr == 0x0 && status == 0x0)
5304 break;
5305 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5306 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5307 addr);
5308 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5309 status);
5310 cayman_vm_decode_fault(rdev, status, addr);
5311 break;
5312 case 176: /* CP_INT in ring buffer */
5313 case 177: /* CP_INT in IB1 */
5314 case 178: /* CP_INT in IB2 */
5315 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5316 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5317 break;
5318 case 181: /* CP EOP event */
5319 DRM_DEBUG("IH: CP EOP\n");
5320 if (rdev->family >= CHIP_CAYMAN) {
5321 switch (src_data) {
5322 case 0:
5323 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5324 break;
5325 case 1:
5326 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5327 break;
5328 case 2:
5329 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5330 break;
5331 }
5332 } else
5333 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5334 break;
5335 case 224: /* DMA trap event */
5336 DRM_DEBUG("IH: DMA trap\n");
5337 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5338 break;
5339 case 230: /* thermal low to high */
5340 DRM_DEBUG("IH: thermal low to high\n");
5341 rdev->pm.dpm.thermal.high_to_low = false;
5342 queue_thermal = true;
5343 break;
5344 case 231: /* thermal high to low */
5345 DRM_DEBUG("IH: thermal high to low\n");
5346 rdev->pm.dpm.thermal.high_to_low = true;
5347 queue_thermal = true;
5348 break;
5349 case 233: /* GUI IDLE */
5350 DRM_DEBUG("IH: GUI idle\n");
5351 break;
5352 case 244: /* DMA trap event */
5353 if (rdev->family >= CHIP_CAYMAN) {
5354 DRM_DEBUG("IH: DMA1 trap\n");
5355 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5356 }
5357 break;
5358 default:
5359 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5360 break;
5361 }
5362
5363 /* wptr/rptr are in bytes! */
5364 rptr += 16;
5365 rptr &= rdev->ih.ptr_mask;
5366 WREG32(IH_RB_RPTR, rptr);
5367 }
5368 if (queue_dp)
5369 schedule_work(&rdev->dp_work);
5370 if (queue_hotplug)
5371 schedule_work(&rdev->hotplug_work);
5372 if (queue_hdmi)
5373 schedule_work(&rdev->audio_work);
5374 if (queue_thermal && rdev->pm.dpm_enabled)
5375 schedule_work(&rdev->pm.dpm.thermal.work);
5376 rdev->ih.rptr = rptr;
5377 atomic_set(&rdev->ih.lock, 0);
5378
5379 /* make sure wptr hasn't changed while processing */
5380 wptr = evergreen_get_ih_wptr(rdev);
5381 if (wptr != rptr)
5382 goto restart_ih;
5383
5384 return IRQ_HANDLED;
5385 }
5386
5387 static int evergreen_startup(struct radeon_device *rdev)
5388 {
5389 struct radeon_ring *ring;
5390 int r;
5391
5392 /* enable pcie gen2 link */
5393 evergreen_pcie_gen2_enable(rdev);
5394 /* enable aspm */
5395 evergreen_program_aspm(rdev);
5396
5397 /* scratch needs to be initialized before MC */
5398 r = r600_vram_scratch_init(rdev);
5399 if (r)
5400 return r;
5401
5402 evergreen_mc_program(rdev);
5403
5404 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5405 r = ni_mc_load_microcode(rdev);
5406 if (r) {
5407 DRM_ERROR("Failed to load MC firmware!\n");
5408 return r;
5409 }
5410 }
5411
5412 if (rdev->flags & RADEON_IS_AGP) {
5413 evergreen_agp_enable(rdev);
5414 } else {
5415 r = evergreen_pcie_gart_enable(rdev);
5416 if (r)
5417 return r;
5418 }
5419 evergreen_gpu_init(rdev);
5420
5421 /* allocate rlc buffers */
5422 if (rdev->flags & RADEON_IS_IGP) {
5423 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5424 rdev->rlc.reg_list_size =
5425 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5426 rdev->rlc.cs_data = evergreen_cs_data;
5427 r = sumo_rlc_init(rdev);
5428 if (r) {
5429 DRM_ERROR("Failed to init rlc BOs!\n");
5430 return r;
5431 }
5432 }
5433
5434 /* allocate wb buffer */
5435 r = radeon_wb_init(rdev);
5436 if (r)
5437 return r;
5438
5439 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5440 if (r) {
5441 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5442 return r;
5443 }
5444
5445 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5446 if (r) {
5447 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5448 return r;
5449 }
5450
5451 r = uvd_v2_2_resume(rdev);
5452 if (!r) {
5453 r = radeon_fence_driver_start_ring(rdev,
5454 R600_RING_TYPE_UVD_INDEX);
5455 if (r)
5456 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5457 }
5458
5459 if (r)
5460 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5461
5462 /* Enable IRQ */
5463 if (!rdev->irq.installed) {
5464 r = radeon_irq_kms_init(rdev);
5465 if (r)
5466 return r;
5467 }
5468
5469 r = r600_irq_init(rdev);
5470 if (r) {
5471 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5472 radeon_irq_kms_fini(rdev);
5473 return r;
5474 }
5475 evergreen_irq_set(rdev);
5476
5477 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5478 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5479 RADEON_CP_PACKET2);
5480 if (r)
5481 return r;
5482
5483 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5484 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5485 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5486 if (r)
5487 return r;
5488
5489 r = evergreen_cp_load_microcode(rdev);
5490 if (r)
5491 return r;
5492 r = evergreen_cp_resume(rdev);
5493 if (r)
5494 return r;
5495 r = r600_dma_resume(rdev);
5496 if (r)
5497 return r;
5498
5499 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5500 if (ring->ring_size) {
5501 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5502 RADEON_CP_PACKET2);
5503 if (!r)
5504 r = uvd_v1_0_init(rdev);
5505
5506 if (r)
5507 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5508 }
5509
5510 r = radeon_ib_pool_init(rdev);
5511 if (r) {
5512 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5513 return r;
5514 }
5515
5516 r = radeon_audio_init(rdev);
5517 if (r) {
5518 DRM_ERROR("radeon: audio init failed\n");
5519 return r;
5520 }
5521
5522 return 0;
5523 }
5524
5525 int evergreen_resume(struct radeon_device *rdev)
5526 {
5527 int r;
5528
5529 /* reset the asic, the gfx blocks are often in a bad state
5530 * after the driver is unloaded or after a resume
5531 */
5532 if (radeon_asic_reset(rdev))
5533 dev_warn(rdev->dev, "GPU reset failed !\n");
5534 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5535 * posting will perform necessary task to bring back GPU into good
5536 * shape.
5537 */
5538 /* post card */
5539 atom_asic_init(rdev->mode_info.atom_context);
5540
5541 /* init golden registers */
5542 evergreen_init_golden_registers(rdev);
5543
5544 if (rdev->pm.pm_method == PM_METHOD_DPM)
5545 radeon_pm_resume(rdev);
5546
5547 rdev->accel_working = true;
5548 r = evergreen_startup(rdev);
5549 if (r) {
5550 DRM_ERROR("evergreen startup failed on resume\n");
5551 rdev->accel_working = false;
5552 return r;
5553 }
5554
5555 return r;
5556
5557 }
5558
5559 int evergreen_suspend(struct radeon_device *rdev)
5560 {
5561 radeon_pm_suspend(rdev);
5562 radeon_audio_fini(rdev);
5563 uvd_v1_0_fini(rdev);
5564 radeon_uvd_suspend(rdev);
5565 r700_cp_stop(rdev);
5566 r600_dma_stop(rdev);
5567 evergreen_irq_suspend(rdev);
5568 radeon_wb_disable(rdev);
5569 evergreen_pcie_gart_disable(rdev);
5570
5571 return 0;
5572 }
5573
5574 /* Plan is to move initialization in that function and use
5575 * helper function so that radeon_device_init pretty much
5576 * do nothing more than calling asic specific function. This
5577 * should also allow to remove a bunch of callback function
5578 * like vram_info.
5579 */
5580 int evergreen_init(struct radeon_device *rdev)
5581 {
5582 int r;
5583
5584 /* Read BIOS */
5585 if (!radeon_get_bios(rdev)) {
5586 if (ASIC_IS_AVIVO(rdev))
5587 return -EINVAL;
5588 }
5589 /* Must be an ATOMBIOS */
5590 if (!rdev->is_atom_bios) {
5591 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5592 return -EINVAL;
5593 }
5594 r = radeon_atombios_init(rdev);
5595 if (r)
5596 return r;
5597 /* reset the asic, the gfx blocks are often in a bad state
5598 * after the driver is unloaded or after a resume
5599 */
5600 if (radeon_asic_reset(rdev))
5601 dev_warn(rdev->dev, "GPU reset failed !\n");
5602 /* Post card if necessary */
5603 if (!radeon_card_posted(rdev)) {
5604 if (!rdev->bios) {
5605 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5606 return -EINVAL;
5607 }
5608 DRM_INFO("GPU not posted. posting now...\n");
5609 atom_asic_init(rdev->mode_info.atom_context);
5610 }
5611 /* init golden registers */
5612 evergreen_init_golden_registers(rdev);
5613 /* Initialize scratch registers */
5614 r600_scratch_init(rdev);
5615 /* Initialize surface registers */
5616 radeon_surface_init(rdev);
5617 /* Initialize clocks */
5618 radeon_get_clock_info(rdev->ddev);
5619 /* Fence driver */
5620 r = radeon_fence_driver_init(rdev);
5621 if (r)
5622 return r;
5623 /* initialize AGP */
5624 if (rdev->flags & RADEON_IS_AGP) {
5625 r = radeon_agp_init(rdev);
5626 if (r)
5627 radeon_agp_disable(rdev);
5628 }
5629 /* initialize memory controller */
5630 r = evergreen_mc_init(rdev);
5631 if (r)
5632 return r;
5633 /* Memory manager */
5634 r = radeon_bo_init(rdev);
5635 if (r)
5636 return r;
5637
5638 if (ASIC_IS_DCE5(rdev)) {
5639 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5640 r = ni_init_microcode(rdev);
5641 if (r) {
5642 DRM_ERROR("Failed to load firmware!\n");
5643 return r;
5644 }
5645 }
5646 } else {
5647 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5648 r = r600_init_microcode(rdev);
5649 if (r) {
5650 DRM_ERROR("Failed to load firmware!\n");
5651 return r;
5652 }
5653 }
5654 }
5655
5656 /* Initialize power management */
5657 radeon_pm_init(rdev);
5658
5659 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5660 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5661
5662 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5663 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5664
5665 r = radeon_uvd_init(rdev);
5666 if (!r) {
5667 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5668 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5669 4096);
5670 }
5671
5672 rdev->ih.ring_obj = NULL;
5673 r600_ih_ring_init(rdev, 64 * 1024);
5674
5675 r = r600_pcie_gart_init(rdev);
5676 if (r)
5677 return r;
5678
5679 rdev->accel_working = true;
5680 r = evergreen_startup(rdev);
5681 if (r) {
5682 dev_err(rdev->dev, "disabling GPU acceleration\n");
5683 r700_cp_fini(rdev);
5684 r600_dma_fini(rdev);
5685 r600_irq_fini(rdev);
5686 if (rdev->flags & RADEON_IS_IGP)
5687 sumo_rlc_fini(rdev);
5688 radeon_wb_fini(rdev);
5689 radeon_ib_pool_fini(rdev);
5690 radeon_irq_kms_fini(rdev);
5691 evergreen_pcie_gart_fini(rdev);
5692 rdev->accel_working = false;
5693 }
5694
5695 /* Don't start up if the MC ucode is missing on BTC parts.
5696 * The default clocks and voltages before the MC ucode
5697 * is loaded are not suffient for advanced operations.
5698 */
5699 if (ASIC_IS_DCE5(rdev)) {
5700 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5701 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5702 return -EINVAL;
5703 }
5704 }
5705
5706 return 0;
5707 }
5708
5709 void evergreen_fini(struct radeon_device *rdev)
5710 {
5711 radeon_pm_fini(rdev);
5712 radeon_audio_fini(rdev);
5713 r700_cp_fini(rdev);
5714 r600_dma_fini(rdev);
5715 r600_irq_fini(rdev);
5716 if (rdev->flags & RADEON_IS_IGP)
5717 sumo_rlc_fini(rdev);
5718 radeon_wb_fini(rdev);
5719 radeon_ib_pool_fini(rdev);
5720 radeon_irq_kms_fini(rdev);
5721 uvd_v1_0_fini(rdev);
5722 radeon_uvd_fini(rdev);
5723 evergreen_pcie_gart_fini(rdev);
5724 r600_vram_scratch_fini(rdev);
5725 radeon_gem_fini(rdev);
5726 radeon_fence_driver_fini(rdev);
5727 radeon_agp_fini(rdev);
5728 radeon_bo_fini(rdev);
5729 radeon_atombios_fini(rdev);
5730 kfree(rdev->bios);
5731 rdev->bios = NULL;
5732 }
5733
5734 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5735 {
5736 u32 link_width_cntl, speed_cntl;
5737
5738 if (radeon_pcie_gen2 == 0)
5739 return;
5740
5741 if (rdev->flags & RADEON_IS_IGP)
5742 return;
5743
5744 if (!(rdev->flags & RADEON_IS_PCIE))
5745 return;
5746
5747 /* x2 cards have a special sequence */
5748 if (ASIC_IS_X2(rdev))
5749 return;
5750
5751 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5752 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5753 return;
5754
5755 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5756 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5757 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5758 return;
5759 }
5760
5761 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5762
5763 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5764 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5765
5766 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5767 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5768 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5769
5770 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5771 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5772 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5773
5774 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5775 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5776 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5777
5778 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5779 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5780 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5781
5782 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5783 speed_cntl |= LC_GEN2_EN_STRAP;
5784 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5785
5786 } else {
5787 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5788 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5789 if (1)
5790 link_width_cntl |= LC_UPCONFIGURE_DIS;
5791 else
5792 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5793 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5794 }
5795 }
5796
5797 void evergreen_program_aspm(struct radeon_device *rdev)
5798 {
5799 u32 data, orig;
5800 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5801 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5802 /* fusion_platform = true
5803 * if the system is a fusion system
5804 * (APU or DGPU in a fusion system).
5805 * todo: check if the system is a fusion platform.
5806 */
5807 bool fusion_platform = false;
5808
5809 if (radeon_aspm == 0)
5810 return;
5811
5812 if (!(rdev->flags & RADEON_IS_PCIE))
5813 return;
5814
5815 switch (rdev->family) {
5816 case CHIP_CYPRESS:
5817 case CHIP_HEMLOCK:
5818 case CHIP_JUNIPER:
5819 case CHIP_REDWOOD:
5820 case CHIP_CEDAR:
5821 case CHIP_SUMO:
5822 case CHIP_SUMO2:
5823 case CHIP_PALM:
5824 case CHIP_ARUBA:
5825 disable_l0s = true;
5826 break;
5827 default:
5828 disable_l0s = false;
5829 break;
5830 }
5831
5832 if (rdev->flags & RADEON_IS_IGP)
5833 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5834
5835 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5836 if (fusion_platform)
5837 data &= ~MULTI_PIF;
5838 else
5839 data |= MULTI_PIF;
5840 if (data != orig)
5841 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5842
5843 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5844 if (fusion_platform)
5845 data &= ~MULTI_PIF;
5846 else
5847 data |= MULTI_PIF;
5848 if (data != orig)
5849 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5850
5851 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5852 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5853 if (!disable_l0s) {
5854 if (rdev->family >= CHIP_BARTS)
5855 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5856 else
5857 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5858 }
5859
5860 if (!disable_l1) {
5861 if (rdev->family >= CHIP_BARTS)
5862 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5863 else
5864 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5865
5866 if (!disable_plloff_in_l1) {
5867 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5868 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5869 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5870 if (data != orig)
5871 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5872
5873 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5874 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5875 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5876 if (data != orig)
5877 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5878
5879 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5880 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5881 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5882 if (data != orig)
5883 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5884
5885 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5886 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5887 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5888 if (data != orig)
5889 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5890
5891 if (rdev->family >= CHIP_BARTS) {
5892 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5893 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5894 data |= PLL_RAMP_UP_TIME_0(4);
5895 if (data != orig)
5896 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5897
5898 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5899 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5900 data |= PLL_RAMP_UP_TIME_1(4);
5901 if (data != orig)
5902 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5903
5904 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5905 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5906 data |= PLL_RAMP_UP_TIME_0(4);
5907 if (data != orig)
5908 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5909
5910 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5911 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5912 data |= PLL_RAMP_UP_TIME_1(4);
5913 if (data != orig)
5914 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5915 }
5916
5917 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5918 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5919 data |= LC_DYN_LANES_PWR_STATE(3);
5920 if (data != orig)
5921 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5922
5923 if (rdev->family >= CHIP_BARTS) {
5924 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5925 data &= ~LS2_EXIT_TIME_MASK;
5926 data |= LS2_EXIT_TIME(1);
5927 if (data != orig)
5928 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5929
5930 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5931 data &= ~LS2_EXIT_TIME_MASK;
5932 data |= LS2_EXIT_TIME(1);
5933 if (data != orig)
5934 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5935 }
5936 }
5937 }
5938
5939 /* evergreen parts only */
5940 if (rdev->family < CHIP_BARTS)
5941 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5942
5943 if (pcie_lc_cntl != pcie_lc_cntl_old)
5944 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5945 }
Linux kernel - Deliverables
This page took 0.147123 seconds and 4 git commands to generate.