projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/kms: add dpm support for rv7xx (v4)
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
50static u32 sumo_rlc_save_restore_register_list[] =
51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
134static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
135
bcc1c2a1
AD		 136static void evergreen_gpu_init(struct radeon_device *rdev);
137void evergreen_fini(struct radeon_device *rdev);
b07759bf 138void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
bcc1c2a1 141
d4788db3
AD		 142static const u32 evergreen_golden_registers[] =
143{
144 0x3f90, 0xffff0000, 0xff000000,
145 0x9148, 0xffff0000, 0xff000000,
146 0x3f94, 0xffff0000, 0xff000000,
147 0x914c, 0xffff0000, 0xff000000,
148 0x9b7c, 0xffffffff, 0x00000000,
149 0x8a14, 0xffffffff, 0x00000007,
150 0x8b10, 0xffffffff, 0x00000000,
151 0x960c, 0xffffffff, 0x54763210,
152 0x88c4, 0xffffffff, 0x000000c2,
153 0x88d4, 0xffffffff, 0x00000010,
154 0x8974, 0xffffffff, 0x00000000,
155 0xc78, 0x00000080, 0x00000080,
156 0x5eb4, 0xffffffff, 0x00000002,
157 0x5e78, 0xffffffff, 0x001000f0,
158 0x6104, 0x01000300, 0x00000000,
159 0x5bc0, 0x00300000, 0x00000000,
160 0x7030, 0xffffffff, 0x00000011,
161 0x7c30, 0xffffffff, 0x00000011,
162 0x10830, 0xffffffff, 0x00000011,
163 0x11430, 0xffffffff, 0x00000011,
164 0x12030, 0xffffffff, 0x00000011,
165 0x12c30, 0xffffffff, 0x00000011,
166 0xd02c, 0xffffffff, 0x08421000,
167 0x240c, 0xffffffff, 0x00000380,
168 0x8b24, 0xffffffff, 0x00ff0fff,
169 0x28a4c, 0x06000000, 0x06000000,
170 0x10c, 0x00000001, 0x00000001,
171 0x8d00, 0xffffffff, 0x100e4848,
172 0x8d04, 0xffffffff, 0x00164745,
173 0x8c00, 0xffffffff, 0xe4000003,
174 0x8c04, 0xffffffff, 0x40600060,
175 0x8c08, 0xffffffff, 0x001c001c,
176 0x8cf0, 0xffffffff, 0x08e00620,
177 0x8c20, 0xffffffff, 0x00800080,
178 0x8c24, 0xffffffff, 0x00800080,
179 0x8c18, 0xffffffff, 0x20202078,
180 0x8c1c, 0xffffffff, 0x00001010,
181 0x28350, 0xffffffff, 0x00000000,
182 0xa008, 0xffffffff, 0x00010000,
183 0x5cc, 0xffffffff, 0x00000001,
184 0x9508, 0xffffffff, 0x00000002,
185 0x913c, 0x0000000f, 0x0000000a
186};
187
188static const u32 evergreen_golden_registers2[] =
189{
190 0x2f4c, 0xffffffff, 0x00000000,
191 0x54f4, 0xffffffff, 0x00000000,
192 0x54f0, 0xffffffff, 0x00000000,
193 0x5498, 0xffffffff, 0x00000000,
194 0x549c, 0xffffffff, 0x00000000,
195 0x5494, 0xffffffff, 0x00000000,
196 0x53cc, 0xffffffff, 0x00000000,
197 0x53c8, 0xffffffff, 0x00000000,
198 0x53c4, 0xffffffff, 0x00000000,
199 0x53c0, 0xffffffff, 0x00000000,
200 0x53bc, 0xffffffff, 0x00000000,
201 0x53b8, 0xffffffff, 0x00000000,
202 0x53b4, 0xffffffff, 0x00000000,
203 0x53b0, 0xffffffff, 0x00000000
204};
205
206static const u32 cypress_mgcg_init[] =
207{
208 0x802c, 0xffffffff, 0xc0000000,
209 0x5448, 0xffffffff, 0x00000100,
210 0x55e4, 0xffffffff, 0x00000100,
211 0x160c, 0xffffffff, 0x00000100,
212 0x5644, 0xffffffff, 0x00000100,
213 0xc164, 0xffffffff, 0x00000100,
214 0x8a18, 0xffffffff, 0x00000100,
215 0x897c, 0xffffffff, 0x06000100,
216 0x8b28, 0xffffffff, 0x00000100,
217 0x9144, 0xffffffff, 0x00000100,
218 0x9a60, 0xffffffff, 0x00000100,
219 0x9868, 0xffffffff, 0x00000100,
220 0x8d58, 0xffffffff, 0x00000100,
221 0x9510, 0xffffffff, 0x00000100,
222 0x949c, 0xffffffff, 0x00000100,
223 0x9654, 0xffffffff, 0x00000100,
224 0x9030, 0xffffffff, 0x00000100,
225 0x9034, 0xffffffff, 0x00000100,
226 0x9038, 0xffffffff, 0x00000100,
227 0x903c, 0xffffffff, 0x00000100,
228 0x9040, 0xffffffff, 0x00000100,
229 0xa200, 0xffffffff, 0x00000100,
230 0xa204, 0xffffffff, 0x00000100,
231 0xa208, 0xffffffff, 0x00000100,
232 0xa20c, 0xffffffff, 0x00000100,
233 0x971c, 0xffffffff, 0x00000100,
234 0x977c, 0xffffffff, 0x00000100,
235 0x3f80, 0xffffffff, 0x00000100,
236 0xa210, 0xffffffff, 0x00000100,
237 0xa214, 0xffffffff, 0x00000100,
238 0x4d8, 0xffffffff, 0x00000100,
239 0x9784, 0xffffffff, 0x00000100,
240 0x9698, 0xffffffff, 0x00000100,
241 0x4d4, 0xffffffff, 0x00000200,
242 0x30cc, 0xffffffff, 0x00000100,
243 0xd0c0, 0xffffffff, 0xff000100,
244 0x802c, 0xffffffff, 0x40000000,
245 0x915c, 0xffffffff, 0x00010000,
246 0x9160, 0xffffffff, 0x00030002,
247 0x9178, 0xffffffff, 0x00070000,
248 0x917c, 0xffffffff, 0x00030002,
249 0x9180, 0xffffffff, 0x00050004,
250 0x918c, 0xffffffff, 0x00010006,
251 0x9190, 0xffffffff, 0x00090008,
252 0x9194, 0xffffffff, 0x00070000,
253 0x9198, 0xffffffff, 0x00030002,
254 0x919c, 0xffffffff, 0x00050004,
255 0x91a8, 0xffffffff, 0x00010006,
256 0x91ac, 0xffffffff, 0x00090008,
257 0x91b0, 0xffffffff, 0x00070000,
258 0x91b4, 0xffffffff, 0x00030002,
259 0x91b8, 0xffffffff, 0x00050004,
260 0x91c4, 0xffffffff, 0x00010006,
261 0x91c8, 0xffffffff, 0x00090008,
262 0x91cc, 0xffffffff, 0x00070000,
263 0x91d0, 0xffffffff, 0x00030002,
264 0x91d4, 0xffffffff, 0x00050004,
265 0x91e0, 0xffffffff, 0x00010006,
266 0x91e4, 0xffffffff, 0x00090008,
267 0x91e8, 0xffffffff, 0x00000000,
268 0x91ec, 0xffffffff, 0x00070000,
269 0x91f0, 0xffffffff, 0x00030002,
270 0x91f4, 0xffffffff, 0x00050004,
271 0x9200, 0xffffffff, 0x00010006,
272 0x9204, 0xffffffff, 0x00090008,
273 0x9208, 0xffffffff, 0x00070000,
274 0x920c, 0xffffffff, 0x00030002,
275 0x9210, 0xffffffff, 0x00050004,
276 0x921c, 0xffffffff, 0x00010006,
277 0x9220, 0xffffffff, 0x00090008,
278 0x9224, 0xffffffff, 0x00070000,
279 0x9228, 0xffffffff, 0x00030002,
280 0x922c, 0xffffffff, 0x00050004,
281 0x9238, 0xffffffff, 0x00010006,
282 0x923c, 0xffffffff, 0x00090008,
283 0x9240, 0xffffffff, 0x00070000,
284 0x9244, 0xffffffff, 0x00030002,
285 0x9248, 0xffffffff, 0x00050004,
286 0x9254, 0xffffffff, 0x00010006,
287 0x9258, 0xffffffff, 0x00090008,
288 0x925c, 0xffffffff, 0x00070000,
289 0x9260, 0xffffffff, 0x00030002,
290 0x9264, 0xffffffff, 0x00050004,
291 0x9270, 0xffffffff, 0x00010006,
292 0x9274, 0xffffffff, 0x00090008,
293 0x9278, 0xffffffff, 0x00070000,
294 0x927c, 0xffffffff, 0x00030002,
295 0x9280, 0xffffffff, 0x00050004,
296 0x928c, 0xffffffff, 0x00010006,
297 0x9290, 0xffffffff, 0x00090008,
298 0x9294, 0xffffffff, 0x00000000,
299 0x929c, 0xffffffff, 0x00000001,
300 0x802c, 0xffffffff, 0x40010000,
301 0x915c, 0xffffffff, 0x00010000,
302 0x9160, 0xffffffff, 0x00030002,
303 0x9178, 0xffffffff, 0x00070000,
304 0x917c, 0xffffffff, 0x00030002,
305 0x9180, 0xffffffff, 0x00050004,
306 0x918c, 0xffffffff, 0x00010006,
307 0x9190, 0xffffffff, 0x00090008,
308 0x9194, 0xffffffff, 0x00070000,
309 0x9198, 0xffffffff, 0x00030002,
310 0x919c, 0xffffffff, 0x00050004,
311 0x91a8, 0xffffffff, 0x00010006,
312 0x91ac, 0xffffffff, 0x00090008,
313 0x91b0, 0xffffffff, 0x00070000,
314 0x91b4, 0xffffffff, 0x00030002,
315 0x91b8, 0xffffffff, 0x00050004,
316 0x91c4, 0xffffffff, 0x00010006,
317 0x91c8, 0xffffffff, 0x00090008,
318 0x91cc, 0xffffffff, 0x00070000,
319 0x91d0, 0xffffffff, 0x00030002,
320 0x91d4, 0xffffffff, 0x00050004,
321 0x91e0, 0xffffffff, 0x00010006,
322 0x91e4, 0xffffffff, 0x00090008,
323 0x91e8, 0xffffffff, 0x00000000,
324 0x91ec, 0xffffffff, 0x00070000,
325 0x91f0, 0xffffffff, 0x00030002,
326 0x91f4, 0xffffffff, 0x00050004,
327 0x9200, 0xffffffff, 0x00010006,
328 0x9204, 0xffffffff, 0x00090008,
329 0x9208, 0xffffffff, 0x00070000,
330 0x920c, 0xffffffff, 0x00030002,
331 0x9210, 0xffffffff, 0x00050004,
332 0x921c, 0xffffffff, 0x00010006,
333 0x9220, 0xffffffff, 0x00090008,
334 0x9224, 0xffffffff, 0x00070000,
335 0x9228, 0xffffffff, 0x00030002,
336 0x922c, 0xffffffff, 0x00050004,
337 0x9238, 0xffffffff, 0x00010006,
338 0x923c, 0xffffffff, 0x00090008,
339 0x9240, 0xffffffff, 0x00070000,
340 0x9244, 0xffffffff, 0x00030002,
341 0x9248, 0xffffffff, 0x00050004,
342 0x9254, 0xffffffff, 0x00010006,
343 0x9258, 0xffffffff, 0x00090008,
344 0x925c, 0xffffffff, 0x00070000,
345 0x9260, 0xffffffff, 0x00030002,
346 0x9264, 0xffffffff, 0x00050004,
347 0x9270, 0xffffffff, 0x00010006,
348 0x9274, 0xffffffff, 0x00090008,
349 0x9278, 0xffffffff, 0x00070000,
350 0x927c, 0xffffffff, 0x00030002,
351 0x9280, 0xffffffff, 0x00050004,
352 0x928c, 0xffffffff, 0x00010006,
353 0x9290, 0xffffffff, 0x00090008,
354 0x9294, 0xffffffff, 0x00000000,
355 0x929c, 0xffffffff, 0x00000001,
356 0x802c, 0xffffffff, 0xc0000000
357};
358
359static const u32 redwood_mgcg_init[] =
360{
361 0x802c, 0xffffffff, 0xc0000000,
362 0x5448, 0xffffffff, 0x00000100,
363 0x55e4, 0xffffffff, 0x00000100,
364 0x160c, 0xffffffff, 0x00000100,
365 0x5644, 0xffffffff, 0x00000100,
366 0xc164, 0xffffffff, 0x00000100,
367 0x8a18, 0xffffffff, 0x00000100,
368 0x897c, 0xffffffff, 0x06000100,
369 0x8b28, 0xffffffff, 0x00000100,
370 0x9144, 0xffffffff, 0x00000100,
371 0x9a60, 0xffffffff, 0x00000100,
372 0x9868, 0xffffffff, 0x00000100,
373 0x8d58, 0xffffffff, 0x00000100,
374 0x9510, 0xffffffff, 0x00000100,
375 0x949c, 0xffffffff, 0x00000100,
376 0x9654, 0xffffffff, 0x00000100,
377 0x9030, 0xffffffff, 0x00000100,
378 0x9034, 0xffffffff, 0x00000100,
379 0x9038, 0xffffffff, 0x00000100,
380 0x903c, 0xffffffff, 0x00000100,
381 0x9040, 0xffffffff, 0x00000100,
382 0xa200, 0xffffffff, 0x00000100,
383 0xa204, 0xffffffff, 0x00000100,
384 0xa208, 0xffffffff, 0x00000100,
385 0xa20c, 0xffffffff, 0x00000100,
386 0x971c, 0xffffffff, 0x00000100,
387 0x977c, 0xffffffff, 0x00000100,
388 0x3f80, 0xffffffff, 0x00000100,
389 0xa210, 0xffffffff, 0x00000100,
390 0xa214, 0xffffffff, 0x00000100,
391 0x4d8, 0xffffffff, 0x00000100,
392 0x9784, 0xffffffff, 0x00000100,
393 0x9698, 0xffffffff, 0x00000100,
394 0x4d4, 0xffffffff, 0x00000200,
395 0x30cc, 0xffffffff, 0x00000100,
396 0xd0c0, 0xffffffff, 0xff000100,
397 0x802c, 0xffffffff, 0x40000000,
398 0x915c, 0xffffffff, 0x00010000,
399 0x9160, 0xffffffff, 0x00030002,
400 0x9178, 0xffffffff, 0x00070000,
401 0x917c, 0xffffffff, 0x00030002,
402 0x9180, 0xffffffff, 0x00050004,
403 0x918c, 0xffffffff, 0x00010006,
404 0x9190, 0xffffffff, 0x00090008,
405 0x9194, 0xffffffff, 0x00070000,
406 0x9198, 0xffffffff, 0x00030002,
407 0x919c, 0xffffffff, 0x00050004,
408 0x91a8, 0xffffffff, 0x00010006,
409 0x91ac, 0xffffffff, 0x00090008,
410 0x91b0, 0xffffffff, 0x00070000,
411 0x91b4, 0xffffffff, 0x00030002,
412 0x91b8, 0xffffffff, 0x00050004,
413 0x91c4, 0xffffffff, 0x00010006,
414 0x91c8, 0xffffffff, 0x00090008,
415 0x91cc, 0xffffffff, 0x00070000,
416 0x91d0, 0xffffffff, 0x00030002,
417 0x91d4, 0xffffffff, 0x00050004,
418 0x91e0, 0xffffffff, 0x00010006,
419 0x91e4, 0xffffffff, 0x00090008,
420 0x91e8, 0xffffffff, 0x00000000,
421 0x91ec, 0xffffffff, 0x00070000,
422 0x91f0, 0xffffffff, 0x00030002,
423 0x91f4, 0xffffffff, 0x00050004,
424 0x9200, 0xffffffff, 0x00010006,
425 0x9204, 0xffffffff, 0x00090008,
426 0x9294, 0xffffffff, 0x00000000,
427 0x929c, 0xffffffff, 0x00000001,
428 0x802c, 0xffffffff, 0xc0000000
429};
430
431static const u32 cedar_golden_registers[] =
432{
433 0x3f90, 0xffff0000, 0xff000000,
434 0x9148, 0xffff0000, 0xff000000,
435 0x3f94, 0xffff0000, 0xff000000,
436 0x914c, 0xffff0000, 0xff000000,
437 0x9b7c, 0xffffffff, 0x00000000,
438 0x8a14, 0xffffffff, 0x00000007,
439 0x8b10, 0xffffffff, 0x00000000,
440 0x960c, 0xffffffff, 0x54763210,
441 0x88c4, 0xffffffff, 0x000000c2,
442 0x88d4, 0xffffffff, 0x00000000,
443 0x8974, 0xffffffff, 0x00000000,
444 0xc78, 0x00000080, 0x00000080,
445 0x5eb4, 0xffffffff, 0x00000002,
446 0x5e78, 0xffffffff, 0x001000f0,
447 0x6104, 0x01000300, 0x00000000,
448 0x5bc0, 0x00300000, 0x00000000,
449 0x7030, 0xffffffff, 0x00000011,
450 0x7c30, 0xffffffff, 0x00000011,
451 0x10830, 0xffffffff, 0x00000011,
452 0x11430, 0xffffffff, 0x00000011,
453 0xd02c, 0xffffffff, 0x08421000,
454 0x240c, 0xffffffff, 0x00000380,
455 0x8b24, 0xffffffff, 0x00ff0fff,
456 0x28a4c, 0x06000000, 0x06000000,
457 0x10c, 0x00000001, 0x00000001,
458 0x8d00, 0xffffffff, 0x100e4848,
459 0x8d04, 0xffffffff, 0x00164745,
460 0x8c00, 0xffffffff, 0xe4000003,
461 0x8c04, 0xffffffff, 0x40600060,
462 0x8c08, 0xffffffff, 0x001c001c,
463 0x8cf0, 0xffffffff, 0x08e00410,
464 0x8c20, 0xffffffff, 0x00800080,
465 0x8c24, 0xffffffff, 0x00800080,
466 0x8c18, 0xffffffff, 0x20202078,
467 0x8c1c, 0xffffffff, 0x00001010,
468 0x28350, 0xffffffff, 0x00000000,
469 0xa008, 0xffffffff, 0x00010000,
470 0x5cc, 0xffffffff, 0x00000001,
471 0x9508, 0xffffffff, 0x00000002
472};
473
474static const u32 cedar_mgcg_init[] =
475{
476 0x802c, 0xffffffff, 0xc0000000,
477 0x5448, 0xffffffff, 0x00000100,
478 0x55e4, 0xffffffff, 0x00000100,
479 0x160c, 0xffffffff, 0x00000100,
480 0x5644, 0xffffffff, 0x00000100,
481 0xc164, 0xffffffff, 0x00000100,
482 0x8a18, 0xffffffff, 0x00000100,
483 0x897c, 0xffffffff, 0x06000100,
484 0x8b28, 0xffffffff, 0x00000100,
485 0x9144, 0xffffffff, 0x00000100,
486 0x9a60, 0xffffffff, 0x00000100,
487 0x9868, 0xffffffff, 0x00000100,
488 0x8d58, 0xffffffff, 0x00000100,
489 0x9510, 0xffffffff, 0x00000100,
490 0x949c, 0xffffffff, 0x00000100,
491 0x9654, 0xffffffff, 0x00000100,
492 0x9030, 0xffffffff, 0x00000100,
493 0x9034, 0xffffffff, 0x00000100,
494 0x9038, 0xffffffff, 0x00000100,
495 0x903c, 0xffffffff, 0x00000100,
496 0x9040, 0xffffffff, 0x00000100,
497 0xa200, 0xffffffff, 0x00000100,
498 0xa204, 0xffffffff, 0x00000100,
499 0xa208, 0xffffffff, 0x00000100,
500 0xa20c, 0xffffffff, 0x00000100,
501 0x971c, 0xffffffff, 0x00000100,
502 0x977c, 0xffffffff, 0x00000100,
503 0x3f80, 0xffffffff, 0x00000100,
504 0xa210, 0xffffffff, 0x00000100,
505 0xa214, 0xffffffff, 0x00000100,
506 0x4d8, 0xffffffff, 0x00000100,
507 0x9784, 0xffffffff, 0x00000100,
508 0x9698, 0xffffffff, 0x00000100,
509 0x4d4, 0xffffffff, 0x00000200,
510 0x30cc, 0xffffffff, 0x00000100,
511 0xd0c0, 0xffffffff, 0xff000100,
512 0x802c, 0xffffffff, 0x40000000,
513 0x915c, 0xffffffff, 0x00010000,
514 0x9178, 0xffffffff, 0x00050000,
515 0x917c, 0xffffffff, 0x00030002,
516 0x918c, 0xffffffff, 0x00010004,
517 0x9190, 0xffffffff, 0x00070006,
518 0x9194, 0xffffffff, 0x00050000,
519 0x9198, 0xffffffff, 0x00030002,
520 0x91a8, 0xffffffff, 0x00010004,
521 0x91ac, 0xffffffff, 0x00070006,
522 0x91e8, 0xffffffff, 0x00000000,
523 0x9294, 0xffffffff, 0x00000000,
524 0x929c, 0xffffffff, 0x00000001,
525 0x802c, 0xffffffff, 0xc0000000
526};
527
528static const u32 juniper_mgcg_init[] =
529{
530 0x802c, 0xffffffff, 0xc0000000,
531 0x5448, 0xffffffff, 0x00000100,
532 0x55e4, 0xffffffff, 0x00000100,
533 0x160c, 0xffffffff, 0x00000100,
534 0x5644, 0xffffffff, 0x00000100,
535 0xc164, 0xffffffff, 0x00000100,
536 0x8a18, 0xffffffff, 0x00000100,
537 0x897c, 0xffffffff, 0x06000100,
538 0x8b28, 0xffffffff, 0x00000100,
539 0x9144, 0xffffffff, 0x00000100,
540 0x9a60, 0xffffffff, 0x00000100,
541 0x9868, 0xffffffff, 0x00000100,
542 0x8d58, 0xffffffff, 0x00000100,
543 0x9510, 0xffffffff, 0x00000100,
544 0x949c, 0xffffffff, 0x00000100,
545 0x9654, 0xffffffff, 0x00000100,
546 0x9030, 0xffffffff, 0x00000100,
547 0x9034, 0xffffffff, 0x00000100,
548 0x9038, 0xffffffff, 0x00000100,
549 0x903c, 0xffffffff, 0x00000100,
550 0x9040, 0xffffffff, 0x00000100,
551 0xa200, 0xffffffff, 0x00000100,
552 0xa204, 0xffffffff, 0x00000100,
553 0xa208, 0xffffffff, 0x00000100,
554 0xa20c, 0xffffffff, 0x00000100,
555 0x971c, 0xffffffff, 0x00000100,
556 0xd0c0, 0xffffffff, 0xff000100,
557 0x802c, 0xffffffff, 0x40000000,
558 0x915c, 0xffffffff, 0x00010000,
559 0x9160, 0xffffffff, 0x00030002,
560 0x9178, 0xffffffff, 0x00070000,
561 0x917c, 0xffffffff, 0x00030002,
562 0x9180, 0xffffffff, 0x00050004,
563 0x918c, 0xffffffff, 0x00010006,
564 0x9190, 0xffffffff, 0x00090008,
565 0x9194, 0xffffffff, 0x00070000,
566 0x9198, 0xffffffff, 0x00030002,
567 0x919c, 0xffffffff, 0x00050004,
568 0x91a8, 0xffffffff, 0x00010006,
569 0x91ac, 0xffffffff, 0x00090008,
570 0x91b0, 0xffffffff, 0x00070000,
571 0x91b4, 0xffffffff, 0x00030002,
572 0x91b8, 0xffffffff, 0x00050004,
573 0x91c4, 0xffffffff, 0x00010006,
574 0x91c8, 0xffffffff, 0x00090008,
575 0x91cc, 0xffffffff, 0x00070000,
576 0x91d0, 0xffffffff, 0x00030002,
577 0x91d4, 0xffffffff, 0x00050004,
578 0x91e0, 0xffffffff, 0x00010006,
579 0x91e4, 0xffffffff, 0x00090008,
580 0x91e8, 0xffffffff, 0x00000000,
581 0x91ec, 0xffffffff, 0x00070000,
582 0x91f0, 0xffffffff, 0x00030002,
583 0x91f4, 0xffffffff, 0x00050004,
584 0x9200, 0xffffffff, 0x00010006,
585 0x9204, 0xffffffff, 0x00090008,
586 0x9208, 0xffffffff, 0x00070000,
587 0x920c, 0xffffffff, 0x00030002,
588 0x9210, 0xffffffff, 0x00050004,
589 0x921c, 0xffffffff, 0x00010006,
590 0x9220, 0xffffffff, 0x00090008,
591 0x9224, 0xffffffff, 0x00070000,
592 0x9228, 0xffffffff, 0x00030002,
593 0x922c, 0xffffffff, 0x00050004,
594 0x9238, 0xffffffff, 0x00010006,
595 0x923c, 0xffffffff, 0x00090008,
596 0x9240, 0xffffffff, 0x00070000,
597 0x9244, 0xffffffff, 0x00030002,
598 0x9248, 0xffffffff, 0x00050004,
599 0x9254, 0xffffffff, 0x00010006,
600 0x9258, 0xffffffff, 0x00090008,
601 0x925c, 0xffffffff, 0x00070000,
602 0x9260, 0xffffffff, 0x00030002,
603 0x9264, 0xffffffff, 0x00050004,
604 0x9270, 0xffffffff, 0x00010006,
605 0x9274, 0xffffffff, 0x00090008,
606 0x9278, 0xffffffff, 0x00070000,
607 0x927c, 0xffffffff, 0x00030002,
608 0x9280, 0xffffffff, 0x00050004,
609 0x928c, 0xffffffff, 0x00010006,
610 0x9290, 0xffffffff, 0x00090008,
611 0x9294, 0xffffffff, 0x00000000,
612 0x929c, 0xffffffff, 0x00000001,
613 0x802c, 0xffffffff, 0xc0000000,
614 0x977c, 0xffffffff, 0x00000100,
615 0x3f80, 0xffffffff, 0x00000100,
616 0xa210, 0xffffffff, 0x00000100,
617 0xa214, 0xffffffff, 0x00000100,
618 0x4d8, 0xffffffff, 0x00000100,
619 0x9784, 0xffffffff, 0x00000100,
620 0x9698, 0xffffffff, 0x00000100,
621 0x4d4, 0xffffffff, 0x00000200,
622 0x30cc, 0xffffffff, 0x00000100,
623 0x802c, 0xffffffff, 0xc0000000
624};
625
626static const u32 supersumo_golden_registers[] =
627{
628 0x5eb4, 0xffffffff, 0x00000002,
629 0x5cc, 0xffffffff, 0x00000001,
630 0x7030, 0xffffffff, 0x00000011,
631 0x7c30, 0xffffffff, 0x00000011,
632 0x6104, 0x01000300, 0x00000000,
633 0x5bc0, 0x00300000, 0x00000000,
634 0x8c04, 0xffffffff, 0x40600060,
635 0x8c08, 0xffffffff, 0x001c001c,
636 0x8c20, 0xffffffff, 0x00800080,
637 0x8c24, 0xffffffff, 0x00800080,
638 0x8c18, 0xffffffff, 0x20202078,
639 0x8c1c, 0xffffffff, 0x00001010,
640 0x918c, 0xffffffff, 0x00010006,
641 0x91a8, 0xffffffff, 0x00010006,
642 0x91c4, 0xffffffff, 0x00010006,
643 0x91e0, 0xffffffff, 0x00010006,
644 0x9200, 0xffffffff, 0x00010006,
645 0x9150, 0xffffffff, 0x6e944040,
646 0x917c, 0xffffffff, 0x00030002,
647 0x9180, 0xffffffff, 0x00050004,
648 0x9198, 0xffffffff, 0x00030002,
649 0x919c, 0xffffffff, 0x00050004,
650 0x91b4, 0xffffffff, 0x00030002,
651 0x91b8, 0xffffffff, 0x00050004,
652 0x91d0, 0xffffffff, 0x00030002,
653 0x91d4, 0xffffffff, 0x00050004,
654 0x91f0, 0xffffffff, 0x00030002,
655 0x91f4, 0xffffffff, 0x00050004,
656 0x915c, 0xffffffff, 0x00010000,
657 0x9160, 0xffffffff, 0x00030002,
658 0x3f90, 0xffff0000, 0xff000000,
659 0x9178, 0xffffffff, 0x00070000,
660 0x9194, 0xffffffff, 0x00070000,
661 0x91b0, 0xffffffff, 0x00070000,
662 0x91cc, 0xffffffff, 0x00070000,
663 0x91ec, 0xffffffff, 0x00070000,
664 0x9148, 0xffff0000, 0xff000000,
665 0x9190, 0xffffffff, 0x00090008,
666 0x91ac, 0xffffffff, 0x00090008,
667 0x91c8, 0xffffffff, 0x00090008,
668 0x91e4, 0xffffffff, 0x00090008,
669 0x9204, 0xffffffff, 0x00090008,
670 0x3f94, 0xffff0000, 0xff000000,
671 0x914c, 0xffff0000, 0xff000000,
672 0x929c, 0xffffffff, 0x00000001,
673 0x8a18, 0xffffffff, 0x00000100,
674 0x8b28, 0xffffffff, 0x00000100,
675 0x9144, 0xffffffff, 0x00000100,
676 0x5644, 0xffffffff, 0x00000100,
677 0x9b7c, 0xffffffff, 0x00000000,
678 0x8030, 0xffffffff, 0x0000100a,
679 0x8a14, 0xffffffff, 0x00000007,
680 0x8b24, 0xffffffff, 0x00ff0fff,
681 0x8b10, 0xffffffff, 0x00000000,
682 0x28a4c, 0x06000000, 0x06000000,
683 0x4d8, 0xffffffff, 0x00000100,
684 0x913c, 0xffff000f, 0x0100000a,
685 0x960c, 0xffffffff, 0x54763210,
686 0x88c4, 0xffffffff, 0x000000c2,
687 0x88d4, 0xffffffff, 0x00000010,
688 0x8974, 0xffffffff, 0x00000000,
689 0xc78, 0x00000080, 0x00000080,
690 0x5e78, 0xffffffff, 0x001000f0,
691 0xd02c, 0xffffffff, 0x08421000,
692 0xa008, 0xffffffff, 0x00010000,
693 0x8d00, 0xffffffff, 0x100e4848,
694 0x8d04, 0xffffffff, 0x00164745,
695 0x8c00, 0xffffffff, 0xe4000003,
696 0x8cf0, 0x1fffffff, 0x08e00620,
697 0x28350, 0xffffffff, 0x00000000,
698 0x9508, 0xffffffff, 0x00000002
699};
700
701static const u32 sumo_golden_registers[] =
702{
703 0x900c, 0x00ffffff, 0x0017071f,
704 0x8c18, 0xffffffff, 0x10101060,
705 0x8c1c, 0xffffffff, 0x00001010,
706 0x8c30, 0x0000000f, 0x00000005,
707 0x9688, 0x0000000f, 0x00000007
708};
709
710static const u32 wrestler_golden_registers[] =
711{
712 0x5eb4, 0xffffffff, 0x00000002,
713 0x5cc, 0xffffffff, 0x00000001,
714 0x7030, 0xffffffff, 0x00000011,
715 0x7c30, 0xffffffff, 0x00000011,
716 0x6104, 0x01000300, 0x00000000,
717 0x5bc0, 0x00300000, 0x00000000,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x9150, 0xffffffff, 0x6e944040,
721 0x917c, 0xffffffff, 0x00030002,
722 0x9198, 0xffffffff, 0x00030002,
723 0x915c, 0xffffffff, 0x00010000,
724 0x3f90, 0xffff0000, 0xff000000,
725 0x9178, 0xffffffff, 0x00070000,
726 0x9194, 0xffffffff, 0x00070000,
727 0x9148, 0xffff0000, 0xff000000,
728 0x9190, 0xffffffff, 0x00090008,
729 0x91ac, 0xffffffff, 0x00090008,
730 0x3f94, 0xffff0000, 0xff000000,
731 0x914c, 0xffff0000, 0xff000000,
732 0x929c, 0xffffffff, 0x00000001,
733 0x8a18, 0xffffffff, 0x00000100,
734 0x8b28, 0xffffffff, 0x00000100,
735 0x9144, 0xffffffff, 0x00000100,
736 0x9b7c, 0xffffffff, 0x00000000,
737 0x8030, 0xffffffff, 0x0000100a,
738 0x8a14, 0xffffffff, 0x00000001,
739 0x8b24, 0xffffffff, 0x00ff0fff,
740 0x8b10, 0xffffffff, 0x00000000,
741 0x28a4c, 0x06000000, 0x06000000,
742 0x4d8, 0xffffffff, 0x00000100,
743 0x913c, 0xffff000f, 0x0100000a,
744 0x960c, 0xffffffff, 0x54763210,
745 0x88c4, 0xffffffff, 0x000000c2,
746 0x88d4, 0xffffffff, 0x00000010,
747 0x8974, 0xffffffff, 0x00000000,
748 0xc78, 0x00000080, 0x00000080,
749 0x5e78, 0xffffffff, 0x001000f0,
750 0xd02c, 0xffffffff, 0x08421000,
751 0xa008, 0xffffffff, 0x00010000,
752 0x8d00, 0xffffffff, 0x100e4848,
753 0x8d04, 0xffffffff, 0x00164745,
754 0x8c00, 0xffffffff, 0xe4000003,
755 0x8cf0, 0x1fffffff, 0x08e00410,
756 0x28350, 0xffffffff, 0x00000000,
757 0x9508, 0xffffffff, 0x00000002,
758 0x900c, 0xffffffff, 0x0017071f,
759 0x8c18, 0xffffffff, 0x10101060,
760 0x8c1c, 0xffffffff, 0x00001010
761};
762
763static const u32 barts_golden_registers[] =
764{
765 0x5eb4, 0xffffffff, 0x00000002,
766 0x5e78, 0x8f311ff1, 0x001000f0,
767 0x3f90, 0xffff0000, 0xff000000,
768 0x9148, 0xffff0000, 0xff000000,
769 0x3f94, 0xffff0000, 0xff000000,
770 0x914c, 0xffff0000, 0xff000000,
771 0xc78, 0x00000080, 0x00000080,
772 0xbd4, 0x70073777, 0x00010001,
773 0xd02c, 0xbfffff1f, 0x08421000,
774 0xd0b8, 0x03773777, 0x02011003,
775 0x5bc0, 0x00200000, 0x50100000,
776 0x98f8, 0x33773777, 0x02011003,
777 0x98fc, 0xffffffff, 0x76543210,
778 0x7030, 0x31000311, 0x00000011,
779 0x2f48, 0x00000007, 0x02011003,
780 0x6b28, 0x00000010, 0x00000012,
781 0x7728, 0x00000010, 0x00000012,
782 0x10328, 0x00000010, 0x00000012,
783 0x10f28, 0x00000010, 0x00000012,
784 0x11b28, 0x00000010, 0x00000012,
785 0x12728, 0x00000010, 0x00000012,
786 0x240c, 0x000007ff, 0x00000380,
787 0x8a14, 0xf000001f, 0x00000007,
788 0x8b24, 0x3fff3fff, 0x00ff0fff,
789 0x8b10, 0x0000ff0f, 0x00000000,
790 0x28a4c, 0x07ffffff, 0x06000000,
791 0x10c, 0x00000001, 0x00010003,
792 0xa02c, 0xffffffff, 0x0000009b,
793 0x913c, 0x0000000f, 0x0100000a,
794 0x8d00, 0xffff7f7f, 0x100e4848,
795 0x8d04, 0x00ffffff, 0x00164745,
796 0x8c00, 0xfffc0003, 0xe4000003,
797 0x8c04, 0xf8ff00ff, 0x40600060,
798 0x8c08, 0x00ff00ff, 0x001c001c,
799 0x8cf0, 0x1fff1fff, 0x08e00620,
800 0x8c20, 0x0fff0fff, 0x00800080,
801 0x8c24, 0x0fff0fff, 0x00800080,
802 0x8c18, 0xffffffff, 0x20202078,
803 0x8c1c, 0x0000ffff, 0x00001010,
804 0x28350, 0x00000f01, 0x00000000,
805 0x9508, 0x3700001f, 0x00000002,
806 0x960c, 0xffffffff, 0x54763210,
807 0x88c4, 0x001f3ae3, 0x000000c2,
808 0x88d4, 0x0000001f, 0x00000010,
809 0x8974, 0xffffffff, 0x00000000
810};
811
812static const u32 turks_golden_registers[] =
813{
814 0x5eb4, 0xffffffff, 0x00000002,
815 0x5e78, 0x8f311ff1, 0x001000f0,
816 0x8c8, 0x00003000, 0x00001070,
817 0x8cc, 0x000fffff, 0x00040035,
818 0x3f90, 0xffff0000, 0xfff00000,
819 0x9148, 0xffff0000, 0xfff00000,
820 0x3f94, 0xffff0000, 0xfff00000,
821 0x914c, 0xffff0000, 0xfff00000,
822 0xc78, 0x00000080, 0x00000080,
823 0xbd4, 0x00073007, 0x00010002,
824 0xd02c, 0xbfffff1f, 0x08421000,
825 0xd0b8, 0x03773777, 0x02010002,
826 0x5bc0, 0x00200000, 0x50100000,
827 0x98f8, 0x33773777, 0x00010002,
828 0x98fc, 0xffffffff, 0x33221100,
829 0x7030, 0x31000311, 0x00000011,
830 0x2f48, 0x33773777, 0x00010002,
831 0x6b28, 0x00000010, 0x00000012,
832 0x7728, 0x00000010, 0x00000012,
833 0x10328, 0x00000010, 0x00000012,
834 0x10f28, 0x00000010, 0x00000012,
835 0x11b28, 0x00000010, 0x00000012,
836 0x12728, 0x00000010, 0x00000012,
837 0x240c, 0x000007ff, 0x00000380,
838 0x8a14, 0xf000001f, 0x00000007,
839 0x8b24, 0x3fff3fff, 0x00ff0fff,
840 0x8b10, 0x0000ff0f, 0x00000000,
841 0x28a4c, 0x07ffffff, 0x06000000,
842 0x10c, 0x00000001, 0x00010003,
843 0xa02c, 0xffffffff, 0x0000009b,
844 0x913c, 0x0000000f, 0x0100000a,
845 0x8d00, 0xffff7f7f, 0x100e4848,
846 0x8d04, 0x00ffffff, 0x00164745,
847 0x8c00, 0xfffc0003, 0xe4000003,
848 0x8c04, 0xf8ff00ff, 0x40600060,
849 0x8c08, 0x00ff00ff, 0x001c001c,
850 0x8cf0, 0x1fff1fff, 0x08e00410,
851 0x8c20, 0x0fff0fff, 0x00800080,
852 0x8c24, 0x0fff0fff, 0x00800080,
853 0x8c18, 0xffffffff, 0x20202078,
854 0x8c1c, 0x0000ffff, 0x00001010,
855 0x28350, 0x00000f01, 0x00000000,
856 0x9508, 0x3700001f, 0x00000002,
857 0x960c, 0xffffffff, 0x54763210,
858 0x88c4, 0x001f3ae3, 0x000000c2,
859 0x88d4, 0x0000001f, 0x00000010,
860 0x8974, 0xffffffff, 0x00000000
861};
862
863static const u32 caicos_golden_registers[] =
864{
865 0x5eb4, 0xffffffff, 0x00000002,
866 0x5e78, 0x8f311ff1, 0x001000f0,
867 0x8c8, 0x00003420, 0x00001450,
868 0x8cc, 0x000fffff, 0x00040035,
869 0x3f90, 0xffff0000, 0xfffc0000,
870 0x9148, 0xffff0000, 0xfffc0000,
871 0x3f94, 0xffff0000, 0xfffc0000,
872 0x914c, 0xffff0000, 0xfffc0000,
873 0xc78, 0x00000080, 0x00000080,
874 0xbd4, 0x00073007, 0x00010001,
875 0xd02c, 0xbfffff1f, 0x08421000,
876 0xd0b8, 0x03773777, 0x02010001,
877 0x5bc0, 0x00200000, 0x50100000,
878 0x98f8, 0x33773777, 0x02010001,
879 0x98fc, 0xffffffff, 0x33221100,
880 0x7030, 0x31000311, 0x00000011,
881 0x2f48, 0x33773777, 0x02010001,
882 0x6b28, 0x00000010, 0x00000012,
883 0x7728, 0x00000010, 0x00000012,
884 0x10328, 0x00000010, 0x00000012,
885 0x10f28, 0x00000010, 0x00000012,
886 0x11b28, 0x00000010, 0x00000012,
887 0x12728, 0x00000010, 0x00000012,
888 0x240c, 0x000007ff, 0x00000380,
889 0x8a14, 0xf000001f, 0x00000001,
890 0x8b24, 0x3fff3fff, 0x00ff0fff,
891 0x8b10, 0x0000ff0f, 0x00000000,
892 0x28a4c, 0x07ffffff, 0x06000000,
893 0x10c, 0x00000001, 0x00010003,
894 0xa02c, 0xffffffff, 0x0000009b,
895 0x913c, 0x0000000f, 0x0100000a,
896 0x8d00, 0xffff7f7f, 0x100e4848,
897 0x8d04, 0x00ffffff, 0x00164745,
898 0x8c00, 0xfffc0003, 0xe4000003,
899 0x8c04, 0xf8ff00ff, 0x40600060,
900 0x8c08, 0x00ff00ff, 0x001c001c,
901 0x8cf0, 0x1fff1fff, 0x08e00410,
902 0x8c20, 0x0fff0fff, 0x00800080,
903 0x8c24, 0x0fff0fff, 0x00800080,
904 0x8c18, 0xffffffff, 0x20202078,
905 0x8c1c, 0x0000ffff, 0x00001010,
906 0x28350, 0x00000f01, 0x00000000,
907 0x9508, 0x3700001f, 0x00000002,
908 0x960c, 0xffffffff, 0x54763210,
909 0x88c4, 0x001f3ae3, 0x000000c2,
910 0x88d4, 0x0000001f, 0x00000010,
911 0x8974, 0xffffffff, 0x00000000
912};
913
914static void evergreen_init_golden_registers(struct radeon_device *rdev)
915{
916 switch (rdev->family) {
917 case CHIP_CYPRESS:
918 case CHIP_HEMLOCK:
919 radeon_program_register_sequence(rdev,
920 evergreen_golden_registers,
921 (const u32)ARRAY_SIZE(evergreen_golden_registers));
922 radeon_program_register_sequence(rdev,
923 evergreen_golden_registers2,
924 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
925 radeon_program_register_sequence(rdev,
926 cypress_mgcg_init,
927 (const u32)ARRAY_SIZE(cypress_mgcg_init));
928 break;
929 case CHIP_JUNIPER:
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers));
933 radeon_program_register_sequence(rdev,
934 evergreen_golden_registers2,
935 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
936 radeon_program_register_sequence(rdev,
937 juniper_mgcg_init,
938 (const u32)ARRAY_SIZE(juniper_mgcg_init));
939 break;
940 case CHIP_REDWOOD:
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers));
944 radeon_program_register_sequence(rdev,
945 evergreen_golden_registers2,
946 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
947 radeon_program_register_sequence(rdev,
948 redwood_mgcg_init,
949 (const u32)ARRAY_SIZE(redwood_mgcg_init));
950 break;
951 case CHIP_CEDAR:
952 radeon_program_register_sequence(rdev,
953 cedar_golden_registers,
954 (const u32)ARRAY_SIZE(cedar_golden_registers));
955 radeon_program_register_sequence(rdev,
956 evergreen_golden_registers2,
957 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
958 radeon_program_register_sequence(rdev,
959 cedar_mgcg_init,
960 (const u32)ARRAY_SIZE(cedar_mgcg_init));
961 break;
962 case CHIP_PALM:
963 radeon_program_register_sequence(rdev,
964 wrestler_golden_registers,
965 (const u32)ARRAY_SIZE(wrestler_golden_registers));
966 break;
967 case CHIP_SUMO:
968 radeon_program_register_sequence(rdev,
969 supersumo_golden_registers,
970 (const u32)ARRAY_SIZE(supersumo_golden_registers));
971 break;
972 case CHIP_SUMO2:
973 radeon_program_register_sequence(rdev,
974 supersumo_golden_registers,
975 (const u32)ARRAY_SIZE(supersumo_golden_registers));
976 radeon_program_register_sequence(rdev,
977 sumo_golden_registers,
978 (const u32)ARRAY_SIZE(sumo_golden_registers));
979 break;
980 case CHIP_BARTS:
981 radeon_program_register_sequence(rdev,
982 barts_golden_registers,
983 (const u32)ARRAY_SIZE(barts_golden_registers));
984 break;
985 case CHIP_TURKS:
986 radeon_program_register_sequence(rdev,
987 turks_golden_registers,
988 (const u32)ARRAY_SIZE(turks_golden_registers));
989 break;
990 case CHIP_CAICOS:
991 radeon_program_register_sequence(rdev,
992 caicos_golden_registers,
993 (const u32)ARRAY_SIZE(caicos_golden_registers));
994 break;
995 default:
996 break;
997 }
998}
999
285484e2
JG		 1000void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1001 unsigned *bankh, unsigned *mtaspect,
1002 unsigned *tile_split)
1003{
1004 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1005 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1006 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1007 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1008 switch (*bankw) {
1009 default:
1010 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1011 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1012 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1013 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1014 }
1015 switch (*bankh) {
1016 default:
1017 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1018 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1019 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1020 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1021 }
1022 switch (*mtaspect) {
1023 default:
1024 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1025 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1026 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1027 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1028 }
1029}
1030
23d33ba3
AD		 1031static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1032 u32 cntl_reg, u32 status_reg)
1033{
1034 int r, i;
1035 struct atom_clock_dividers dividers;
1036
1037 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1038 clock, false, &dividers);
1039 if (r)
1040 return r;
1041
1042 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1043
1044 for (i = 0; i < 100; i++) {
1045 if (RREG32(status_reg) & DCLK_STATUS)
1046 break;
1047 mdelay(10);
1048 }
1049 if (i == 100)
1050 return -ETIMEDOUT;
1051
1052 return 0;
1053}
1054
1055int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1056{
1057 int r = 0;
1058 u32 cg_scratch = RREG32(CG_SCRATCH1);
1059
1060 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1061 if (r)
1062 goto done;
1063 cg_scratch &= 0xffff0000;
1064 cg_scratch |= vclk / 100; /* Mhz */
1065
1066 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1067 if (r)
1068 goto done;
1069 cg_scratch &= 0x0000ffff;
1070 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1071
1072done:
1073 WREG32(CG_SCRATCH1, cg_scratch);
1074
1075 return r;
1076}
1077
a8b4925c
AD		 1078int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1079{
1080 /* start off with something large */
facd112d 1081 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1082 int r;
1083
4ed10835
CK		 1084 /* bypass vclk and dclk with bclk */
1085 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1086 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1087 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1088
1089 /* put PLL in bypass mode */
1090 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1091
1092 if (!vclk || !dclk) {
1093 /* keep the Bypass mode, put PLL to sleep */
1094 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1095 return 0;
1096 }
1097
facd112d
CK		 1098 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1099 16384, 0x03FFFFFF, 0, 128, 5,
1100 &fb_div, &vclk_div, &dclk_div);
1101 if (r)
1102 return r;
a8b4925c
AD		 1103
1104 /* set VCO_MODE to 1 */
1105 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1106
1107 /* toggle UPLL_SLEEP to 1 then back to 0 */
1108 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1109 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1110
1111 /* deassert UPLL_RESET */
1112 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1113
1114 mdelay(1);
1115
facd112d 1116 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1117 if (r)
1118 return r;
1119
1120 /* assert UPLL_RESET again */
1121 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1122
1123 /* disable spread spectrum. */
1124 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1125
1126 /* set feedback divider */
facd112d 1127 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1128
1129 /* set ref divider to 0 */
1130 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1131
facd112d 1132 if (fb_div < 307200)
a8b4925c
AD		 1133 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1134 else
1135 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1136
1137 /* set PDIV_A and PDIV_B */
1138 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1139 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1140 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1141
1142 /* give the PLL some time to settle */
1143 mdelay(15);
1144
1145 /* deassert PLL_RESET */
1146 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1147
1148 mdelay(15);
1149
1150 /* switch from bypass mode to normal mode */
1151 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1152
facd112d 1153 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1154 if (r)
1155 return r;
1156
1157 /* switch VCLK and DCLK selection */
1158 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1159 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1160 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1161
1162 mdelay(100);
1163
1164 return 0;
1165}
1166
d054ac16
AD		 1167void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1168{
1169 u16 ctl, v;
32195aec 1170 int err;
d054ac16 1171
32195aec 1172 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 1173 if (err)
1174 return;
1175
1176 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1177
1178 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1179 * to avoid hangs or perfomance issues
1180 */
1181 if ((v == 0) || (v == 6) || (v == 7)) {
1182 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1183 ctl |= (2 << 12);
32195aec 1184 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 1185 }
1186}
1187
10257a6d
AD		 1188static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1189{
1190 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1191 return true;
1192 else
1193 return false;
1194}
1195
1196static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1197{
1198 u32 pos1, pos2;
1199
1200 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1201 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1202
1203 if (pos1 != pos2)
1204 return true;
1205 else
1206 return false;
1207}
1208
377edc8b
AD		 1209/**
1210 * dce4_wait_for_vblank - vblank wait asic callback.
1211 *
1212 * @rdev: radeon_device pointer
1213 * @crtc: crtc to wait for vblank on
1214 *
1215 * Wait for vblank on the requested crtc (evergreen+).
1216 */
3ae19b75
AD		 1217void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1218{
10257a6d 1219 unsigned i = 0;
3ae19b75 1220
4a15903d
AD		 1221 if (crtc >= rdev->num_crtc)
1222 return;
1223
10257a6d
AD		 1224 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1225 return;
1226
1227 /* depending on when we hit vblank, we may be close to active; if so,
1228 * wait for another frame.
1229 */
1230 while (dce4_is_in_vblank(rdev, crtc)) {
1231 if (i++ % 100 == 0) {
1232 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1233 break;
3ae19b75 1234 }
10257a6d
AD		 1235 }
1236
1237 while (!dce4_is_in_vblank(rdev, crtc)) {
1238 if (i++ % 100 == 0) {
1239 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1240 break;
3ae19b75
AD		 1241 }
1242 }
1243}
1244
377edc8b
AD		 1245/**
1246 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1247 *
1248 * @rdev: radeon_device pointer
1249 * @crtc: crtc to prepare for pageflip on
1250 *
1251 * Pre-pageflip callback (evergreen+).
1252 * Enables the pageflip irq (vblank irq).
1253 */
6f34be50
AD		 1254void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1255{
6f34be50
AD		 1256 /* enable the pflip int */
1257 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1258}
1259
377edc8b
AD		 1260/**
1261 * evergreen_post_page_flip - pos-pageflip callback.
1262 *
1263 * @rdev: radeon_device pointer
1264 * @crtc: crtc to cleanup pageflip on
1265 *
1266 * Post-pageflip callback (evergreen+).
1267 * Disables the pageflip irq (vblank irq).
1268 */
6f34be50
AD		 1269void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1270{
1271 /* disable the pflip int */
1272 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1273}
1274
377edc8b
AD		 1275/**
1276 * evergreen_page_flip - pageflip callback.
1277 *
1278 * @rdev: radeon_device pointer
1279 * @crtc_id: crtc to cleanup pageflip on
1280 * @crtc_base: new address of the crtc (GPU MC address)
1281 *
1282 * Does the actual pageflip (evergreen+).
1283 * During vblank we take the crtc lock and wait for the update_pending
1284 * bit to go high, when it does, we release the lock, and allow the
1285 * double buffered update to take place.
1286 * Returns the current update pending status.
1287 */
6f34be50
AD		 1288u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1289{
1290 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1291 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1292 int i;
6f34be50
AD		 1293
1294 /* Lock the graphics update lock */
1295 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1296 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1297
1298 /* update the scanout addresses */
1299 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1300 upper_32_bits(crtc_base));
1301 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1302 (u32)crtc_base);
1303
1304 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1305 upper_32_bits(crtc_base));
1306 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1307 (u32)crtc_base);
1308
1309 /* Wait for update_pending to go high. */
f6496479
AD		 1310 for (i = 0; i < rdev->usec_timeout; i++) {
1311 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1312 break;
1313 udelay(1);
1314 }
6f34be50
AD		 1315 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1316
1317 /* Unlock the lock, so double-buffering can take place inside vblank */
1318 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1319 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1320
1321 /* Return current update_pending status: */
1322 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1323}
1324
21a8122a 1325/* get temperature in millidegrees */
20d391d7 1326int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1327{
1c88d74f
AD		 1328 u32 temp, toffset;
1329 int actual_temp = 0;
67b3f823
AD		 1330
1331 if (rdev->family == CHIP_JUNIPER) {
1332 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1333 TOFFSET_SHIFT;
1334 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1335 TS0_ADC_DOUT_SHIFT;
1336
1337 if (toffset & 0x100)
1338 actual_temp = temp / 2 - (0x200 - toffset);
1339 else
1340 actual_temp = temp / 2 + toffset;
1341
1342 actual_temp = actual_temp * 1000;
1343
1344 } else {
1345 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1346 ASIC_T_SHIFT;
1347
1348 if (temp & 0x400)
1349 actual_temp = -256;
1350 else if (temp & 0x200)
1351 actual_temp = 255;
1352 else if (temp & 0x100) {
1353 actual_temp = temp & 0x1ff;
1354 actual_temp |= ~0x1ff;
1355 } else
1356 actual_temp = temp & 0xff;
1357
1358 actual_temp = (actual_temp * 1000) / 2;
1359 }
21a8122a 1360
67b3f823 1361 return actual_temp;
21a8122a
AD		 1362}
1363
20d391d7 1364int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1365{
1366 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1367 int actual_temp = temp - 49;
e33df25f
AD		 1368
1369 return actual_temp * 1000;
1370}
1371
377edc8b
AD		 1372/**
1373 * sumo_pm_init_profile - Initialize power profiles callback.
1374 *
1375 * @rdev: radeon_device pointer
1376 *
1377 * Initialize the power states used in profile mode
1378 * (sumo, trinity, SI).
1379 * Used for profile mode only.
1380 */
a4c9e2ee
AD		 1381void sumo_pm_init_profile(struct radeon_device *rdev)
1382{
1383 int idx;
1384
1385 /* default */
1386 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1387 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1390
1391 /* low,mid sh/mh */
1392 if (rdev->flags & RADEON_IS_MOBILITY)
1393 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1394 else
1395 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1396
1397 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1398 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1399 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1401
1402 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1403 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1404 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1406
1407 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1408 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1409 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1411
1412 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1413 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1414 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1416
1417 /* high sh/mh */
1418 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1419 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1420 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1421 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1423 rdev->pm.power_state[idx].num_clock_modes - 1;
1424
1425 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1426 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1427 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1429 rdev->pm.power_state[idx].num_clock_modes - 1;
1430}
1431
27810fb2
AD		 1432/**
1433 * btc_pm_init_profile - Initialize power profiles callback.
1434 *
1435 * @rdev: radeon_device pointer
1436 *
1437 * Initialize the power states used in profile mode
1438 * (BTC, cayman).
1439 * Used for profile mode only.
1440 */
1441void btc_pm_init_profile(struct radeon_device *rdev)
1442{
1443 int idx;
1444
1445 /* default */
1446 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1450 /* starting with BTC, there is one state that is used for both
1451 * MH and SH. Difference is that we always use the high clock index for
1452 * mclk.
1453 */
1454 if (rdev->flags & RADEON_IS_MOBILITY)
1455 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1456 else
1457 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1458 /* low sh */
1459 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1460 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1463 /* mid sh */
1464 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1468 /* high sh */
1469 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1470 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1473 /* low mh */
1474 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1475 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1476 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1478 /* mid mh */
1479 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1480 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1483 /* high mh */
1484 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1488}
1489
377edc8b
AD		 1490/**
1491 * evergreen_pm_misc - set additional pm hw parameters callback.
1492 *
1493 * @rdev: radeon_device pointer
1494 *
1495 * Set non-clock parameters associated with a power state
1496 * (voltage, etc.) (evergreen+).
1497 */
49e02b73
AD		 1498void evergreen_pm_misc(struct radeon_device *rdev)
1499{
a081a9d6
RM		 1500 int req_ps_idx = rdev->pm.requested_power_state_index;
1501 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1502 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1503 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1504
2feea49a 1505 if (voltage->type == VOLTAGE_SW) {
a377e187
AD		 1506 /* 0xff01 is a flag rather then an actual voltage */
1507 if (voltage->voltage == 0xff01)
1508 return;
2feea49a 1509 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1510 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1511 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1512 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1513 }
7ae764b1
AD		 1514
1515 /* starting with BTC, there is one state that is used for both
1516 * MH and SH. Difference is that we always use the high clock index for
1517 * mclk and vddci.
1518 */
1519 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1520 (rdev->family >= CHIP_BARTS) &&
1521 rdev->pm.active_crtc_count &&
1522 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1523 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1524 voltage = &rdev->pm.power_state[req_ps_idx].
1525 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1526
a377e187
AD		 1527 /* 0xff01 is a flag rather then an actual voltage */
1528 if (voltage->vddci == 0xff01)
1529 return;
2feea49a
AD		 1530 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1531 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1532 rdev->pm.current_vddci = voltage->vddci;
1533 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1534 }
1535 }
49e02b73
AD		 1536}
1537
377edc8b
AD		 1538/**
1539 * evergreen_pm_prepare - pre-power state change callback.
1540 *
1541 * @rdev: radeon_device pointer
1542 *
1543 * Prepare for a power state change (evergreen+).
1544 */
49e02b73
AD		 1545void evergreen_pm_prepare(struct radeon_device *rdev)
1546{
1547 struct drm_device *ddev = rdev->ddev;
1548 struct drm_crtc *crtc;
1549 struct radeon_crtc *radeon_crtc;
1550 u32 tmp;
1551
1552 /* disable any active CRTCs */
1553 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1554 radeon_crtc = to_radeon_crtc(crtc);
1555 if (radeon_crtc->enabled) {
1556 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1557 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1558 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1559 }
1560 }
1561}
1562
377edc8b
AD		 1563/**
1564 * evergreen_pm_finish - post-power state change callback.
1565 *
1566 * @rdev: radeon_device pointer
1567 *
1568 * Clean up after a power state change (evergreen+).
1569 */
49e02b73
AD		 1570void evergreen_pm_finish(struct radeon_device *rdev)
1571{
1572 struct drm_device *ddev = rdev->ddev;
1573 struct drm_crtc *crtc;
1574 struct radeon_crtc *radeon_crtc;
1575 u32 tmp;
1576
1577 /* enable any active CRTCs */
1578 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1579 radeon_crtc = to_radeon_crtc(crtc);
1580 if (radeon_crtc->enabled) {
1581 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1582 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1583 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1584 }
1585 }
1586}
1587
377edc8b
AD		 1588/**
1589 * evergreen_hpd_sense - hpd sense callback.
1590 *
1591 * @rdev: radeon_device pointer
1592 * @hpd: hpd (hotplug detect) pin
1593 *
1594 * Checks if a digital monitor is connected (evergreen+).
1595 * Returns true if connected, false if not connected.
1596 */
bcc1c2a1
AD		 1597bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1598{
1599 bool connected = false;
0ca2ab52
AD		 1600
1601 switch (hpd) {
1602 case RADEON_HPD_1:
1603 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1604 connected = true;
1605 break;
1606 case RADEON_HPD_2:
1607 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1608 connected = true;
1609 break;
1610 case RADEON_HPD_3:
1611 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1612 connected = true;
1613 break;
1614 case RADEON_HPD_4:
1615 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1616 connected = true;
1617 break;
1618 case RADEON_HPD_5:
1619 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1620 connected = true;
1621 break;
1622 case RADEON_HPD_6:
1623 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1624 connected = true;
1625 break;
1626 default:
1627 break;
1628 }
1629
bcc1c2a1
AD		 1630 return connected;
1631}
1632
377edc8b
AD		 1633/**
1634 * evergreen_hpd_set_polarity - hpd set polarity callback.
1635 *
1636 * @rdev: radeon_device pointer
1637 * @hpd: hpd (hotplug detect) pin
1638 *
1639 * Set the polarity of the hpd pin (evergreen+).
1640 */
bcc1c2a1
AD		 1641void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1642 enum radeon_hpd_id hpd)
1643{
0ca2ab52
AD		 1644 u32 tmp;
1645 bool connected = evergreen_hpd_sense(rdev, hpd);
1646
1647 switch (hpd) {
1648 case RADEON_HPD_1:
1649 tmp = RREG32(DC_HPD1_INT_CONTROL);
1650 if (connected)
1651 tmp &= ~DC_HPDx_INT_POLARITY;
1652 else
1653 tmp |= DC_HPDx_INT_POLARITY;
1654 WREG32(DC_HPD1_INT_CONTROL, tmp);
1655 break;
1656 case RADEON_HPD_2:
1657 tmp = RREG32(DC_HPD2_INT_CONTROL);
1658 if (connected)
1659 tmp &= ~DC_HPDx_INT_POLARITY;
1660 else
1661 tmp |= DC_HPDx_INT_POLARITY;
1662 WREG32(DC_HPD2_INT_CONTROL, tmp);
1663 break;
1664 case RADEON_HPD_3:
1665 tmp = RREG32(DC_HPD3_INT_CONTROL);
1666 if (connected)
1667 tmp &= ~DC_HPDx_INT_POLARITY;
1668 else
1669 tmp |= DC_HPDx_INT_POLARITY;
1670 WREG32(DC_HPD3_INT_CONTROL, tmp);
1671 break;
1672 case RADEON_HPD_4:
1673 tmp = RREG32(DC_HPD4_INT_CONTROL);
1674 if (connected)
1675 tmp &= ~DC_HPDx_INT_POLARITY;
1676 else
1677 tmp |= DC_HPDx_INT_POLARITY;
1678 WREG32(DC_HPD4_INT_CONTROL, tmp);
1679 break;
1680 case RADEON_HPD_5:
1681 tmp = RREG32(DC_HPD5_INT_CONTROL);
1682 if (connected)
1683 tmp &= ~DC_HPDx_INT_POLARITY;
1684 else
1685 tmp |= DC_HPDx_INT_POLARITY;
1686 WREG32(DC_HPD5_INT_CONTROL, tmp);
1687 break;
1688 case RADEON_HPD_6:
1689 tmp = RREG32(DC_HPD6_INT_CONTROL);
1690 if (connected)
1691 tmp &= ~DC_HPDx_INT_POLARITY;
1692 else
1693 tmp |= DC_HPDx_INT_POLARITY;
1694 WREG32(DC_HPD6_INT_CONTROL, tmp);
1695 break;
1696 default:
1697 break;
1698 }
bcc1c2a1
AD		 1699}
1700
377edc8b
AD		 1701/**
1702 * evergreen_hpd_init - hpd setup callback.
1703 *
1704 * @rdev: radeon_device pointer
1705 *
1706 * Setup the hpd pins used by the card (evergreen+).
1707 * Enable the pin, set the polarity, and enable the hpd interrupts.
1708 */
bcc1c2a1
AD		 1709void evergreen_hpd_init(struct radeon_device *rdev)
1710{
0ca2ab52
AD		 1711 struct drm_device *dev = rdev->ddev;
1712 struct drm_connector *connector;
fb98257a 1713 unsigned enabled = 0;
0ca2ab52
AD		 1714 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1715 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1716
0ca2ab52
AD		 1717 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1718 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1719
1720 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1721 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1722 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1723 * aux dp channel on imac and help (but not completely fix)
1724 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1725 * also avoid interrupt storms during dpms.
1726 */
1727 continue;
1728 }
0ca2ab52
AD		 1729 switch (radeon_connector->hpd.hpd) {
1730 case RADEON_HPD_1:
1731 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1732 break;
1733 case RADEON_HPD_2:
1734 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1735 break;
1736 case RADEON_HPD_3:
1737 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1738 break;
1739 case RADEON_HPD_4:
1740 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1741 break;
1742 case RADEON_HPD_5:
1743 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1744 break;
1745 case RADEON_HPD_6:
1746 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1747 break;
1748 default:
1749 break;
1750 }
64912e99 1751 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1752 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1753 }
fb98257a 1754 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1755}
1756
377edc8b
AD		 1757/**
1758 * evergreen_hpd_fini - hpd tear down callback.
1759 *
1760 * @rdev: radeon_device pointer
1761 *
1762 * Tear down the hpd pins used by the card (evergreen+).
1763 * Disable the hpd interrupts.
1764 */
0ca2ab52 1765void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1766{
0ca2ab52
AD		 1767 struct drm_device *dev = rdev->ddev;
1768 struct drm_connector *connector;
fb98257a 1769 unsigned disabled = 0;
0ca2ab52
AD		 1770
1771 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1772 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1773 switch (radeon_connector->hpd.hpd) {
1774 case RADEON_HPD_1:
1775 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1776 break;
1777 case RADEON_HPD_2:
1778 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1779 break;
1780 case RADEON_HPD_3:
1781 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1782 break;
1783 case RADEON_HPD_4:
1784 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1785 break;
1786 case RADEON_HPD_5:
1787 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1788 break;
1789 case RADEON_HPD_6:
1790 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1791 break;
1792 default:
1793 break;
1794 }
fb98257a 1795 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1796 }
fb98257a 1797 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1798}
1799
f9d9c362
AD		 1800/* watermark setup */
1801
1802static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1803 struct radeon_crtc *radeon_crtc,
1804 struct drm_display_mode *mode,
1805 struct drm_display_mode *other_mode)
1806{
12dfc843 1807 u32 tmp;
f9d9c362
AD		 1808 /*
1809 * Line Buffer Setup
1810 * There are 3 line buffers, each one shared by 2 display controllers.
1811 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1812 * the display controllers. The paritioning is done via one of four
1813 * preset allocations specified in bits 2:0:
1814 * first display controller
1815 * 0 - first half of lb (3840 * 2)
1816 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1817 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1818 * 3 - first 1/4 of lb (1920 * 2)
1819 * second display controller
1820 * 4 - second half of lb (3840 * 2)
1821 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1822 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1823 * 7 - last 1/4 of lb (1920 * 2)
1824 */
12dfc843
AD		 1825 /* this can get tricky if we have two large displays on a paired group
1826 * of crtcs. Ideally for multiple large displays we'd assign them to
1827 * non-linked crtcs for maximum line buffer allocation.
1828 */
1829 if (radeon_crtc->base.enabled && mode) {
1830 if (other_mode)
f9d9c362 1831 tmp = 0; /* 1/2 */
12dfc843
AD		 1832 else
1833 tmp = 2; /* whole */
1834 } else
1835 tmp = 0;
f9d9c362
AD		 1836
1837 /* second controller of the pair uses second half of the lb */
1838 if (radeon_crtc->crtc_id % 2)
1839 tmp += 4;
1840 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1841
12dfc843
AD		 1842 if (radeon_crtc->base.enabled && mode) {
1843 switch (tmp) {
1844 case 0:
1845 case 4:
1846 default:
1847 if (ASIC_IS_DCE5(rdev))
1848 return 4096 * 2;
1849 else
1850 return 3840 * 2;
1851 case 1:
1852 case 5:
1853 if (ASIC_IS_DCE5(rdev))
1854 return 6144 * 2;
1855 else
1856 return 5760 * 2;
1857 case 2:
1858 case 6:
1859 if (ASIC_IS_DCE5(rdev))
1860 return 8192 * 2;
1861 else
1862 return 7680 * 2;
1863 case 3:
1864 case 7:
1865 if (ASIC_IS_DCE5(rdev))
1866 return 2048 * 2;
1867 else
1868 return 1920 * 2;
1869 }
f9d9c362 1870 }
12dfc843
AD		 1871
1872 /* controller not enabled, so no lb used */
1873 return 0;
f9d9c362
AD		 1874}
1875
ca7db22b 1876u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1877{
1878 u32 tmp = RREG32(MC_SHARED_CHMAP);
1879
1880 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1881 case 0:
1882 default:
1883 return 1;
1884 case 1:
1885 return 2;
1886 case 2:
1887 return 4;
1888 case 3:
1889 return 8;
1890 }
1891}
1892
1893struct evergreen_wm_params {
1894 u32 dram_channels; /* number of dram channels */
1895 u32 yclk; /* bandwidth per dram data pin in kHz */
1896 u32 sclk; /* engine clock in kHz */
1897 u32 disp_clk; /* display clock in kHz */
1898 u32 src_width; /* viewport width */
1899 u32 active_time; /* active display time in ns */
1900 u32 blank_time; /* blank time in ns */
1901 bool interlaced; /* mode is interlaced */
1902 fixed20_12 vsc; /* vertical scale ratio */
1903 u32 num_heads; /* number of active crtcs */
1904 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1905 u32 lb_size; /* line buffer allocated to pipe */
1906 u32 vtaps; /* vertical scaler taps */
1907};
1908
1909static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1910{
1911 /* Calculate DRAM Bandwidth and the part allocated to display. */
1912 fixed20_12 dram_efficiency; /* 0.7 */
1913 fixed20_12 yclk, dram_channels, bandwidth;
1914 fixed20_12 a;
1915
1916 a.full = dfixed_const(1000);
1917 yclk.full = dfixed_const(wm->yclk);
1918 yclk.full = dfixed_div(yclk, a);
1919 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1920 a.full = dfixed_const(10);
1921 dram_efficiency.full = dfixed_const(7);
1922 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1923 bandwidth.full = dfixed_mul(dram_channels, yclk);
1924 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1925
1926 return dfixed_trunc(bandwidth);
1927}
1928
1929static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1930{
1931 /* Calculate DRAM Bandwidth and the part allocated to display. */
1932 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1933 fixed20_12 yclk, dram_channels, bandwidth;
1934 fixed20_12 a;
1935
1936 a.full = dfixed_const(1000);
1937 yclk.full = dfixed_const(wm->yclk);
1938 yclk.full = dfixed_div(yclk, a);
1939 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1940 a.full = dfixed_const(10);
1941 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1942 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1943 bandwidth.full = dfixed_mul(dram_channels, yclk);
1944 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1945
1946 return dfixed_trunc(bandwidth);
1947}
1948
1949static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1950{
1951 /* Calculate the display Data return Bandwidth */
1952 fixed20_12 return_efficiency; /* 0.8 */
1953 fixed20_12 sclk, bandwidth;
1954 fixed20_12 a;
1955
1956 a.full = dfixed_const(1000);
1957 sclk.full = dfixed_const(wm->sclk);
1958 sclk.full = dfixed_div(sclk, a);
1959 a.full = dfixed_const(10);
1960 return_efficiency.full = dfixed_const(8);
1961 return_efficiency.full = dfixed_div(return_efficiency, a);
1962 a.full = dfixed_const(32);
1963 bandwidth.full = dfixed_mul(a, sclk);
1964 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1965
1966 return dfixed_trunc(bandwidth);
1967}
1968
1969static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1970{
1971 /* Calculate the DMIF Request Bandwidth */
1972 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1973 fixed20_12 disp_clk, bandwidth;
1974 fixed20_12 a;
1975
1976 a.full = dfixed_const(1000);
1977 disp_clk.full = dfixed_const(wm->disp_clk);
1978 disp_clk.full = dfixed_div(disp_clk, a);
1979 a.full = dfixed_const(10);
1980 disp_clk_request_efficiency.full = dfixed_const(8);
1981 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1982 a.full = dfixed_const(32);
1983 bandwidth.full = dfixed_mul(a, disp_clk);
1984 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1985
1986 return dfixed_trunc(bandwidth);
1987}
1988
1989static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1990{
1991 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1992 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1993 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1994 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1995
1996 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1997}
1998
1999static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2000{
2001 /* Calculate the display mode Average Bandwidth
2002 * DisplayMode should contain the source and destination dimensions,
2003 * timing, etc.
2004 */
2005 fixed20_12 bpp;
2006 fixed20_12 line_time;
2007 fixed20_12 src_width;
2008 fixed20_12 bandwidth;
2009 fixed20_12 a;
2010
2011 a.full = dfixed_const(1000);
2012 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2013 line_time.full = dfixed_div(line_time, a);
2014 bpp.full = dfixed_const(wm->bytes_per_pixel);
2015 src_width.full = dfixed_const(wm->src_width);
2016 bandwidth.full = dfixed_mul(src_width, bpp);
2017 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2018 bandwidth.full = dfixed_div(bandwidth, line_time);
2019
2020 return dfixed_trunc(bandwidth);
2021}
2022
2023static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2024{
2025 /* First calcualte the latency in ns */
2026 u32 mc_latency = 2000; /* 2000 ns. */
2027 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2028 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2029 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2030 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2031 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2032 (wm->num_heads * cursor_line_pair_return_time);
2033 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2034 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2035 fixed20_12 a, b, c;
2036
2037 if (wm->num_heads == 0)
2038 return 0;
2039
2040 a.full = dfixed_const(2);
2041 b.full = dfixed_const(1);
2042 if ((wm->vsc.full > a.full) ||
2043 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2044 (wm->vtaps >= 5) ||
2045 ((wm->vsc.full >= a.full) && wm->interlaced))
2046 max_src_lines_per_dst_line = 4;
2047 else
2048 max_src_lines_per_dst_line = 2;
2049
2050 a.full = dfixed_const(available_bandwidth);
2051 b.full = dfixed_const(wm->num_heads);
2052 a.full = dfixed_div(a, b);
2053
2054 b.full = dfixed_const(1000);
2055 c.full = dfixed_const(wm->disp_clk);
2056 b.full = dfixed_div(c, b);
2057 c.full = dfixed_const(wm->bytes_per_pixel);
2058 b.full = dfixed_mul(b, c);
2059
2060 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2061
2062 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2063 b.full = dfixed_const(1000);
2064 c.full = dfixed_const(lb_fill_bw);
2065 b.full = dfixed_div(c, b);
2066 a.full = dfixed_div(a, b);
2067 line_fill_time = dfixed_trunc(a);
2068
2069 if (line_fill_time < wm->active_time)
2070 return latency;
2071 else
2072 return latency + (line_fill_time - wm->active_time);
2073
2074}
2075
2076static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2077{
2078 if (evergreen_average_bandwidth(wm) <=
2079 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2080 return true;
2081 else
2082 return false;
2083};
2084
2085static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2086{
2087 if (evergreen_average_bandwidth(wm) <=
2088 (evergreen_available_bandwidth(wm) / wm->num_heads))
2089 return true;
2090 else
2091 return false;
2092};
2093
2094static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2095{
2096 u32 lb_partitions = wm->lb_size / wm->src_width;
2097 u32 line_time = wm->active_time + wm->blank_time;
2098 u32 latency_tolerant_lines;
2099 u32 latency_hiding;
2100 fixed20_12 a;
2101
2102 a.full = dfixed_const(1);
2103 if (wm->vsc.full > a.full)
2104 latency_tolerant_lines = 1;
2105 else {
2106 if (lb_partitions <= (wm->vtaps + 1))
2107 latency_tolerant_lines = 1;
2108 else
2109 latency_tolerant_lines = 2;
2110 }
2111
2112 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2113
2114 if (evergreen_latency_watermark(wm) <= latency_hiding)
2115 return true;
2116 else
2117 return false;
2118}
2119
2120static void evergreen_program_watermarks(struct radeon_device *rdev,
2121 struct radeon_crtc *radeon_crtc,
2122 u32 lb_size, u32 num_heads)
2123{
2124 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2125 struct evergreen_wm_params wm_low, wm_high;
2126 u32 dram_channels;
f9d9c362
AD		 2127 u32 pixel_period;
2128 u32 line_time = 0;
2129 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2130 u32 priority_a_mark = 0, priority_b_mark = 0;
2131 u32 priority_a_cnt = PRIORITY_OFF;
2132 u32 priority_b_cnt = PRIORITY_OFF;
2133 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2134 u32 tmp, arb_control3;
2135 fixed20_12 a, b, c;
2136
2137 if (radeon_crtc->base.enabled && num_heads && mode) {
2138 pixel_period = 1000000 / (u32)mode->clock;
2139 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2140 priority_a_cnt = 0;
2141 priority_b_cnt = 0;
cf0cfdd7
AD		 2142 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2143
2144 /* watermark for high clocks */
2145 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2146 wm_high.yclk =
2147 radeon_dpm_get_mclk(rdev, false) * 10;
2148 wm_high.sclk =
2149 radeon_dpm_get_sclk(rdev, false) * 10;
2150 } else {
2151 wm_high.yclk = rdev->pm.current_mclk * 10;
2152 wm_high.sclk = rdev->pm.current_sclk * 10;
2153 }
f9d9c362 2154
cf0cfdd7
AD		 2155 wm_high.disp_clk = mode->clock;
2156 wm_high.src_width = mode->crtc_hdisplay;
2157 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2158 wm_high.blank_time = line_time - wm_high.active_time;
2159 wm_high.interlaced = false;
f9d9c362 2160 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2161 wm_high.interlaced = true;
2162 wm_high.vsc = radeon_crtc->vsc;
2163 wm_high.vtaps = 1;
f9d9c362 2164 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2165 wm_high.vtaps = 2;
2166 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2167 wm_high.lb_size = lb_size;
2168 wm_high.dram_channels = dram_channels;
2169 wm_high.num_heads = num_heads;
2170
2171 /* watermark for low clocks */
2172 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2173 wm_low.yclk =
2174 radeon_dpm_get_mclk(rdev, true) * 10;
2175 wm_low.sclk =
2176 radeon_dpm_get_sclk(rdev, true) * 10;
2177 } else {
2178 wm_low.yclk = rdev->pm.current_mclk * 10;
2179 wm_low.sclk = rdev->pm.current_sclk * 10;
2180 }
2181
2182 wm_low.disp_clk = mode->clock;
2183 wm_low.src_width = mode->crtc_hdisplay;
2184 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2185 wm_low.blank_time = line_time - wm_low.active_time;
2186 wm_low.interlaced = false;
2187 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2188 wm_low.interlaced = true;
2189 wm_low.vsc = radeon_crtc->vsc;
2190 wm_low.vtaps = 1;
2191 if (radeon_crtc->rmx_type != RMX_OFF)
2192 wm_low.vtaps = 2;
2193 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2194 wm_low.lb_size = lb_size;
2195 wm_low.dram_channels = dram_channels;
2196 wm_low.num_heads = num_heads;
f9d9c362
AD		 2197
2198 /* set for high clocks */
cf0cfdd7 2199 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2200 /* set for low clocks */
cf0cfdd7 2201 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2202
2203 /* possibly force display priority to high */
2204 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2205 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2206 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2207 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2208 (rdev->disp_priority == 2)) {
cf0cfdd7 2209 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2210 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2211 }
2212 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2213 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2214 !evergreen_check_latency_hiding(&wm_low) ||
2215 (rdev->disp_priority == 2)) {
2216 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2217 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2218 }
2219
2220 a.full = dfixed_const(1000);
2221 b.full = dfixed_const(mode->clock);
2222 b.full = dfixed_div(b, a);
2223 c.full = dfixed_const(latency_watermark_a);
2224 c.full = dfixed_mul(c, b);
2225 c.full = dfixed_mul(c, radeon_crtc->hsc);
2226 c.full = dfixed_div(c, a);
2227 a.full = dfixed_const(16);
2228 c.full = dfixed_div(c, a);
2229 priority_a_mark = dfixed_trunc(c);
2230 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2231
2232 a.full = dfixed_const(1000);
2233 b.full = dfixed_const(mode->clock);
2234 b.full = dfixed_div(b, a);
2235 c.full = dfixed_const(latency_watermark_b);
2236 c.full = dfixed_mul(c, b);
2237 c.full = dfixed_mul(c, radeon_crtc->hsc);
2238 c.full = dfixed_div(c, a);
2239 a.full = dfixed_const(16);
2240 c.full = dfixed_div(c, a);
2241 priority_b_mark = dfixed_trunc(c);
2242 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2243 }
2244
2245 /* select wm A */
2246 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2247 tmp = arb_control3;
2248 tmp &= ~LATENCY_WATERMARK_MASK(3);
2249 tmp |= LATENCY_WATERMARK_MASK(1);
2250 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2251 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2252 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2253 LATENCY_HIGH_WATERMARK(line_time)));
2254 /* select wm B */
2255 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2256 tmp &= ~LATENCY_WATERMARK_MASK(3);
2257 tmp |= LATENCY_WATERMARK_MASK(2);
2258 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2259 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2260 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2261 LATENCY_HIGH_WATERMARK(line_time)));
2262 /* restore original selection */
2263 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2264
2265 /* write the priority marks */
2266 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2267 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2268
2269}
2270
377edc8b
AD		 2271/**
2272 * evergreen_bandwidth_update - update display watermarks callback.
2273 *
2274 * @rdev: radeon_device pointer
2275 *
2276 * Update the display watermarks based on the requested mode(s)
2277 * (evergreen+).
2278 */
0ca2ab52 2279void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2280{
f9d9c362
AD		 2281 struct drm_display_mode *mode0 = NULL;
2282 struct drm_display_mode *mode1 = NULL;
2283 u32 num_heads = 0, lb_size;
2284 int i;
2285
2286 radeon_update_display_priority(rdev);
2287
2288 for (i = 0; i < rdev->num_crtc; i++) {
2289 if (rdev->mode_info.crtcs[i]->base.enabled)
2290 num_heads++;
2291 }
2292 for (i = 0; i < rdev->num_crtc; i += 2) {
2293 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2294 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2295 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2296 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2297 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2298 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2299 }
bcc1c2a1
AD		 2300}
2301
377edc8b
AD		 2302/**
2303 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2304 *
2305 * @rdev: radeon_device pointer
2306 *
2307 * Wait for the MC (memory controller) to be idle.
2308 * (evergreen+).
2309 * Returns 0 if the MC is idle, -1 if not.
2310 */
b9952a8a 2311int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2312{
2313 unsigned i;
2314 u32 tmp;
2315
2316 for (i = 0; i < rdev->usec_timeout; i++) {
2317 /* read MC_STATUS */
2318 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2319 if (!tmp)
2320 return 0;
2321 udelay(1);
2322 }
2323 return -1;
2324}
2325
2326/*
2327 * GART
2328 */
0fcdb61e
AD		 2329void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2330{
2331 unsigned i;
2332 u32 tmp;
2333
6f2f48a9
AD		 2334 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2335
0fcdb61e
AD		 2336 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2337 for (i = 0; i < rdev->usec_timeout; i++) {
2338 /* read MC_STATUS */
2339 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2340 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2341 if (tmp == 2) {
2342 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2343 return;
2344 }
2345 if (tmp) {
2346 return;
2347 }
2348 udelay(1);
2349 }
2350}
2351
1109ca09 2352static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2353{
2354 u32 tmp;
0fcdb61e 2355 int r;
bcc1c2a1 2356
c9a1be96 2357 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2358 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2359 return -EINVAL;
2360 }
2361 r = radeon_gart_table_vram_pin(rdev);
2362 if (r)
2363 return r;
82568565 2364 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2365 /* Setup L2 cache */
2366 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2367 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2368 EFFECTIVE_L2_QUEUE_SIZE(7));
2369 WREG32(VM_L2_CNTL2, 0);
2370 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2371 /* Setup TLB control */
2372 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2373 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2374 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2375 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2376 if (rdev->flags & RADEON_IS_IGP) {
2377 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2378 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2379 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2380 } else {
2381 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2382 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2383 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2384 if ((rdev->family == CHIP_JUNIPER) ||
2385 (rdev->family == CHIP_CYPRESS) ||
2386 (rdev->family == CHIP_HEMLOCK) ||
2387 (rdev->family == CHIP_BARTS))
2388 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2389 }
bcc1c2a1
AD		 2390 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2391 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2392 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2393 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2394 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2395 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2396 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2397 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2398 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2399 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2400 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2401 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2402
0fcdb61e 2403 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2404 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2405 (unsigned)(rdev->mc.gtt_size >> 20),
2406 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2407 rdev->gart.ready = true;
2408 return 0;
2409}
2410
1109ca09 2411static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2412{
2413 u32 tmp;
bcc1c2a1
AD		 2414
2415 /* Disable all tables */
0fcdb61e
AD		 2416 WREG32(VM_CONTEXT0_CNTL, 0);
2417 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2418
2419 /* Setup L2 cache */
2420 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2421 EFFECTIVE_L2_QUEUE_SIZE(7));
2422 WREG32(VM_L2_CNTL2, 0);
2423 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2424 /* Setup TLB control */
2425 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2426 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2427 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2428 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2429 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2430 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2431 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2432 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2433 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2434}
2435
1109ca09 2436static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2437{
2438 evergreen_pcie_gart_disable(rdev);
2439 radeon_gart_table_vram_free(rdev);
2440 radeon_gart_fini(rdev);
2441}
2442
2443
1109ca09 2444static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2445{
2446 u32 tmp;
bcc1c2a1
AD		 2447
2448 /* Setup L2 cache */
2449 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2450 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2451 EFFECTIVE_L2_QUEUE_SIZE(7));
2452 WREG32(VM_L2_CNTL2, 0);
2453 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2454 /* Setup TLB control */
2455 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2456 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2457 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2458 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2459 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2460 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2461 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2462 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2463 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2464 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2465 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2466 WREG32(VM_CONTEXT0_CNTL, 0);
2467 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2468}
2469
b9952a8a 2470void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2471{
62444b74
AD		 2472 u32 crtc_enabled, tmp, frame_count, blackout;
2473 int i, j;
2474
5153550a
AD		 2475 if (!ASIC_IS_NODCE(rdev)) {
2476 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2477 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2478
5153550a
AD		 2479 /* disable VGA render */
2480 WREG32(VGA_RENDER_CONTROL, 0);
2481 }
62444b74
AD		 2482 /* blank the display controllers */
2483 for (i = 0; i < rdev->num_crtc; i++) {
2484 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2485 if (crtc_enabled) {
2486 save->crtc_enabled[i] = true;
2487 if (ASIC_IS_DCE6(rdev)) {
2488 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2489 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2490 radeon_wait_for_vblank(rdev, i);
abf1457b 2491 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2492 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2493 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2494 }
2495 } else {
2496 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2497 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2498 radeon_wait_for_vblank(rdev, i);
abf1457b 2499 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2500 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2501 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2502 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2503 }
2504 }
2505 /* wait for the next frame */
2506 frame_count = radeon_get_vblank_counter(rdev, i);
2507 for (j = 0; j < rdev->usec_timeout; j++) {
2508 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2509 break;
2510 udelay(1);
2511 }
abf1457b
AD		 2512
2513 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2514 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2515 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2516 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2517 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2518 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2519 save->crtc_enabled[i] = false;
2520 /* ***** */
804cc4a0
AD		 2521 } else {
2522 save->crtc_enabled[i] = false;
62444b74 2523 }
18007401 2524 }
bcc1c2a1 2525
62444b74
AD		 2526 radeon_mc_wait_for_idle(rdev);
2527
2528 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2529 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2530 /* Block CPU access */
2531 WREG32(BIF_FB_EN, 0);
2532 /* blackout the MC */
2533 blackout &= ~BLACKOUT_MODE_MASK;
2534 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2535 }
ed39fadd
AD		 2536 /* wait for the MC to settle */
2537 udelay(100);
968c0166
AD		 2538
2539 /* lock double buffered regs */
2540 for (i = 0; i < rdev->num_crtc; i++) {
2541 if (save->crtc_enabled[i]) {
2542 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2543 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2544 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2545 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2546 }
2547 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2548 if (!(tmp & 1)) {
2549 tmp |= 1;
2550 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2551 }
2552 }
2553 }
bcc1c2a1
AD		 2554}
2555
b9952a8a 2556void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2557{
62444b74
AD		 2558 u32 tmp, frame_count;
2559 int i, j;
18007401 2560
62444b74
AD		 2561 /* update crtc base addresses */
2562 for (i = 0; i < rdev->num_crtc; i++) {
2563 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2564 upper_32_bits(rdev->mc.vram_start));
62444b74 2565 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2566 upper_32_bits(rdev->mc.vram_start));
62444b74 2567 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2568 (u32)rdev->mc.vram_start);
62444b74 2569 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2570 (u32)rdev->mc.vram_start);
2571 }
5153550a
AD		 2572
2573 if (!ASIC_IS_NODCE(rdev)) {
2574 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2575 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2576 }
62444b74 2577
968c0166
AD		 2578 /* unlock regs and wait for update */
2579 for (i = 0; i < rdev->num_crtc; i++) {
2580 if (save->crtc_enabled[i]) {
2581 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2582 if ((tmp & 0x3) != 0) {
2583 tmp &= ~0x3;
2584 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2585 }
2586 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2587 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2588 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2589 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2590 }
2591 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2592 if (tmp & 1) {
2593 tmp &= ~1;
2594 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2595 }
2596 for (j = 0; j < rdev->usec_timeout; j++) {
2597 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2598 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2599 break;
2600 udelay(1);
2601 }
2602 }
2603 }
2604
62444b74
AD		 2605 /* unblackout the MC */
2606 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2607 tmp &= ~BLACKOUT_MODE_MASK;
2608 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2609 /* allow CPU access */
2610 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2611
2612 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2613 if (save->crtc_enabled[i]) {
62444b74
AD		 2614 if (ASIC_IS_DCE6(rdev)) {
2615 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2616 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2617 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2618 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2619 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2620 } else {
2621 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2622 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2623 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2624 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2625 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2626 }
2627 /* wait for the next frame */
2628 frame_count = radeon_get_vblank_counter(rdev, i);
2629 for (j = 0; j < rdev->usec_timeout; j++) {
2630 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2631 break;
2632 udelay(1);
2633 }
2634 }
2635 }
5153550a
AD		 2636 if (!ASIC_IS_NODCE(rdev)) {
2637 /* Unlock vga access */
2638 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2639 mdelay(1);
2640 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2641 }
bcc1c2a1
AD		 2642}
2643
755d819e 2644void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2645{
2646 struct evergreen_mc_save save;
2647 u32 tmp;
2648 int i, j;
2649
2650 /* Initialize HDP */
2651 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2652 WREG32((0x2c14 + j), 0x00000000);
2653 WREG32((0x2c18 + j), 0x00000000);
2654 WREG32((0x2c1c + j), 0x00000000);
2655 WREG32((0x2c20 + j), 0x00000000);
2656 WREG32((0x2c24 + j), 0x00000000);
2657 }
2658 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2659
2660 evergreen_mc_stop(rdev, &save);
2661 if (evergreen_mc_wait_for_idle(rdev)) {
2662 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2663 }
2664 /* Lockout access through VGA aperture*/
2665 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2666 /* Update configuration */
2667 if (rdev->flags & RADEON_IS_AGP) {
2668 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2669 /* VRAM before AGP */
2670 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2671 rdev->mc.vram_start >> 12);
2672 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2673 rdev->mc.gtt_end >> 12);
2674 } else {
2675 /* VRAM after AGP */
2676 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2677 rdev->mc.gtt_start >> 12);
2678 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2679 rdev->mc.vram_end >> 12);
2680 }
2681 } else {
2682 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2683 rdev->mc.vram_start >> 12);
2684 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2685 rdev->mc.vram_end >> 12);
2686 }
3b9832f6 2687 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2688 /* llano/ontario only */
2689 if ((rdev->family == CHIP_PALM) ||
2690 (rdev->family == CHIP_SUMO) ||
2691 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2692 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2693 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2694 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2695 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2696 }
bcc1c2a1
AD		 2697 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2698 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2699 WREG32(MC_VM_FB_LOCATION, tmp);
2700 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2701 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2702 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2703 if (rdev->flags & RADEON_IS_AGP) {
2704 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2705 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2706 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2707 } else {
2708 WREG32(MC_VM_AGP_BASE, 0);
2709 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2710 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2711 }
2712 if (evergreen_mc_wait_for_idle(rdev)) {
2713 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2714 }
2715 evergreen_mc_resume(rdev, &save);
2716 /* we need to own VRAM, so turn off the VGA renderer here
2717 * to stop it overwriting our objects */
2718 rv515_vga_render_disable(rdev);
2719}
2720
bcc1c2a1
AD		 2721/*
2722 * CP.
2723 */
12920591
AD		 2724void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2725{
876dc9f3 2726 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2727 u32 next_rptr;
7b1f2485 2728
12920591 2729 /* set to DX10/11 mode */
e32eb50d
CK		 2730 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2731 radeon_ring_write(ring, 1);
45df6803
CK		 2732
2733 if (ring->rptr_save_reg) {
89d35807 2734 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2735 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2736 radeon_ring_write(ring, ((ring->rptr_save_reg -
2737 PACKET3_SET_CONFIG_REG_START) >> 2));
2738 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2739 } else if (rdev->wb.enabled) {
2740 next_rptr = ring->wptr + 5 + 4;
2741 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2742 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2743 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2744 radeon_ring_write(ring, next_rptr);
2745 radeon_ring_write(ring, 0);
45df6803
CK		 2746 }
2747
e32eb50d
CK		 2748 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2749 radeon_ring_write(ring,
0f234f5f
AD		 2750#ifdef __BIG_ENDIAN
2751 (2 << 0) |
2752#endif
2753 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2754 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2755 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2756}
2757
bcc1c2a1
AD		 2758
2759static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2760{
fe251e2f
AD		 2761 const __be32 *fw_data;
2762 int i;
2763
2764 if (!rdev->me_fw || !rdev->pfp_fw)
2765 return -EINVAL;
bcc1c2a1 2766
fe251e2f 2767 r700_cp_stop(rdev);
0f234f5f
AD		 2768 WREG32(CP_RB_CNTL,
2769#ifdef __BIG_ENDIAN
2770 BUF_SWAP_32BIT |
2771#endif
2772 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2773
2774 fw_data = (const __be32 *)rdev->pfp_fw->data;
2775 WREG32(CP_PFP_UCODE_ADDR, 0);
2776 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2777 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2778 WREG32(CP_PFP_UCODE_ADDR, 0);
2779
2780 fw_data = (const __be32 *)rdev->me_fw->data;
2781 WREG32(CP_ME_RAM_WADDR, 0);
2782 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2783 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2784
2785 WREG32(CP_PFP_UCODE_ADDR, 0);
2786 WREG32(CP_ME_RAM_WADDR, 0);
2787 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2788 return 0;
2789}
2790
7e7b41d2
AD		 2791static int evergreen_cp_start(struct radeon_device *rdev)
2792{
e32eb50d 2793 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2794 int r, i;
7e7b41d2
AD		 2795 uint32_t cp_me;
2796
e32eb50d 2797 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2798 if (r) {
2799 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2800 return r;
2801 }
e32eb50d
CK		 2802 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2803 radeon_ring_write(ring, 0x1);
2804 radeon_ring_write(ring, 0x0);
2805 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2806 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2807 radeon_ring_write(ring, 0);
2808 radeon_ring_write(ring, 0);
2809 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2810
2811 cp_me = 0xff;
2812 WREG32(CP_ME_CNTL, cp_me);
2813
e32eb50d 2814 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2815 if (r) {
2816 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2817 return r;
2818 }
2281a378
AD		 2819
2820 /* setup clear context state */
e32eb50d
CK		 2821 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2822 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2823
2824 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2825 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2826
e32eb50d
CK		 2827 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2828 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2829
2830 /* set clear context state */
e32eb50d
CK		 2831 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2832 radeon_ring_write(ring, 0);
2281a378
AD		 2833
2834 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2835 radeon_ring_write(ring, 0xc0026f00);
2836 radeon_ring_write(ring, 0x00000000);
2837 radeon_ring_write(ring, 0x00000000);
2838 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2839
2840 /* Clear consts */
e32eb50d
CK		 2841 radeon_ring_write(ring, 0xc0036f00);
2842 radeon_ring_write(ring, 0x00000bc4);
2843 radeon_ring_write(ring, 0xffffffff);
2844 radeon_ring_write(ring, 0xffffffff);
2845 radeon_ring_write(ring, 0xffffffff);
2281a378 2846
e32eb50d
CK		 2847 radeon_ring_write(ring, 0xc0026900);
2848 radeon_ring_write(ring, 0x00000316);
2849 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2850 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2851
e32eb50d 2852 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2853
2854 return 0;
2855}
2856
1109ca09 2857static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2858{
e32eb50d 2859 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2860 u32 tmp;
2861 u32 rb_bufsz;
2862 int r;
2863
2864 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2865 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2866 SOFT_RESET_PA |
2867 SOFT_RESET_SH |
2868 SOFT_RESET_VGT |
a49a50da 2869 SOFT_RESET_SPI |
fe251e2f
AD		 2870 SOFT_RESET_SX));
2871 RREG32(GRBM_SOFT_RESET);
2872 mdelay(15);
2873 WREG32(GRBM_SOFT_RESET, 0);
2874 RREG32(GRBM_SOFT_RESET);
2875
2876 /* Set ring buffer size */
e32eb50d 2877 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 2878 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2879#ifdef __BIG_ENDIAN
2880 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2881#endif
fe251e2f 2882 WREG32(CP_RB_CNTL, tmp);
15d3332f 2883 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2884 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2885
2886 /* Set the write pointer delay */
2887 WREG32(CP_RB_WPTR_DELAY, 0);
2888
2889 /* Initialize the ring buffer's read and write pointers */
2890 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2891 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2892 ring->wptr = 0;
2893 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2894
48fc7f7e 2895 /* set the wb address whether it's enabled or not */
0f234f5f 2896 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2897 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2898 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2899 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2900
2901 if (rdev->wb.enabled)
2902 WREG32(SCRATCH_UMSK, 0xff);
2903 else {
2904 tmp |= RB_NO_UPDATE;
2905 WREG32(SCRATCH_UMSK, 0);
2906 }
2907
fe251e2f
AD		 2908 mdelay(1);
2909 WREG32(CP_RB_CNTL, tmp);
2910
e32eb50d 2911 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2912 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2913
e32eb50d 2914 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2915
7e7b41d2 2916 evergreen_cp_start(rdev);
e32eb50d 2917 ring->ready = true;
f712812e 2918 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2919 if (r) {
e32eb50d 2920 ring->ready = false;
fe251e2f
AD		 2921 return r;
2922 }
2923 return 0;
2924}
bcc1c2a1
AD		 2925
2926/*
2927 * Core functions
2928 */
bcc1c2a1
AD		 2929static void evergreen_gpu_init(struct radeon_device *rdev)
2930{
416a2bd2 2931 u32 gb_addr_config;
32fcdbf4 2932 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 2933 u32 sx_debug_1;
2934 u32 smx_dc_ctl0;
2935 u32 sq_config;
2936 u32 sq_lds_resource_mgmt;
2937 u32 sq_gpr_resource_mgmt_1;
2938 u32 sq_gpr_resource_mgmt_2;
2939 u32 sq_gpr_resource_mgmt_3;
2940 u32 sq_thread_resource_mgmt;
2941 u32 sq_thread_resource_mgmt_2;
2942 u32 sq_stack_resource_mgmt_1;
2943 u32 sq_stack_resource_mgmt_2;
2944 u32 sq_stack_resource_mgmt_3;
2945 u32 vgt_cache_invalidation;
f25a5c63 2946 u32 hdp_host_path_cntl, tmp;
416a2bd2 2947 u32 disabled_rb_mask;
32fcdbf4
AD		 2948 int i, j, num_shader_engines, ps_thread_count;
2949
2950 switch (rdev->family) {
2951 case CHIP_CYPRESS:
2952 case CHIP_HEMLOCK:
2953 rdev->config.evergreen.num_ses = 2;
2954 rdev->config.evergreen.max_pipes = 4;
2955 rdev->config.evergreen.max_tile_pipes = 8;
2956 rdev->config.evergreen.max_simds = 10;
2957 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2958 rdev->config.evergreen.max_gprs = 256;
2959 rdev->config.evergreen.max_threads = 248;
2960 rdev->config.evergreen.max_gs_threads = 32;
2961 rdev->config.evergreen.max_stack_entries = 512;
2962 rdev->config.evergreen.sx_num_of_sets = 4;
2963 rdev->config.evergreen.sx_max_export_size = 256;
2964 rdev->config.evergreen.sx_max_export_pos_size = 64;
2965 rdev->config.evergreen.sx_max_export_smx_size = 192;
2966 rdev->config.evergreen.max_hw_contexts = 8;
2967 rdev->config.evergreen.sq_num_cf_insts = 2;
2968
2969 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2970 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2971 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2972 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2973 break;
2974 case CHIP_JUNIPER:
2975 rdev->config.evergreen.num_ses = 1;
2976 rdev->config.evergreen.max_pipes = 4;
2977 rdev->config.evergreen.max_tile_pipes = 4;
2978 rdev->config.evergreen.max_simds = 10;
2979 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2980 rdev->config.evergreen.max_gprs = 256;
2981 rdev->config.evergreen.max_threads = 248;
2982 rdev->config.evergreen.max_gs_threads = 32;
2983 rdev->config.evergreen.max_stack_entries = 512;
2984 rdev->config.evergreen.sx_num_of_sets = 4;
2985 rdev->config.evergreen.sx_max_export_size = 256;
2986 rdev->config.evergreen.sx_max_export_pos_size = 64;
2987 rdev->config.evergreen.sx_max_export_smx_size = 192;
2988 rdev->config.evergreen.max_hw_contexts = 8;
2989 rdev->config.evergreen.sq_num_cf_insts = 2;
2990
2991 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2992 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2993 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2994 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2995 break;
2996 case CHIP_REDWOOD:
2997 rdev->config.evergreen.num_ses = 1;
2998 rdev->config.evergreen.max_pipes = 4;
2999 rdev->config.evergreen.max_tile_pipes = 4;
3000 rdev->config.evergreen.max_simds = 5;
3001 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3002 rdev->config.evergreen.max_gprs = 256;
3003 rdev->config.evergreen.max_threads = 248;
3004 rdev->config.evergreen.max_gs_threads = 32;
3005 rdev->config.evergreen.max_stack_entries = 256;
3006 rdev->config.evergreen.sx_num_of_sets = 4;
3007 rdev->config.evergreen.sx_max_export_size = 256;
3008 rdev->config.evergreen.sx_max_export_pos_size = 64;
3009 rdev->config.evergreen.sx_max_export_smx_size = 192;
3010 rdev->config.evergreen.max_hw_contexts = 8;
3011 rdev->config.evergreen.sq_num_cf_insts = 2;
3012
3013 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3014 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3015 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3016 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3017 break;
3018 case CHIP_CEDAR:
3019 default:
3020 rdev->config.evergreen.num_ses = 1;
3021 rdev->config.evergreen.max_pipes = 2;
3022 rdev->config.evergreen.max_tile_pipes = 2;
3023 rdev->config.evergreen.max_simds = 2;
3024 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3025 rdev->config.evergreen.max_gprs = 256;
3026 rdev->config.evergreen.max_threads = 192;
3027 rdev->config.evergreen.max_gs_threads = 16;
3028 rdev->config.evergreen.max_stack_entries = 256;
3029 rdev->config.evergreen.sx_num_of_sets = 4;
3030 rdev->config.evergreen.sx_max_export_size = 128;
3031 rdev->config.evergreen.sx_max_export_pos_size = 32;
3032 rdev->config.evergreen.sx_max_export_smx_size = 96;
3033 rdev->config.evergreen.max_hw_contexts = 4;
3034 rdev->config.evergreen.sq_num_cf_insts = 1;
3035
d5e455e4
AD		 3036 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3037 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3038 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3039 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3040 break;
3041 case CHIP_PALM:
3042 rdev->config.evergreen.num_ses = 1;
3043 rdev->config.evergreen.max_pipes = 2;
3044 rdev->config.evergreen.max_tile_pipes = 2;
3045 rdev->config.evergreen.max_simds = 2;
3046 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3047 rdev->config.evergreen.max_gprs = 256;
3048 rdev->config.evergreen.max_threads = 192;
3049 rdev->config.evergreen.max_gs_threads = 16;
3050 rdev->config.evergreen.max_stack_entries = 256;
3051 rdev->config.evergreen.sx_num_of_sets = 4;
3052 rdev->config.evergreen.sx_max_export_size = 128;
3053 rdev->config.evergreen.sx_max_export_pos_size = 32;
3054 rdev->config.evergreen.sx_max_export_smx_size = 96;
3055 rdev->config.evergreen.max_hw_contexts = 4;
3056 rdev->config.evergreen.sq_num_cf_insts = 1;
3057
d5c5a72f
AD		 3058 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3059 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3060 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3061 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3062 break;
3063 case CHIP_SUMO:
3064 rdev->config.evergreen.num_ses = 1;
3065 rdev->config.evergreen.max_pipes = 4;
bd25f078 3066 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3067 if (rdev->pdev->device == 0x9648)
3068 rdev->config.evergreen.max_simds = 3;
3069 else if ((rdev->pdev->device == 0x9647) ||
3070 (rdev->pdev->device == 0x964a))
3071 rdev->config.evergreen.max_simds = 4;
3072 else
3073 rdev->config.evergreen.max_simds = 5;
3074 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3075 rdev->config.evergreen.max_gprs = 256;
3076 rdev->config.evergreen.max_threads = 248;
3077 rdev->config.evergreen.max_gs_threads = 32;
3078 rdev->config.evergreen.max_stack_entries = 256;
3079 rdev->config.evergreen.sx_num_of_sets = 4;
3080 rdev->config.evergreen.sx_max_export_size = 256;
3081 rdev->config.evergreen.sx_max_export_pos_size = 64;
3082 rdev->config.evergreen.sx_max_export_smx_size = 192;
3083 rdev->config.evergreen.max_hw_contexts = 8;
3084 rdev->config.evergreen.sq_num_cf_insts = 2;
3085
3086 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3087 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3088 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3089 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3090 break;
3091 case CHIP_SUMO2:
3092 rdev->config.evergreen.num_ses = 1;
3093 rdev->config.evergreen.max_pipes = 4;
3094 rdev->config.evergreen.max_tile_pipes = 4;
3095 rdev->config.evergreen.max_simds = 2;
3096 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3097 rdev->config.evergreen.max_gprs = 256;
3098 rdev->config.evergreen.max_threads = 248;
3099 rdev->config.evergreen.max_gs_threads = 32;
3100 rdev->config.evergreen.max_stack_entries = 512;
3101 rdev->config.evergreen.sx_num_of_sets = 4;
3102 rdev->config.evergreen.sx_max_export_size = 256;
3103 rdev->config.evergreen.sx_max_export_pos_size = 64;
3104 rdev->config.evergreen.sx_max_export_smx_size = 192;
3105 rdev->config.evergreen.max_hw_contexts = 8;
3106 rdev->config.evergreen.sq_num_cf_insts = 2;
3107
adb68fa2
AD		 3108 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3109 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3110 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3111 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3112 break;
3113 case CHIP_BARTS:
3114 rdev->config.evergreen.num_ses = 2;
3115 rdev->config.evergreen.max_pipes = 4;
3116 rdev->config.evergreen.max_tile_pipes = 8;
3117 rdev->config.evergreen.max_simds = 7;
3118 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3119 rdev->config.evergreen.max_gprs = 256;
3120 rdev->config.evergreen.max_threads = 248;
3121 rdev->config.evergreen.max_gs_threads = 32;
3122 rdev->config.evergreen.max_stack_entries = 512;
3123 rdev->config.evergreen.sx_num_of_sets = 4;
3124 rdev->config.evergreen.sx_max_export_size = 256;
3125 rdev->config.evergreen.sx_max_export_pos_size = 64;
3126 rdev->config.evergreen.sx_max_export_smx_size = 192;
3127 rdev->config.evergreen.max_hw_contexts = 8;
3128 rdev->config.evergreen.sq_num_cf_insts = 2;
3129
3130 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3131 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3132 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3133 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3134 break;
3135 case CHIP_TURKS:
3136 rdev->config.evergreen.num_ses = 1;
3137 rdev->config.evergreen.max_pipes = 4;
3138 rdev->config.evergreen.max_tile_pipes = 4;
3139 rdev->config.evergreen.max_simds = 6;
3140 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3141 rdev->config.evergreen.max_gprs = 256;
3142 rdev->config.evergreen.max_threads = 248;
3143 rdev->config.evergreen.max_gs_threads = 32;
3144 rdev->config.evergreen.max_stack_entries = 256;
3145 rdev->config.evergreen.sx_num_of_sets = 4;
3146 rdev->config.evergreen.sx_max_export_size = 256;
3147 rdev->config.evergreen.sx_max_export_pos_size = 64;
3148 rdev->config.evergreen.sx_max_export_smx_size = 192;
3149 rdev->config.evergreen.max_hw_contexts = 8;
3150 rdev->config.evergreen.sq_num_cf_insts = 2;
3151
3152 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3153 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3154 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3155 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3156 break;
3157 case CHIP_CAICOS:
3158 rdev->config.evergreen.num_ses = 1;
bd25f078 3159 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3160 rdev->config.evergreen.max_tile_pipes = 2;
3161 rdev->config.evergreen.max_simds = 2;
3162 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3163 rdev->config.evergreen.max_gprs = 256;
3164 rdev->config.evergreen.max_threads = 192;
3165 rdev->config.evergreen.max_gs_threads = 16;
3166 rdev->config.evergreen.max_stack_entries = 256;
3167 rdev->config.evergreen.sx_num_of_sets = 4;
3168 rdev->config.evergreen.sx_max_export_size = 128;
3169 rdev->config.evergreen.sx_max_export_pos_size = 32;
3170 rdev->config.evergreen.sx_max_export_smx_size = 96;
3171 rdev->config.evergreen.max_hw_contexts = 4;
3172 rdev->config.evergreen.sq_num_cf_insts = 1;
3173
32fcdbf4
AD		 3174 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3175 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3176 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3177 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3178 break;
3179 }
3180
3181 /* Initialize HDP */
3182 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3183 WREG32((0x2c14 + j), 0x00000000);
3184 WREG32((0x2c18 + j), 0x00000000);
3185 WREG32((0x2c1c + j), 0x00000000);
3186 WREG32((0x2c20 + j), 0x00000000);
3187 WREG32((0x2c24 + j), 0x00000000);
3188 }
3189
3190 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3191
d054ac16
AD		 3192 evergreen_fix_pci_max_read_req_size(rdev);
3193
32fcdbf4 3194 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3195 if ((rdev->family == CHIP_PALM) ||
3196 (rdev->family == CHIP_SUMO) ||
3197 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3198 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3199 else
3200 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3201
1aa52bd3
AD		 3202 /* setup tiling info dword. gb_addr_config is not adequate since it does
3203 * not have bank info, so create a custom tiling dword.
3204 * bits 3:0 num_pipes
3205 * bits 7:4 num_banks
3206 * bits 11:8 group_size
3207 * bits 15:12 row_size
3208 */
3209 rdev->config.evergreen.tile_config = 0;
3210 switch (rdev->config.evergreen.max_tile_pipes) {
3211 case 1:
3212 default:
3213 rdev->config.evergreen.tile_config |= (0 << 0);
3214 break;
3215 case 2:
3216 rdev->config.evergreen.tile_config |= (1 << 0);
3217 break;
3218 case 4:
3219 rdev->config.evergreen.tile_config |= (2 << 0);
3220 break;
3221 case 8:
3222 rdev->config.evergreen.tile_config |= (3 << 0);
3223 break;
3224 }
d698a34d 3225 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3226 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3227 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3228 else {
c8d15edc
AD		 3229 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3230 case 0: /* four banks */
29d65406 3231 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3232 break;
3233 case 1: /* eight banks */
3234 rdev->config.evergreen.tile_config |= 1 << 4;
3235 break;
3236 case 2: /* sixteen banks */
3237 default:
3238 rdev->config.evergreen.tile_config |= 2 << 4;
3239 break;
3240 }
29d65406 3241 }
416a2bd2 3242 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3243 rdev->config.evergreen.tile_config |=
3244 ((gb_addr_config & 0x30000000) >> 28) << 12;
3245
416a2bd2 3246 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3247
416a2bd2
AD		 3248 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3249 u32 efuse_straps_4;
3250 u32 efuse_straps_3;
32fcdbf4 3251
ff82bbc4
AD		 3252 efuse_straps_4 = RREG32_RCU(0x204);
3253 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3254 tmp = (((efuse_straps_4 & 0xf) << 4) |
3255 ((efuse_straps_3 & 0xf0000000) >> 28));
3256 } else {
3257 tmp = 0;
3258 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3259 u32 rb_disable_bitmap;
3260
3261 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3262 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3263 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3264 tmp <<= 4;
3265 tmp |= rb_disable_bitmap;
32fcdbf4 3266 }
416a2bd2
AD		 3267 }
3268 /* enabled rb are just the one not disabled :) */
3269 disabled_rb_mask = tmp;
cedb655a
AD		 3270 tmp = 0;
3271 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3272 tmp |= (1 << i);
3273 /* if all the backends are disabled, fix it up here */
3274 if ((disabled_rb_mask & tmp) == tmp) {
3275 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3276 disabled_rb_mask &= ~(1 << i);
3277 }
32fcdbf4 3278
416a2bd2
AD		 3279 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3280 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3281
416a2bd2
AD		 3282 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3283 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3284 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3285 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3286 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3287 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3288 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3289
f7eb9730
AD		 3290 if ((rdev->config.evergreen.max_backends == 1) &&
3291 (rdev->flags & RADEON_IS_IGP)) {
3292 if ((disabled_rb_mask & 3) == 1) {
3293 /* RB0 disabled, RB1 enabled */
3294 tmp = 0x11111111;
3295 } else {
3296 /* RB1 disabled, RB0 enabled */
3297 tmp = 0x00000000;
3298 }
3299 } else {
3300 tmp = gb_addr_config & NUM_PIPES_MASK;
3301 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3302 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3303 }
416a2bd2 3304 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3305
3306 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3307 WREG32(CGTS_TCC_DISABLE, 0);
3308 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3309 WREG32(CGTS_USER_TCC_DISABLE, 0);
3310
3311 /* set HW defaults for 3D engine */
3312 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3313 ROQ_IB2_START(0x2b)));
3314
3315 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3316
3317 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3318 SYNC_GRADIENT |
3319 SYNC_WALKER |
3320 SYNC_ALIGNER));
3321
3322 sx_debug_1 = RREG32(SX_DEBUG_1);
3323 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3324 WREG32(SX_DEBUG_1, sx_debug_1);
3325
3326
3327 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3328 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3329 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3330 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3331
b866d133
AD		 3332 if (rdev->family <= CHIP_SUMO2)
3333 WREG32(SMX_SAR_CTL0, 0x00010000);
3334
32fcdbf4
AD		 3335 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3336 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3337 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3338
3339 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3340 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3341 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3342
3343 WREG32(VGT_NUM_INSTANCES, 1);
3344 WREG32(SPI_CONFIG_CNTL, 0);
3345 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3346 WREG32(CP_PERFMON_CNTL, 0);
3347
3348 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3349 FETCH_FIFO_HIWATER(0x4) |
3350 DONE_FIFO_HIWATER(0xe0) |
3351 ALU_UPDATE_FIFO_HIWATER(0x8)));
3352
3353 sq_config = RREG32(SQ_CONFIG);
3354 sq_config &= ~(PS_PRIO(3) |
3355 VS_PRIO(3) |
3356 GS_PRIO(3) |
3357 ES_PRIO(3));
3358 sq_config |= (VC_ENABLE |
3359 EXPORT_SRC_C |
3360 PS_PRIO(0) |
3361 VS_PRIO(1) |
3362 GS_PRIO(2) |
3363 ES_PRIO(3));
3364
d5e455e4
AD		 3365 switch (rdev->family) {
3366 case CHIP_CEDAR:
3367 case CHIP_PALM:
d5c5a72f
AD		 3368 case CHIP_SUMO:
3369 case CHIP_SUMO2:
adb68fa2 3370 case CHIP_CAICOS:
32fcdbf4
AD		 3371 /* no vertex cache */
3372 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3373 break;
3374 default:
3375 break;
3376 }
32fcdbf4
AD		 3377
3378 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3379
3380 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3381 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3382 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3383 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3384 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3385 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3386 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3387
d5e455e4
AD		 3388 switch (rdev->family) {
3389 case CHIP_CEDAR:
3390 case CHIP_PALM:
d5c5a72f
AD		 3391 case CHIP_SUMO:
3392 case CHIP_SUMO2:
32fcdbf4 3393 ps_thread_count = 96;
d5e455e4
AD		 3394 break;
3395 default:
32fcdbf4 3396 ps_thread_count = 128;
d5e455e4
AD		 3397 break;
3398 }
32fcdbf4
AD		 3399
3400 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3401 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3402 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3403 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3404 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3405 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3406
3407 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3408 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3409 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3410 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3411 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3412 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3413
3414 WREG32(SQ_CONFIG, sq_config);
3415 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3416 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3417 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3418 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3419 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3420 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3421 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3422 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3423 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3424 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3425
3426 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3427 FORCE_EOV_MAX_REZ_CNT(255)));
3428
d5e455e4
AD		 3429 switch (rdev->family) {
3430 case CHIP_CEDAR:
3431 case CHIP_PALM:
d5c5a72f
AD		 3432 case CHIP_SUMO:
3433 case CHIP_SUMO2:
adb68fa2 3434 case CHIP_CAICOS:
32fcdbf4 3435 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3436 break;
3437 default:
32fcdbf4 3438 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3439 break;
3440 }
32fcdbf4
AD		 3441 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3442 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3443
3444 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3445 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3446 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3447
60a4a3e0
AD		 3448 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3449 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3450
32fcdbf4
AD		 3451 WREG32(CB_PERF_CTR0_SEL_0, 0);
3452 WREG32(CB_PERF_CTR0_SEL_1, 0);
3453 WREG32(CB_PERF_CTR1_SEL_0, 0);
3454 WREG32(CB_PERF_CTR1_SEL_1, 0);
3455 WREG32(CB_PERF_CTR2_SEL_0, 0);
3456 WREG32(CB_PERF_CTR2_SEL_1, 0);
3457 WREG32(CB_PERF_CTR3_SEL_0, 0);
3458 WREG32(CB_PERF_CTR3_SEL_1, 0);
3459
60a4a3e0
AD		 3460 /* clear render buffer base addresses */
3461 WREG32(CB_COLOR0_BASE, 0);
3462 WREG32(CB_COLOR1_BASE, 0);
3463 WREG32(CB_COLOR2_BASE, 0);
3464 WREG32(CB_COLOR3_BASE, 0);
3465 WREG32(CB_COLOR4_BASE, 0);
3466 WREG32(CB_COLOR5_BASE, 0);
3467 WREG32(CB_COLOR6_BASE, 0);
3468 WREG32(CB_COLOR7_BASE, 0);
3469 WREG32(CB_COLOR8_BASE, 0);
3470 WREG32(CB_COLOR9_BASE, 0);
3471 WREG32(CB_COLOR10_BASE, 0);
3472 WREG32(CB_COLOR11_BASE, 0);
3473
3474 /* set the shader const cache sizes to 0 */
3475 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3476 WREG32(i, 0);
3477 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3478 WREG32(i, 0);
3479
f25a5c63
AD		 3480 tmp = RREG32(HDP_MISC_CNTL);
3481 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3482 WREG32(HDP_MISC_CNTL, tmp);
3483
32fcdbf4
AD		 3484 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3485 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3486
3487 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3488
3489 udelay(50);
3490
bcc1c2a1
AD		 3491}
3492
3493int evergreen_mc_init(struct radeon_device *rdev)
3494{
bcc1c2a1
AD		 3495 u32 tmp;
3496 int chansize, numchan;
bcc1c2a1
AD		 3497
3498 /* Get VRAM informations */
3499 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3500 if ((rdev->family == CHIP_PALM) ||
3501 (rdev->family == CHIP_SUMO) ||
3502 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3503 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3504 else
3505 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3506 if (tmp & CHANSIZE_OVERRIDE) {
3507 chansize = 16;
3508 } else if (tmp & CHANSIZE_MASK) {
3509 chansize = 64;
3510 } else {
3511 chansize = 32;
3512 }
3513 tmp = RREG32(MC_SHARED_CHMAP);
3514 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3515 case 0:
3516 default:
3517 numchan = 1;
3518 break;
3519 case 1:
3520 numchan = 2;
3521 break;
3522 case 2:
3523 numchan = 4;
3524 break;
3525 case 3:
3526 numchan = 8;
3527 break;
3528 }
3529 rdev->mc.vram_width = numchan * chansize;
3530 /* Could aper size report 0 ? */
01d73a69
JC		 3531 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3532 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3533 /* Setup GPU memory space */
05b3ef69
AD		 3534 if ((rdev->family == CHIP_PALM) ||
3535 (rdev->family == CHIP_SUMO) ||
3536 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3537 /* size in bytes on fusion */
3538 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3539 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3540 } else {
05b3ef69 3541 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3542 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3543 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3544 }
51e5fcd3 3545 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3546 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3547 radeon_update_bandwidth_info(rdev);
3548
bcc1c2a1
AD		 3549 return 0;
3550}
d594e46a 3551
187e3593 3552void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3553{
64c56e8c 3554 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3555 RREG32(GRBM_STATUS));
64c56e8c 3556 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3557 RREG32(GRBM_STATUS_SE0));
64c56e8c 3558 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3559 RREG32(GRBM_STATUS_SE1));
64c56e8c 3560 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3561 RREG32(SRBM_STATUS));
a65a4369
AD		 3562 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3563 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3564 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3565 RREG32(CP_STALLED_STAT1));
3566 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3567 RREG32(CP_STALLED_STAT2));
3568 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3569 RREG32(CP_BUSY_STAT));
3570 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3571 RREG32(CP_STAT));
eaaa6983
JG		 3572 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3573 RREG32(DMA_STATUS_REG));
168757ea
AD		 3574 if (rdev->family >= CHIP_CAYMAN) {
3575 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3576 RREG32(DMA_STATUS_REG + 0x800));
3577 }
0ecebb9e
AD		 3578}
3579
168757ea 3580bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3581{
a65a4369
AD		 3582 u32 crtc_hung = 0;
3583 u32 crtc_status[6];
3584 u32 i, j, tmp;
3585
3586 for (i = 0; i < rdev->num_crtc; i++) {
3587 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3588 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3589 crtc_hung |= (1 << i);
3590 }
3591 }
3592
3593 for (j = 0; j < 10; j++) {
3594 for (i = 0; i < rdev->num_crtc; i++) {
3595 if (crtc_hung & (1 << i)) {
3596 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3597 if (tmp != crtc_status[i])
3598 crtc_hung &= ~(1 << i);
3599 }
3600 }
3601 if (crtc_hung == 0)
3602 return false;
3603 udelay(100);
3604 }
3605
3606 return true;
3607}
3608
3609static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3610{
3611 u32 reset_mask = 0;
b7630473 3612 u32 tmp;
0ecebb9e 3613
a65a4369
AD		 3614 /* GRBM_STATUS */
3615 tmp = RREG32(GRBM_STATUS);
3616 if (tmp & (PA_BUSY | SC_BUSY |
3617 SH_BUSY | SX_BUSY |
3618 TA_BUSY | VGT_BUSY |
3619 DB_BUSY | CB_BUSY |
3620 SPI_BUSY | VGT_BUSY_NO_DMA))
3621 reset_mask |= RADEON_RESET_GFX;
3622
3623 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3624 CP_BUSY | CP_COHERENCY_BUSY))
3625 reset_mask |= RADEON_RESET_CP;
3626
3627 if (tmp & GRBM_EE_BUSY)
3628 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3629
a65a4369
AD		 3630 /* DMA_STATUS_REG */
3631 tmp = RREG32(DMA_STATUS_REG);
3632 if (!(tmp & DMA_IDLE))
3633 reset_mask |= RADEON_RESET_DMA;
3634
3635 /* SRBM_STATUS2 */
3636 tmp = RREG32(SRBM_STATUS2);
3637 if (tmp & DMA_BUSY)
3638 reset_mask |= RADEON_RESET_DMA;
3639
3640 /* SRBM_STATUS */
3641 tmp = RREG32(SRBM_STATUS);
3642 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3643 reset_mask |= RADEON_RESET_RLC;
3644
3645 if (tmp & IH_BUSY)
3646 reset_mask |= RADEON_RESET_IH;
3647
3648 if (tmp & SEM_BUSY)
3649 reset_mask |= RADEON_RESET_SEM;
3650
3651 if (tmp & GRBM_RQ_PENDING)
3652 reset_mask |= RADEON_RESET_GRBM;
3653
3654 if (tmp & VMC_BUSY)
3655 reset_mask |= RADEON_RESET_VMC;
3656
3657 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3658 MCC_BUSY | MCD_BUSY))
3659 reset_mask |= RADEON_RESET_MC;
3660
3661 if (evergreen_is_display_hung(rdev))
3662 reset_mask |= RADEON_RESET_DISPLAY;
3663
3664 /* VM_L2_STATUS */
3665 tmp = RREG32(VM_L2_STATUS);
3666 if (tmp & L2_BUSY)
3667 reset_mask |= RADEON_RESET_VMC;
3668
d808fc88
AD		 3669 /* Skip MC reset as it's mostly likely not hung, just busy */
3670 if (reset_mask & RADEON_RESET_MC) {
3671 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3672 reset_mask &= ~RADEON_RESET_MC;
3673 }
3674
a65a4369
AD		 3675 return reset_mask;
3676}
3677
3678static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3679{
3680 struct evergreen_mc_save save;
3681 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3682 u32 tmp;
19fc42ed 3683
0ecebb9e 3684 if (reset_mask == 0)
a65a4369 3685 return;
0ecebb9e
AD		 3686
3687 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3688
b7630473
AD		 3689 evergreen_print_gpu_status_regs(rdev);
3690
b7630473
AD		 3691 /* Disable CP parsing/prefetching */
3692 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3693
3694 if (reset_mask & RADEON_RESET_DMA) {
3695 /* Disable DMA */
3696 tmp = RREG32(DMA_RB_CNTL);
3697 tmp &= ~DMA_RB_ENABLE;
3698 WREG32(DMA_RB_CNTL, tmp);
3699 }
3700
b21b6e7a
AD		 3701 udelay(50);
3702
3703 evergreen_mc_stop(rdev, &save);
3704 if (evergreen_mc_wait_for_idle(rdev)) {
3705 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3706 }
3707
b7630473
AD		 3708 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3709 grbm_soft_reset |= SOFT_RESET_DB |
3710 SOFT_RESET_CB |
3711 SOFT_RESET_PA |
3712 SOFT_RESET_SC |
3713 SOFT_RESET_SPI |
3714 SOFT_RESET_SX |
3715 SOFT_RESET_SH |
3716 SOFT_RESET_TC |
3717 SOFT_RESET_TA |
3718 SOFT_RESET_VC |
3719 SOFT_RESET_VGT;
3720 }
3721
3722 if (reset_mask & RADEON_RESET_CP) {
3723 grbm_soft_reset |= SOFT_RESET_CP |
3724 SOFT_RESET_VGT;
3725
3726 srbm_soft_reset |= SOFT_RESET_GRBM;
3727 }
0ecebb9e
AD		 3728
3729 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3730 srbm_soft_reset |= SOFT_RESET_DMA;
3731
a65a4369
AD		 3732 if (reset_mask & RADEON_RESET_DISPLAY)
3733 srbm_soft_reset |= SOFT_RESET_DC;
3734
3735 if (reset_mask & RADEON_RESET_RLC)
3736 srbm_soft_reset |= SOFT_RESET_RLC;
3737
3738 if (reset_mask & RADEON_RESET_SEM)
3739 srbm_soft_reset |= SOFT_RESET_SEM;
3740
3741 if (reset_mask & RADEON_RESET_IH)
3742 srbm_soft_reset |= SOFT_RESET_IH;
3743
3744 if (reset_mask & RADEON_RESET_GRBM)
3745 srbm_soft_reset |= SOFT_RESET_GRBM;
3746
3747 if (reset_mask & RADEON_RESET_VMC)
3748 srbm_soft_reset |= SOFT_RESET_VMC;
3749
24178ec4
AD		 3750 if (!(rdev->flags & RADEON_IS_IGP)) {
3751 if (reset_mask & RADEON_RESET_MC)
3752 srbm_soft_reset |= SOFT_RESET_MC;
3753 }
a65a4369 3754
b7630473
AD		 3755 if (grbm_soft_reset) {
3756 tmp = RREG32(GRBM_SOFT_RESET);
3757 tmp |= grbm_soft_reset;
3758 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3759 WREG32(GRBM_SOFT_RESET, tmp);
3760 tmp = RREG32(GRBM_SOFT_RESET);
3761
3762 udelay(50);
3763
3764 tmp &= ~grbm_soft_reset;
3765 WREG32(GRBM_SOFT_RESET, tmp);
3766 tmp = RREG32(GRBM_SOFT_RESET);
3767 }
3768
3769 if (srbm_soft_reset) {
3770 tmp = RREG32(SRBM_SOFT_RESET);
3771 tmp |= srbm_soft_reset;
3772 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3773 WREG32(SRBM_SOFT_RESET, tmp);
3774 tmp = RREG32(SRBM_SOFT_RESET);
3775
3776 udelay(50);
3777
3778 tmp &= ~srbm_soft_reset;
3779 WREG32(SRBM_SOFT_RESET, tmp);
3780 tmp = RREG32(SRBM_SOFT_RESET);
3781 }
0ecebb9e
AD		 3782
3783 /* Wait a little for things to settle down */
3784 udelay(50);
3785
747943ea 3786 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3787 udelay(50);
3788
b7630473 3789 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3790}
3791
a2d07b74 3792int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3793{
a65a4369
AD		 3794 u32 reset_mask;
3795
3796 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3797
3798 if (reset_mask)
3799 r600_set_bios_scratch_engine_hung(rdev, true);
3800
3801 evergreen_gpu_soft_reset(rdev, reset_mask);
3802
3803 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3804
3805 if (!reset_mask)
3806 r600_set_bios_scratch_engine_hung(rdev, false);
3807
3808 return 0;
747943ea
AD		 3809}
3810
123bc183
AD		 3811/**
3812 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3813 *
3814 * @rdev: radeon_device pointer
3815 * @ring: radeon_ring structure holding ring information
3816 *
3817 * Check if the GFX engine is locked up.
3818 * Returns true if the engine appears to be locked up, false if not.
3819 */
3820bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3821{
3822 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3823
3824 if (!(reset_mask & (RADEON_RESET_GFX |
3825 RADEON_RESET_COMPUTE |
3826 RADEON_RESET_CP))) {
3827 radeon_ring_lockup_update(ring);
3828 return false;
3829 }
3830 /* force CP activities */
3831 radeon_ring_force_activity(rdev, ring);
3832 return radeon_ring_test_lockup(rdev, ring);
3833}
3834
3835/**
3836 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3837 *
3838 * @rdev: radeon_device pointer
3839 * @ring: radeon_ring structure holding ring information
3840 *
3841 * Check if the async DMA engine is locked up.
3842 * Returns true if the engine appears to be locked up, false if not.
3843 */
3844bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3845{
3846 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3847
3848 if (!(reset_mask & RADEON_RESET_DMA)) {
3849 radeon_ring_lockup_update(ring);
3850 return false;
3851 }
3852 /* force ring activities */
3853 radeon_ring_force_activity(rdev, ring);
3854 return radeon_ring_test_lockup(rdev, ring);
3855}
3856
2948f5e6
AD		 3857/*
3858 * RLC
3859 */
3860#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3861#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3862
3863void sumo_rlc_fini(struct radeon_device *rdev)
3864{
3865 int r;
3866
3867 /* save restore block */
3868 if (rdev->rlc.save_restore_obj) {
3869 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3870 if (unlikely(r != 0))
3871 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3872 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3873 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3874
3875 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3876 rdev->rlc.save_restore_obj = NULL;
3877 }
3878
3879 /* clear state block */
3880 if (rdev->rlc.clear_state_obj) {
3881 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3882 if (unlikely(r != 0))
3883 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3884 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3885 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3886
3887 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3888 rdev->rlc.clear_state_obj = NULL;
3889 }
3890}
3891
3892int sumo_rlc_init(struct radeon_device *rdev)
3893{
3894 u32 *src_ptr;
3895 volatile u32 *dst_ptr;
3896 u32 dws, data, i, j, k, reg_num;
3897 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3898 u64 reg_list_mc_addr;
3899 struct cs_section_def *cs_data;
3900 int r;
3901
3902 src_ptr = rdev->rlc.reg_list;
3903 dws = rdev->rlc.reg_list_size;
3904 cs_data = rdev->rlc.cs_data;
3905
3906 /* save restore block */
3907 if (rdev->rlc.save_restore_obj == NULL) {
3908 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3909 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3910 if (r) {
3911 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3912 return r;
3913 }
3914 }
3915
3916 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3917 if (unlikely(r != 0)) {
3918 sumo_rlc_fini(rdev);
3919 return r;
3920 }
3921 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3922 &rdev->rlc.save_restore_gpu_addr);
3923 if (r) {
3924 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3925 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3926 sumo_rlc_fini(rdev);
3927 return r;
3928 }
3929 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3930 if (r) {
3931 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3932 sumo_rlc_fini(rdev);
3933 return r;
3934 }
3935 /* write the sr buffer */
3936 dst_ptr = rdev->rlc.sr_ptr;
3937 /* format:
3938 * dw0: (reg2 << 16) | reg1
3939 * dw1: reg1 save space
3940 * dw2: reg2 save space
3941 */
3942 for (i = 0; i < dws; i++) {
3943 data = src_ptr[i] >> 2;
3944 i++;
3945 if (i < dws)
3946 data |= (src_ptr[i] >> 2) << 16;
3947 j = (((i - 1) * 3) / 2);
3948 dst_ptr[j] = data;
3949 }
3950 j = ((i * 3) / 2);
3951 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3952
3953 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3954 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3955
3956 /* clear state block */
3957 reg_list_num = 0;
3958 dws = 0;
3959 for (i = 0; cs_data[i].section != NULL; i++) {
3960 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3961 reg_list_num++;
3962 dws += cs_data[i].section[j].reg_count;
3963 }
3964 }
3965 reg_list_blk_index = (3 * reg_list_num + 2);
3966 dws += reg_list_blk_index;
3967
3968 if (rdev->rlc.clear_state_obj == NULL) {
3969 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3970 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3971 if (r) {
3972 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3973 sumo_rlc_fini(rdev);
3974 return r;
3975 }
3976 }
3977 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3978 if (unlikely(r != 0)) {
3979 sumo_rlc_fini(rdev);
3980 return r;
3981 }
3982 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3983 &rdev->rlc.clear_state_gpu_addr);
3984 if (r) {
3985
3986 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3987 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3988 sumo_rlc_fini(rdev);
3989 return r;
3990 }
3991 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
3992 if (r) {
3993 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
3994 sumo_rlc_fini(rdev);
3995 return r;
3996 }
3997 /* set up the cs buffer */
3998 dst_ptr = rdev->rlc.cs_ptr;
3999 reg_list_hdr_blk_index = 0;
4000 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4001 data = upper_32_bits(reg_list_mc_addr);
4002 dst_ptr[reg_list_hdr_blk_index] = data;
4003 reg_list_hdr_blk_index++;
4004 for (i = 0; cs_data[i].section != NULL; i++) {
4005 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4006 reg_num = cs_data[i].section[j].reg_count;
4007 data = reg_list_mc_addr & 0xffffffff;
4008 dst_ptr[reg_list_hdr_blk_index] = data;
4009 reg_list_hdr_blk_index++;
4010
4011 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4012 dst_ptr[reg_list_hdr_blk_index] = data;
4013 reg_list_hdr_blk_index++;
4014
4015 data = 0x08000000 | (reg_num * 4);
4016 dst_ptr[reg_list_hdr_blk_index] = data;
4017 reg_list_hdr_blk_index++;
4018
4019 for (k = 0; k < reg_num; k++) {
4020 data = cs_data[i].section[j].extent[k];
4021 dst_ptr[reg_list_blk_index + k] = data;
4022 }
4023 reg_list_mc_addr += reg_num * 4;
4024 reg_list_blk_index += reg_num;
4025 }
4026 }
4027 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4028
4029 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4030 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4031
4032 return 0;
4033}
4034
4035static void evergreen_rlc_start(struct radeon_device *rdev)
4036{
4037 if (rdev->flags & RADEON_IS_IGP)
4038 WREG32(RLC_CNTL, RLC_ENABLE | GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC);
4039 else
4040 WREG32(RLC_CNTL, RLC_ENABLE);
4041}
4042
4043int evergreen_rlc_resume(struct radeon_device *rdev)
4044{
4045 u32 i;
4046 const __be32 *fw_data;
4047
4048 if (!rdev->rlc_fw)
4049 return -EINVAL;
4050
4051 r600_rlc_stop(rdev);
4052
4053 WREG32(RLC_HB_CNTL, 0);
4054
4055 if (rdev->flags & RADEON_IS_IGP) {
4056 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4057 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4058 } else {
4059 WREG32(RLC_HB_BASE, 0);
4060 WREG32(RLC_HB_RPTR, 0);
4061 WREG32(RLC_HB_WPTR, 0);
4062 }
4063 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4064 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4065 WREG32(RLC_MC_CNTL, 0);
4066 WREG32(RLC_UCODE_CNTL, 0);
4067
4068 fw_data = (const __be32 *)rdev->rlc_fw->data;
4069 if (rdev->family >= CHIP_ARUBA) {
4070 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4071 WREG32(RLC_UCODE_ADDR, i);
4072 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4073 }
4074 } else if (rdev->family >= CHIP_CAYMAN) {
4075 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4076 WREG32(RLC_UCODE_ADDR, i);
4077 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4078 }
4079 } else {
4080 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4081 WREG32(RLC_UCODE_ADDR, i);
4082 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4083 }
4084 }
4085 WREG32(RLC_UCODE_ADDR, 0);
4086
4087 evergreen_rlc_start(rdev);
4088
4089 return 0;
4090}
4091
45f9a39b
AD		 4092/* Interrupts */
4093
4094u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4095{
46437057 4096 if (crtc >= rdev->num_crtc)
45f9a39b 4097 return 0;
46437057
AD		 4098 else
4099 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4100}
4101
4102void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4103{
4104 u32 tmp;
4105
1b37078b
AD		 4106 if (rdev->family >= CHIP_CAYMAN) {
4107 cayman_cp_int_cntl_setup(rdev, 0,
4108 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4109 cayman_cp_int_cntl_setup(rdev, 1, 0);
4110 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4111 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4112 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4113 } else
4114 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4115 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4116 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4117 WREG32(GRBM_INT_CNTL, 0);
4118 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4119 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4120 if (rdev->num_crtc >= 4) {
18007401
AD		 4121 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4122 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4123 }
4124 if (rdev->num_crtc >= 6) {
18007401
AD		 4125 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4126 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4127 }
45f9a39b
AD		 4128
4129 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4130 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4131 if (rdev->num_crtc >= 4) {
18007401
AD		 4132 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4133 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4134 }
4135 if (rdev->num_crtc >= 6) {
18007401
AD		 4136 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4137 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4138 }
45f9a39b 4139
05b3ef69
AD		 4140 /* only one DAC on DCE6 */
4141 if (!ASIC_IS_DCE6(rdev))
4142 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4143 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4144
4145 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4146 WREG32(DC_HPD1_INT_CONTROL, tmp);
4147 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4148 WREG32(DC_HPD2_INT_CONTROL, tmp);
4149 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4150 WREG32(DC_HPD3_INT_CONTROL, tmp);
4151 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4152 WREG32(DC_HPD4_INT_CONTROL, tmp);
4153 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4154 WREG32(DC_HPD5_INT_CONTROL, tmp);
4155 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4156 WREG32(DC_HPD6_INT_CONTROL, tmp);
4157
4158}
4159
4160int evergreen_irq_set(struct radeon_device *rdev)
4161{
4162 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4163 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4164 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4165 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4166 u32 grbm_int_cntl = 0;
6f34be50 4167 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4168 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4169 u32 dma_cntl, dma_cntl1 = 0;
45f9a39b
AD		 4170
4171 if (!rdev->irq.installed) {
fce7d61b 4172 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4173 return -EINVAL;
4174 }
4175 /* don't enable anything if the ih is disabled */
4176 if (!rdev->ih.enabled) {
4177 r600_disable_interrupts(rdev);
4178 /* force the active interrupt state to all disabled */
4179 evergreen_disable_interrupt_state(rdev);
4180 return 0;
4181 }
4182
4183 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4184 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4185 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4186 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4187 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4188 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4189
f122c610
AD		 4190 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4191 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4192 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4193 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4194 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4195 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4196
233d1ad5
AD		 4197 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4198
1b37078b
AD		 4199 if (rdev->family >= CHIP_CAYMAN) {
4200 /* enable CP interrupts on all rings */
736fc37f 4201 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4202 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4203 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4204 }
736fc37f 4205 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4206 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4207 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4208 }
736fc37f 4209 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4210 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4211 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4212 }
4213 } else {
736fc37f 4214 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4215 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4216 cp_int_cntl |= RB_INT_ENABLE;
4217 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4218 }
45f9a39b 4219 }
1b37078b 4220
233d1ad5
AD		 4221 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4222 DRM_DEBUG("r600_irq_set: sw int dma\n");
4223 dma_cntl |= TRAP_ENABLE;
4224 }
4225
f60cbd11
AD		 4226 if (rdev->family >= CHIP_CAYMAN) {
4227 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4228 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4229 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4230 dma_cntl1 |= TRAP_ENABLE;
4231 }
4232 }
4233
6f34be50 4234 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4235 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4236 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4237 crtc1 |= VBLANK_INT_MASK;
4238 }
6f34be50 4239 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4240 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4241 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4242 crtc2 |= VBLANK_INT_MASK;
4243 }
6f34be50 4244 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4245 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4246 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4247 crtc3 |= VBLANK_INT_MASK;
4248 }
6f34be50 4249 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4250 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4251 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4252 crtc4 |= VBLANK_INT_MASK;
4253 }
6f34be50 4254 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4255 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4256 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4257 crtc5 |= VBLANK_INT_MASK;
4258 }
6f34be50 4259 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4260 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4261 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4262 crtc6 |= VBLANK_INT_MASK;
4263 }
4264 if (rdev->irq.hpd[0]) {
4265 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4266 hpd1 |= DC_HPDx_INT_EN;
4267 }
4268 if (rdev->irq.hpd[1]) {
4269 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4270 hpd2 |= DC_HPDx_INT_EN;
4271 }
4272 if (rdev->irq.hpd[2]) {
4273 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4274 hpd3 |= DC_HPDx_INT_EN;
4275 }
4276 if (rdev->irq.hpd[3]) {
4277 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4278 hpd4 |= DC_HPDx_INT_EN;
4279 }
4280 if (rdev->irq.hpd[4]) {
4281 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4282 hpd5 |= DC_HPDx_INT_EN;
4283 }
4284 if (rdev->irq.hpd[5]) {
4285 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4286 hpd6 |= DC_HPDx_INT_EN;
4287 }
f122c610
AD		 4288 if (rdev->irq.afmt[0]) {
4289 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4290 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4291 }
4292 if (rdev->irq.afmt[1]) {
4293 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4294 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4295 }
4296 if (rdev->irq.afmt[2]) {
4297 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4298 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4299 }
4300 if (rdev->irq.afmt[3]) {
4301 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4302 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4303 }
4304 if (rdev->irq.afmt[4]) {
4305 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4306 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4307 }
4308 if (rdev->irq.afmt[5]) {
4309 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4310 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4311 }
45f9a39b 4312
1b37078b
AD		 4313 if (rdev->family >= CHIP_CAYMAN) {
4314 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4315 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4316 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4317 } else
4318 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4319
4320 WREG32(DMA_CNTL, dma_cntl);
4321
f60cbd11
AD		 4322 if (rdev->family >= CHIP_CAYMAN)
4323 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4324
2031f77c 4325 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4326
4327 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4328 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4329 if (rdev->num_crtc >= 4) {
18007401
AD		 4330 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4331 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4332 }
4333 if (rdev->num_crtc >= 6) {
18007401
AD		 4334 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4335 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4336 }
45f9a39b 4337
6f34be50
AD		 4338 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4339 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4340 if (rdev->num_crtc >= 4) {
4341 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4342 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4343 }
4344 if (rdev->num_crtc >= 6) {
4345 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4346 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4347 }
6f34be50 4348
45f9a39b
AD		 4349 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4350 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4351 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4352 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4353 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4354 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4355
f122c610
AD		 4356 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4357 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4358 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4359 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4360 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4361 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4362
bcc1c2a1
AD		 4363 return 0;
4364}
4365
cbdd4501 4366static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4367{
4368 u32 tmp;
4369
6f34be50
AD		 4370 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4371 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4372 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4373 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4374 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4375 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4376 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4377 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4378 if (rdev->num_crtc >= 4) {
4379 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4380 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4381 }
4382 if (rdev->num_crtc >= 6) {
4383 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4384 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4385 }
6f34be50 4386
f122c610
AD		 4387 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4388 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4389 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4390 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4391 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4392 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4393
6f34be50
AD		 4394 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4395 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4396 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4397 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4398 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4399 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4400 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4401 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4402 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4403 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4404 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4405 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4406
b7eff394
AD		 4407 if (rdev->num_crtc >= 4) {
4408 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4409 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4410 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4411 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4412 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4413 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4414 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4415 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4416 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4417 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4418 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4419 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4420 }
4421
4422 if (rdev->num_crtc >= 6) {
4423 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4424 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4425 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4426 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4427 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4428 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4429 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4430 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4431 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4432 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4433 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4434 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4435 }
45f9a39b 4436
6f34be50 4437 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4438 tmp = RREG32(DC_HPD1_INT_CONTROL);
4439 tmp |= DC_HPDx_INT_ACK;
4440 WREG32(DC_HPD1_INT_CONTROL, tmp);
4441 }
6f34be50 4442 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4443 tmp = RREG32(DC_HPD2_INT_CONTROL);
4444 tmp |= DC_HPDx_INT_ACK;
4445 WREG32(DC_HPD2_INT_CONTROL, tmp);
4446 }
6f34be50 4447 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4448 tmp = RREG32(DC_HPD3_INT_CONTROL);
4449 tmp |= DC_HPDx_INT_ACK;
4450 WREG32(DC_HPD3_INT_CONTROL, tmp);
4451 }
6f34be50 4452 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4453 tmp = RREG32(DC_HPD4_INT_CONTROL);
4454 tmp |= DC_HPDx_INT_ACK;
4455 WREG32(DC_HPD4_INT_CONTROL, tmp);
4456 }
6f34be50 4457 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4458 tmp = RREG32(DC_HPD5_INT_CONTROL);
4459 tmp |= DC_HPDx_INT_ACK;
4460 WREG32(DC_HPD5_INT_CONTROL, tmp);
4461 }
6f34be50 4462 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4463 tmp = RREG32(DC_HPD5_INT_CONTROL);
4464 tmp |= DC_HPDx_INT_ACK;
4465 WREG32(DC_HPD6_INT_CONTROL, tmp);
4466 }
f122c610
AD		 4467 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4468 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4469 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4470 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4471 }
4472 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4473 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4474 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4475 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4476 }
4477 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4478 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4479 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4480 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4481 }
4482 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4483 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4484 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4485 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4486 }
4487 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4488 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4489 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4490 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4491 }
4492 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4493 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4494 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4495 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4496 }
45f9a39b
AD		 4497}
4498
1109ca09 4499static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4500{
45f9a39b
AD		 4501 r600_disable_interrupts(rdev);
4502 /* Wait and acknowledge irq */
4503 mdelay(1);
6f34be50 4504 evergreen_irq_ack(rdev);
45f9a39b
AD		 4505 evergreen_disable_interrupt_state(rdev);
4506}
4507
755d819e 4508void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4509{
4510 evergreen_irq_disable(rdev);
4511 r600_rlc_stop(rdev);
4512}
4513
cbdd4501 4514static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4515{
4516 u32 wptr, tmp;
4517
724c80e1 4518 if (rdev->wb.enabled)
204ae24d 4519 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4520 else
4521 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4522
4523 if (wptr & RB_OVERFLOW) {
4524 /* When a ring buffer overflow happen start parsing interrupt
4525 * from the last not overwritten vector (wptr + 16). Hopefully
4526 * this should allow us to catchup.
4527 */
4528 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4529 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4530 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4531 tmp = RREG32(IH_RB_CNTL);
4532 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4533 WREG32(IH_RB_CNTL, tmp);
4534 }
4535 return (wptr & rdev->ih.ptr_mask);
4536}
4537
4538int evergreen_irq_process(struct radeon_device *rdev)
4539{
682f1a54
DA		 4540 u32 wptr;
4541 u32 rptr;
45f9a39b
AD		 4542 u32 src_id, src_data;
4543 u32 ring_index;
45f9a39b 4544 bool queue_hotplug = false;
f122c610 4545 bool queue_hdmi = false;
45f9a39b 4546
682f1a54 4547 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4548 return IRQ_NONE;
4549
682f1a54 4550 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4551
4552restart_ih:
4553 /* is somebody else already processing irqs? */
4554 if (atomic_xchg(&rdev->ih.lock, 1))
4555 return IRQ_NONE;
4556
682f1a54
DA		 4557 rptr = rdev->ih.rptr;
4558 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4559
964f6645
BH		 4560 /* Order reading of wptr vs. reading of IH ring data */
4561 rmb();
4562
45f9a39b 4563 /* display interrupts */
6f34be50 4564 evergreen_irq_ack(rdev);
45f9a39b 4565
45f9a39b
AD		 4566 while (rptr != wptr) {
4567 /* wptr/rptr are in bytes! */
4568 ring_index = rptr / 4;
0f234f5f
AD		 4569 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4570 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4571
4572 switch (src_id) {
4573 case 1: /* D1 vblank/vline */
4574 switch (src_data) {
4575 case 0: /* D1 vblank */
6f34be50 4576 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4577 if (rdev->irq.crtc_vblank_int[0]) {
4578 drm_handle_vblank(rdev->ddev, 0);
4579 rdev->pm.vblank_sync = true;
4580 wake_up(&rdev->irq.vblank_queue);
4581 }
736fc37f 4582 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4583 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4584 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4585 DRM_DEBUG("IH: D1 vblank\n");
4586 }
4587 break;
4588 case 1: /* D1 vline */
6f34be50
AD		 4589 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4590 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4591 DRM_DEBUG("IH: D1 vline\n");
4592 }
4593 break;
4594 default:
4595 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4596 break;
4597 }
4598 break;
4599 case 2: /* D2 vblank/vline */
4600 switch (src_data) {
4601 case 0: /* D2 vblank */
6f34be50 4602 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4603 if (rdev->irq.crtc_vblank_int[1]) {
4604 drm_handle_vblank(rdev->ddev, 1);
4605 rdev->pm.vblank_sync = true;
4606 wake_up(&rdev->irq.vblank_queue);
4607 }
736fc37f 4608 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4609 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4610 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4611 DRM_DEBUG("IH: D2 vblank\n");
4612 }
4613 break;
4614 case 1: /* D2 vline */
6f34be50
AD		 4615 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4616 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4617 DRM_DEBUG("IH: D2 vline\n");
4618 }
4619 break;
4620 default:
4621 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4622 break;
4623 }
4624 break;
4625 case 3: /* D3 vblank/vline */
4626 switch (src_data) {
4627 case 0: /* D3 vblank */
6f34be50
AD		 4628 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4629 if (rdev->irq.crtc_vblank_int[2]) {
4630 drm_handle_vblank(rdev->ddev, 2);
4631 rdev->pm.vblank_sync = true;
4632 wake_up(&rdev->irq.vblank_queue);
4633 }
736fc37f 4634 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4635 radeon_crtc_handle_flip(rdev, 2);
4636 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4637 DRM_DEBUG("IH: D3 vblank\n");
4638 }
4639 break;
4640 case 1: /* D3 vline */
6f34be50
AD		 4641 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4642 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4643 DRM_DEBUG("IH: D3 vline\n");
4644 }
4645 break;
4646 default:
4647 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4648 break;
4649 }
4650 break;
4651 case 4: /* D4 vblank/vline */
4652 switch (src_data) {
4653 case 0: /* D4 vblank */
6f34be50
AD		 4654 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4655 if (rdev->irq.crtc_vblank_int[3]) {
4656 drm_handle_vblank(rdev->ddev, 3);
4657 rdev->pm.vblank_sync = true;
4658 wake_up(&rdev->irq.vblank_queue);
4659 }
736fc37f 4660 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4661 radeon_crtc_handle_flip(rdev, 3);
4662 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4663 DRM_DEBUG("IH: D4 vblank\n");
4664 }
4665 break;
4666 case 1: /* D4 vline */
6f34be50
AD		 4667 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4668 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4669 DRM_DEBUG("IH: D4 vline\n");
4670 }
4671 break;
4672 default:
4673 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4674 break;
4675 }
4676 break;
4677 case 5: /* D5 vblank/vline */
4678 switch (src_data) {
4679 case 0: /* D5 vblank */
6f34be50
AD		 4680 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4681 if (rdev->irq.crtc_vblank_int[4]) {
4682 drm_handle_vblank(rdev->ddev, 4);
4683 rdev->pm.vblank_sync = true;
4684 wake_up(&rdev->irq.vblank_queue);
4685 }
736fc37f 4686 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4687 radeon_crtc_handle_flip(rdev, 4);
4688 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4689 DRM_DEBUG("IH: D5 vblank\n");
4690 }
4691 break;
4692 case 1: /* D5 vline */
6f34be50
AD		 4693 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4694 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4695 DRM_DEBUG("IH: D5 vline\n");
4696 }
4697 break;
4698 default:
4699 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4700 break;
4701 }
4702 break;
4703 case 6: /* D6 vblank/vline */
4704 switch (src_data) {
4705 case 0: /* D6 vblank */
6f34be50
AD		 4706 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4707 if (rdev->irq.crtc_vblank_int[5]) {
4708 drm_handle_vblank(rdev->ddev, 5);
4709 rdev->pm.vblank_sync = true;
4710 wake_up(&rdev->irq.vblank_queue);
4711 }
736fc37f 4712 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4713 radeon_crtc_handle_flip(rdev, 5);
4714 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4715 DRM_DEBUG("IH: D6 vblank\n");
4716 }
4717 break;
4718 case 1: /* D6 vline */
6f34be50
AD		 4719 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4720 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4721 DRM_DEBUG("IH: D6 vline\n");
4722 }
4723 break;
4724 default:
4725 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4726 break;
4727 }
4728 break;
4729 case 42: /* HPD hotplug */
4730 switch (src_data) {
4731 case 0:
6f34be50
AD		 4732 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4733 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4734 queue_hotplug = true;
4735 DRM_DEBUG("IH: HPD1\n");
4736 }
4737 break;
4738 case 1:
6f34be50
AD		 4739 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4740 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4741 queue_hotplug = true;
4742 DRM_DEBUG("IH: HPD2\n");
4743 }
4744 break;
4745 case 2:
6f34be50
AD		 4746 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4747 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4748 queue_hotplug = true;
4749 DRM_DEBUG("IH: HPD3\n");
4750 }
4751 break;
4752 case 3:
6f34be50
AD		 4753 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4754 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4755 queue_hotplug = true;
4756 DRM_DEBUG("IH: HPD4\n");
4757 }
4758 break;
4759 case 4:
6f34be50
AD		 4760 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4761 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4762 queue_hotplug = true;
4763 DRM_DEBUG("IH: HPD5\n");
4764 }
4765 break;
4766 case 5:
6f34be50
AD		 4767 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4768 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4769 queue_hotplug = true;
4770 DRM_DEBUG("IH: HPD6\n");
4771 }
4772 break;
4773 default:
4774 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4775 break;
4776 }
4777 break;
f122c610
AD		 4778 case 44: /* hdmi */
4779 switch (src_data) {
4780 case 0:
4781 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4782 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4783 queue_hdmi = true;
4784 DRM_DEBUG("IH: HDMI0\n");
4785 }
4786 break;
4787 case 1:
4788 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4789 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4790 queue_hdmi = true;
4791 DRM_DEBUG("IH: HDMI1\n");
4792 }
4793 break;
4794 case 2:
4795 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4796 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4797 queue_hdmi = true;
4798 DRM_DEBUG("IH: HDMI2\n");
4799 }
4800 break;
4801 case 3:
4802 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4803 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4804 queue_hdmi = true;
4805 DRM_DEBUG("IH: HDMI3\n");
4806 }
4807 break;
4808 case 4:
4809 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4810 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4811 queue_hdmi = true;
4812 DRM_DEBUG("IH: HDMI4\n");
4813 }
4814 break;
4815 case 5:
4816 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4817 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4818 queue_hdmi = true;
4819 DRM_DEBUG("IH: HDMI5\n");
4820 }
4821 break;
4822 default:
4823 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4824 break;
4825 }
f2ba57b5
CK		 4826 case 124: /* UVD */
4827 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4828 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 4829 break;
ae133a11
CK		 4830 case 146:
4831 case 147:
4832 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4833 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4834 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4835 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4836 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4837 /* reset addr and status */
4838 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4839 break;
45f9a39b
AD		 4840 case 176: /* CP_INT in ring buffer */
4841 case 177: /* CP_INT in IB1 */
4842 case 178: /* CP_INT in IB2 */
4843 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 4844 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 4845 break;
4846 case 181: /* CP EOP event */
4847 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 4848 if (rdev->family >= CHIP_CAYMAN) {
4849 switch (src_data) {
4850 case 0:
4851 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4852 break;
4853 case 1:
4854 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4855 break;
4856 case 2:
4857 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4858 break;
4859 }
4860 } else
4861 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 4862 break;
233d1ad5
AD		 4863 case 224: /* DMA trap event */
4864 DRM_DEBUG("IH: DMA trap\n");
4865 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4866 break;
2031f77c 4867 case 233: /* GUI IDLE */
303c805c 4868 DRM_DEBUG("IH: GUI idle\n");
2031f77c 4869 break;
f60cbd11
AD		 4870 case 244: /* DMA trap event */
4871 if (rdev->family >= CHIP_CAYMAN) {
4872 DRM_DEBUG("IH: DMA1 trap\n");
4873 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4874 }
4875 break;
45f9a39b
AD		 4876 default:
4877 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4878 break;
4879 }
4880
4881 /* wptr/rptr are in bytes! */
4882 rptr += 16;
4883 rptr &= rdev->ih.ptr_mask;
4884 }
45f9a39b 4885 if (queue_hotplug)
32c87fca 4886 schedule_work(&rdev->hotplug_work);
f122c610
AD		 4887 if (queue_hdmi)
4888 schedule_work(&rdev->audio_work);
45f9a39b
AD		 4889 rdev->ih.rptr = rptr;
4890 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4891 atomic_set(&rdev->ih.lock, 0);
4892
4893 /* make sure wptr hasn't changed while processing */
4894 wptr = evergreen_get_ih_wptr(rdev);
4895 if (wptr != rptr)
4896 goto restart_ih;
4897
45f9a39b
AD		 4898 return IRQ_HANDLED;
4899}
4900
233d1ad5
AD		 4901/**
4902 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4903 *
4904 * @rdev: radeon_device pointer
4905 * @fence: radeon fence object
4906 *
4907 * Add a DMA fence packet to the ring to write
4908 * the fence seq number and DMA trap packet to generate
4909 * an interrupt if needed (evergreen-SI).
4910 */
4911void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4912 struct radeon_fence *fence)
4913{
4914 struct radeon_ring *ring = &rdev->ring[fence->ring];
4915 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4916 /* write the fence */
0fcb6155 4917 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
233d1ad5
AD		 4918 radeon_ring_write(ring, addr & 0xfffffffc);
4919 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4920 radeon_ring_write(ring, fence->seq);
4921 /* generate an interrupt */
0fcb6155 4922 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
233d1ad5 4923 /* flush HDP */
0fcb6155 4924 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4b681c28 4925 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
233d1ad5
AD		 4926 radeon_ring_write(ring, 1);
4927}
4928
4929/**
4930 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4931 *
4932 * @rdev: radeon_device pointer
4933 * @ib: IB object to schedule
4934 *
4935 * Schedule an IB in the DMA ring (evergreen).
4936 */
4937void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4938 struct radeon_ib *ib)
4939{
4940 struct radeon_ring *ring = &rdev->ring[ib->ring];
4941
4942 if (rdev->wb.enabled) {
4943 u32 next_rptr = ring->wptr + 4;
4944 while ((next_rptr & 7) != 5)
4945 next_rptr++;
4946 next_rptr += 3;
0fcb6155 4947 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
233d1ad5
AD		 4948 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4949 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
4950 radeon_ring_write(ring, next_rptr);
4951 }
4952
4953 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
4954 * Pad as necessary with NOPs.
4955 */
4956 while ((ring->wptr & 7) != 5)
0fcb6155
JG		 4957 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
4958 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
233d1ad5
AD		 4959 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
4960 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
4961
4962}
4963
4964/**
4965 * evergreen_copy_dma - copy pages using the DMA engine
4966 *
4967 * @rdev: radeon_device pointer
4968 * @src_offset: src GPU address
4969 * @dst_offset: dst GPU address
4970 * @num_gpu_pages: number of GPU pages to xfer
4971 * @fence: radeon fence object
4972 *
4973 * Copy GPU paging using the DMA engine (evergreen-cayman).
4974 * Used by the radeon ttm implementation to move pages if
4975 * registered as the asic copy callback.
4976 */
4977int evergreen_copy_dma(struct radeon_device *rdev,
4978 uint64_t src_offset, uint64_t dst_offset,
4979 unsigned num_gpu_pages,
4980 struct radeon_fence **fence)
4981{
4982 struct radeon_semaphore *sem = NULL;
4983 int ring_index = rdev->asic->copy.dma_ring_index;
4984 struct radeon_ring *ring = &rdev->ring[ring_index];
4985 u32 size_in_dw, cur_size_in_dw;
4986 int i, num_loops;
4987 int r = 0;
4988
4989 r = radeon_semaphore_create(rdev, &sem);
4990 if (r) {
4991 DRM_ERROR("radeon: moving bo (%d).\n", r);
4992 return r;
4993 }
4994
4995 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
4996 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
4997 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
4998 if (r) {
4999 DRM_ERROR("radeon: moving bo (%d).\n", r);
5000 radeon_semaphore_free(rdev, &sem, NULL);
5001 return r;
5002 }
5003
5004 if (radeon_fence_need_sync(*fence, ring->idx)) {
5005 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5006 ring->idx);
5007 radeon_fence_note_sync(*fence, ring->idx);
5008 } else {
5009 radeon_semaphore_free(rdev, &sem, NULL);
5010 }
5011
5012 for (i = 0; i < num_loops; i++) {
5013 cur_size_in_dw = size_in_dw;
5014 if (cur_size_in_dw > 0xFFFFF)
5015 cur_size_in_dw = 0xFFFFF;
5016 size_in_dw -= cur_size_in_dw;
0fcb6155 5017 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
233d1ad5
AD		 5018 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5019 radeon_ring_write(ring, src_offset & 0xfffffffc);
5020 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5021 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5022 src_offset += cur_size_in_dw * 4;
5023 dst_offset += cur_size_in_dw * 4;
5024 }
5025
5026 r = radeon_fence_emit(rdev, fence, ring->idx);
5027 if (r) {
5028 radeon_ring_unlock_undo(rdev, ring);
5029 return r;
5030 }
5031
5032 radeon_ring_unlock_commit(rdev, ring);
5033 radeon_semaphore_free(rdev, &sem, *fence);
5034
5035 return r;
5036}
5037
bcc1c2a1
AD		 5038static int evergreen_startup(struct radeon_device *rdev)
5039{
f2ba57b5 5040 struct radeon_ring *ring;
bcc1c2a1
AD		 5041 int r;
5042
9e46a48d 5043 /* enable pcie gen2 link */
cd54033a 5044 evergreen_pcie_gen2_enable(rdev);
9e46a48d 5045
0af62b01
AD		 5046 if (ASIC_IS_DCE5(rdev)) {
5047 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5048 r = ni_init_microcode(rdev);
5049 if (r) {
5050 DRM_ERROR("Failed to load firmware!\n");
5051 return r;
5052 }
5053 }
755d819e 5054 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5055 if (r) {
0af62b01 5056 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5057 return r;
5058 }
0af62b01
AD		 5059 } else {
5060 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5061 r = r600_init_microcode(rdev);
5062 if (r) {
5063 DRM_ERROR("Failed to load firmware!\n");
5064 return r;
5065 }
5066 }
bcc1c2a1 5067 }
fe251e2f 5068
16cdf04d
AD		 5069 r = r600_vram_scratch_init(rdev);
5070 if (r)
5071 return r;
5072
bcc1c2a1 5073 evergreen_mc_program(rdev);
bcc1c2a1 5074 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5075 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5076 } else {
5077 r = evergreen_pcie_gart_enable(rdev);
5078 if (r)
5079 return r;
5080 }
bcc1c2a1 5081 evergreen_gpu_init(rdev);
bcc1c2a1 5082
d7ccd8fc 5083 r = evergreen_blit_init(rdev);
bcc1c2a1 5084 if (r) {
fb3d9e97 5085 r600_blit_fini(rdev);
27cd7769 5086 rdev->asic->copy.copy = NULL;
d7ccd8fc 5087 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
bcc1c2a1
AD		 5088 }
5089
2948f5e6
AD		 5090 /* allocate rlc buffers */
5091 if (rdev->flags & RADEON_IS_IGP) {
5092 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5093 rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
5094 rdev->rlc.cs_data = evergreen_cs_data;
5095 r = sumo_rlc_init(rdev);
5096 if (r) {
5097 DRM_ERROR("Failed to init rlc BOs!\n");
5098 return r;
5099 }
5100 }
5101
724c80e1
AD		 5102 /* allocate wb buffer */
5103 r = radeon_wb_init(rdev);
5104 if (r)
5105 return r;
5106
30eb77f4
JG		 5107 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5108 if (r) {
5109 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5110 return r;
5111 }
5112
233d1ad5
AD		 5113 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5114 if (r) {
5115 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5116 return r;
5117 }
5118
f2ba57b5
CK		 5119 r = rv770_uvd_resume(rdev);
5120 if (!r) {
5121 r = radeon_fence_driver_start_ring(rdev,
5122 R600_RING_TYPE_UVD_INDEX);
5123 if (r)
5124 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5125 }
5126
5127 if (r)
5128 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5129
bcc1c2a1 5130 /* Enable IRQ */
e49f3959
AH		 5131 if (!rdev->irq.installed) {
5132 r = radeon_irq_kms_init(rdev);
5133 if (r)
5134 return r;
5135 }
5136
bcc1c2a1
AD		 5137 r = r600_irq_init(rdev);
5138 if (r) {
5139 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5140 radeon_irq_kms_fini(rdev);
5141 return r;
5142 }
45f9a39b 5143 evergreen_irq_set(rdev);
bcc1c2a1 5144
f2ba57b5 5145 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5146 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 5147 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5148 0, 0xfffff, RADEON_CP_PACKET2);
bcc1c2a1
AD		 5149 if (r)
5150 return r;
233d1ad5
AD		 5151
5152 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5153 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5154 DMA_RB_RPTR, DMA_RB_WPTR,
0fcb6155 5155 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5156 if (r)
5157 return r;
5158
bcc1c2a1
AD		 5159 r = evergreen_cp_load_microcode(rdev);
5160 if (r)
5161 return r;
fe251e2f 5162 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5163 if (r)
5164 return r;
5165 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5166 if (r)
5167 return r;
fe251e2f 5168
f2ba57b5
CK		 5169 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5170 if (ring->ring_size) {
5171 r = radeon_ring_init(rdev, ring, ring->ring_size,
5172 R600_WB_UVD_RPTR_OFFSET,
5173 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5174 0, 0xfffff, RADEON_CP_PACKET2);
5175 if (!r)
5176 r = r600_uvd_init(rdev);
5177
5178 if (r)
5179 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5180 }
5181
2898c348
CK		 5182 r = radeon_ib_pool_init(rdev);
5183 if (r) {
5184 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5185 return r;
2898c348 5186 }
b15ba512 5187
69d2ae57
RM		 5188 r = r600_audio_init(rdev);
5189 if (r) {
5190 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5191 return r;
5192 }
5193
bcc1c2a1
AD		 5194 return 0;
5195}
5196
5197int evergreen_resume(struct radeon_device *rdev)
5198{
5199 int r;
5200
86f5c9ed
AD		 5201 /* reset the asic, the gfx blocks are often in a bad state
5202 * after the driver is unloaded or after a resume
5203 */
5204 if (radeon_asic_reset(rdev))
5205 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5206 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5207 * posting will perform necessary task to bring back GPU into good
5208 * shape.
5209 */
5210 /* post card */
5211 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5212
d4788db3
AD		 5213 /* init golden registers */
5214 evergreen_init_golden_registers(rdev);
5215
b15ba512 5216 rdev->accel_working = true;
bcc1c2a1
AD		 5217 r = evergreen_startup(rdev);
5218 if (r) {
755d819e 5219 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5220 rdev->accel_working = false;
bcc1c2a1
AD		 5221 return r;
5222 }
fe251e2f 5223
bcc1c2a1
AD		 5224 return r;
5225
5226}
5227
5228int evergreen_suspend(struct radeon_device *rdev)
5229{
69d2ae57 5230 r600_audio_fini(rdev);
f2ba57b5 5231 radeon_uvd_suspend(rdev);
bcc1c2a1 5232 r700_cp_stop(rdev);
233d1ad5 5233 r600_dma_stop(rdev);
f2ba57b5 5234 r600_uvd_rbc_stop(rdev);
45f9a39b 5235 evergreen_irq_suspend(rdev);
724c80e1 5236 radeon_wb_disable(rdev);
bcc1c2a1 5237 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5238
5239 return 0;
5240}
5241
bcc1c2a1
AD		 5242/* Plan is to move initialization in that function and use
5243 * helper function so that radeon_device_init pretty much
5244 * do nothing more than calling asic specific function. This
5245 * should also allow to remove a bunch of callback function
5246 * like vram_info.
5247 */
5248int evergreen_init(struct radeon_device *rdev)
5249{
5250 int r;
5251
bcc1c2a1
AD		 5252 /* Read BIOS */
5253 if (!radeon_get_bios(rdev)) {
5254 if (ASIC_IS_AVIVO(rdev))
5255 return -EINVAL;
5256 }
5257 /* Must be an ATOMBIOS */
5258 if (!rdev->is_atom_bios) {
755d819e 5259 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5260 return -EINVAL;
5261 }
5262 r = radeon_atombios_init(rdev);
5263 if (r)
5264 return r;
86f5c9ed
AD		 5265 /* reset the asic, the gfx blocks are often in a bad state
5266 * after the driver is unloaded or after a resume
5267 */
5268 if (radeon_asic_reset(rdev))
5269 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5270 /* Post card if necessary */
fd909c37 5271 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5272 if (!rdev->bios) {
5273 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5274 return -EINVAL;
5275 }
5276 DRM_INFO("GPU not posted. posting now...\n");
5277 atom_asic_init(rdev->mode_info.atom_context);
5278 }
d4788db3
AD		 5279 /* init golden registers */
5280 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5281 /* Initialize scratch registers */
5282 r600_scratch_init(rdev);
5283 /* Initialize surface registers */
5284 radeon_surface_init(rdev);
5285 /* Initialize clocks */
5286 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5287 /* Fence driver */
5288 r = radeon_fence_driver_init(rdev);
5289 if (r)
5290 return r;
d594e46a
JG		 5291 /* initialize AGP */
5292 if (rdev->flags & RADEON_IS_AGP) {
5293 r = radeon_agp_init(rdev);
5294 if (r)
5295 radeon_agp_disable(rdev);
5296 }
5297 /* initialize memory controller */
bcc1c2a1
AD		 5298 r = evergreen_mc_init(rdev);
5299 if (r)
5300 return r;
5301 /* Memory manager */
5302 r = radeon_bo_init(rdev);
5303 if (r)
5304 return r;
45f9a39b 5305
e32eb50d
CK		 5306 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5307 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5308
233d1ad5
AD		 5309 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5310 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5311
f2ba57b5
CK		 5312 r = radeon_uvd_init(rdev);
5313 if (!r) {
5314 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5315 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5316 4096);
5317 }
5318
bcc1c2a1
AD		 5319 rdev->ih.ring_obj = NULL;
5320 r600_ih_ring_init(rdev, 64 * 1024);
5321
5322 r = r600_pcie_gart_init(rdev);
5323 if (r)
5324 return r;
0fcdb61e 5325
148a03bc 5326 rdev->accel_working = true;
bcc1c2a1
AD		 5327 r = evergreen_startup(rdev);
5328 if (r) {
fe251e2f
AD		 5329 dev_err(rdev->dev, "disabling GPU acceleration\n");
5330 r700_cp_fini(rdev);
233d1ad5 5331 r600_dma_fini(rdev);
fe251e2f 5332 r600_irq_fini(rdev);
2948f5e6
AD		 5333 if (rdev->flags & RADEON_IS_IGP)
5334 sumo_rlc_fini(rdev);
724c80e1 5335 radeon_wb_fini(rdev);
2898c348 5336 radeon_ib_pool_fini(rdev);
fe251e2f 5337 radeon_irq_kms_fini(rdev);
0fcdb61e 5338 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5339 rdev->accel_working = false;
5340 }
77e00f2e
AD		 5341
5342 /* Don't start up if the MC ucode is missing on BTC parts.
5343 * The default clocks and voltages before the MC ucode
5344 * is loaded are not suffient for advanced operations.
5345 */
5346 if (ASIC_IS_DCE5(rdev)) {
5347 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5348 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5349 return -EINVAL;
5350 }
5351 }
5352
bcc1c2a1
AD		 5353 return 0;
5354}
5355
5356void evergreen_fini(struct radeon_device *rdev)
5357{
69d2ae57 5358 r600_audio_fini(rdev);
fb3d9e97 5359 r600_blit_fini(rdev);
45f9a39b 5360 r700_cp_fini(rdev);
233d1ad5 5361 r600_dma_fini(rdev);
bcc1c2a1 5362 r600_irq_fini(rdev);
2948f5e6
AD		 5363 if (rdev->flags & RADEON_IS_IGP)
5364 sumo_rlc_fini(rdev);
724c80e1 5365 radeon_wb_fini(rdev);
2898c348 5366 radeon_ib_pool_fini(rdev);
bcc1c2a1 5367 radeon_irq_kms_fini(rdev);
bcc1c2a1 5368 evergreen_pcie_gart_fini(rdev);
f2ba57b5 5369 radeon_uvd_fini(rdev);
16cdf04d 5370 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5371 radeon_gem_fini(rdev);
5372 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5373 radeon_agp_fini(rdev);
5374 radeon_bo_fini(rdev);
5375 radeon_atombios_fini(rdev);
5376 kfree(rdev->bios);
5377 rdev->bios = NULL;
bcc1c2a1 5378}
9e46a48d 5379
b07759bf 5380void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5381{
7e0e4196 5382 u32 link_width_cntl, speed_cntl;
9e46a48d 5383
d42dd579
AD		 5384 if (radeon_pcie_gen2 == 0)
5385 return;
5386
9e46a48d
AD		 5387 if (rdev->flags & RADEON_IS_IGP)
5388 return;
5389
5390 if (!(rdev->flags & RADEON_IS_PCIE))
5391 return;
5392
5393 /* x2 cards have a special sequence */
5394 if (ASIC_IS_X2(rdev))
5395 return;
5396
7e0e4196
KSS		 5397 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5398 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5399 return;
5400
492d2b61 5401 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5402 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5403 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5404 return;
5405 }
5406
197bbb3d
DA		 5407 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5408
9e46a48d
AD		 5409 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5410 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5411
492d2b61 5412 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5413 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5414 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5415
492d2b61 5416 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5417 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5418 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5419
492d2b61 5420 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5421 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5422 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5423
492d2b61 5424 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5425 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5426 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5427
492d2b61 5428 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5429 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5430 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5431
5432 } else {
492d2b61 5433 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5434 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5435 if (1)
5436 link_width_cntl |= LC_UPCONFIGURE_DIS;
5437 else
5438 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5439 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5440 }
5441}
Linux kernel - Deliverables
This page took 0.707078 seconds and 5 git commands to generate.