projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/kms: fix up 6xx/7xx display watermark calc for dpm
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
5a0e3ad6 26#include <linux/slab.h>
760285e7 27#include <drm/drmP.h>
bcc1c2a1 28#include "radeon.h"
e6990375 29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
0fcdb61e 31#include "evergreend.h"
bcc1c2a1
AD		 32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
2281a378 35#include "evergreen_blit_shaders.h"
138e4e16 36#include "radeon_ucode.h"
fe251e2f 37
4a15903d
AD		 38static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
2948f5e6
AD		 48#include "clearstate_evergreen.h"
49
50static u32 sumo_rlc_save_restore_register_list[] =
51{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
134static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
135
bcc1c2a1
AD		 136static void evergreen_gpu_init(struct radeon_device *rdev);
137void evergreen_fini(struct radeon_device *rdev);
b07759bf 138void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1b37078b
AD		 139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
bcc1c2a1 141
d4788db3
AD		 142static const u32 evergreen_golden_registers[] =
143{
144 0x3f90, 0xffff0000, 0xff000000,
145 0x9148, 0xffff0000, 0xff000000,
146 0x3f94, 0xffff0000, 0xff000000,
147 0x914c, 0xffff0000, 0xff000000,
148 0x9b7c, 0xffffffff, 0x00000000,
149 0x8a14, 0xffffffff, 0x00000007,
150 0x8b10, 0xffffffff, 0x00000000,
151 0x960c, 0xffffffff, 0x54763210,
152 0x88c4, 0xffffffff, 0x000000c2,
153 0x88d4, 0xffffffff, 0x00000010,
154 0x8974, 0xffffffff, 0x00000000,
155 0xc78, 0x00000080, 0x00000080,
156 0x5eb4, 0xffffffff, 0x00000002,
157 0x5e78, 0xffffffff, 0x001000f0,
158 0x6104, 0x01000300, 0x00000000,
159 0x5bc0, 0x00300000, 0x00000000,
160 0x7030, 0xffffffff, 0x00000011,
161 0x7c30, 0xffffffff, 0x00000011,
162 0x10830, 0xffffffff, 0x00000011,
163 0x11430, 0xffffffff, 0x00000011,
164 0x12030, 0xffffffff, 0x00000011,
165 0x12c30, 0xffffffff, 0x00000011,
166 0xd02c, 0xffffffff, 0x08421000,
167 0x240c, 0xffffffff, 0x00000380,
168 0x8b24, 0xffffffff, 0x00ff0fff,
169 0x28a4c, 0x06000000, 0x06000000,
170 0x10c, 0x00000001, 0x00000001,
171 0x8d00, 0xffffffff, 0x100e4848,
172 0x8d04, 0xffffffff, 0x00164745,
173 0x8c00, 0xffffffff, 0xe4000003,
174 0x8c04, 0xffffffff, 0x40600060,
175 0x8c08, 0xffffffff, 0x001c001c,
176 0x8cf0, 0xffffffff, 0x08e00620,
177 0x8c20, 0xffffffff, 0x00800080,
178 0x8c24, 0xffffffff, 0x00800080,
179 0x8c18, 0xffffffff, 0x20202078,
180 0x8c1c, 0xffffffff, 0x00001010,
181 0x28350, 0xffffffff, 0x00000000,
182 0xa008, 0xffffffff, 0x00010000,
183 0x5cc, 0xffffffff, 0x00000001,
184 0x9508, 0xffffffff, 0x00000002,
185 0x913c, 0x0000000f, 0x0000000a
186};
187
188static const u32 evergreen_golden_registers2[] =
189{
190 0x2f4c, 0xffffffff, 0x00000000,
191 0x54f4, 0xffffffff, 0x00000000,
192 0x54f0, 0xffffffff, 0x00000000,
193 0x5498, 0xffffffff, 0x00000000,
194 0x549c, 0xffffffff, 0x00000000,
195 0x5494, 0xffffffff, 0x00000000,
196 0x53cc, 0xffffffff, 0x00000000,
197 0x53c8, 0xffffffff, 0x00000000,
198 0x53c4, 0xffffffff, 0x00000000,
199 0x53c0, 0xffffffff, 0x00000000,
200 0x53bc, 0xffffffff, 0x00000000,
201 0x53b8, 0xffffffff, 0x00000000,
202 0x53b4, 0xffffffff, 0x00000000,
203 0x53b0, 0xffffffff, 0x00000000
204};
205
206static const u32 cypress_mgcg_init[] =
207{
208 0x802c, 0xffffffff, 0xc0000000,
209 0x5448, 0xffffffff, 0x00000100,
210 0x55e4, 0xffffffff, 0x00000100,
211 0x160c, 0xffffffff, 0x00000100,
212 0x5644, 0xffffffff, 0x00000100,
213 0xc164, 0xffffffff, 0x00000100,
214 0x8a18, 0xffffffff, 0x00000100,
215 0x897c, 0xffffffff, 0x06000100,
216 0x8b28, 0xffffffff, 0x00000100,
217 0x9144, 0xffffffff, 0x00000100,
218 0x9a60, 0xffffffff, 0x00000100,
219 0x9868, 0xffffffff, 0x00000100,
220 0x8d58, 0xffffffff, 0x00000100,
221 0x9510, 0xffffffff, 0x00000100,
222 0x949c, 0xffffffff, 0x00000100,
223 0x9654, 0xffffffff, 0x00000100,
224 0x9030, 0xffffffff, 0x00000100,
225 0x9034, 0xffffffff, 0x00000100,
226 0x9038, 0xffffffff, 0x00000100,
227 0x903c, 0xffffffff, 0x00000100,
228 0x9040, 0xffffffff, 0x00000100,
229 0xa200, 0xffffffff, 0x00000100,
230 0xa204, 0xffffffff, 0x00000100,
231 0xa208, 0xffffffff, 0x00000100,
232 0xa20c, 0xffffffff, 0x00000100,
233 0x971c, 0xffffffff, 0x00000100,
234 0x977c, 0xffffffff, 0x00000100,
235 0x3f80, 0xffffffff, 0x00000100,
236 0xa210, 0xffffffff, 0x00000100,
237 0xa214, 0xffffffff, 0x00000100,
238 0x4d8, 0xffffffff, 0x00000100,
239 0x9784, 0xffffffff, 0x00000100,
240 0x9698, 0xffffffff, 0x00000100,
241 0x4d4, 0xffffffff, 0x00000200,
242 0x30cc, 0xffffffff, 0x00000100,
243 0xd0c0, 0xffffffff, 0xff000100,
244 0x802c, 0xffffffff, 0x40000000,
245 0x915c, 0xffffffff, 0x00010000,
246 0x9160, 0xffffffff, 0x00030002,
247 0x9178, 0xffffffff, 0x00070000,
248 0x917c, 0xffffffff, 0x00030002,
249 0x9180, 0xffffffff, 0x00050004,
250 0x918c, 0xffffffff, 0x00010006,
251 0x9190, 0xffffffff, 0x00090008,
252 0x9194, 0xffffffff, 0x00070000,
253 0x9198, 0xffffffff, 0x00030002,
254 0x919c, 0xffffffff, 0x00050004,
255 0x91a8, 0xffffffff, 0x00010006,
256 0x91ac, 0xffffffff, 0x00090008,
257 0x91b0, 0xffffffff, 0x00070000,
258 0x91b4, 0xffffffff, 0x00030002,
259 0x91b8, 0xffffffff, 0x00050004,
260 0x91c4, 0xffffffff, 0x00010006,
261 0x91c8, 0xffffffff, 0x00090008,
262 0x91cc, 0xffffffff, 0x00070000,
263 0x91d0, 0xffffffff, 0x00030002,
264 0x91d4, 0xffffffff, 0x00050004,
265 0x91e0, 0xffffffff, 0x00010006,
266 0x91e4, 0xffffffff, 0x00090008,
267 0x91e8, 0xffffffff, 0x00000000,
268 0x91ec, 0xffffffff, 0x00070000,
269 0x91f0, 0xffffffff, 0x00030002,
270 0x91f4, 0xffffffff, 0x00050004,
271 0x9200, 0xffffffff, 0x00010006,
272 0x9204, 0xffffffff, 0x00090008,
273 0x9208, 0xffffffff, 0x00070000,
274 0x920c, 0xffffffff, 0x00030002,
275 0x9210, 0xffffffff, 0x00050004,
276 0x921c, 0xffffffff, 0x00010006,
277 0x9220, 0xffffffff, 0x00090008,
278 0x9224, 0xffffffff, 0x00070000,
279 0x9228, 0xffffffff, 0x00030002,
280 0x922c, 0xffffffff, 0x00050004,
281 0x9238, 0xffffffff, 0x00010006,
282 0x923c, 0xffffffff, 0x00090008,
283 0x9240, 0xffffffff, 0x00070000,
284 0x9244, 0xffffffff, 0x00030002,
285 0x9248, 0xffffffff, 0x00050004,
286 0x9254, 0xffffffff, 0x00010006,
287 0x9258, 0xffffffff, 0x00090008,
288 0x925c, 0xffffffff, 0x00070000,
289 0x9260, 0xffffffff, 0x00030002,
290 0x9264, 0xffffffff, 0x00050004,
291 0x9270, 0xffffffff, 0x00010006,
292 0x9274, 0xffffffff, 0x00090008,
293 0x9278, 0xffffffff, 0x00070000,
294 0x927c, 0xffffffff, 0x00030002,
295 0x9280, 0xffffffff, 0x00050004,
296 0x928c, 0xffffffff, 0x00010006,
297 0x9290, 0xffffffff, 0x00090008,
298 0x9294, 0xffffffff, 0x00000000,
299 0x929c, 0xffffffff, 0x00000001,
300 0x802c, 0xffffffff, 0x40010000,
301 0x915c, 0xffffffff, 0x00010000,
302 0x9160, 0xffffffff, 0x00030002,
303 0x9178, 0xffffffff, 0x00070000,
304 0x917c, 0xffffffff, 0x00030002,
305 0x9180, 0xffffffff, 0x00050004,
306 0x918c, 0xffffffff, 0x00010006,
307 0x9190, 0xffffffff, 0x00090008,
308 0x9194, 0xffffffff, 0x00070000,
309 0x9198, 0xffffffff, 0x00030002,
310 0x919c, 0xffffffff, 0x00050004,
311 0x91a8, 0xffffffff, 0x00010006,
312 0x91ac, 0xffffffff, 0x00090008,
313 0x91b0, 0xffffffff, 0x00070000,
314 0x91b4, 0xffffffff, 0x00030002,
315 0x91b8, 0xffffffff, 0x00050004,
316 0x91c4, 0xffffffff, 0x00010006,
317 0x91c8, 0xffffffff, 0x00090008,
318 0x91cc, 0xffffffff, 0x00070000,
319 0x91d0, 0xffffffff, 0x00030002,
320 0x91d4, 0xffffffff, 0x00050004,
321 0x91e0, 0xffffffff, 0x00010006,
322 0x91e4, 0xffffffff, 0x00090008,
323 0x91e8, 0xffffffff, 0x00000000,
324 0x91ec, 0xffffffff, 0x00070000,
325 0x91f0, 0xffffffff, 0x00030002,
326 0x91f4, 0xffffffff, 0x00050004,
327 0x9200, 0xffffffff, 0x00010006,
328 0x9204, 0xffffffff, 0x00090008,
329 0x9208, 0xffffffff, 0x00070000,
330 0x920c, 0xffffffff, 0x00030002,
331 0x9210, 0xffffffff, 0x00050004,
332 0x921c, 0xffffffff, 0x00010006,
333 0x9220, 0xffffffff, 0x00090008,
334 0x9224, 0xffffffff, 0x00070000,
335 0x9228, 0xffffffff, 0x00030002,
336 0x922c, 0xffffffff, 0x00050004,
337 0x9238, 0xffffffff, 0x00010006,
338 0x923c, 0xffffffff, 0x00090008,
339 0x9240, 0xffffffff, 0x00070000,
340 0x9244, 0xffffffff, 0x00030002,
341 0x9248, 0xffffffff, 0x00050004,
342 0x9254, 0xffffffff, 0x00010006,
343 0x9258, 0xffffffff, 0x00090008,
344 0x925c, 0xffffffff, 0x00070000,
345 0x9260, 0xffffffff, 0x00030002,
346 0x9264, 0xffffffff, 0x00050004,
347 0x9270, 0xffffffff, 0x00010006,
348 0x9274, 0xffffffff, 0x00090008,
349 0x9278, 0xffffffff, 0x00070000,
350 0x927c, 0xffffffff, 0x00030002,
351 0x9280, 0xffffffff, 0x00050004,
352 0x928c, 0xffffffff, 0x00010006,
353 0x9290, 0xffffffff, 0x00090008,
354 0x9294, 0xffffffff, 0x00000000,
355 0x929c, 0xffffffff, 0x00000001,
356 0x802c, 0xffffffff, 0xc0000000
357};
358
359static const u32 redwood_mgcg_init[] =
360{
361 0x802c, 0xffffffff, 0xc0000000,
362 0x5448, 0xffffffff, 0x00000100,
363 0x55e4, 0xffffffff, 0x00000100,
364 0x160c, 0xffffffff, 0x00000100,
365 0x5644, 0xffffffff, 0x00000100,
366 0xc164, 0xffffffff, 0x00000100,
367 0x8a18, 0xffffffff, 0x00000100,
368 0x897c, 0xffffffff, 0x06000100,
369 0x8b28, 0xffffffff, 0x00000100,
370 0x9144, 0xffffffff, 0x00000100,
371 0x9a60, 0xffffffff, 0x00000100,
372 0x9868, 0xffffffff, 0x00000100,
373 0x8d58, 0xffffffff, 0x00000100,
374 0x9510, 0xffffffff, 0x00000100,
375 0x949c, 0xffffffff, 0x00000100,
376 0x9654, 0xffffffff, 0x00000100,
377 0x9030, 0xffffffff, 0x00000100,
378 0x9034, 0xffffffff, 0x00000100,
379 0x9038, 0xffffffff, 0x00000100,
380 0x903c, 0xffffffff, 0x00000100,
381 0x9040, 0xffffffff, 0x00000100,
382 0xa200, 0xffffffff, 0x00000100,
383 0xa204, 0xffffffff, 0x00000100,
384 0xa208, 0xffffffff, 0x00000100,
385 0xa20c, 0xffffffff, 0x00000100,
386 0x971c, 0xffffffff, 0x00000100,
387 0x977c, 0xffffffff, 0x00000100,
388 0x3f80, 0xffffffff, 0x00000100,
389 0xa210, 0xffffffff, 0x00000100,
390 0xa214, 0xffffffff, 0x00000100,
391 0x4d8, 0xffffffff, 0x00000100,
392 0x9784, 0xffffffff, 0x00000100,
393 0x9698, 0xffffffff, 0x00000100,
394 0x4d4, 0xffffffff, 0x00000200,
395 0x30cc, 0xffffffff, 0x00000100,
396 0xd0c0, 0xffffffff, 0xff000100,
397 0x802c, 0xffffffff, 0x40000000,
398 0x915c, 0xffffffff, 0x00010000,
399 0x9160, 0xffffffff, 0x00030002,
400 0x9178, 0xffffffff, 0x00070000,
401 0x917c, 0xffffffff, 0x00030002,
402 0x9180, 0xffffffff, 0x00050004,
403 0x918c, 0xffffffff, 0x00010006,
404 0x9190, 0xffffffff, 0x00090008,
405 0x9194, 0xffffffff, 0x00070000,
406 0x9198, 0xffffffff, 0x00030002,
407 0x919c, 0xffffffff, 0x00050004,
408 0x91a8, 0xffffffff, 0x00010006,
409 0x91ac, 0xffffffff, 0x00090008,
410 0x91b0, 0xffffffff, 0x00070000,
411 0x91b4, 0xffffffff, 0x00030002,
412 0x91b8, 0xffffffff, 0x00050004,
413 0x91c4, 0xffffffff, 0x00010006,
414 0x91c8, 0xffffffff, 0x00090008,
415 0x91cc, 0xffffffff, 0x00070000,
416 0x91d0, 0xffffffff, 0x00030002,
417 0x91d4, 0xffffffff, 0x00050004,
418 0x91e0, 0xffffffff, 0x00010006,
419 0x91e4, 0xffffffff, 0x00090008,
420 0x91e8, 0xffffffff, 0x00000000,
421 0x91ec, 0xffffffff, 0x00070000,
422 0x91f0, 0xffffffff, 0x00030002,
423 0x91f4, 0xffffffff, 0x00050004,
424 0x9200, 0xffffffff, 0x00010006,
425 0x9204, 0xffffffff, 0x00090008,
426 0x9294, 0xffffffff, 0x00000000,
427 0x929c, 0xffffffff, 0x00000001,
428 0x802c, 0xffffffff, 0xc0000000
429};
430
431static const u32 cedar_golden_registers[] =
432{
433 0x3f90, 0xffff0000, 0xff000000,
434 0x9148, 0xffff0000, 0xff000000,
435 0x3f94, 0xffff0000, 0xff000000,
436 0x914c, 0xffff0000, 0xff000000,
437 0x9b7c, 0xffffffff, 0x00000000,
438 0x8a14, 0xffffffff, 0x00000007,
439 0x8b10, 0xffffffff, 0x00000000,
440 0x960c, 0xffffffff, 0x54763210,
441 0x88c4, 0xffffffff, 0x000000c2,
442 0x88d4, 0xffffffff, 0x00000000,
443 0x8974, 0xffffffff, 0x00000000,
444 0xc78, 0x00000080, 0x00000080,
445 0x5eb4, 0xffffffff, 0x00000002,
446 0x5e78, 0xffffffff, 0x001000f0,
447 0x6104, 0x01000300, 0x00000000,
448 0x5bc0, 0x00300000, 0x00000000,
449 0x7030, 0xffffffff, 0x00000011,
450 0x7c30, 0xffffffff, 0x00000011,
451 0x10830, 0xffffffff, 0x00000011,
452 0x11430, 0xffffffff, 0x00000011,
453 0xd02c, 0xffffffff, 0x08421000,
454 0x240c, 0xffffffff, 0x00000380,
455 0x8b24, 0xffffffff, 0x00ff0fff,
456 0x28a4c, 0x06000000, 0x06000000,
457 0x10c, 0x00000001, 0x00000001,
458 0x8d00, 0xffffffff, 0x100e4848,
459 0x8d04, 0xffffffff, 0x00164745,
460 0x8c00, 0xffffffff, 0xe4000003,
461 0x8c04, 0xffffffff, 0x40600060,
462 0x8c08, 0xffffffff, 0x001c001c,
463 0x8cf0, 0xffffffff, 0x08e00410,
464 0x8c20, 0xffffffff, 0x00800080,
465 0x8c24, 0xffffffff, 0x00800080,
466 0x8c18, 0xffffffff, 0x20202078,
467 0x8c1c, 0xffffffff, 0x00001010,
468 0x28350, 0xffffffff, 0x00000000,
469 0xa008, 0xffffffff, 0x00010000,
470 0x5cc, 0xffffffff, 0x00000001,
471 0x9508, 0xffffffff, 0x00000002
472};
473
474static const u32 cedar_mgcg_init[] =
475{
476 0x802c, 0xffffffff, 0xc0000000,
477 0x5448, 0xffffffff, 0x00000100,
478 0x55e4, 0xffffffff, 0x00000100,
479 0x160c, 0xffffffff, 0x00000100,
480 0x5644, 0xffffffff, 0x00000100,
481 0xc164, 0xffffffff, 0x00000100,
482 0x8a18, 0xffffffff, 0x00000100,
483 0x897c, 0xffffffff, 0x06000100,
484 0x8b28, 0xffffffff, 0x00000100,
485 0x9144, 0xffffffff, 0x00000100,
486 0x9a60, 0xffffffff, 0x00000100,
487 0x9868, 0xffffffff, 0x00000100,
488 0x8d58, 0xffffffff, 0x00000100,
489 0x9510, 0xffffffff, 0x00000100,
490 0x949c, 0xffffffff, 0x00000100,
491 0x9654, 0xffffffff, 0x00000100,
492 0x9030, 0xffffffff, 0x00000100,
493 0x9034, 0xffffffff, 0x00000100,
494 0x9038, 0xffffffff, 0x00000100,
495 0x903c, 0xffffffff, 0x00000100,
496 0x9040, 0xffffffff, 0x00000100,
497 0xa200, 0xffffffff, 0x00000100,
498 0xa204, 0xffffffff, 0x00000100,
499 0xa208, 0xffffffff, 0x00000100,
500 0xa20c, 0xffffffff, 0x00000100,
501 0x971c, 0xffffffff, 0x00000100,
502 0x977c, 0xffffffff, 0x00000100,
503 0x3f80, 0xffffffff, 0x00000100,
504 0xa210, 0xffffffff, 0x00000100,
505 0xa214, 0xffffffff, 0x00000100,
506 0x4d8, 0xffffffff, 0x00000100,
507 0x9784, 0xffffffff, 0x00000100,
508 0x9698, 0xffffffff, 0x00000100,
509 0x4d4, 0xffffffff, 0x00000200,
510 0x30cc, 0xffffffff, 0x00000100,
511 0xd0c0, 0xffffffff, 0xff000100,
512 0x802c, 0xffffffff, 0x40000000,
513 0x915c, 0xffffffff, 0x00010000,
514 0x9178, 0xffffffff, 0x00050000,
515 0x917c, 0xffffffff, 0x00030002,
516 0x918c, 0xffffffff, 0x00010004,
517 0x9190, 0xffffffff, 0x00070006,
518 0x9194, 0xffffffff, 0x00050000,
519 0x9198, 0xffffffff, 0x00030002,
520 0x91a8, 0xffffffff, 0x00010004,
521 0x91ac, 0xffffffff, 0x00070006,
522 0x91e8, 0xffffffff, 0x00000000,
523 0x9294, 0xffffffff, 0x00000000,
524 0x929c, 0xffffffff, 0x00000001,
525 0x802c, 0xffffffff, 0xc0000000
526};
527
528static const u32 juniper_mgcg_init[] =
529{
530 0x802c, 0xffffffff, 0xc0000000,
531 0x5448, 0xffffffff, 0x00000100,
532 0x55e4, 0xffffffff, 0x00000100,
533 0x160c, 0xffffffff, 0x00000100,
534 0x5644, 0xffffffff, 0x00000100,
535 0xc164, 0xffffffff, 0x00000100,
536 0x8a18, 0xffffffff, 0x00000100,
537 0x897c, 0xffffffff, 0x06000100,
538 0x8b28, 0xffffffff, 0x00000100,
539 0x9144, 0xffffffff, 0x00000100,
540 0x9a60, 0xffffffff, 0x00000100,
541 0x9868, 0xffffffff, 0x00000100,
542 0x8d58, 0xffffffff, 0x00000100,
543 0x9510, 0xffffffff, 0x00000100,
544 0x949c, 0xffffffff, 0x00000100,
545 0x9654, 0xffffffff, 0x00000100,
546 0x9030, 0xffffffff, 0x00000100,
547 0x9034, 0xffffffff, 0x00000100,
548 0x9038, 0xffffffff, 0x00000100,
549 0x903c, 0xffffffff, 0x00000100,
550 0x9040, 0xffffffff, 0x00000100,
551 0xa200, 0xffffffff, 0x00000100,
552 0xa204, 0xffffffff, 0x00000100,
553 0xa208, 0xffffffff, 0x00000100,
554 0xa20c, 0xffffffff, 0x00000100,
555 0x971c, 0xffffffff, 0x00000100,
556 0xd0c0, 0xffffffff, 0xff000100,
557 0x802c, 0xffffffff, 0x40000000,
558 0x915c, 0xffffffff, 0x00010000,
559 0x9160, 0xffffffff, 0x00030002,
560 0x9178, 0xffffffff, 0x00070000,
561 0x917c, 0xffffffff, 0x00030002,
562 0x9180, 0xffffffff, 0x00050004,
563 0x918c, 0xffffffff, 0x00010006,
564 0x9190, 0xffffffff, 0x00090008,
565 0x9194, 0xffffffff, 0x00070000,
566 0x9198, 0xffffffff, 0x00030002,
567 0x919c, 0xffffffff, 0x00050004,
568 0x91a8, 0xffffffff, 0x00010006,
569 0x91ac, 0xffffffff, 0x00090008,
570 0x91b0, 0xffffffff, 0x00070000,
571 0x91b4, 0xffffffff, 0x00030002,
572 0x91b8, 0xffffffff, 0x00050004,
573 0x91c4, 0xffffffff, 0x00010006,
574 0x91c8, 0xffffffff, 0x00090008,
575 0x91cc, 0xffffffff, 0x00070000,
576 0x91d0, 0xffffffff, 0x00030002,
577 0x91d4, 0xffffffff, 0x00050004,
578 0x91e0, 0xffffffff, 0x00010006,
579 0x91e4, 0xffffffff, 0x00090008,
580 0x91e8, 0xffffffff, 0x00000000,
581 0x91ec, 0xffffffff, 0x00070000,
582 0x91f0, 0xffffffff, 0x00030002,
583 0x91f4, 0xffffffff, 0x00050004,
584 0x9200, 0xffffffff, 0x00010006,
585 0x9204, 0xffffffff, 0x00090008,
586 0x9208, 0xffffffff, 0x00070000,
587 0x920c, 0xffffffff, 0x00030002,
588 0x9210, 0xffffffff, 0x00050004,
589 0x921c, 0xffffffff, 0x00010006,
590 0x9220, 0xffffffff, 0x00090008,
591 0x9224, 0xffffffff, 0x00070000,
592 0x9228, 0xffffffff, 0x00030002,
593 0x922c, 0xffffffff, 0x00050004,
594 0x9238, 0xffffffff, 0x00010006,
595 0x923c, 0xffffffff, 0x00090008,
596 0x9240, 0xffffffff, 0x00070000,
597 0x9244, 0xffffffff, 0x00030002,
598 0x9248, 0xffffffff, 0x00050004,
599 0x9254, 0xffffffff, 0x00010006,
600 0x9258, 0xffffffff, 0x00090008,
601 0x925c, 0xffffffff, 0x00070000,
602 0x9260, 0xffffffff, 0x00030002,
603 0x9264, 0xffffffff, 0x00050004,
604 0x9270, 0xffffffff, 0x00010006,
605 0x9274, 0xffffffff, 0x00090008,
606 0x9278, 0xffffffff, 0x00070000,
607 0x927c, 0xffffffff, 0x00030002,
608 0x9280, 0xffffffff, 0x00050004,
609 0x928c, 0xffffffff, 0x00010006,
610 0x9290, 0xffffffff, 0x00090008,
611 0x9294, 0xffffffff, 0x00000000,
612 0x929c, 0xffffffff, 0x00000001,
613 0x802c, 0xffffffff, 0xc0000000,
614 0x977c, 0xffffffff, 0x00000100,
615 0x3f80, 0xffffffff, 0x00000100,
616 0xa210, 0xffffffff, 0x00000100,
617 0xa214, 0xffffffff, 0x00000100,
618 0x4d8, 0xffffffff, 0x00000100,
619 0x9784, 0xffffffff, 0x00000100,
620 0x9698, 0xffffffff, 0x00000100,
621 0x4d4, 0xffffffff, 0x00000200,
622 0x30cc, 0xffffffff, 0x00000100,
623 0x802c, 0xffffffff, 0xc0000000
624};
625
626static const u32 supersumo_golden_registers[] =
627{
628 0x5eb4, 0xffffffff, 0x00000002,
629 0x5cc, 0xffffffff, 0x00000001,
630 0x7030, 0xffffffff, 0x00000011,
631 0x7c30, 0xffffffff, 0x00000011,
632 0x6104, 0x01000300, 0x00000000,
633 0x5bc0, 0x00300000, 0x00000000,
634 0x8c04, 0xffffffff, 0x40600060,
635 0x8c08, 0xffffffff, 0x001c001c,
636 0x8c20, 0xffffffff, 0x00800080,
637 0x8c24, 0xffffffff, 0x00800080,
638 0x8c18, 0xffffffff, 0x20202078,
639 0x8c1c, 0xffffffff, 0x00001010,
640 0x918c, 0xffffffff, 0x00010006,
641 0x91a8, 0xffffffff, 0x00010006,
642 0x91c4, 0xffffffff, 0x00010006,
643 0x91e0, 0xffffffff, 0x00010006,
644 0x9200, 0xffffffff, 0x00010006,
645 0x9150, 0xffffffff, 0x6e944040,
646 0x917c, 0xffffffff, 0x00030002,
647 0x9180, 0xffffffff, 0x00050004,
648 0x9198, 0xffffffff, 0x00030002,
649 0x919c, 0xffffffff, 0x00050004,
650 0x91b4, 0xffffffff, 0x00030002,
651 0x91b8, 0xffffffff, 0x00050004,
652 0x91d0, 0xffffffff, 0x00030002,
653 0x91d4, 0xffffffff, 0x00050004,
654 0x91f0, 0xffffffff, 0x00030002,
655 0x91f4, 0xffffffff, 0x00050004,
656 0x915c, 0xffffffff, 0x00010000,
657 0x9160, 0xffffffff, 0x00030002,
658 0x3f90, 0xffff0000, 0xff000000,
659 0x9178, 0xffffffff, 0x00070000,
660 0x9194, 0xffffffff, 0x00070000,
661 0x91b0, 0xffffffff, 0x00070000,
662 0x91cc, 0xffffffff, 0x00070000,
663 0x91ec, 0xffffffff, 0x00070000,
664 0x9148, 0xffff0000, 0xff000000,
665 0x9190, 0xffffffff, 0x00090008,
666 0x91ac, 0xffffffff, 0x00090008,
667 0x91c8, 0xffffffff, 0x00090008,
668 0x91e4, 0xffffffff, 0x00090008,
669 0x9204, 0xffffffff, 0x00090008,
670 0x3f94, 0xffff0000, 0xff000000,
671 0x914c, 0xffff0000, 0xff000000,
672 0x929c, 0xffffffff, 0x00000001,
673 0x8a18, 0xffffffff, 0x00000100,
674 0x8b28, 0xffffffff, 0x00000100,
675 0x9144, 0xffffffff, 0x00000100,
676 0x5644, 0xffffffff, 0x00000100,
677 0x9b7c, 0xffffffff, 0x00000000,
678 0x8030, 0xffffffff, 0x0000100a,
679 0x8a14, 0xffffffff, 0x00000007,
680 0x8b24, 0xffffffff, 0x00ff0fff,
681 0x8b10, 0xffffffff, 0x00000000,
682 0x28a4c, 0x06000000, 0x06000000,
683 0x4d8, 0xffffffff, 0x00000100,
684 0x913c, 0xffff000f, 0x0100000a,
685 0x960c, 0xffffffff, 0x54763210,
686 0x88c4, 0xffffffff, 0x000000c2,
687 0x88d4, 0xffffffff, 0x00000010,
688 0x8974, 0xffffffff, 0x00000000,
689 0xc78, 0x00000080, 0x00000080,
690 0x5e78, 0xffffffff, 0x001000f0,
691 0xd02c, 0xffffffff, 0x08421000,
692 0xa008, 0xffffffff, 0x00010000,
693 0x8d00, 0xffffffff, 0x100e4848,
694 0x8d04, 0xffffffff, 0x00164745,
695 0x8c00, 0xffffffff, 0xe4000003,
696 0x8cf0, 0x1fffffff, 0x08e00620,
697 0x28350, 0xffffffff, 0x00000000,
698 0x9508, 0xffffffff, 0x00000002
699};
700
701static const u32 sumo_golden_registers[] =
702{
703 0x900c, 0x00ffffff, 0x0017071f,
704 0x8c18, 0xffffffff, 0x10101060,
705 0x8c1c, 0xffffffff, 0x00001010,
706 0x8c30, 0x0000000f, 0x00000005,
707 0x9688, 0x0000000f, 0x00000007
708};
709
710static const u32 wrestler_golden_registers[] =
711{
712 0x5eb4, 0xffffffff, 0x00000002,
713 0x5cc, 0xffffffff, 0x00000001,
714 0x7030, 0xffffffff, 0x00000011,
715 0x7c30, 0xffffffff, 0x00000011,
716 0x6104, 0x01000300, 0x00000000,
717 0x5bc0, 0x00300000, 0x00000000,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x9150, 0xffffffff, 0x6e944040,
721 0x917c, 0xffffffff, 0x00030002,
722 0x9198, 0xffffffff, 0x00030002,
723 0x915c, 0xffffffff, 0x00010000,
724 0x3f90, 0xffff0000, 0xff000000,
725 0x9178, 0xffffffff, 0x00070000,
726 0x9194, 0xffffffff, 0x00070000,
727 0x9148, 0xffff0000, 0xff000000,
728 0x9190, 0xffffffff, 0x00090008,
729 0x91ac, 0xffffffff, 0x00090008,
730 0x3f94, 0xffff0000, 0xff000000,
731 0x914c, 0xffff0000, 0xff000000,
732 0x929c, 0xffffffff, 0x00000001,
733 0x8a18, 0xffffffff, 0x00000100,
734 0x8b28, 0xffffffff, 0x00000100,
735 0x9144, 0xffffffff, 0x00000100,
736 0x9b7c, 0xffffffff, 0x00000000,
737 0x8030, 0xffffffff, 0x0000100a,
738 0x8a14, 0xffffffff, 0x00000001,
739 0x8b24, 0xffffffff, 0x00ff0fff,
740 0x8b10, 0xffffffff, 0x00000000,
741 0x28a4c, 0x06000000, 0x06000000,
742 0x4d8, 0xffffffff, 0x00000100,
743 0x913c, 0xffff000f, 0x0100000a,
744 0x960c, 0xffffffff, 0x54763210,
745 0x88c4, 0xffffffff, 0x000000c2,
746 0x88d4, 0xffffffff, 0x00000010,
747 0x8974, 0xffffffff, 0x00000000,
748 0xc78, 0x00000080, 0x00000080,
749 0x5e78, 0xffffffff, 0x001000f0,
750 0xd02c, 0xffffffff, 0x08421000,
751 0xa008, 0xffffffff, 0x00010000,
752 0x8d00, 0xffffffff, 0x100e4848,
753 0x8d04, 0xffffffff, 0x00164745,
754 0x8c00, 0xffffffff, 0xe4000003,
755 0x8cf0, 0x1fffffff, 0x08e00410,
756 0x28350, 0xffffffff, 0x00000000,
757 0x9508, 0xffffffff, 0x00000002,
758 0x900c, 0xffffffff, 0x0017071f,
759 0x8c18, 0xffffffff, 0x10101060,
760 0x8c1c, 0xffffffff, 0x00001010
761};
762
763static const u32 barts_golden_registers[] =
764{
765 0x5eb4, 0xffffffff, 0x00000002,
766 0x5e78, 0x8f311ff1, 0x001000f0,
767 0x3f90, 0xffff0000, 0xff000000,
768 0x9148, 0xffff0000, 0xff000000,
769 0x3f94, 0xffff0000, 0xff000000,
770 0x914c, 0xffff0000, 0xff000000,
771 0xc78, 0x00000080, 0x00000080,
772 0xbd4, 0x70073777, 0x00010001,
773 0xd02c, 0xbfffff1f, 0x08421000,
774 0xd0b8, 0x03773777, 0x02011003,
775 0x5bc0, 0x00200000, 0x50100000,
776 0x98f8, 0x33773777, 0x02011003,
777 0x98fc, 0xffffffff, 0x76543210,
778 0x7030, 0x31000311, 0x00000011,
779 0x2f48, 0x00000007, 0x02011003,
780 0x6b28, 0x00000010, 0x00000012,
781 0x7728, 0x00000010, 0x00000012,
782 0x10328, 0x00000010, 0x00000012,
783 0x10f28, 0x00000010, 0x00000012,
784 0x11b28, 0x00000010, 0x00000012,
785 0x12728, 0x00000010, 0x00000012,
786 0x240c, 0x000007ff, 0x00000380,
787 0x8a14, 0xf000001f, 0x00000007,
788 0x8b24, 0x3fff3fff, 0x00ff0fff,
789 0x8b10, 0x0000ff0f, 0x00000000,
790 0x28a4c, 0x07ffffff, 0x06000000,
791 0x10c, 0x00000001, 0x00010003,
792 0xa02c, 0xffffffff, 0x0000009b,
793 0x913c, 0x0000000f, 0x0100000a,
794 0x8d00, 0xffff7f7f, 0x100e4848,
795 0x8d04, 0x00ffffff, 0x00164745,
796 0x8c00, 0xfffc0003, 0xe4000003,
797 0x8c04, 0xf8ff00ff, 0x40600060,
798 0x8c08, 0x00ff00ff, 0x001c001c,
799 0x8cf0, 0x1fff1fff, 0x08e00620,
800 0x8c20, 0x0fff0fff, 0x00800080,
801 0x8c24, 0x0fff0fff, 0x00800080,
802 0x8c18, 0xffffffff, 0x20202078,
803 0x8c1c, 0x0000ffff, 0x00001010,
804 0x28350, 0x00000f01, 0x00000000,
805 0x9508, 0x3700001f, 0x00000002,
806 0x960c, 0xffffffff, 0x54763210,
807 0x88c4, 0x001f3ae3, 0x000000c2,
808 0x88d4, 0x0000001f, 0x00000010,
809 0x8974, 0xffffffff, 0x00000000
810};
811
812static const u32 turks_golden_registers[] =
813{
814 0x5eb4, 0xffffffff, 0x00000002,
815 0x5e78, 0x8f311ff1, 0x001000f0,
816 0x8c8, 0x00003000, 0x00001070,
817 0x8cc, 0x000fffff, 0x00040035,
818 0x3f90, 0xffff0000, 0xfff00000,
819 0x9148, 0xffff0000, 0xfff00000,
820 0x3f94, 0xffff0000, 0xfff00000,
821 0x914c, 0xffff0000, 0xfff00000,
822 0xc78, 0x00000080, 0x00000080,
823 0xbd4, 0x00073007, 0x00010002,
824 0xd02c, 0xbfffff1f, 0x08421000,
825 0xd0b8, 0x03773777, 0x02010002,
826 0x5bc0, 0x00200000, 0x50100000,
827 0x98f8, 0x33773777, 0x00010002,
828 0x98fc, 0xffffffff, 0x33221100,
829 0x7030, 0x31000311, 0x00000011,
830 0x2f48, 0x33773777, 0x00010002,
831 0x6b28, 0x00000010, 0x00000012,
832 0x7728, 0x00000010, 0x00000012,
833 0x10328, 0x00000010, 0x00000012,
834 0x10f28, 0x00000010, 0x00000012,
835 0x11b28, 0x00000010, 0x00000012,
836 0x12728, 0x00000010, 0x00000012,
837 0x240c, 0x000007ff, 0x00000380,
838 0x8a14, 0xf000001f, 0x00000007,
839 0x8b24, 0x3fff3fff, 0x00ff0fff,
840 0x8b10, 0x0000ff0f, 0x00000000,
841 0x28a4c, 0x07ffffff, 0x06000000,
842 0x10c, 0x00000001, 0x00010003,
843 0xa02c, 0xffffffff, 0x0000009b,
844 0x913c, 0x0000000f, 0x0100000a,
845 0x8d00, 0xffff7f7f, 0x100e4848,
846 0x8d04, 0x00ffffff, 0x00164745,
847 0x8c00, 0xfffc0003, 0xe4000003,
848 0x8c04, 0xf8ff00ff, 0x40600060,
849 0x8c08, 0x00ff00ff, 0x001c001c,
850 0x8cf0, 0x1fff1fff, 0x08e00410,
851 0x8c20, 0x0fff0fff, 0x00800080,
852 0x8c24, 0x0fff0fff, 0x00800080,
853 0x8c18, 0xffffffff, 0x20202078,
854 0x8c1c, 0x0000ffff, 0x00001010,
855 0x28350, 0x00000f01, 0x00000000,
856 0x9508, 0x3700001f, 0x00000002,
857 0x960c, 0xffffffff, 0x54763210,
858 0x88c4, 0x001f3ae3, 0x000000c2,
859 0x88d4, 0x0000001f, 0x00000010,
860 0x8974, 0xffffffff, 0x00000000
861};
862
863static const u32 caicos_golden_registers[] =
864{
865 0x5eb4, 0xffffffff, 0x00000002,
866 0x5e78, 0x8f311ff1, 0x001000f0,
867 0x8c8, 0x00003420, 0x00001450,
868 0x8cc, 0x000fffff, 0x00040035,
869 0x3f90, 0xffff0000, 0xfffc0000,
870 0x9148, 0xffff0000, 0xfffc0000,
871 0x3f94, 0xffff0000, 0xfffc0000,
872 0x914c, 0xffff0000, 0xfffc0000,
873 0xc78, 0x00000080, 0x00000080,
874 0xbd4, 0x00073007, 0x00010001,
875 0xd02c, 0xbfffff1f, 0x08421000,
876 0xd0b8, 0x03773777, 0x02010001,
877 0x5bc0, 0x00200000, 0x50100000,
878 0x98f8, 0x33773777, 0x02010001,
879 0x98fc, 0xffffffff, 0x33221100,
880 0x7030, 0x31000311, 0x00000011,
881 0x2f48, 0x33773777, 0x02010001,
882 0x6b28, 0x00000010, 0x00000012,
883 0x7728, 0x00000010, 0x00000012,
884 0x10328, 0x00000010, 0x00000012,
885 0x10f28, 0x00000010, 0x00000012,
886 0x11b28, 0x00000010, 0x00000012,
887 0x12728, 0x00000010, 0x00000012,
888 0x240c, 0x000007ff, 0x00000380,
889 0x8a14, 0xf000001f, 0x00000001,
890 0x8b24, 0x3fff3fff, 0x00ff0fff,
891 0x8b10, 0x0000ff0f, 0x00000000,
892 0x28a4c, 0x07ffffff, 0x06000000,
893 0x10c, 0x00000001, 0x00010003,
894 0xa02c, 0xffffffff, 0x0000009b,
895 0x913c, 0x0000000f, 0x0100000a,
896 0x8d00, 0xffff7f7f, 0x100e4848,
897 0x8d04, 0x00ffffff, 0x00164745,
898 0x8c00, 0xfffc0003, 0xe4000003,
899 0x8c04, 0xf8ff00ff, 0x40600060,
900 0x8c08, 0x00ff00ff, 0x001c001c,
901 0x8cf0, 0x1fff1fff, 0x08e00410,
902 0x8c20, 0x0fff0fff, 0x00800080,
903 0x8c24, 0x0fff0fff, 0x00800080,
904 0x8c18, 0xffffffff, 0x20202078,
905 0x8c1c, 0x0000ffff, 0x00001010,
906 0x28350, 0x00000f01, 0x00000000,
907 0x9508, 0x3700001f, 0x00000002,
908 0x960c, 0xffffffff, 0x54763210,
909 0x88c4, 0x001f3ae3, 0x000000c2,
910 0x88d4, 0x0000001f, 0x00000010,
911 0x8974, 0xffffffff, 0x00000000
912};
913
914static void evergreen_init_golden_registers(struct radeon_device *rdev)
915{
916 switch (rdev->family) {
917 case CHIP_CYPRESS:
918 case CHIP_HEMLOCK:
919 radeon_program_register_sequence(rdev,
920 evergreen_golden_registers,
921 (const u32)ARRAY_SIZE(evergreen_golden_registers));
922 radeon_program_register_sequence(rdev,
923 evergreen_golden_registers2,
924 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
925 radeon_program_register_sequence(rdev,
926 cypress_mgcg_init,
927 (const u32)ARRAY_SIZE(cypress_mgcg_init));
928 break;
929 case CHIP_JUNIPER:
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers));
933 radeon_program_register_sequence(rdev,
934 evergreen_golden_registers2,
935 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
936 radeon_program_register_sequence(rdev,
937 juniper_mgcg_init,
938 (const u32)ARRAY_SIZE(juniper_mgcg_init));
939 break;
940 case CHIP_REDWOOD:
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers));
944 radeon_program_register_sequence(rdev,
945 evergreen_golden_registers2,
946 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
947 radeon_program_register_sequence(rdev,
948 redwood_mgcg_init,
949 (const u32)ARRAY_SIZE(redwood_mgcg_init));
950 break;
951 case CHIP_CEDAR:
952 radeon_program_register_sequence(rdev,
953 cedar_golden_registers,
954 (const u32)ARRAY_SIZE(cedar_golden_registers));
955 radeon_program_register_sequence(rdev,
956 evergreen_golden_registers2,
957 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
958 radeon_program_register_sequence(rdev,
959 cedar_mgcg_init,
960 (const u32)ARRAY_SIZE(cedar_mgcg_init));
961 break;
962 case CHIP_PALM:
963 radeon_program_register_sequence(rdev,
964 wrestler_golden_registers,
965 (const u32)ARRAY_SIZE(wrestler_golden_registers));
966 break;
967 case CHIP_SUMO:
968 radeon_program_register_sequence(rdev,
969 supersumo_golden_registers,
970 (const u32)ARRAY_SIZE(supersumo_golden_registers));
971 break;
972 case CHIP_SUMO2:
973 radeon_program_register_sequence(rdev,
974 supersumo_golden_registers,
975 (const u32)ARRAY_SIZE(supersumo_golden_registers));
976 radeon_program_register_sequence(rdev,
977 sumo_golden_registers,
978 (const u32)ARRAY_SIZE(sumo_golden_registers));
979 break;
980 case CHIP_BARTS:
981 radeon_program_register_sequence(rdev,
982 barts_golden_registers,
983 (const u32)ARRAY_SIZE(barts_golden_registers));
984 break;
985 case CHIP_TURKS:
986 radeon_program_register_sequence(rdev,
987 turks_golden_registers,
988 (const u32)ARRAY_SIZE(turks_golden_registers));
989 break;
990 case CHIP_CAICOS:
991 radeon_program_register_sequence(rdev,
992 caicos_golden_registers,
993 (const u32)ARRAY_SIZE(caicos_golden_registers));
994 break;
995 default:
996 break;
997 }
998}
999
285484e2
JG		 1000void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1001 unsigned *bankh, unsigned *mtaspect,
1002 unsigned *tile_split)
1003{
1004 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1005 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1006 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1007 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1008 switch (*bankw) {
1009 default:
1010 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1011 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1012 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1013 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1014 }
1015 switch (*bankh) {
1016 default:
1017 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1018 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1019 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1020 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1021 }
1022 switch (*mtaspect) {
1023 default:
1024 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1025 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1026 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1027 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1028 }
1029}
1030
23d33ba3
AD		 1031static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1032 u32 cntl_reg, u32 status_reg)
1033{
1034 int r, i;
1035 struct atom_clock_dividers dividers;
1036
1037 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1038 clock, false, &dividers);
1039 if (r)
1040 return r;
1041
1042 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1043
1044 for (i = 0; i < 100; i++) {
1045 if (RREG32(status_reg) & DCLK_STATUS)
1046 break;
1047 mdelay(10);
1048 }
1049 if (i == 100)
1050 return -ETIMEDOUT;
1051
1052 return 0;
1053}
1054
1055int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1056{
1057 int r = 0;
1058 u32 cg_scratch = RREG32(CG_SCRATCH1);
1059
1060 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1061 if (r)
1062 goto done;
1063 cg_scratch &= 0xffff0000;
1064 cg_scratch |= vclk / 100; /* Mhz */
1065
1066 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1067 if (r)
1068 goto done;
1069 cg_scratch &= 0x0000ffff;
1070 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1071
1072done:
1073 WREG32(CG_SCRATCH1, cg_scratch);
1074
1075 return r;
1076}
1077
a8b4925c
AD		 1078int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1079{
1080 /* start off with something large */
facd112d 1081 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1082 int r;
1083
4ed10835
CK		 1084 /* bypass vclk and dclk with bclk */
1085 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1086 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1087 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1088
1089 /* put PLL in bypass mode */
1090 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1091
1092 if (!vclk || !dclk) {
1093 /* keep the Bypass mode, put PLL to sleep */
1094 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1095 return 0;
1096 }
1097
facd112d
CK		 1098 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1099 16384, 0x03FFFFFF, 0, 128, 5,
1100 &fb_div, &vclk_div, &dclk_div);
1101 if (r)
1102 return r;
a8b4925c
AD		 1103
1104 /* set VCO_MODE to 1 */
1105 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1106
1107 /* toggle UPLL_SLEEP to 1 then back to 0 */
1108 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1109 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1110
1111 /* deassert UPLL_RESET */
1112 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1113
1114 mdelay(1);
1115
facd112d 1116 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1117 if (r)
1118 return r;
1119
1120 /* assert UPLL_RESET again */
1121 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1122
1123 /* disable spread spectrum. */
1124 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1125
1126 /* set feedback divider */
facd112d 1127 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1128
1129 /* set ref divider to 0 */
1130 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1131
facd112d 1132 if (fb_div < 307200)
a8b4925c
AD		 1133 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1134 else
1135 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1136
1137 /* set PDIV_A and PDIV_B */
1138 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1139 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1140 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1141
1142 /* give the PLL some time to settle */
1143 mdelay(15);
1144
1145 /* deassert PLL_RESET */
1146 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1147
1148 mdelay(15);
1149
1150 /* switch from bypass mode to normal mode */
1151 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1152
facd112d 1153 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1154 if (r)
1155 return r;
1156
1157 /* switch VCLK and DCLK selection */
1158 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1159 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1160 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1161
1162 mdelay(100);
1163
1164 return 0;
1165}
1166
d054ac16
AD		 1167void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1168{
1169 u16 ctl, v;
32195aec 1170 int err;
d054ac16 1171
32195aec 1172 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
d054ac16
AD		 1173 if (err)
1174 return;
1175
1176 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1177
1178 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1179 * to avoid hangs or perfomance issues
1180 */
1181 if ((v == 0) || (v == 6) || (v == 7)) {
1182 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1183 ctl |= (2 << 12);
32195aec 1184 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
d054ac16
AD		 1185 }
1186}
1187
10257a6d
AD		 1188static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1189{
1190 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1191 return true;
1192 else
1193 return false;
1194}
1195
1196static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1197{
1198 u32 pos1, pos2;
1199
1200 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1201 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1202
1203 if (pos1 != pos2)
1204 return true;
1205 else
1206 return false;
1207}
1208
377edc8b
AD		 1209/**
1210 * dce4_wait_for_vblank - vblank wait asic callback.
1211 *
1212 * @rdev: radeon_device pointer
1213 * @crtc: crtc to wait for vblank on
1214 *
1215 * Wait for vblank on the requested crtc (evergreen+).
1216 */
3ae19b75
AD		 1217void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1218{
10257a6d 1219 unsigned i = 0;
3ae19b75 1220
4a15903d
AD		 1221 if (crtc >= rdev->num_crtc)
1222 return;
1223
10257a6d
AD		 1224 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1225 return;
1226
1227 /* depending on when we hit vblank, we may be close to active; if so,
1228 * wait for another frame.
1229 */
1230 while (dce4_is_in_vblank(rdev, crtc)) {
1231 if (i++ % 100 == 0) {
1232 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1233 break;
3ae19b75 1234 }
10257a6d
AD		 1235 }
1236
1237 while (!dce4_is_in_vblank(rdev, crtc)) {
1238 if (i++ % 100 == 0) {
1239 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1240 break;
3ae19b75
AD		 1241 }
1242 }
1243}
1244
377edc8b
AD		 1245/**
1246 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1247 *
1248 * @rdev: radeon_device pointer
1249 * @crtc: crtc to prepare for pageflip on
1250 *
1251 * Pre-pageflip callback (evergreen+).
1252 * Enables the pageflip irq (vblank irq).
1253 */
6f34be50
AD		 1254void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1255{
6f34be50
AD		 1256 /* enable the pflip int */
1257 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1258}
1259
377edc8b
AD		 1260/**
1261 * evergreen_post_page_flip - pos-pageflip callback.
1262 *
1263 * @rdev: radeon_device pointer
1264 * @crtc: crtc to cleanup pageflip on
1265 *
1266 * Post-pageflip callback (evergreen+).
1267 * Disables the pageflip irq (vblank irq).
1268 */
6f34be50
AD		 1269void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1270{
1271 /* disable the pflip int */
1272 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1273}
1274
377edc8b
AD		 1275/**
1276 * evergreen_page_flip - pageflip callback.
1277 *
1278 * @rdev: radeon_device pointer
1279 * @crtc_id: crtc to cleanup pageflip on
1280 * @crtc_base: new address of the crtc (GPU MC address)
1281 *
1282 * Does the actual pageflip (evergreen+).
1283 * During vblank we take the crtc lock and wait for the update_pending
1284 * bit to go high, when it does, we release the lock, and allow the
1285 * double buffered update to take place.
1286 * Returns the current update pending status.
1287 */
6f34be50
AD		 1288u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1289{
1290 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1291 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1292 int i;
6f34be50
AD		 1293
1294 /* Lock the graphics update lock */
1295 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1296 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1297
1298 /* update the scanout addresses */
1299 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1300 upper_32_bits(crtc_base));
1301 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1302 (u32)crtc_base);
1303
1304 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1305 upper_32_bits(crtc_base));
1306 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1307 (u32)crtc_base);
1308
1309 /* Wait for update_pending to go high. */
f6496479
AD		 1310 for (i = 0; i < rdev->usec_timeout; i++) {
1311 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1312 break;
1313 udelay(1);
1314 }
6f34be50
AD		 1315 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1316
1317 /* Unlock the lock, so double-buffering can take place inside vblank */
1318 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1319 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1320
1321 /* Return current update_pending status: */
1322 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1323}
1324
21a8122a 1325/* get temperature in millidegrees */
20d391d7 1326int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1327{
1c88d74f
AD		 1328 u32 temp, toffset;
1329 int actual_temp = 0;
67b3f823
AD		 1330
1331 if (rdev->family == CHIP_JUNIPER) {
1332 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1333 TOFFSET_SHIFT;
1334 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1335 TS0_ADC_DOUT_SHIFT;
1336
1337 if (toffset & 0x100)
1338 actual_temp = temp / 2 - (0x200 - toffset);
1339 else
1340 actual_temp = temp / 2 + toffset;
1341
1342 actual_temp = actual_temp * 1000;
1343
1344 } else {
1345 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1346 ASIC_T_SHIFT;
1347
1348 if (temp & 0x400)
1349 actual_temp = -256;
1350 else if (temp & 0x200)
1351 actual_temp = 255;
1352 else if (temp & 0x100) {
1353 actual_temp = temp & 0x1ff;
1354 actual_temp |= ~0x1ff;
1355 } else
1356 actual_temp = temp & 0xff;
1357
1358 actual_temp = (actual_temp * 1000) / 2;
1359 }
21a8122a 1360
67b3f823 1361 return actual_temp;
21a8122a
AD		 1362}
1363
20d391d7 1364int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1365{
1366 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1367 int actual_temp = temp - 49;
e33df25f
AD		 1368
1369 return actual_temp * 1000;
1370}
1371
377edc8b
AD		 1372/**
1373 * sumo_pm_init_profile - Initialize power profiles callback.
1374 *
1375 * @rdev: radeon_device pointer
1376 *
1377 * Initialize the power states used in profile mode
1378 * (sumo, trinity, SI).
1379 * Used for profile mode only.
1380 */
a4c9e2ee
AD		 1381void sumo_pm_init_profile(struct radeon_device *rdev)
1382{
1383 int idx;
1384
1385 /* default */
1386 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1387 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1388 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1390
1391 /* low,mid sh/mh */
1392 if (rdev->flags & RADEON_IS_MOBILITY)
1393 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1394 else
1395 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1396
1397 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1398 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1399 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1401
1402 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1403 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1404 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1406
1407 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1408 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1409 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1411
1412 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1413 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1414 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1416
1417 /* high sh/mh */
1418 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1419 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1420 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1421 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1423 rdev->pm.power_state[idx].num_clock_modes - 1;
1424
1425 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1426 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1427 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1429 rdev->pm.power_state[idx].num_clock_modes - 1;
1430}
1431
27810fb2
AD		 1432/**
1433 * btc_pm_init_profile - Initialize power profiles callback.
1434 *
1435 * @rdev: radeon_device pointer
1436 *
1437 * Initialize the power states used in profile mode
1438 * (BTC, cayman).
1439 * Used for profile mode only.
1440 */
1441void btc_pm_init_profile(struct radeon_device *rdev)
1442{
1443 int idx;
1444
1445 /* default */
1446 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1448 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1450 /* starting with BTC, there is one state that is used for both
1451 * MH and SH. Difference is that we always use the high clock index for
1452 * mclk.
1453 */
1454 if (rdev->flags & RADEON_IS_MOBILITY)
1455 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1456 else
1457 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1458 /* low sh */
1459 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1460 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1463 /* mid sh */
1464 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1468 /* high sh */
1469 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1470 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1473 /* low mh */
1474 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1475 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1476 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1478 /* mid mh */
1479 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1480 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1481 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1483 /* high mh */
1484 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1488}
1489
377edc8b
AD		 1490/**
1491 * evergreen_pm_misc - set additional pm hw parameters callback.
1492 *
1493 * @rdev: radeon_device pointer
1494 *
1495 * Set non-clock parameters associated with a power state
1496 * (voltage, etc.) (evergreen+).
1497 */
49e02b73
AD		 1498void evergreen_pm_misc(struct radeon_device *rdev)
1499{
a081a9d6
RM		 1500 int req_ps_idx = rdev->pm.requested_power_state_index;
1501 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1502 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1503 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1504
2feea49a 1505 if (voltage->type == VOLTAGE_SW) {
a377e187
AD		 1506 /* 0xff01 is a flag rather then an actual voltage */
1507 if (voltage->voltage == 0xff01)
1508 return;
2feea49a 1509 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1510 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1511 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1512 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1513 }
7ae764b1
AD		 1514
1515 /* starting with BTC, there is one state that is used for both
1516 * MH and SH. Difference is that we always use the high clock index for
1517 * mclk and vddci.
1518 */
1519 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1520 (rdev->family >= CHIP_BARTS) &&
1521 rdev->pm.active_crtc_count &&
1522 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1523 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1524 voltage = &rdev->pm.power_state[req_ps_idx].
1525 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1526
a377e187
AD		 1527 /* 0xff01 is a flag rather then an actual voltage */
1528 if (voltage->vddci == 0xff01)
1529 return;
2feea49a
AD		 1530 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1531 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1532 rdev->pm.current_vddci = voltage->vddci;
1533 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1534 }
1535 }
49e02b73
AD		 1536}
1537
377edc8b
AD		 1538/**
1539 * evergreen_pm_prepare - pre-power state change callback.
1540 *
1541 * @rdev: radeon_device pointer
1542 *
1543 * Prepare for a power state change (evergreen+).
1544 */
49e02b73
AD		 1545void evergreen_pm_prepare(struct radeon_device *rdev)
1546{
1547 struct drm_device *ddev = rdev->ddev;
1548 struct drm_crtc *crtc;
1549 struct radeon_crtc *radeon_crtc;
1550 u32 tmp;
1551
1552 /* disable any active CRTCs */
1553 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1554 radeon_crtc = to_radeon_crtc(crtc);
1555 if (radeon_crtc->enabled) {
1556 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1557 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1558 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1559 }
1560 }
1561}
1562
377edc8b
AD		 1563/**
1564 * evergreen_pm_finish - post-power state change callback.
1565 *
1566 * @rdev: radeon_device pointer
1567 *
1568 * Clean up after a power state change (evergreen+).
1569 */
49e02b73
AD		 1570void evergreen_pm_finish(struct radeon_device *rdev)
1571{
1572 struct drm_device *ddev = rdev->ddev;
1573 struct drm_crtc *crtc;
1574 struct radeon_crtc *radeon_crtc;
1575 u32 tmp;
1576
1577 /* enable any active CRTCs */
1578 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1579 radeon_crtc = to_radeon_crtc(crtc);
1580 if (radeon_crtc->enabled) {
1581 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1582 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1583 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1584 }
1585 }
1586}
1587
377edc8b
AD		 1588/**
1589 * evergreen_hpd_sense - hpd sense callback.
1590 *
1591 * @rdev: radeon_device pointer
1592 * @hpd: hpd (hotplug detect) pin
1593 *
1594 * Checks if a digital monitor is connected (evergreen+).
1595 * Returns true if connected, false if not connected.
1596 */
bcc1c2a1
AD		 1597bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1598{
1599 bool connected = false;
0ca2ab52
AD		 1600
1601 switch (hpd) {
1602 case RADEON_HPD_1:
1603 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1604 connected = true;
1605 break;
1606 case RADEON_HPD_2:
1607 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1608 connected = true;
1609 break;
1610 case RADEON_HPD_3:
1611 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1612 connected = true;
1613 break;
1614 case RADEON_HPD_4:
1615 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1616 connected = true;
1617 break;
1618 case RADEON_HPD_5:
1619 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1620 connected = true;
1621 break;
1622 case RADEON_HPD_6:
1623 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1624 connected = true;
1625 break;
1626 default:
1627 break;
1628 }
1629
bcc1c2a1
AD		 1630 return connected;
1631}
1632
377edc8b
AD		 1633/**
1634 * evergreen_hpd_set_polarity - hpd set polarity callback.
1635 *
1636 * @rdev: radeon_device pointer
1637 * @hpd: hpd (hotplug detect) pin
1638 *
1639 * Set the polarity of the hpd pin (evergreen+).
1640 */
bcc1c2a1
AD		 1641void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1642 enum radeon_hpd_id hpd)
1643{
0ca2ab52
AD		 1644 u32 tmp;
1645 bool connected = evergreen_hpd_sense(rdev, hpd);
1646
1647 switch (hpd) {
1648 case RADEON_HPD_1:
1649 tmp = RREG32(DC_HPD1_INT_CONTROL);
1650 if (connected)
1651 tmp &= ~DC_HPDx_INT_POLARITY;
1652 else
1653 tmp |= DC_HPDx_INT_POLARITY;
1654 WREG32(DC_HPD1_INT_CONTROL, tmp);
1655 break;
1656 case RADEON_HPD_2:
1657 tmp = RREG32(DC_HPD2_INT_CONTROL);
1658 if (connected)
1659 tmp &= ~DC_HPDx_INT_POLARITY;
1660 else
1661 tmp |= DC_HPDx_INT_POLARITY;
1662 WREG32(DC_HPD2_INT_CONTROL, tmp);
1663 break;
1664 case RADEON_HPD_3:
1665 tmp = RREG32(DC_HPD3_INT_CONTROL);
1666 if (connected)
1667 tmp &= ~DC_HPDx_INT_POLARITY;
1668 else
1669 tmp |= DC_HPDx_INT_POLARITY;
1670 WREG32(DC_HPD3_INT_CONTROL, tmp);
1671 break;
1672 case RADEON_HPD_4:
1673 tmp = RREG32(DC_HPD4_INT_CONTROL);
1674 if (connected)
1675 tmp &= ~DC_HPDx_INT_POLARITY;
1676 else
1677 tmp |= DC_HPDx_INT_POLARITY;
1678 WREG32(DC_HPD4_INT_CONTROL, tmp);
1679 break;
1680 case RADEON_HPD_5:
1681 tmp = RREG32(DC_HPD5_INT_CONTROL);
1682 if (connected)
1683 tmp &= ~DC_HPDx_INT_POLARITY;
1684 else
1685 tmp |= DC_HPDx_INT_POLARITY;
1686 WREG32(DC_HPD5_INT_CONTROL, tmp);
1687 break;
1688 case RADEON_HPD_6:
1689 tmp = RREG32(DC_HPD6_INT_CONTROL);
1690 if (connected)
1691 tmp &= ~DC_HPDx_INT_POLARITY;
1692 else
1693 tmp |= DC_HPDx_INT_POLARITY;
1694 WREG32(DC_HPD6_INT_CONTROL, tmp);
1695 break;
1696 default:
1697 break;
1698 }
bcc1c2a1
AD		 1699}
1700
377edc8b
AD		 1701/**
1702 * evergreen_hpd_init - hpd setup callback.
1703 *
1704 * @rdev: radeon_device pointer
1705 *
1706 * Setup the hpd pins used by the card (evergreen+).
1707 * Enable the pin, set the polarity, and enable the hpd interrupts.
1708 */
bcc1c2a1
AD		 1709void evergreen_hpd_init(struct radeon_device *rdev)
1710{
0ca2ab52
AD		 1711 struct drm_device *dev = rdev->ddev;
1712 struct drm_connector *connector;
fb98257a 1713 unsigned enabled = 0;
0ca2ab52
AD		 1714 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1715 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1716
0ca2ab52
AD		 1717 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1718 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1719
1720 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1721 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1722 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1723 * aux dp channel on imac and help (but not completely fix)
1724 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1725 * also avoid interrupt storms during dpms.
1726 */
1727 continue;
1728 }
0ca2ab52
AD		 1729 switch (radeon_connector->hpd.hpd) {
1730 case RADEON_HPD_1:
1731 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1732 break;
1733 case RADEON_HPD_2:
1734 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1735 break;
1736 case RADEON_HPD_3:
1737 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1738 break;
1739 case RADEON_HPD_4:
1740 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1741 break;
1742 case RADEON_HPD_5:
1743 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1744 break;
1745 case RADEON_HPD_6:
1746 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1747 break;
1748 default:
1749 break;
1750 }
64912e99 1751 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1752 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1753 }
fb98257a 1754 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1755}
1756
377edc8b
AD		 1757/**
1758 * evergreen_hpd_fini - hpd tear down callback.
1759 *
1760 * @rdev: radeon_device pointer
1761 *
1762 * Tear down the hpd pins used by the card (evergreen+).
1763 * Disable the hpd interrupts.
1764 */
0ca2ab52 1765void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1766{
0ca2ab52
AD		 1767 struct drm_device *dev = rdev->ddev;
1768 struct drm_connector *connector;
fb98257a 1769 unsigned disabled = 0;
0ca2ab52
AD		 1770
1771 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1772 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1773 switch (radeon_connector->hpd.hpd) {
1774 case RADEON_HPD_1:
1775 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1776 break;
1777 case RADEON_HPD_2:
1778 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1779 break;
1780 case RADEON_HPD_3:
1781 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1782 break;
1783 case RADEON_HPD_4:
1784 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1785 break;
1786 case RADEON_HPD_5:
1787 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1788 break;
1789 case RADEON_HPD_6:
1790 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1791 break;
1792 default:
1793 break;
1794 }
fb98257a 1795 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1796 }
fb98257a 1797 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1798}
1799
f9d9c362
AD		 1800/* watermark setup */
1801
1802static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1803 struct radeon_crtc *radeon_crtc,
1804 struct drm_display_mode *mode,
1805 struct drm_display_mode *other_mode)
1806{
12dfc843 1807 u32 tmp;
f9d9c362
AD		 1808 /*
1809 * Line Buffer Setup
1810 * There are 3 line buffers, each one shared by 2 display controllers.
1811 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1812 * the display controllers. The paritioning is done via one of four
1813 * preset allocations specified in bits 2:0:
1814 * first display controller
1815 * 0 - first half of lb (3840 * 2)
1816 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1817 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1818 * 3 - first 1/4 of lb (1920 * 2)
1819 * second display controller
1820 * 4 - second half of lb (3840 * 2)
1821 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1822 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1823 * 7 - last 1/4 of lb (1920 * 2)
1824 */
12dfc843
AD		 1825 /* this can get tricky if we have two large displays on a paired group
1826 * of crtcs. Ideally for multiple large displays we'd assign them to
1827 * non-linked crtcs for maximum line buffer allocation.
1828 */
1829 if (radeon_crtc->base.enabled && mode) {
1830 if (other_mode)
f9d9c362 1831 tmp = 0; /* 1/2 */
12dfc843
AD		 1832 else
1833 tmp = 2; /* whole */
1834 } else
1835 tmp = 0;
f9d9c362
AD		 1836
1837 /* second controller of the pair uses second half of the lb */
1838 if (radeon_crtc->crtc_id % 2)
1839 tmp += 4;
1840 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1841
12dfc843
AD		 1842 if (radeon_crtc->base.enabled && mode) {
1843 switch (tmp) {
1844 case 0:
1845 case 4:
1846 default:
1847 if (ASIC_IS_DCE5(rdev))
1848 return 4096 * 2;
1849 else
1850 return 3840 * 2;
1851 case 1:
1852 case 5:
1853 if (ASIC_IS_DCE5(rdev))
1854 return 6144 * 2;
1855 else
1856 return 5760 * 2;
1857 case 2:
1858 case 6:
1859 if (ASIC_IS_DCE5(rdev))
1860 return 8192 * 2;
1861 else
1862 return 7680 * 2;
1863 case 3:
1864 case 7:
1865 if (ASIC_IS_DCE5(rdev))
1866 return 2048 * 2;
1867 else
1868 return 1920 * 2;
1869 }
f9d9c362 1870 }
12dfc843
AD		 1871
1872 /* controller not enabled, so no lb used */
1873 return 0;
f9d9c362
AD		 1874}
1875
ca7db22b 1876u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1877{
1878 u32 tmp = RREG32(MC_SHARED_CHMAP);
1879
1880 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1881 case 0:
1882 default:
1883 return 1;
1884 case 1:
1885 return 2;
1886 case 2:
1887 return 4;
1888 case 3:
1889 return 8;
1890 }
1891}
1892
1893struct evergreen_wm_params {
1894 u32 dram_channels; /* number of dram channels */
1895 u32 yclk; /* bandwidth per dram data pin in kHz */
1896 u32 sclk; /* engine clock in kHz */
1897 u32 disp_clk; /* display clock in kHz */
1898 u32 src_width; /* viewport width */
1899 u32 active_time; /* active display time in ns */
1900 u32 blank_time; /* blank time in ns */
1901 bool interlaced; /* mode is interlaced */
1902 fixed20_12 vsc; /* vertical scale ratio */
1903 u32 num_heads; /* number of active crtcs */
1904 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1905 u32 lb_size; /* line buffer allocated to pipe */
1906 u32 vtaps; /* vertical scaler taps */
1907};
1908
1909static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1910{
1911 /* Calculate DRAM Bandwidth and the part allocated to display. */
1912 fixed20_12 dram_efficiency; /* 0.7 */
1913 fixed20_12 yclk, dram_channels, bandwidth;
1914 fixed20_12 a;
1915
1916 a.full = dfixed_const(1000);
1917 yclk.full = dfixed_const(wm->yclk);
1918 yclk.full = dfixed_div(yclk, a);
1919 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1920 a.full = dfixed_const(10);
1921 dram_efficiency.full = dfixed_const(7);
1922 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1923 bandwidth.full = dfixed_mul(dram_channels, yclk);
1924 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1925
1926 return dfixed_trunc(bandwidth);
1927}
1928
1929static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1930{
1931 /* Calculate DRAM Bandwidth and the part allocated to display. */
1932 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1933 fixed20_12 yclk, dram_channels, bandwidth;
1934 fixed20_12 a;
1935
1936 a.full = dfixed_const(1000);
1937 yclk.full = dfixed_const(wm->yclk);
1938 yclk.full = dfixed_div(yclk, a);
1939 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1940 a.full = dfixed_const(10);
1941 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1942 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1943 bandwidth.full = dfixed_mul(dram_channels, yclk);
1944 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1945
1946 return dfixed_trunc(bandwidth);
1947}
1948
1949static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1950{
1951 /* Calculate the display Data return Bandwidth */
1952 fixed20_12 return_efficiency; /* 0.8 */
1953 fixed20_12 sclk, bandwidth;
1954 fixed20_12 a;
1955
1956 a.full = dfixed_const(1000);
1957 sclk.full = dfixed_const(wm->sclk);
1958 sclk.full = dfixed_div(sclk, a);
1959 a.full = dfixed_const(10);
1960 return_efficiency.full = dfixed_const(8);
1961 return_efficiency.full = dfixed_div(return_efficiency, a);
1962 a.full = dfixed_const(32);
1963 bandwidth.full = dfixed_mul(a, sclk);
1964 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1965
1966 return dfixed_trunc(bandwidth);
1967}
1968
1969static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1970{
1971 /* Calculate the DMIF Request Bandwidth */
1972 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1973 fixed20_12 disp_clk, bandwidth;
1974 fixed20_12 a;
1975
1976 a.full = dfixed_const(1000);
1977 disp_clk.full = dfixed_const(wm->disp_clk);
1978 disp_clk.full = dfixed_div(disp_clk, a);
1979 a.full = dfixed_const(10);
1980 disp_clk_request_efficiency.full = dfixed_const(8);
1981 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1982 a.full = dfixed_const(32);
1983 bandwidth.full = dfixed_mul(a, disp_clk);
1984 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1985
1986 return dfixed_trunc(bandwidth);
1987}
1988
1989static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1990{
1991 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1992 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1993 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1994 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1995
1996 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1997}
1998
1999static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2000{
2001 /* Calculate the display mode Average Bandwidth
2002 * DisplayMode should contain the source and destination dimensions,
2003 * timing, etc.
2004 */
2005 fixed20_12 bpp;
2006 fixed20_12 line_time;
2007 fixed20_12 src_width;
2008 fixed20_12 bandwidth;
2009 fixed20_12 a;
2010
2011 a.full = dfixed_const(1000);
2012 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2013 line_time.full = dfixed_div(line_time, a);
2014 bpp.full = dfixed_const(wm->bytes_per_pixel);
2015 src_width.full = dfixed_const(wm->src_width);
2016 bandwidth.full = dfixed_mul(src_width, bpp);
2017 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2018 bandwidth.full = dfixed_div(bandwidth, line_time);
2019
2020 return dfixed_trunc(bandwidth);
2021}
2022
2023static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2024{
2025 /* First calcualte the latency in ns */
2026 u32 mc_latency = 2000; /* 2000 ns. */
2027 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2028 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2029 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2030 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2031 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2032 (wm->num_heads * cursor_line_pair_return_time);
2033 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2034 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2035 fixed20_12 a, b, c;
2036
2037 if (wm->num_heads == 0)
2038 return 0;
2039
2040 a.full = dfixed_const(2);
2041 b.full = dfixed_const(1);
2042 if ((wm->vsc.full > a.full) ||
2043 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2044 (wm->vtaps >= 5) ||
2045 ((wm->vsc.full >= a.full) && wm->interlaced))
2046 max_src_lines_per_dst_line = 4;
2047 else
2048 max_src_lines_per_dst_line = 2;
2049
2050 a.full = dfixed_const(available_bandwidth);
2051 b.full = dfixed_const(wm->num_heads);
2052 a.full = dfixed_div(a, b);
2053
2054 b.full = dfixed_const(1000);
2055 c.full = dfixed_const(wm->disp_clk);
2056 b.full = dfixed_div(c, b);
2057 c.full = dfixed_const(wm->bytes_per_pixel);
2058 b.full = dfixed_mul(b, c);
2059
2060 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2061
2062 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2063 b.full = dfixed_const(1000);
2064 c.full = dfixed_const(lb_fill_bw);
2065 b.full = dfixed_div(c, b);
2066 a.full = dfixed_div(a, b);
2067 line_fill_time = dfixed_trunc(a);
2068
2069 if (line_fill_time < wm->active_time)
2070 return latency;
2071 else
2072 return latency + (line_fill_time - wm->active_time);
2073
2074}
2075
2076static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2077{
2078 if (evergreen_average_bandwidth(wm) <=
2079 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2080 return true;
2081 else
2082 return false;
2083};
2084
2085static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2086{
2087 if (evergreen_average_bandwidth(wm) <=
2088 (evergreen_available_bandwidth(wm) / wm->num_heads))
2089 return true;
2090 else
2091 return false;
2092};
2093
2094static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2095{
2096 u32 lb_partitions = wm->lb_size / wm->src_width;
2097 u32 line_time = wm->active_time + wm->blank_time;
2098 u32 latency_tolerant_lines;
2099 u32 latency_hiding;
2100 fixed20_12 a;
2101
2102 a.full = dfixed_const(1);
2103 if (wm->vsc.full > a.full)
2104 latency_tolerant_lines = 1;
2105 else {
2106 if (lb_partitions <= (wm->vtaps + 1))
2107 latency_tolerant_lines = 1;
2108 else
2109 latency_tolerant_lines = 2;
2110 }
2111
2112 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2113
2114 if (evergreen_latency_watermark(wm) <= latency_hiding)
2115 return true;
2116 else
2117 return false;
2118}
2119
2120static void evergreen_program_watermarks(struct radeon_device *rdev,
2121 struct radeon_crtc *radeon_crtc,
2122 u32 lb_size, u32 num_heads)
2123{
2124 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2125 struct evergreen_wm_params wm;
2126 u32 pixel_period;
2127 u32 line_time = 0;
2128 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2129 u32 priority_a_mark = 0, priority_b_mark = 0;
2130 u32 priority_a_cnt = PRIORITY_OFF;
2131 u32 priority_b_cnt = PRIORITY_OFF;
2132 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2133 u32 tmp, arb_control3;
2134 fixed20_12 a, b, c;
2135
2136 if (radeon_crtc->base.enabled && num_heads && mode) {
2137 pixel_period = 1000000 / (u32)mode->clock;
2138 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2139 priority_a_cnt = 0;
2140 priority_b_cnt = 0;
2141
2142 wm.yclk = rdev->pm.current_mclk * 10;
2143 wm.sclk = rdev->pm.current_sclk * 10;
2144 wm.disp_clk = mode->clock;
2145 wm.src_width = mode->crtc_hdisplay;
2146 wm.active_time = mode->crtc_hdisplay * pixel_period;
2147 wm.blank_time = line_time - wm.active_time;
2148 wm.interlaced = false;
2149 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2150 wm.interlaced = true;
2151 wm.vsc = radeon_crtc->vsc;
2152 wm.vtaps = 1;
2153 if (radeon_crtc->rmx_type != RMX_OFF)
2154 wm.vtaps = 2;
2155 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
2156 wm.lb_size = lb_size;
2157 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
2158 wm.num_heads = num_heads;
2159
2160 /* set for high clocks */
2161 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
2162 /* set for low clocks */
2163 /* wm.yclk = low clk; wm.sclk = low clk */
2164 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
2165
2166 /* possibly force display priority to high */
2167 /* should really do this at mode validation time... */
2168 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
2169 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
2170 !evergreen_check_latency_hiding(&wm) ||
2171 (rdev->disp_priority == 2)) {
92bdfd4a 2172 DRM_DEBUG_KMS("force priority to high\n");
f9d9c362
AD		 2173 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2174 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2175 }
2176
2177 a.full = dfixed_const(1000);
2178 b.full = dfixed_const(mode->clock);
2179 b.full = dfixed_div(b, a);
2180 c.full = dfixed_const(latency_watermark_a);
2181 c.full = dfixed_mul(c, b);
2182 c.full = dfixed_mul(c, radeon_crtc->hsc);
2183 c.full = dfixed_div(c, a);
2184 a.full = dfixed_const(16);
2185 c.full = dfixed_div(c, a);
2186 priority_a_mark = dfixed_trunc(c);
2187 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2188
2189 a.full = dfixed_const(1000);
2190 b.full = dfixed_const(mode->clock);
2191 b.full = dfixed_div(b, a);
2192 c.full = dfixed_const(latency_watermark_b);
2193 c.full = dfixed_mul(c, b);
2194 c.full = dfixed_mul(c, radeon_crtc->hsc);
2195 c.full = dfixed_div(c, a);
2196 a.full = dfixed_const(16);
2197 c.full = dfixed_div(c, a);
2198 priority_b_mark = dfixed_trunc(c);
2199 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2200 }
2201
2202 /* select wm A */
2203 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2204 tmp = arb_control3;
2205 tmp &= ~LATENCY_WATERMARK_MASK(3);
2206 tmp |= LATENCY_WATERMARK_MASK(1);
2207 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2208 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2209 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2210 LATENCY_HIGH_WATERMARK(line_time)));
2211 /* select wm B */
2212 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2213 tmp &= ~LATENCY_WATERMARK_MASK(3);
2214 tmp |= LATENCY_WATERMARK_MASK(2);
2215 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2216 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2217 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2218 LATENCY_HIGH_WATERMARK(line_time)));
2219 /* restore original selection */
2220 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2221
2222 /* write the priority marks */
2223 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2224 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2225
2226}
2227
377edc8b
AD		 2228/**
2229 * evergreen_bandwidth_update - update display watermarks callback.
2230 *
2231 * @rdev: radeon_device pointer
2232 *
2233 * Update the display watermarks based on the requested mode(s)
2234 * (evergreen+).
2235 */
0ca2ab52 2236void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2237{
f9d9c362
AD		 2238 struct drm_display_mode *mode0 = NULL;
2239 struct drm_display_mode *mode1 = NULL;
2240 u32 num_heads = 0, lb_size;
2241 int i;
2242
2243 radeon_update_display_priority(rdev);
2244
2245 for (i = 0; i < rdev->num_crtc; i++) {
2246 if (rdev->mode_info.crtcs[i]->base.enabled)
2247 num_heads++;
2248 }
2249 for (i = 0; i < rdev->num_crtc; i += 2) {
2250 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2251 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2252 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2253 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2254 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2255 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2256 }
bcc1c2a1
AD		 2257}
2258
377edc8b
AD		 2259/**
2260 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2261 *
2262 * @rdev: radeon_device pointer
2263 *
2264 * Wait for the MC (memory controller) to be idle.
2265 * (evergreen+).
2266 * Returns 0 if the MC is idle, -1 if not.
2267 */
b9952a8a 2268int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2269{
2270 unsigned i;
2271 u32 tmp;
2272
2273 for (i = 0; i < rdev->usec_timeout; i++) {
2274 /* read MC_STATUS */
2275 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2276 if (!tmp)
2277 return 0;
2278 udelay(1);
2279 }
2280 return -1;
2281}
2282
2283/*
2284 * GART
2285 */
0fcdb61e
AD		 2286void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2287{
2288 unsigned i;
2289 u32 tmp;
2290
6f2f48a9
AD		 2291 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2292
0fcdb61e
AD		 2293 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2294 for (i = 0; i < rdev->usec_timeout; i++) {
2295 /* read MC_STATUS */
2296 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2297 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2298 if (tmp == 2) {
2299 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2300 return;
2301 }
2302 if (tmp) {
2303 return;
2304 }
2305 udelay(1);
2306 }
2307}
2308
1109ca09 2309static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2310{
2311 u32 tmp;
0fcdb61e 2312 int r;
bcc1c2a1 2313
c9a1be96 2314 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2315 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2316 return -EINVAL;
2317 }
2318 r = radeon_gart_table_vram_pin(rdev);
2319 if (r)
2320 return r;
82568565 2321 radeon_gart_restore(rdev);
bcc1c2a1
AD		 2322 /* Setup L2 cache */
2323 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2324 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2325 EFFECTIVE_L2_QUEUE_SIZE(7));
2326 WREG32(VM_L2_CNTL2, 0);
2327 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2328 /* Setup TLB control */
2329 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2330 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2331 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2332 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2333 if (rdev->flags & RADEON_IS_IGP) {
2334 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2335 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2336 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2337 } else {
2338 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2339 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2340 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2341 if ((rdev->family == CHIP_JUNIPER) ||
2342 (rdev->family == CHIP_CYPRESS) ||
2343 (rdev->family == CHIP_HEMLOCK) ||
2344 (rdev->family == CHIP_BARTS))
2345 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2346 }
bcc1c2a1
AD		 2347 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2348 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2349 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2350 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2351 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2352 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2353 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2354 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2355 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2356 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2357 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2358 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2359
0fcdb61e 2360 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2361 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2362 (unsigned)(rdev->mc.gtt_size >> 20),
2363 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2364 rdev->gart.ready = true;
2365 return 0;
2366}
2367
1109ca09 2368static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2369{
2370 u32 tmp;
bcc1c2a1
AD		 2371
2372 /* Disable all tables */
0fcdb61e
AD		 2373 WREG32(VM_CONTEXT0_CNTL, 0);
2374 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2375
2376 /* Setup L2 cache */
2377 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2378 EFFECTIVE_L2_QUEUE_SIZE(7));
2379 WREG32(VM_L2_CNTL2, 0);
2380 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2381 /* Setup TLB control */
2382 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2383 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2384 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2385 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2386 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2387 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2388 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2389 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2390 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2391}
2392
1109ca09 2393static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2394{
2395 evergreen_pcie_gart_disable(rdev);
2396 radeon_gart_table_vram_free(rdev);
2397 radeon_gart_fini(rdev);
2398}
2399
2400
1109ca09 2401static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2402{
2403 u32 tmp;
bcc1c2a1
AD		 2404
2405 /* Setup L2 cache */
2406 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2407 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2408 EFFECTIVE_L2_QUEUE_SIZE(7));
2409 WREG32(VM_L2_CNTL2, 0);
2410 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2411 /* Setup TLB control */
2412 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2413 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2414 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2415 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2416 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2417 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2418 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2419 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2420 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2421 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2422 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2423 WREG32(VM_CONTEXT0_CNTL, 0);
2424 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2425}
2426
b9952a8a 2427void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2428{
62444b74
AD		 2429 u32 crtc_enabled, tmp, frame_count, blackout;
2430 int i, j;
2431
5153550a
AD		 2432 if (!ASIC_IS_NODCE(rdev)) {
2433 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2434 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2435
5153550a
AD		 2436 /* disable VGA render */
2437 WREG32(VGA_RENDER_CONTROL, 0);
2438 }
62444b74
AD		 2439 /* blank the display controllers */
2440 for (i = 0; i < rdev->num_crtc; i++) {
2441 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2442 if (crtc_enabled) {
2443 save->crtc_enabled[i] = true;
2444 if (ASIC_IS_DCE6(rdev)) {
2445 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2446 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2447 radeon_wait_for_vblank(rdev, i);
abf1457b 2448 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2449 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2450 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2451 }
2452 } else {
2453 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2454 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2455 radeon_wait_for_vblank(rdev, i);
abf1457b 2456 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2457 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2458 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2459 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2460 }
2461 }
2462 /* wait for the next frame */
2463 frame_count = radeon_get_vblank_counter(rdev, i);
2464 for (j = 0; j < rdev->usec_timeout; j++) {
2465 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2466 break;
2467 udelay(1);
2468 }
abf1457b
AD		 2469
2470 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2471 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2472 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2473 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2474 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2475 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2476 save->crtc_enabled[i] = false;
2477 /* ***** */
804cc4a0
AD		 2478 } else {
2479 save->crtc_enabled[i] = false;
62444b74 2480 }
18007401 2481 }
bcc1c2a1 2482
62444b74
AD		 2483 radeon_mc_wait_for_idle(rdev);
2484
2485 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2486 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2487 /* Block CPU access */
2488 WREG32(BIF_FB_EN, 0);
2489 /* blackout the MC */
2490 blackout &= ~BLACKOUT_MODE_MASK;
2491 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2492 }
ed39fadd
AD		 2493 /* wait for the MC to settle */
2494 udelay(100);
968c0166
AD		 2495
2496 /* lock double buffered regs */
2497 for (i = 0; i < rdev->num_crtc; i++) {
2498 if (save->crtc_enabled[i]) {
2499 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2500 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2501 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2502 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2503 }
2504 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2505 if (!(tmp & 1)) {
2506 tmp |= 1;
2507 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2508 }
2509 }
2510 }
bcc1c2a1
AD		 2511}
2512
b9952a8a 2513void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2514{
62444b74
AD		 2515 u32 tmp, frame_count;
2516 int i, j;
18007401 2517
62444b74
AD		 2518 /* update crtc base addresses */
2519 for (i = 0; i < rdev->num_crtc; i++) {
2520 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2521 upper_32_bits(rdev->mc.vram_start));
62444b74 2522 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2523 upper_32_bits(rdev->mc.vram_start));
62444b74 2524 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2525 (u32)rdev->mc.vram_start);
62444b74 2526 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2527 (u32)rdev->mc.vram_start);
2528 }
5153550a
AD		 2529
2530 if (!ASIC_IS_NODCE(rdev)) {
2531 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2532 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2533 }
62444b74 2534
968c0166
AD		 2535 /* unlock regs and wait for update */
2536 for (i = 0; i < rdev->num_crtc; i++) {
2537 if (save->crtc_enabled[i]) {
2538 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2539 if ((tmp & 0x3) != 0) {
2540 tmp &= ~0x3;
2541 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2542 }
2543 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2544 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2545 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2546 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2547 }
2548 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2549 if (tmp & 1) {
2550 tmp &= ~1;
2551 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2552 }
2553 for (j = 0; j < rdev->usec_timeout; j++) {
2554 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2555 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2556 break;
2557 udelay(1);
2558 }
2559 }
2560 }
2561
62444b74
AD		 2562 /* unblackout the MC */
2563 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2564 tmp &= ~BLACKOUT_MODE_MASK;
2565 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2566 /* allow CPU access */
2567 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2568
2569 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2570 if (save->crtc_enabled[i]) {
62444b74
AD		 2571 if (ASIC_IS_DCE6(rdev)) {
2572 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2573 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2574 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2575 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2576 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2577 } else {
2578 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2579 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2580 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2581 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2582 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2583 }
2584 /* wait for the next frame */
2585 frame_count = radeon_get_vblank_counter(rdev, i);
2586 for (j = 0; j < rdev->usec_timeout; j++) {
2587 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2588 break;
2589 udelay(1);
2590 }
2591 }
2592 }
5153550a
AD		 2593 if (!ASIC_IS_NODCE(rdev)) {
2594 /* Unlock vga access */
2595 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2596 mdelay(1);
2597 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2598 }
bcc1c2a1
AD		 2599}
2600
755d819e 2601void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2602{
2603 struct evergreen_mc_save save;
2604 u32 tmp;
2605 int i, j;
2606
2607 /* Initialize HDP */
2608 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2609 WREG32((0x2c14 + j), 0x00000000);
2610 WREG32((0x2c18 + j), 0x00000000);
2611 WREG32((0x2c1c + j), 0x00000000);
2612 WREG32((0x2c20 + j), 0x00000000);
2613 WREG32((0x2c24 + j), 0x00000000);
2614 }
2615 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2616
2617 evergreen_mc_stop(rdev, &save);
2618 if (evergreen_mc_wait_for_idle(rdev)) {
2619 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2620 }
2621 /* Lockout access through VGA aperture*/
2622 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2623 /* Update configuration */
2624 if (rdev->flags & RADEON_IS_AGP) {
2625 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2626 /* VRAM before AGP */
2627 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2628 rdev->mc.vram_start >> 12);
2629 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2630 rdev->mc.gtt_end >> 12);
2631 } else {
2632 /* VRAM after AGP */
2633 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2634 rdev->mc.gtt_start >> 12);
2635 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2636 rdev->mc.vram_end >> 12);
2637 }
2638 } else {
2639 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2640 rdev->mc.vram_start >> 12);
2641 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2642 rdev->mc.vram_end >> 12);
2643 }
3b9832f6 2644 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2645 /* llano/ontario only */
2646 if ((rdev->family == CHIP_PALM) ||
2647 (rdev->family == CHIP_SUMO) ||
2648 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2649 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2650 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2651 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2652 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2653 }
bcc1c2a1
AD		 2654 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2655 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2656 WREG32(MC_VM_FB_LOCATION, tmp);
2657 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2658 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2659 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2660 if (rdev->flags & RADEON_IS_AGP) {
2661 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2662 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2663 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2664 } else {
2665 WREG32(MC_VM_AGP_BASE, 0);
2666 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2667 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2668 }
2669 if (evergreen_mc_wait_for_idle(rdev)) {
2670 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2671 }
2672 evergreen_mc_resume(rdev, &save);
2673 /* we need to own VRAM, so turn off the VGA renderer here
2674 * to stop it overwriting our objects */
2675 rv515_vga_render_disable(rdev);
2676}
2677
bcc1c2a1
AD		 2678/*
2679 * CP.
2680 */
12920591
AD		 2681void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2682{
876dc9f3 2683 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2684 u32 next_rptr;
7b1f2485 2685
12920591 2686 /* set to DX10/11 mode */
e32eb50d
CK		 2687 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2688 radeon_ring_write(ring, 1);
45df6803
CK		 2689
2690 if (ring->rptr_save_reg) {
89d35807 2691 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2692 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2693 radeon_ring_write(ring, ((ring->rptr_save_reg -
2694 PACKET3_SET_CONFIG_REG_START) >> 2));
2695 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2696 } else if (rdev->wb.enabled) {
2697 next_rptr = ring->wptr + 5 + 4;
2698 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2699 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2700 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2701 radeon_ring_write(ring, next_rptr);
2702 radeon_ring_write(ring, 0);
45df6803
CK		 2703 }
2704
e32eb50d
CK		 2705 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2706 radeon_ring_write(ring,
0f234f5f
AD		 2707#ifdef __BIG_ENDIAN
2708 (2 << 0) |
2709#endif
2710 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2711 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2712 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2713}
2714
bcc1c2a1
AD		 2715
2716static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2717{
fe251e2f
AD		 2718 const __be32 *fw_data;
2719 int i;
2720
2721 if (!rdev->me_fw || !rdev->pfp_fw)
2722 return -EINVAL;
bcc1c2a1 2723
fe251e2f 2724 r700_cp_stop(rdev);
0f234f5f
AD		 2725 WREG32(CP_RB_CNTL,
2726#ifdef __BIG_ENDIAN
2727 BUF_SWAP_32BIT |
2728#endif
2729 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2730
2731 fw_data = (const __be32 *)rdev->pfp_fw->data;
2732 WREG32(CP_PFP_UCODE_ADDR, 0);
2733 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2734 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2735 WREG32(CP_PFP_UCODE_ADDR, 0);
2736
2737 fw_data = (const __be32 *)rdev->me_fw->data;
2738 WREG32(CP_ME_RAM_WADDR, 0);
2739 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2740 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2741
2742 WREG32(CP_PFP_UCODE_ADDR, 0);
2743 WREG32(CP_ME_RAM_WADDR, 0);
2744 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2745 return 0;
2746}
2747
7e7b41d2
AD		 2748static int evergreen_cp_start(struct radeon_device *rdev)
2749{
e32eb50d 2750 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2751 int r, i;
7e7b41d2
AD		 2752 uint32_t cp_me;
2753
e32eb50d 2754 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2755 if (r) {
2756 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2757 return r;
2758 }
e32eb50d
CK		 2759 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2760 radeon_ring_write(ring, 0x1);
2761 radeon_ring_write(ring, 0x0);
2762 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2763 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2764 radeon_ring_write(ring, 0);
2765 radeon_ring_write(ring, 0);
2766 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2767
2768 cp_me = 0xff;
2769 WREG32(CP_ME_CNTL, cp_me);
2770
e32eb50d 2771 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2772 if (r) {
2773 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2774 return r;
2775 }
2281a378
AD		 2776
2777 /* setup clear context state */
e32eb50d
CK		 2778 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2779 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2780
2781 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2782 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2783
e32eb50d
CK		 2784 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2785 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2786
2787 /* set clear context state */
e32eb50d
CK		 2788 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2789 radeon_ring_write(ring, 0);
2281a378
AD		 2790
2791 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2792 radeon_ring_write(ring, 0xc0026f00);
2793 radeon_ring_write(ring, 0x00000000);
2794 radeon_ring_write(ring, 0x00000000);
2795 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2796
2797 /* Clear consts */
e32eb50d
CK		 2798 radeon_ring_write(ring, 0xc0036f00);
2799 radeon_ring_write(ring, 0x00000bc4);
2800 radeon_ring_write(ring, 0xffffffff);
2801 radeon_ring_write(ring, 0xffffffff);
2802 radeon_ring_write(ring, 0xffffffff);
2281a378 2803
e32eb50d
CK		 2804 radeon_ring_write(ring, 0xc0026900);
2805 radeon_ring_write(ring, 0x00000316);
2806 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2807 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2808
e32eb50d 2809 radeon_ring_unlock_commit(rdev, ring);
7e7b41d2
AD		 2810
2811 return 0;
2812}
2813
1109ca09 2814static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2815{
e32eb50d 2816 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2817 u32 tmp;
2818 u32 rb_bufsz;
2819 int r;
2820
2821 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2822 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2823 SOFT_RESET_PA |
2824 SOFT_RESET_SH |
2825 SOFT_RESET_VGT |
a49a50da 2826 SOFT_RESET_SPI |
fe251e2f
AD		 2827 SOFT_RESET_SX));
2828 RREG32(GRBM_SOFT_RESET);
2829 mdelay(15);
2830 WREG32(GRBM_SOFT_RESET, 0);
2831 RREG32(GRBM_SOFT_RESET);
2832
2833 /* Set ring buffer size */
e32eb50d 2834 rb_bufsz = drm_order(ring->ring_size / 8);
724c80e1 2835 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2836#ifdef __BIG_ENDIAN
2837 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2838#endif
fe251e2f 2839 WREG32(CP_RB_CNTL, tmp);
15d3332f 2840 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2841 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2842
2843 /* Set the write pointer delay */
2844 WREG32(CP_RB_WPTR_DELAY, 0);
2845
2846 /* Initialize the ring buffer's read and write pointers */
2847 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2848 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2849 ring->wptr = 0;
2850 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2851
48fc7f7e 2852 /* set the wb address whether it's enabled or not */
0f234f5f 2853 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2854 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2855 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2856 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2857
2858 if (rdev->wb.enabled)
2859 WREG32(SCRATCH_UMSK, 0xff);
2860 else {
2861 tmp |= RB_NO_UPDATE;
2862 WREG32(SCRATCH_UMSK, 0);
2863 }
2864
fe251e2f
AD		 2865 mdelay(1);
2866 WREG32(CP_RB_CNTL, tmp);
2867
e32eb50d 2868 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2869 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2870
e32eb50d 2871 ring->rptr = RREG32(CP_RB_RPTR);
fe251e2f 2872
7e7b41d2 2873 evergreen_cp_start(rdev);
e32eb50d 2874 ring->ready = true;
f712812e 2875 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2876 if (r) {
e32eb50d 2877 ring->ready = false;
fe251e2f
AD		 2878 return r;
2879 }
2880 return 0;
2881}
bcc1c2a1
AD		 2882
2883/*
2884 * Core functions
2885 */
bcc1c2a1
AD		 2886static void evergreen_gpu_init(struct radeon_device *rdev)
2887{
416a2bd2 2888 u32 gb_addr_config;
32fcdbf4 2889 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 2890 u32 sx_debug_1;
2891 u32 smx_dc_ctl0;
2892 u32 sq_config;
2893 u32 sq_lds_resource_mgmt;
2894 u32 sq_gpr_resource_mgmt_1;
2895 u32 sq_gpr_resource_mgmt_2;
2896 u32 sq_gpr_resource_mgmt_3;
2897 u32 sq_thread_resource_mgmt;
2898 u32 sq_thread_resource_mgmt_2;
2899 u32 sq_stack_resource_mgmt_1;
2900 u32 sq_stack_resource_mgmt_2;
2901 u32 sq_stack_resource_mgmt_3;
2902 u32 vgt_cache_invalidation;
f25a5c63 2903 u32 hdp_host_path_cntl, tmp;
416a2bd2 2904 u32 disabled_rb_mask;
32fcdbf4
AD		 2905 int i, j, num_shader_engines, ps_thread_count;
2906
2907 switch (rdev->family) {
2908 case CHIP_CYPRESS:
2909 case CHIP_HEMLOCK:
2910 rdev->config.evergreen.num_ses = 2;
2911 rdev->config.evergreen.max_pipes = 4;
2912 rdev->config.evergreen.max_tile_pipes = 8;
2913 rdev->config.evergreen.max_simds = 10;
2914 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2915 rdev->config.evergreen.max_gprs = 256;
2916 rdev->config.evergreen.max_threads = 248;
2917 rdev->config.evergreen.max_gs_threads = 32;
2918 rdev->config.evergreen.max_stack_entries = 512;
2919 rdev->config.evergreen.sx_num_of_sets = 4;
2920 rdev->config.evergreen.sx_max_export_size = 256;
2921 rdev->config.evergreen.sx_max_export_pos_size = 64;
2922 rdev->config.evergreen.sx_max_export_smx_size = 192;
2923 rdev->config.evergreen.max_hw_contexts = 8;
2924 rdev->config.evergreen.sq_num_cf_insts = 2;
2925
2926 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2927 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2928 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2929 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2930 break;
2931 case CHIP_JUNIPER:
2932 rdev->config.evergreen.num_ses = 1;
2933 rdev->config.evergreen.max_pipes = 4;
2934 rdev->config.evergreen.max_tile_pipes = 4;
2935 rdev->config.evergreen.max_simds = 10;
2936 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2937 rdev->config.evergreen.max_gprs = 256;
2938 rdev->config.evergreen.max_threads = 248;
2939 rdev->config.evergreen.max_gs_threads = 32;
2940 rdev->config.evergreen.max_stack_entries = 512;
2941 rdev->config.evergreen.sx_num_of_sets = 4;
2942 rdev->config.evergreen.sx_max_export_size = 256;
2943 rdev->config.evergreen.sx_max_export_pos_size = 64;
2944 rdev->config.evergreen.sx_max_export_smx_size = 192;
2945 rdev->config.evergreen.max_hw_contexts = 8;
2946 rdev->config.evergreen.sq_num_cf_insts = 2;
2947
2948 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2949 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2950 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2951 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2952 break;
2953 case CHIP_REDWOOD:
2954 rdev->config.evergreen.num_ses = 1;
2955 rdev->config.evergreen.max_pipes = 4;
2956 rdev->config.evergreen.max_tile_pipes = 4;
2957 rdev->config.evergreen.max_simds = 5;
2958 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
2959 rdev->config.evergreen.max_gprs = 256;
2960 rdev->config.evergreen.max_threads = 248;
2961 rdev->config.evergreen.max_gs_threads = 32;
2962 rdev->config.evergreen.max_stack_entries = 256;
2963 rdev->config.evergreen.sx_num_of_sets = 4;
2964 rdev->config.evergreen.sx_max_export_size = 256;
2965 rdev->config.evergreen.sx_max_export_pos_size = 64;
2966 rdev->config.evergreen.sx_max_export_smx_size = 192;
2967 rdev->config.evergreen.max_hw_contexts = 8;
2968 rdev->config.evergreen.sq_num_cf_insts = 2;
2969
2970 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2971 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2972 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2973 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 2974 break;
2975 case CHIP_CEDAR:
2976 default:
2977 rdev->config.evergreen.num_ses = 1;
2978 rdev->config.evergreen.max_pipes = 2;
2979 rdev->config.evergreen.max_tile_pipes = 2;
2980 rdev->config.evergreen.max_simds = 2;
2981 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
2982 rdev->config.evergreen.max_gprs = 256;
2983 rdev->config.evergreen.max_threads = 192;
2984 rdev->config.evergreen.max_gs_threads = 16;
2985 rdev->config.evergreen.max_stack_entries = 256;
2986 rdev->config.evergreen.sx_num_of_sets = 4;
2987 rdev->config.evergreen.sx_max_export_size = 128;
2988 rdev->config.evergreen.sx_max_export_pos_size = 32;
2989 rdev->config.evergreen.sx_max_export_smx_size = 96;
2990 rdev->config.evergreen.max_hw_contexts = 4;
2991 rdev->config.evergreen.sq_num_cf_insts = 1;
2992
d5e455e4
AD		 2993 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
2994 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2995 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 2996 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 2997 break;
2998 case CHIP_PALM:
2999 rdev->config.evergreen.num_ses = 1;
3000 rdev->config.evergreen.max_pipes = 2;
3001 rdev->config.evergreen.max_tile_pipes = 2;
3002 rdev->config.evergreen.max_simds = 2;
3003 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3004 rdev->config.evergreen.max_gprs = 256;
3005 rdev->config.evergreen.max_threads = 192;
3006 rdev->config.evergreen.max_gs_threads = 16;
3007 rdev->config.evergreen.max_stack_entries = 256;
3008 rdev->config.evergreen.sx_num_of_sets = 4;
3009 rdev->config.evergreen.sx_max_export_size = 128;
3010 rdev->config.evergreen.sx_max_export_pos_size = 32;
3011 rdev->config.evergreen.sx_max_export_smx_size = 96;
3012 rdev->config.evergreen.max_hw_contexts = 4;
3013 rdev->config.evergreen.sq_num_cf_insts = 1;
3014
d5c5a72f
AD		 3015 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3016 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3017 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3018 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3019 break;
3020 case CHIP_SUMO:
3021 rdev->config.evergreen.num_ses = 1;
3022 rdev->config.evergreen.max_pipes = 4;
bd25f078 3023 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3024 if (rdev->pdev->device == 0x9648)
3025 rdev->config.evergreen.max_simds = 3;
3026 else if ((rdev->pdev->device == 0x9647) ||
3027 (rdev->pdev->device == 0x964a))
3028 rdev->config.evergreen.max_simds = 4;
3029 else
3030 rdev->config.evergreen.max_simds = 5;
3031 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3032 rdev->config.evergreen.max_gprs = 256;
3033 rdev->config.evergreen.max_threads = 248;
3034 rdev->config.evergreen.max_gs_threads = 32;
3035 rdev->config.evergreen.max_stack_entries = 256;
3036 rdev->config.evergreen.sx_num_of_sets = 4;
3037 rdev->config.evergreen.sx_max_export_size = 256;
3038 rdev->config.evergreen.sx_max_export_pos_size = 64;
3039 rdev->config.evergreen.sx_max_export_smx_size = 192;
3040 rdev->config.evergreen.max_hw_contexts = 8;
3041 rdev->config.evergreen.sq_num_cf_insts = 2;
3042
3043 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3044 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3045 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3046 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3047 break;
3048 case CHIP_SUMO2:
3049 rdev->config.evergreen.num_ses = 1;
3050 rdev->config.evergreen.max_pipes = 4;
3051 rdev->config.evergreen.max_tile_pipes = 4;
3052 rdev->config.evergreen.max_simds = 2;
3053 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3054 rdev->config.evergreen.max_gprs = 256;
3055 rdev->config.evergreen.max_threads = 248;
3056 rdev->config.evergreen.max_gs_threads = 32;
3057 rdev->config.evergreen.max_stack_entries = 512;
3058 rdev->config.evergreen.sx_num_of_sets = 4;
3059 rdev->config.evergreen.sx_max_export_size = 256;
3060 rdev->config.evergreen.sx_max_export_pos_size = 64;
3061 rdev->config.evergreen.sx_max_export_smx_size = 192;
3062 rdev->config.evergreen.max_hw_contexts = 8;
3063 rdev->config.evergreen.sq_num_cf_insts = 2;
3064
adb68fa2
AD		 3065 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3066 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3067 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3068 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3069 break;
3070 case CHIP_BARTS:
3071 rdev->config.evergreen.num_ses = 2;
3072 rdev->config.evergreen.max_pipes = 4;
3073 rdev->config.evergreen.max_tile_pipes = 8;
3074 rdev->config.evergreen.max_simds = 7;
3075 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3076 rdev->config.evergreen.max_gprs = 256;
3077 rdev->config.evergreen.max_threads = 248;
3078 rdev->config.evergreen.max_gs_threads = 32;
3079 rdev->config.evergreen.max_stack_entries = 512;
3080 rdev->config.evergreen.sx_num_of_sets = 4;
3081 rdev->config.evergreen.sx_max_export_size = 256;
3082 rdev->config.evergreen.sx_max_export_pos_size = 64;
3083 rdev->config.evergreen.sx_max_export_smx_size = 192;
3084 rdev->config.evergreen.max_hw_contexts = 8;
3085 rdev->config.evergreen.sq_num_cf_insts = 2;
3086
3087 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3088 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3089 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3090 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3091 break;
3092 case CHIP_TURKS:
3093 rdev->config.evergreen.num_ses = 1;
3094 rdev->config.evergreen.max_pipes = 4;
3095 rdev->config.evergreen.max_tile_pipes = 4;
3096 rdev->config.evergreen.max_simds = 6;
3097 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3098 rdev->config.evergreen.max_gprs = 256;
3099 rdev->config.evergreen.max_threads = 248;
3100 rdev->config.evergreen.max_gs_threads = 32;
3101 rdev->config.evergreen.max_stack_entries = 256;
3102 rdev->config.evergreen.sx_num_of_sets = 4;
3103 rdev->config.evergreen.sx_max_export_size = 256;
3104 rdev->config.evergreen.sx_max_export_pos_size = 64;
3105 rdev->config.evergreen.sx_max_export_smx_size = 192;
3106 rdev->config.evergreen.max_hw_contexts = 8;
3107 rdev->config.evergreen.sq_num_cf_insts = 2;
3108
3109 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3110 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3111 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3112 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3113 break;
3114 case CHIP_CAICOS:
3115 rdev->config.evergreen.num_ses = 1;
bd25f078 3116 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3117 rdev->config.evergreen.max_tile_pipes = 2;
3118 rdev->config.evergreen.max_simds = 2;
3119 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3120 rdev->config.evergreen.max_gprs = 256;
3121 rdev->config.evergreen.max_threads = 192;
3122 rdev->config.evergreen.max_gs_threads = 16;
3123 rdev->config.evergreen.max_stack_entries = 256;
3124 rdev->config.evergreen.sx_num_of_sets = 4;
3125 rdev->config.evergreen.sx_max_export_size = 128;
3126 rdev->config.evergreen.sx_max_export_pos_size = 32;
3127 rdev->config.evergreen.sx_max_export_smx_size = 96;
3128 rdev->config.evergreen.max_hw_contexts = 4;
3129 rdev->config.evergreen.sq_num_cf_insts = 1;
3130
32fcdbf4
AD		 3131 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3132 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3133 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3134 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3135 break;
3136 }
3137
3138 /* Initialize HDP */
3139 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3140 WREG32((0x2c14 + j), 0x00000000);
3141 WREG32((0x2c18 + j), 0x00000000);
3142 WREG32((0x2c1c + j), 0x00000000);
3143 WREG32((0x2c20 + j), 0x00000000);
3144 WREG32((0x2c24 + j), 0x00000000);
3145 }
3146
3147 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3148
d054ac16
AD		 3149 evergreen_fix_pci_max_read_req_size(rdev);
3150
32fcdbf4 3151 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3152 if ((rdev->family == CHIP_PALM) ||
3153 (rdev->family == CHIP_SUMO) ||
3154 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3155 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3156 else
3157 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3158
1aa52bd3
AD		 3159 /* setup tiling info dword. gb_addr_config is not adequate since it does
3160 * not have bank info, so create a custom tiling dword.
3161 * bits 3:0 num_pipes
3162 * bits 7:4 num_banks
3163 * bits 11:8 group_size
3164 * bits 15:12 row_size
3165 */
3166 rdev->config.evergreen.tile_config = 0;
3167 switch (rdev->config.evergreen.max_tile_pipes) {
3168 case 1:
3169 default:
3170 rdev->config.evergreen.tile_config |= (0 << 0);
3171 break;
3172 case 2:
3173 rdev->config.evergreen.tile_config |= (1 << 0);
3174 break;
3175 case 4:
3176 rdev->config.evergreen.tile_config |= (2 << 0);
3177 break;
3178 case 8:
3179 rdev->config.evergreen.tile_config |= (3 << 0);
3180 break;
3181 }
d698a34d 3182 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3183 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3184 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3185 else {
c8d15edc
AD		 3186 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3187 case 0: /* four banks */
29d65406 3188 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3189 break;
3190 case 1: /* eight banks */
3191 rdev->config.evergreen.tile_config |= 1 << 4;
3192 break;
3193 case 2: /* sixteen banks */
3194 default:
3195 rdev->config.evergreen.tile_config |= 2 << 4;
3196 break;
3197 }
29d65406 3198 }
416a2bd2 3199 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3200 rdev->config.evergreen.tile_config |=
3201 ((gb_addr_config & 0x30000000) >> 28) << 12;
3202
416a2bd2 3203 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
32fcdbf4 3204
416a2bd2
AD		 3205 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3206 u32 efuse_straps_4;
3207 u32 efuse_straps_3;
32fcdbf4 3208
ff82bbc4
AD		 3209 efuse_straps_4 = RREG32_RCU(0x204);
3210 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3211 tmp = (((efuse_straps_4 & 0xf) << 4) |
3212 ((efuse_straps_3 & 0xf0000000) >> 28));
3213 } else {
3214 tmp = 0;
3215 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3216 u32 rb_disable_bitmap;
3217
3218 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3219 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3220 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3221 tmp <<= 4;
3222 tmp |= rb_disable_bitmap;
32fcdbf4 3223 }
416a2bd2
AD		 3224 }
3225 /* enabled rb are just the one not disabled :) */
3226 disabled_rb_mask = tmp;
cedb655a
AD		 3227 tmp = 0;
3228 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3229 tmp |= (1 << i);
3230 /* if all the backends are disabled, fix it up here */
3231 if ((disabled_rb_mask & tmp) == tmp) {
3232 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3233 disabled_rb_mask &= ~(1 << i);
3234 }
32fcdbf4 3235
416a2bd2
AD		 3236 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3237 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3238
416a2bd2
AD		 3239 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3240 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3241 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3242 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3243 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3244 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3245 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3246
f7eb9730
AD		 3247 if ((rdev->config.evergreen.max_backends == 1) &&
3248 (rdev->flags & RADEON_IS_IGP)) {
3249 if ((disabled_rb_mask & 3) == 1) {
3250 /* RB0 disabled, RB1 enabled */
3251 tmp = 0x11111111;
3252 } else {
3253 /* RB1 disabled, RB0 enabled */
3254 tmp = 0x00000000;
3255 }
3256 } else {
3257 tmp = gb_addr_config & NUM_PIPES_MASK;
3258 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3259 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3260 }
416a2bd2 3261 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3262
3263 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3264 WREG32(CGTS_TCC_DISABLE, 0);
3265 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3266 WREG32(CGTS_USER_TCC_DISABLE, 0);
3267
3268 /* set HW defaults for 3D engine */
3269 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3270 ROQ_IB2_START(0x2b)));
3271
3272 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3273
3274 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3275 SYNC_GRADIENT |
3276 SYNC_WALKER |
3277 SYNC_ALIGNER));
3278
3279 sx_debug_1 = RREG32(SX_DEBUG_1);
3280 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3281 WREG32(SX_DEBUG_1, sx_debug_1);
3282
3283
3284 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3285 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3286 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3287 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3288
b866d133
AD		 3289 if (rdev->family <= CHIP_SUMO2)
3290 WREG32(SMX_SAR_CTL0, 0x00010000);
3291
32fcdbf4
AD		 3292 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3293 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3294 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3295
3296 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3297 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3298 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3299
3300 WREG32(VGT_NUM_INSTANCES, 1);
3301 WREG32(SPI_CONFIG_CNTL, 0);
3302 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3303 WREG32(CP_PERFMON_CNTL, 0);
3304
3305 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3306 FETCH_FIFO_HIWATER(0x4) |
3307 DONE_FIFO_HIWATER(0xe0) |
3308 ALU_UPDATE_FIFO_HIWATER(0x8)));
3309
3310 sq_config = RREG32(SQ_CONFIG);
3311 sq_config &= ~(PS_PRIO(3) |
3312 VS_PRIO(3) |
3313 GS_PRIO(3) |
3314 ES_PRIO(3));
3315 sq_config |= (VC_ENABLE |
3316 EXPORT_SRC_C |
3317 PS_PRIO(0) |
3318 VS_PRIO(1) |
3319 GS_PRIO(2) |
3320 ES_PRIO(3));
3321
d5e455e4
AD		 3322 switch (rdev->family) {
3323 case CHIP_CEDAR:
3324 case CHIP_PALM:
d5c5a72f
AD		 3325 case CHIP_SUMO:
3326 case CHIP_SUMO2:
adb68fa2 3327 case CHIP_CAICOS:
32fcdbf4
AD		 3328 /* no vertex cache */
3329 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3330 break;
3331 default:
3332 break;
3333 }
32fcdbf4
AD		 3334
3335 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3336
3337 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3338 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3339 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3340 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3341 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3342 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3343 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3344
d5e455e4
AD		 3345 switch (rdev->family) {
3346 case CHIP_CEDAR:
3347 case CHIP_PALM:
d5c5a72f
AD		 3348 case CHIP_SUMO:
3349 case CHIP_SUMO2:
32fcdbf4 3350 ps_thread_count = 96;
d5e455e4
AD		 3351 break;
3352 default:
32fcdbf4 3353 ps_thread_count = 128;
d5e455e4
AD		 3354 break;
3355 }
32fcdbf4
AD		 3356
3357 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3358 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3359 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3360 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3361 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3362 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3363
3364 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3365 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3366 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3367 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3368 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3369 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3370
3371 WREG32(SQ_CONFIG, sq_config);
3372 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3373 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3374 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3375 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3376 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3377 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3378 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3379 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3380 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3381 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3382
3383 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3384 FORCE_EOV_MAX_REZ_CNT(255)));
3385
d5e455e4
AD		 3386 switch (rdev->family) {
3387 case CHIP_CEDAR:
3388 case CHIP_PALM:
d5c5a72f
AD		 3389 case CHIP_SUMO:
3390 case CHIP_SUMO2:
adb68fa2 3391 case CHIP_CAICOS:
32fcdbf4 3392 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3393 break;
3394 default:
32fcdbf4 3395 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3396 break;
3397 }
32fcdbf4
AD		 3398 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3399 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3400
3401 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3402 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3403 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3404
60a4a3e0
AD		 3405 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3406 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3407
32fcdbf4
AD		 3408 WREG32(CB_PERF_CTR0_SEL_0, 0);
3409 WREG32(CB_PERF_CTR0_SEL_1, 0);
3410 WREG32(CB_PERF_CTR1_SEL_0, 0);
3411 WREG32(CB_PERF_CTR1_SEL_1, 0);
3412 WREG32(CB_PERF_CTR2_SEL_0, 0);
3413 WREG32(CB_PERF_CTR2_SEL_1, 0);
3414 WREG32(CB_PERF_CTR3_SEL_0, 0);
3415 WREG32(CB_PERF_CTR3_SEL_1, 0);
3416
60a4a3e0
AD		 3417 /* clear render buffer base addresses */
3418 WREG32(CB_COLOR0_BASE, 0);
3419 WREG32(CB_COLOR1_BASE, 0);
3420 WREG32(CB_COLOR2_BASE, 0);
3421 WREG32(CB_COLOR3_BASE, 0);
3422 WREG32(CB_COLOR4_BASE, 0);
3423 WREG32(CB_COLOR5_BASE, 0);
3424 WREG32(CB_COLOR6_BASE, 0);
3425 WREG32(CB_COLOR7_BASE, 0);
3426 WREG32(CB_COLOR8_BASE, 0);
3427 WREG32(CB_COLOR9_BASE, 0);
3428 WREG32(CB_COLOR10_BASE, 0);
3429 WREG32(CB_COLOR11_BASE, 0);
3430
3431 /* set the shader const cache sizes to 0 */
3432 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3433 WREG32(i, 0);
3434 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3435 WREG32(i, 0);
3436
f25a5c63
AD		 3437 tmp = RREG32(HDP_MISC_CNTL);
3438 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3439 WREG32(HDP_MISC_CNTL, tmp);
3440
32fcdbf4
AD		 3441 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3442 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3443
3444 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3445
3446 udelay(50);
3447
bcc1c2a1
AD		 3448}
3449
3450int evergreen_mc_init(struct radeon_device *rdev)
3451{
bcc1c2a1
AD		 3452 u32 tmp;
3453 int chansize, numchan;
bcc1c2a1
AD		 3454
3455 /* Get VRAM informations */
3456 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3457 if ((rdev->family == CHIP_PALM) ||
3458 (rdev->family == CHIP_SUMO) ||
3459 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3460 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3461 else
3462 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3463 if (tmp & CHANSIZE_OVERRIDE) {
3464 chansize = 16;
3465 } else if (tmp & CHANSIZE_MASK) {
3466 chansize = 64;
3467 } else {
3468 chansize = 32;
3469 }
3470 tmp = RREG32(MC_SHARED_CHMAP);
3471 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3472 case 0:
3473 default:
3474 numchan = 1;
3475 break;
3476 case 1:
3477 numchan = 2;
3478 break;
3479 case 2:
3480 numchan = 4;
3481 break;
3482 case 3:
3483 numchan = 8;
3484 break;
3485 }
3486 rdev->mc.vram_width = numchan * chansize;
3487 /* Could aper size report 0 ? */
01d73a69
JC		 3488 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3489 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3490 /* Setup GPU memory space */
05b3ef69
AD		 3491 if ((rdev->family == CHIP_PALM) ||
3492 (rdev->family == CHIP_SUMO) ||
3493 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3494 /* size in bytes on fusion */
3495 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3496 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3497 } else {
05b3ef69 3498 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3499 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3500 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3501 }
51e5fcd3 3502 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3503 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3504 radeon_update_bandwidth_info(rdev);
3505
bcc1c2a1
AD		 3506 return 0;
3507}
d594e46a 3508
187e3593 3509void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3510{
64c56e8c 3511 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3512 RREG32(GRBM_STATUS));
64c56e8c 3513 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3514 RREG32(GRBM_STATUS_SE0));
64c56e8c 3515 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3516 RREG32(GRBM_STATUS_SE1));
64c56e8c 3517 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3518 RREG32(SRBM_STATUS));
a65a4369
AD		 3519 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3520 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3521 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3522 RREG32(CP_STALLED_STAT1));
3523 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3524 RREG32(CP_STALLED_STAT2));
3525 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3526 RREG32(CP_BUSY_STAT));
3527 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3528 RREG32(CP_STAT));
eaaa6983
JG		 3529 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3530 RREG32(DMA_STATUS_REG));
168757ea
AD		 3531 if (rdev->family >= CHIP_CAYMAN) {
3532 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3533 RREG32(DMA_STATUS_REG + 0x800));
3534 }
0ecebb9e
AD		 3535}
3536
168757ea 3537bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3538{
a65a4369
AD		 3539 u32 crtc_hung = 0;
3540 u32 crtc_status[6];
3541 u32 i, j, tmp;
3542
3543 for (i = 0; i < rdev->num_crtc; i++) {
3544 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3545 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3546 crtc_hung |= (1 << i);
3547 }
3548 }
3549
3550 for (j = 0; j < 10; j++) {
3551 for (i = 0; i < rdev->num_crtc; i++) {
3552 if (crtc_hung & (1 << i)) {
3553 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3554 if (tmp != crtc_status[i])
3555 crtc_hung &= ~(1 << i);
3556 }
3557 }
3558 if (crtc_hung == 0)
3559 return false;
3560 udelay(100);
3561 }
3562
3563 return true;
3564}
3565
3566static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3567{
3568 u32 reset_mask = 0;
b7630473 3569 u32 tmp;
0ecebb9e 3570
a65a4369
AD		 3571 /* GRBM_STATUS */
3572 tmp = RREG32(GRBM_STATUS);
3573 if (tmp & (PA_BUSY | SC_BUSY |
3574 SH_BUSY | SX_BUSY |
3575 TA_BUSY | VGT_BUSY |
3576 DB_BUSY | CB_BUSY |
3577 SPI_BUSY | VGT_BUSY_NO_DMA))
3578 reset_mask |= RADEON_RESET_GFX;
3579
3580 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3581 CP_BUSY | CP_COHERENCY_BUSY))
3582 reset_mask |= RADEON_RESET_CP;
3583
3584 if (tmp & GRBM_EE_BUSY)
3585 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3586
a65a4369
AD		 3587 /* DMA_STATUS_REG */
3588 tmp = RREG32(DMA_STATUS_REG);
3589 if (!(tmp & DMA_IDLE))
3590 reset_mask |= RADEON_RESET_DMA;
3591
3592 /* SRBM_STATUS2 */
3593 tmp = RREG32(SRBM_STATUS2);
3594 if (tmp & DMA_BUSY)
3595 reset_mask |= RADEON_RESET_DMA;
3596
3597 /* SRBM_STATUS */
3598 tmp = RREG32(SRBM_STATUS);
3599 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3600 reset_mask |= RADEON_RESET_RLC;
3601
3602 if (tmp & IH_BUSY)
3603 reset_mask |= RADEON_RESET_IH;
3604
3605 if (tmp & SEM_BUSY)
3606 reset_mask |= RADEON_RESET_SEM;
3607
3608 if (tmp & GRBM_RQ_PENDING)
3609 reset_mask |= RADEON_RESET_GRBM;
3610
3611 if (tmp & VMC_BUSY)
3612 reset_mask |= RADEON_RESET_VMC;
3613
3614 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3615 MCC_BUSY | MCD_BUSY))
3616 reset_mask |= RADEON_RESET_MC;
3617
3618 if (evergreen_is_display_hung(rdev))
3619 reset_mask |= RADEON_RESET_DISPLAY;
3620
3621 /* VM_L2_STATUS */
3622 tmp = RREG32(VM_L2_STATUS);
3623 if (tmp & L2_BUSY)
3624 reset_mask |= RADEON_RESET_VMC;
3625
d808fc88
AD		 3626 /* Skip MC reset as it's mostly likely not hung, just busy */
3627 if (reset_mask & RADEON_RESET_MC) {
3628 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3629 reset_mask &= ~RADEON_RESET_MC;
3630 }
3631
a65a4369
AD		 3632 return reset_mask;
3633}
3634
3635static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3636{
3637 struct evergreen_mc_save save;
3638 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3639 u32 tmp;
19fc42ed 3640
0ecebb9e 3641 if (reset_mask == 0)
a65a4369 3642 return;
0ecebb9e
AD		 3643
3644 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3645
b7630473
AD		 3646 evergreen_print_gpu_status_regs(rdev);
3647
b7630473
AD		 3648 /* Disable CP parsing/prefetching */
3649 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3650
3651 if (reset_mask & RADEON_RESET_DMA) {
3652 /* Disable DMA */
3653 tmp = RREG32(DMA_RB_CNTL);
3654 tmp &= ~DMA_RB_ENABLE;
3655 WREG32(DMA_RB_CNTL, tmp);
3656 }
3657
b21b6e7a
AD		 3658 udelay(50);
3659
3660 evergreen_mc_stop(rdev, &save);
3661 if (evergreen_mc_wait_for_idle(rdev)) {
3662 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3663 }
3664
b7630473
AD		 3665 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3666 grbm_soft_reset |= SOFT_RESET_DB |
3667 SOFT_RESET_CB |
3668 SOFT_RESET_PA |
3669 SOFT_RESET_SC |
3670 SOFT_RESET_SPI |
3671 SOFT_RESET_SX |
3672 SOFT_RESET_SH |
3673 SOFT_RESET_TC |
3674 SOFT_RESET_TA |
3675 SOFT_RESET_VC |
3676 SOFT_RESET_VGT;
3677 }
3678
3679 if (reset_mask & RADEON_RESET_CP) {
3680 grbm_soft_reset |= SOFT_RESET_CP |
3681 SOFT_RESET_VGT;
3682
3683 srbm_soft_reset |= SOFT_RESET_GRBM;
3684 }
0ecebb9e
AD		 3685
3686 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3687 srbm_soft_reset |= SOFT_RESET_DMA;
3688
a65a4369
AD		 3689 if (reset_mask & RADEON_RESET_DISPLAY)
3690 srbm_soft_reset |= SOFT_RESET_DC;
3691
3692 if (reset_mask & RADEON_RESET_RLC)
3693 srbm_soft_reset |= SOFT_RESET_RLC;
3694
3695 if (reset_mask & RADEON_RESET_SEM)
3696 srbm_soft_reset |= SOFT_RESET_SEM;
3697
3698 if (reset_mask & RADEON_RESET_IH)
3699 srbm_soft_reset |= SOFT_RESET_IH;
3700
3701 if (reset_mask & RADEON_RESET_GRBM)
3702 srbm_soft_reset |= SOFT_RESET_GRBM;
3703
3704 if (reset_mask & RADEON_RESET_VMC)
3705 srbm_soft_reset |= SOFT_RESET_VMC;
3706
24178ec4
AD		 3707 if (!(rdev->flags & RADEON_IS_IGP)) {
3708 if (reset_mask & RADEON_RESET_MC)
3709 srbm_soft_reset |= SOFT_RESET_MC;
3710 }
a65a4369 3711
b7630473
AD		 3712 if (grbm_soft_reset) {
3713 tmp = RREG32(GRBM_SOFT_RESET);
3714 tmp |= grbm_soft_reset;
3715 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3716 WREG32(GRBM_SOFT_RESET, tmp);
3717 tmp = RREG32(GRBM_SOFT_RESET);
3718
3719 udelay(50);
3720
3721 tmp &= ~grbm_soft_reset;
3722 WREG32(GRBM_SOFT_RESET, tmp);
3723 tmp = RREG32(GRBM_SOFT_RESET);
3724 }
3725
3726 if (srbm_soft_reset) {
3727 tmp = RREG32(SRBM_SOFT_RESET);
3728 tmp |= srbm_soft_reset;
3729 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3730 WREG32(SRBM_SOFT_RESET, tmp);
3731 tmp = RREG32(SRBM_SOFT_RESET);
3732
3733 udelay(50);
3734
3735 tmp &= ~srbm_soft_reset;
3736 WREG32(SRBM_SOFT_RESET, tmp);
3737 tmp = RREG32(SRBM_SOFT_RESET);
3738 }
0ecebb9e
AD		 3739
3740 /* Wait a little for things to settle down */
3741 udelay(50);
3742
747943ea 3743 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3744 udelay(50);
3745
b7630473 3746 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3747}
3748
a2d07b74 3749int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3750{
a65a4369
AD		 3751 u32 reset_mask;
3752
3753 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3754
3755 if (reset_mask)
3756 r600_set_bios_scratch_engine_hung(rdev, true);
3757
3758 evergreen_gpu_soft_reset(rdev, reset_mask);
3759
3760 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3761
3762 if (!reset_mask)
3763 r600_set_bios_scratch_engine_hung(rdev, false);
3764
3765 return 0;
747943ea
AD		 3766}
3767
123bc183
AD		 3768/**
3769 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3770 *
3771 * @rdev: radeon_device pointer
3772 * @ring: radeon_ring structure holding ring information
3773 *
3774 * Check if the GFX engine is locked up.
3775 * Returns true if the engine appears to be locked up, false if not.
3776 */
3777bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3778{
3779 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3780
3781 if (!(reset_mask & (RADEON_RESET_GFX |
3782 RADEON_RESET_COMPUTE |
3783 RADEON_RESET_CP))) {
3784 radeon_ring_lockup_update(ring);
3785 return false;
3786 }
3787 /* force CP activities */
3788 radeon_ring_force_activity(rdev, ring);
3789 return radeon_ring_test_lockup(rdev, ring);
3790}
3791
3792/**
3793 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3794 *
3795 * @rdev: radeon_device pointer
3796 * @ring: radeon_ring structure holding ring information
3797 *
3798 * Check if the async DMA engine is locked up.
3799 * Returns true if the engine appears to be locked up, false if not.
3800 */
3801bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3802{
3803 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3804
3805 if (!(reset_mask & RADEON_RESET_DMA)) {
3806 radeon_ring_lockup_update(ring);
3807 return false;
3808 }
3809 /* force ring activities */
3810 radeon_ring_force_activity(rdev, ring);
3811 return radeon_ring_test_lockup(rdev, ring);
3812}
3813
2948f5e6
AD		 3814/*
3815 * RLC
3816 */
3817#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3818#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3819
3820void sumo_rlc_fini(struct radeon_device *rdev)
3821{
3822 int r;
3823
3824 /* save restore block */
3825 if (rdev->rlc.save_restore_obj) {
3826 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3827 if (unlikely(r != 0))
3828 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3829 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3830 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3831
3832 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3833 rdev->rlc.save_restore_obj = NULL;
3834 }
3835
3836 /* clear state block */
3837 if (rdev->rlc.clear_state_obj) {
3838 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3839 if (unlikely(r != 0))
3840 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3841 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3842 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3843
3844 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3845 rdev->rlc.clear_state_obj = NULL;
3846 }
3847}
3848
3849int sumo_rlc_init(struct radeon_device *rdev)
3850{
3851 u32 *src_ptr;
3852 volatile u32 *dst_ptr;
3853 u32 dws, data, i, j, k, reg_num;
3854 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3855 u64 reg_list_mc_addr;
3856 struct cs_section_def *cs_data;
3857 int r;
3858
3859 src_ptr = rdev->rlc.reg_list;
3860 dws = rdev->rlc.reg_list_size;
3861 cs_data = rdev->rlc.cs_data;
3862
3863 /* save restore block */
3864 if (rdev->rlc.save_restore_obj == NULL) {
3865 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3866 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3867 if (r) {
3868 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3869 return r;
3870 }
3871 }
3872
3873 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3874 if (unlikely(r != 0)) {
3875 sumo_rlc_fini(rdev);
3876 return r;
3877 }
3878 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3879 &rdev->rlc.save_restore_gpu_addr);
3880 if (r) {
3881 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3882 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3883 sumo_rlc_fini(rdev);
3884 return r;
3885 }
3886 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3887 if (r) {
3888 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3889 sumo_rlc_fini(rdev);
3890 return r;
3891 }
3892 /* write the sr buffer */
3893 dst_ptr = rdev->rlc.sr_ptr;
3894 /* format:
3895 * dw0: (reg2 << 16) | reg1
3896 * dw1: reg1 save space
3897 * dw2: reg2 save space
3898 */
3899 for (i = 0; i < dws; i++) {
3900 data = src_ptr[i] >> 2;
3901 i++;
3902 if (i < dws)
3903 data |= (src_ptr[i] >> 2) << 16;
3904 j = (((i - 1) * 3) / 2);
3905 dst_ptr[j] = data;
3906 }
3907 j = ((i * 3) / 2);
3908 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3909
3910 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3911 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3912
3913 /* clear state block */
3914 reg_list_num = 0;
3915 dws = 0;
3916 for (i = 0; cs_data[i].section != NULL; i++) {
3917 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3918 reg_list_num++;
3919 dws += cs_data[i].section[j].reg_count;
3920 }
3921 }
3922 reg_list_blk_index = (3 * reg_list_num + 2);
3923 dws += reg_list_blk_index;
3924
3925 if (rdev->rlc.clear_state_obj == NULL) {
3926 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3927 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3928 if (r) {
3929 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3930 sumo_rlc_fini(rdev);
3931 return r;
3932 }
3933 }
3934 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3935 if (unlikely(r != 0)) {
3936 sumo_rlc_fini(rdev);
3937 return r;
3938 }
3939 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3940 &rdev->rlc.clear_state_gpu_addr);
3941 if (r) {
3942
3943 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3944 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3945 sumo_rlc_fini(rdev);
3946 return r;
3947 }
3948 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
3949 if (r) {
3950 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
3951 sumo_rlc_fini(rdev);
3952 return r;
3953 }
3954 /* set up the cs buffer */
3955 dst_ptr = rdev->rlc.cs_ptr;
3956 reg_list_hdr_blk_index = 0;
3957 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
3958 data = upper_32_bits(reg_list_mc_addr);
3959 dst_ptr[reg_list_hdr_blk_index] = data;
3960 reg_list_hdr_blk_index++;
3961 for (i = 0; cs_data[i].section != NULL; i++) {
3962 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3963 reg_num = cs_data[i].section[j].reg_count;
3964 data = reg_list_mc_addr & 0xffffffff;
3965 dst_ptr[reg_list_hdr_blk_index] = data;
3966 reg_list_hdr_blk_index++;
3967
3968 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
3969 dst_ptr[reg_list_hdr_blk_index] = data;
3970 reg_list_hdr_blk_index++;
3971
3972 data = 0x08000000 | (reg_num * 4);
3973 dst_ptr[reg_list_hdr_blk_index] = data;
3974 reg_list_hdr_blk_index++;
3975
3976 for (k = 0; k < reg_num; k++) {
3977 data = cs_data[i].section[j].extent[k];
3978 dst_ptr[reg_list_blk_index + k] = data;
3979 }
3980 reg_list_mc_addr += reg_num * 4;
3981 reg_list_blk_index += reg_num;
3982 }
3983 }
3984 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
3985
3986 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
3987 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3988
3989 return 0;
3990}
3991
3992static void evergreen_rlc_start(struct radeon_device *rdev)
3993{
3994 if (rdev->flags & RADEON_IS_IGP)
3995 WREG32(RLC_CNTL, RLC_ENABLE | GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC);
3996 else
3997 WREG32(RLC_CNTL, RLC_ENABLE);
3998}
3999
4000int evergreen_rlc_resume(struct radeon_device *rdev)
4001{
4002 u32 i;
4003 const __be32 *fw_data;
4004
4005 if (!rdev->rlc_fw)
4006 return -EINVAL;
4007
4008 r600_rlc_stop(rdev);
4009
4010 WREG32(RLC_HB_CNTL, 0);
4011
4012 if (rdev->flags & RADEON_IS_IGP) {
4013 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4014 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4015 } else {
4016 WREG32(RLC_HB_BASE, 0);
4017 WREG32(RLC_HB_RPTR, 0);
4018 WREG32(RLC_HB_WPTR, 0);
4019 }
4020 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4021 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4022 WREG32(RLC_MC_CNTL, 0);
4023 WREG32(RLC_UCODE_CNTL, 0);
4024
4025 fw_data = (const __be32 *)rdev->rlc_fw->data;
4026 if (rdev->family >= CHIP_ARUBA) {
4027 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4028 WREG32(RLC_UCODE_ADDR, i);
4029 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4030 }
4031 } else if (rdev->family >= CHIP_CAYMAN) {
4032 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4033 WREG32(RLC_UCODE_ADDR, i);
4034 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4035 }
4036 } else {
4037 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4038 WREG32(RLC_UCODE_ADDR, i);
4039 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4040 }
4041 }
4042 WREG32(RLC_UCODE_ADDR, 0);
4043
4044 evergreen_rlc_start(rdev);
4045
4046 return 0;
4047}
4048
45f9a39b
AD		 4049/* Interrupts */
4050
4051u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4052{
46437057 4053 if (crtc >= rdev->num_crtc)
45f9a39b 4054 return 0;
46437057
AD		 4055 else
4056 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4057}
4058
4059void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4060{
4061 u32 tmp;
4062
1b37078b
AD		 4063 if (rdev->family >= CHIP_CAYMAN) {
4064 cayman_cp_int_cntl_setup(rdev, 0,
4065 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4066 cayman_cp_int_cntl_setup(rdev, 1, 0);
4067 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4068 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4069 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4070 } else
4071 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4072 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4073 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4074 WREG32(GRBM_INT_CNTL, 0);
4075 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4076 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4077 if (rdev->num_crtc >= 4) {
18007401
AD		 4078 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4079 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4080 }
4081 if (rdev->num_crtc >= 6) {
18007401
AD		 4082 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4083 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4084 }
45f9a39b
AD		 4085
4086 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4087 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4088 if (rdev->num_crtc >= 4) {
18007401
AD		 4089 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4090 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4091 }
4092 if (rdev->num_crtc >= 6) {
18007401
AD		 4093 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4094 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4095 }
45f9a39b 4096
05b3ef69
AD		 4097 /* only one DAC on DCE6 */
4098 if (!ASIC_IS_DCE6(rdev))
4099 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4100 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4101
4102 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4103 WREG32(DC_HPD1_INT_CONTROL, tmp);
4104 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4105 WREG32(DC_HPD2_INT_CONTROL, tmp);
4106 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4107 WREG32(DC_HPD3_INT_CONTROL, tmp);
4108 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4109 WREG32(DC_HPD4_INT_CONTROL, tmp);
4110 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4111 WREG32(DC_HPD5_INT_CONTROL, tmp);
4112 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4113 WREG32(DC_HPD6_INT_CONTROL, tmp);
4114
4115}
4116
4117int evergreen_irq_set(struct radeon_device *rdev)
4118{
4119 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4120 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4121 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4122 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4123 u32 grbm_int_cntl = 0;
6f34be50 4124 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
f122c610 4125 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4126 u32 dma_cntl, dma_cntl1 = 0;
45f9a39b
AD		 4127
4128 if (!rdev->irq.installed) {
fce7d61b 4129 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4130 return -EINVAL;
4131 }
4132 /* don't enable anything if the ih is disabled */
4133 if (!rdev->ih.enabled) {
4134 r600_disable_interrupts(rdev);
4135 /* force the active interrupt state to all disabled */
4136 evergreen_disable_interrupt_state(rdev);
4137 return 0;
4138 }
4139
4140 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4141 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4142 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4143 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4144 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4145 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4146
f122c610
AD		 4147 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4148 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4149 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4150 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4151 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4152 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4153
233d1ad5
AD		 4154 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4155
1b37078b
AD		 4156 if (rdev->family >= CHIP_CAYMAN) {
4157 /* enable CP interrupts on all rings */
736fc37f 4158 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4159 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4160 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4161 }
736fc37f 4162 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4163 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4164 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4165 }
736fc37f 4166 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4167 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4168 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4169 }
4170 } else {
736fc37f 4171 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4172 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4173 cp_int_cntl |= RB_INT_ENABLE;
4174 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4175 }
45f9a39b 4176 }
1b37078b 4177
233d1ad5
AD		 4178 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4179 DRM_DEBUG("r600_irq_set: sw int dma\n");
4180 dma_cntl |= TRAP_ENABLE;
4181 }
4182
f60cbd11
AD		 4183 if (rdev->family >= CHIP_CAYMAN) {
4184 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4185 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4186 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4187 dma_cntl1 |= TRAP_ENABLE;
4188 }
4189 }
4190
6f34be50 4191 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4192 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4193 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4194 crtc1 |= VBLANK_INT_MASK;
4195 }
6f34be50 4196 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4197 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4198 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4199 crtc2 |= VBLANK_INT_MASK;
4200 }
6f34be50 4201 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4202 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4203 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4204 crtc3 |= VBLANK_INT_MASK;
4205 }
6f34be50 4206 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4207 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4208 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4209 crtc4 |= VBLANK_INT_MASK;
4210 }
6f34be50 4211 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4212 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4213 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4214 crtc5 |= VBLANK_INT_MASK;
4215 }
6f34be50 4216 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4217 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4218 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4219 crtc6 |= VBLANK_INT_MASK;
4220 }
4221 if (rdev->irq.hpd[0]) {
4222 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4223 hpd1 |= DC_HPDx_INT_EN;
4224 }
4225 if (rdev->irq.hpd[1]) {
4226 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4227 hpd2 |= DC_HPDx_INT_EN;
4228 }
4229 if (rdev->irq.hpd[2]) {
4230 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4231 hpd3 |= DC_HPDx_INT_EN;
4232 }
4233 if (rdev->irq.hpd[3]) {
4234 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4235 hpd4 |= DC_HPDx_INT_EN;
4236 }
4237 if (rdev->irq.hpd[4]) {
4238 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4239 hpd5 |= DC_HPDx_INT_EN;
4240 }
4241 if (rdev->irq.hpd[5]) {
4242 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4243 hpd6 |= DC_HPDx_INT_EN;
4244 }
f122c610
AD		 4245 if (rdev->irq.afmt[0]) {
4246 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4247 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4248 }
4249 if (rdev->irq.afmt[1]) {
4250 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4251 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4252 }
4253 if (rdev->irq.afmt[2]) {
4254 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4255 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4256 }
4257 if (rdev->irq.afmt[3]) {
4258 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4259 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4260 }
4261 if (rdev->irq.afmt[4]) {
4262 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4263 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4264 }
4265 if (rdev->irq.afmt[5]) {
4266 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4267 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4268 }
45f9a39b 4269
1b37078b
AD		 4270 if (rdev->family >= CHIP_CAYMAN) {
4271 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4272 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4273 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4274 } else
4275 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4276
4277 WREG32(DMA_CNTL, dma_cntl);
4278
f60cbd11
AD		 4279 if (rdev->family >= CHIP_CAYMAN)
4280 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4281
2031f77c 4282 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4283
4284 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4285 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4286 if (rdev->num_crtc >= 4) {
18007401
AD		 4287 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4288 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4289 }
4290 if (rdev->num_crtc >= 6) {
18007401
AD		 4291 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4292 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4293 }
45f9a39b 4294
6f34be50
AD		 4295 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4296 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
b7eff394
AD		 4297 if (rdev->num_crtc >= 4) {
4298 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4299 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4300 }
4301 if (rdev->num_crtc >= 6) {
4302 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4303 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4304 }
6f34be50 4305
45f9a39b
AD		 4306 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4307 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4308 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4309 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4310 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4311 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4312
f122c610
AD		 4313 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4314 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4315 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4316 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4317 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4318 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4319
bcc1c2a1
AD		 4320 return 0;
4321}
4322
cbdd4501 4323static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4324{
4325 u32 tmp;
4326
6f34be50
AD		 4327 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4328 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4329 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4330 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4331 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4332 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4333 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4334 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4335 if (rdev->num_crtc >= 4) {
4336 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4337 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4338 }
4339 if (rdev->num_crtc >= 6) {
4340 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4341 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4342 }
6f34be50 4343
f122c610
AD		 4344 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4345 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4346 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4347 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4348 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4349 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4350
6f34be50
AD		 4351 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4352 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4353 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4354 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4355 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4356 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4357 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4358 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4359 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4360 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4361 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4362 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4363
b7eff394
AD		 4364 if (rdev->num_crtc >= 4) {
4365 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4366 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4367 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4368 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4369 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4370 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4371 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4372 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4373 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4374 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4375 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4376 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4377 }
4378
4379 if (rdev->num_crtc >= 6) {
4380 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4381 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4382 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4383 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4384 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4385 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4386 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4387 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4388 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4389 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4390 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4391 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4392 }
45f9a39b 4393
6f34be50 4394 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4395 tmp = RREG32(DC_HPD1_INT_CONTROL);
4396 tmp |= DC_HPDx_INT_ACK;
4397 WREG32(DC_HPD1_INT_CONTROL, tmp);
4398 }
6f34be50 4399 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4400 tmp = RREG32(DC_HPD2_INT_CONTROL);
4401 tmp |= DC_HPDx_INT_ACK;
4402 WREG32(DC_HPD2_INT_CONTROL, tmp);
4403 }
6f34be50 4404 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4405 tmp = RREG32(DC_HPD3_INT_CONTROL);
4406 tmp |= DC_HPDx_INT_ACK;
4407 WREG32(DC_HPD3_INT_CONTROL, tmp);
4408 }
6f34be50 4409 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4410 tmp = RREG32(DC_HPD4_INT_CONTROL);
4411 tmp |= DC_HPDx_INT_ACK;
4412 WREG32(DC_HPD4_INT_CONTROL, tmp);
4413 }
6f34be50 4414 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4415 tmp = RREG32(DC_HPD5_INT_CONTROL);
4416 tmp |= DC_HPDx_INT_ACK;
4417 WREG32(DC_HPD5_INT_CONTROL, tmp);
4418 }
6f34be50 4419 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4420 tmp = RREG32(DC_HPD5_INT_CONTROL);
4421 tmp |= DC_HPDx_INT_ACK;
4422 WREG32(DC_HPD6_INT_CONTROL, tmp);
4423 }
f122c610
AD		 4424 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4425 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4426 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4427 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4428 }
4429 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4430 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4431 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4432 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4433 }
4434 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4435 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4436 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4437 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4438 }
4439 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4440 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4441 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4442 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4443 }
4444 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4445 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4446 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4447 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4448 }
4449 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4450 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4451 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4452 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4453 }
45f9a39b
AD		 4454}
4455
1109ca09 4456static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4457{
45f9a39b
AD		 4458 r600_disable_interrupts(rdev);
4459 /* Wait and acknowledge irq */
4460 mdelay(1);
6f34be50 4461 evergreen_irq_ack(rdev);
45f9a39b
AD		 4462 evergreen_disable_interrupt_state(rdev);
4463}
4464
755d819e 4465void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4466{
4467 evergreen_irq_disable(rdev);
4468 r600_rlc_stop(rdev);
4469}
4470
cbdd4501 4471static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4472{
4473 u32 wptr, tmp;
4474
724c80e1 4475 if (rdev->wb.enabled)
204ae24d 4476 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4477 else
4478 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4479
4480 if (wptr & RB_OVERFLOW) {
4481 /* When a ring buffer overflow happen start parsing interrupt
4482 * from the last not overwritten vector (wptr + 16). Hopefully
4483 * this should allow us to catchup.
4484 */
4485 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4486 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4487 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4488 tmp = RREG32(IH_RB_CNTL);
4489 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4490 WREG32(IH_RB_CNTL, tmp);
4491 }
4492 return (wptr & rdev->ih.ptr_mask);
4493}
4494
4495int evergreen_irq_process(struct radeon_device *rdev)
4496{
682f1a54
DA		 4497 u32 wptr;
4498 u32 rptr;
45f9a39b
AD		 4499 u32 src_id, src_data;
4500 u32 ring_index;
45f9a39b 4501 bool queue_hotplug = false;
f122c610 4502 bool queue_hdmi = false;
45f9a39b 4503
682f1a54 4504 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4505 return IRQ_NONE;
4506
682f1a54 4507 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4508
4509restart_ih:
4510 /* is somebody else already processing irqs? */
4511 if (atomic_xchg(&rdev->ih.lock, 1))
4512 return IRQ_NONE;
4513
682f1a54
DA		 4514 rptr = rdev->ih.rptr;
4515 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4516
964f6645
BH		 4517 /* Order reading of wptr vs. reading of IH ring data */
4518 rmb();
4519
45f9a39b 4520 /* display interrupts */
6f34be50 4521 evergreen_irq_ack(rdev);
45f9a39b 4522
45f9a39b
AD		 4523 while (rptr != wptr) {
4524 /* wptr/rptr are in bytes! */
4525 ring_index = rptr / 4;
0f234f5f
AD		 4526 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4527 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4528
4529 switch (src_id) {
4530 case 1: /* D1 vblank/vline */
4531 switch (src_data) {
4532 case 0: /* D1 vblank */
6f34be50 4533 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4534 if (rdev->irq.crtc_vblank_int[0]) {
4535 drm_handle_vblank(rdev->ddev, 0);
4536 rdev->pm.vblank_sync = true;
4537 wake_up(&rdev->irq.vblank_queue);
4538 }
736fc37f 4539 if (atomic_read(&rdev->irq.pflip[0]))
3e4ea742 4540 radeon_crtc_handle_flip(rdev, 0);
6f34be50 4541 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4542 DRM_DEBUG("IH: D1 vblank\n");
4543 }
4544 break;
4545 case 1: /* D1 vline */
6f34be50
AD		 4546 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4547 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4548 DRM_DEBUG("IH: D1 vline\n");
4549 }
4550 break;
4551 default:
4552 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4553 break;
4554 }
4555 break;
4556 case 2: /* D2 vblank/vline */
4557 switch (src_data) {
4558 case 0: /* D2 vblank */
6f34be50 4559 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4560 if (rdev->irq.crtc_vblank_int[1]) {
4561 drm_handle_vblank(rdev->ddev, 1);
4562 rdev->pm.vblank_sync = true;
4563 wake_up(&rdev->irq.vblank_queue);
4564 }
736fc37f 4565 if (atomic_read(&rdev->irq.pflip[1]))
3e4ea742 4566 radeon_crtc_handle_flip(rdev, 1);
6f34be50 4567 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4568 DRM_DEBUG("IH: D2 vblank\n");
4569 }
4570 break;
4571 case 1: /* D2 vline */
6f34be50
AD		 4572 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4573 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4574 DRM_DEBUG("IH: D2 vline\n");
4575 }
4576 break;
4577 default:
4578 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4579 break;
4580 }
4581 break;
4582 case 3: /* D3 vblank/vline */
4583 switch (src_data) {
4584 case 0: /* D3 vblank */
6f34be50
AD		 4585 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4586 if (rdev->irq.crtc_vblank_int[2]) {
4587 drm_handle_vblank(rdev->ddev, 2);
4588 rdev->pm.vblank_sync = true;
4589 wake_up(&rdev->irq.vblank_queue);
4590 }
736fc37f 4591 if (atomic_read(&rdev->irq.pflip[2]))
6f34be50
AD		 4592 radeon_crtc_handle_flip(rdev, 2);
4593 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4594 DRM_DEBUG("IH: D3 vblank\n");
4595 }
4596 break;
4597 case 1: /* D3 vline */
6f34be50
AD		 4598 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4599 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4600 DRM_DEBUG("IH: D3 vline\n");
4601 }
4602 break;
4603 default:
4604 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4605 break;
4606 }
4607 break;
4608 case 4: /* D4 vblank/vline */
4609 switch (src_data) {
4610 case 0: /* D4 vblank */
6f34be50
AD		 4611 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4612 if (rdev->irq.crtc_vblank_int[3]) {
4613 drm_handle_vblank(rdev->ddev, 3);
4614 rdev->pm.vblank_sync = true;
4615 wake_up(&rdev->irq.vblank_queue);
4616 }
736fc37f 4617 if (atomic_read(&rdev->irq.pflip[3]))
6f34be50
AD		 4618 radeon_crtc_handle_flip(rdev, 3);
4619 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4620 DRM_DEBUG("IH: D4 vblank\n");
4621 }
4622 break;
4623 case 1: /* D4 vline */
6f34be50
AD		 4624 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4625 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4626 DRM_DEBUG("IH: D4 vline\n");
4627 }
4628 break;
4629 default:
4630 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4631 break;
4632 }
4633 break;
4634 case 5: /* D5 vblank/vline */
4635 switch (src_data) {
4636 case 0: /* D5 vblank */
6f34be50
AD		 4637 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4638 if (rdev->irq.crtc_vblank_int[4]) {
4639 drm_handle_vblank(rdev->ddev, 4);
4640 rdev->pm.vblank_sync = true;
4641 wake_up(&rdev->irq.vblank_queue);
4642 }
736fc37f 4643 if (atomic_read(&rdev->irq.pflip[4]))
6f34be50
AD		 4644 radeon_crtc_handle_flip(rdev, 4);
4645 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4646 DRM_DEBUG("IH: D5 vblank\n");
4647 }
4648 break;
4649 case 1: /* D5 vline */
6f34be50
AD		 4650 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4651 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4652 DRM_DEBUG("IH: D5 vline\n");
4653 }
4654 break;
4655 default:
4656 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4657 break;
4658 }
4659 break;
4660 case 6: /* D6 vblank/vline */
4661 switch (src_data) {
4662 case 0: /* D6 vblank */
6f34be50
AD		 4663 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4664 if (rdev->irq.crtc_vblank_int[5]) {
4665 drm_handle_vblank(rdev->ddev, 5);
4666 rdev->pm.vblank_sync = true;
4667 wake_up(&rdev->irq.vblank_queue);
4668 }
736fc37f 4669 if (atomic_read(&rdev->irq.pflip[5]))
6f34be50
AD		 4670 radeon_crtc_handle_flip(rdev, 5);
4671 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4672 DRM_DEBUG("IH: D6 vblank\n");
4673 }
4674 break;
4675 case 1: /* D6 vline */
6f34be50
AD		 4676 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4677 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4678 DRM_DEBUG("IH: D6 vline\n");
4679 }
4680 break;
4681 default:
4682 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4683 break;
4684 }
4685 break;
4686 case 42: /* HPD hotplug */
4687 switch (src_data) {
4688 case 0:
6f34be50
AD		 4689 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4690 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4691 queue_hotplug = true;
4692 DRM_DEBUG("IH: HPD1\n");
4693 }
4694 break;
4695 case 1:
6f34be50
AD		 4696 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4697 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4698 queue_hotplug = true;
4699 DRM_DEBUG("IH: HPD2\n");
4700 }
4701 break;
4702 case 2:
6f34be50
AD		 4703 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4704 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4705 queue_hotplug = true;
4706 DRM_DEBUG("IH: HPD3\n");
4707 }
4708 break;
4709 case 3:
6f34be50
AD		 4710 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4711 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4712 queue_hotplug = true;
4713 DRM_DEBUG("IH: HPD4\n");
4714 }
4715 break;
4716 case 4:
6f34be50
AD		 4717 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4718 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 4719 queue_hotplug = true;
4720 DRM_DEBUG("IH: HPD5\n");
4721 }
4722 break;
4723 case 5:
6f34be50
AD		 4724 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4725 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 4726 queue_hotplug = true;
4727 DRM_DEBUG("IH: HPD6\n");
4728 }
4729 break;
4730 default:
4731 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4732 break;
4733 }
4734 break;
f122c610
AD		 4735 case 44: /* hdmi */
4736 switch (src_data) {
4737 case 0:
4738 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4739 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4740 queue_hdmi = true;
4741 DRM_DEBUG("IH: HDMI0\n");
4742 }
4743 break;
4744 case 1:
4745 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4746 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4747 queue_hdmi = true;
4748 DRM_DEBUG("IH: HDMI1\n");
4749 }
4750 break;
4751 case 2:
4752 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4753 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4754 queue_hdmi = true;
4755 DRM_DEBUG("IH: HDMI2\n");
4756 }
4757 break;
4758 case 3:
4759 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4760 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4761 queue_hdmi = true;
4762 DRM_DEBUG("IH: HDMI3\n");
4763 }
4764 break;
4765 case 4:
4766 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4767 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4768 queue_hdmi = true;
4769 DRM_DEBUG("IH: HDMI4\n");
4770 }
4771 break;
4772 case 5:
4773 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4774 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4775 queue_hdmi = true;
4776 DRM_DEBUG("IH: HDMI5\n");
4777 }
4778 break;
4779 default:
4780 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4781 break;
4782 }
f2ba57b5
CK		 4783 case 124: /* UVD */
4784 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4785 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 4786 break;
ae133a11
CK		 4787 case 146:
4788 case 147:
4789 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4790 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4791 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4792 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4793 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4794 /* reset addr and status */
4795 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4796 break;
45f9a39b
AD		 4797 case 176: /* CP_INT in ring buffer */
4798 case 177: /* CP_INT in IB1 */
4799 case 178: /* CP_INT in IB2 */
4800 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 4801 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 4802 break;
4803 case 181: /* CP EOP event */
4804 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 4805 if (rdev->family >= CHIP_CAYMAN) {
4806 switch (src_data) {
4807 case 0:
4808 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4809 break;
4810 case 1:
4811 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4812 break;
4813 case 2:
4814 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4815 break;
4816 }
4817 } else
4818 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 4819 break;
233d1ad5
AD		 4820 case 224: /* DMA trap event */
4821 DRM_DEBUG("IH: DMA trap\n");
4822 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4823 break;
2031f77c 4824 case 233: /* GUI IDLE */
303c805c 4825 DRM_DEBUG("IH: GUI idle\n");
2031f77c 4826 break;
f60cbd11
AD		 4827 case 244: /* DMA trap event */
4828 if (rdev->family >= CHIP_CAYMAN) {
4829 DRM_DEBUG("IH: DMA1 trap\n");
4830 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4831 }
4832 break;
45f9a39b
AD		 4833 default:
4834 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4835 break;
4836 }
4837
4838 /* wptr/rptr are in bytes! */
4839 rptr += 16;
4840 rptr &= rdev->ih.ptr_mask;
4841 }
45f9a39b 4842 if (queue_hotplug)
32c87fca 4843 schedule_work(&rdev->hotplug_work);
f122c610
AD		 4844 if (queue_hdmi)
4845 schedule_work(&rdev->audio_work);
45f9a39b
AD		 4846 rdev->ih.rptr = rptr;
4847 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 4848 atomic_set(&rdev->ih.lock, 0);
4849
4850 /* make sure wptr hasn't changed while processing */
4851 wptr = evergreen_get_ih_wptr(rdev);
4852 if (wptr != rptr)
4853 goto restart_ih;
4854
45f9a39b
AD		 4855 return IRQ_HANDLED;
4856}
4857
233d1ad5
AD		 4858/**
4859 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4860 *
4861 * @rdev: radeon_device pointer
4862 * @fence: radeon fence object
4863 *
4864 * Add a DMA fence packet to the ring to write
4865 * the fence seq number and DMA trap packet to generate
4866 * an interrupt if needed (evergreen-SI).
4867 */
4868void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4869 struct radeon_fence *fence)
4870{
4871 struct radeon_ring *ring = &rdev->ring[fence->ring];
4872 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4873 /* write the fence */
0fcb6155 4874 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
233d1ad5
AD		 4875 radeon_ring_write(ring, addr & 0xfffffffc);
4876 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4877 radeon_ring_write(ring, fence->seq);
4878 /* generate an interrupt */
0fcb6155 4879 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
233d1ad5 4880 /* flush HDP */
0fcb6155 4881 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4b681c28 4882 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
233d1ad5
AD		 4883 radeon_ring_write(ring, 1);
4884}
4885
4886/**
4887 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4888 *
4889 * @rdev: radeon_device pointer
4890 * @ib: IB object to schedule
4891 *
4892 * Schedule an IB in the DMA ring (evergreen).
4893 */
4894void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4895 struct radeon_ib *ib)
4896{
4897 struct radeon_ring *ring = &rdev->ring[ib->ring];
4898
4899 if (rdev->wb.enabled) {
4900 u32 next_rptr = ring->wptr + 4;
4901 while ((next_rptr & 7) != 5)
4902 next_rptr++;
4903 next_rptr += 3;
0fcb6155 4904 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
233d1ad5
AD		 4905 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4906 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
4907 radeon_ring_write(ring, next_rptr);
4908 }
4909
4910 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
4911 * Pad as necessary with NOPs.
4912 */
4913 while ((ring->wptr & 7) != 5)
0fcb6155
JG		 4914 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
4915 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
233d1ad5
AD		 4916 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
4917 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
4918
4919}
4920
4921/**
4922 * evergreen_copy_dma - copy pages using the DMA engine
4923 *
4924 * @rdev: radeon_device pointer
4925 * @src_offset: src GPU address
4926 * @dst_offset: dst GPU address
4927 * @num_gpu_pages: number of GPU pages to xfer
4928 * @fence: radeon fence object
4929 *
4930 * Copy GPU paging using the DMA engine (evergreen-cayman).
4931 * Used by the radeon ttm implementation to move pages if
4932 * registered as the asic copy callback.
4933 */
4934int evergreen_copy_dma(struct radeon_device *rdev,
4935 uint64_t src_offset, uint64_t dst_offset,
4936 unsigned num_gpu_pages,
4937 struct radeon_fence **fence)
4938{
4939 struct radeon_semaphore *sem = NULL;
4940 int ring_index = rdev->asic->copy.dma_ring_index;
4941 struct radeon_ring *ring = &rdev->ring[ring_index];
4942 u32 size_in_dw, cur_size_in_dw;
4943 int i, num_loops;
4944 int r = 0;
4945
4946 r = radeon_semaphore_create(rdev, &sem);
4947 if (r) {
4948 DRM_ERROR("radeon: moving bo (%d).\n", r);
4949 return r;
4950 }
4951
4952 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
4953 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
4954 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
4955 if (r) {
4956 DRM_ERROR("radeon: moving bo (%d).\n", r);
4957 radeon_semaphore_free(rdev, &sem, NULL);
4958 return r;
4959 }
4960
4961 if (radeon_fence_need_sync(*fence, ring->idx)) {
4962 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
4963 ring->idx);
4964 radeon_fence_note_sync(*fence, ring->idx);
4965 } else {
4966 radeon_semaphore_free(rdev, &sem, NULL);
4967 }
4968
4969 for (i = 0; i < num_loops; i++) {
4970 cur_size_in_dw = size_in_dw;
4971 if (cur_size_in_dw > 0xFFFFF)
4972 cur_size_in_dw = 0xFFFFF;
4973 size_in_dw -= cur_size_in_dw;
0fcb6155 4974 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
233d1ad5
AD		 4975 radeon_ring_write(ring, dst_offset & 0xfffffffc);
4976 radeon_ring_write(ring, src_offset & 0xfffffffc);
4977 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
4978 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
4979 src_offset += cur_size_in_dw * 4;
4980 dst_offset += cur_size_in_dw * 4;
4981 }
4982
4983 r = radeon_fence_emit(rdev, fence, ring->idx);
4984 if (r) {
4985 radeon_ring_unlock_undo(rdev, ring);
4986 return r;
4987 }
4988
4989 radeon_ring_unlock_commit(rdev, ring);
4990 radeon_semaphore_free(rdev, &sem, *fence);
4991
4992 return r;
4993}
4994
bcc1c2a1
AD		 4995static int evergreen_startup(struct radeon_device *rdev)
4996{
f2ba57b5 4997 struct radeon_ring *ring;
bcc1c2a1
AD		 4998 int r;
4999
9e46a48d 5000 /* enable pcie gen2 link */
cd54033a 5001 evergreen_pcie_gen2_enable(rdev);
9e46a48d 5002
0af62b01
AD		 5003 if (ASIC_IS_DCE5(rdev)) {
5004 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5005 r = ni_init_microcode(rdev);
5006 if (r) {
5007 DRM_ERROR("Failed to load firmware!\n");
5008 return r;
5009 }
5010 }
755d819e 5011 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5012 if (r) {
0af62b01 5013 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5014 return r;
5015 }
0af62b01
AD		 5016 } else {
5017 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5018 r = r600_init_microcode(rdev);
5019 if (r) {
5020 DRM_ERROR("Failed to load firmware!\n");
5021 return r;
5022 }
5023 }
bcc1c2a1 5024 }
fe251e2f 5025
16cdf04d
AD		 5026 r = r600_vram_scratch_init(rdev);
5027 if (r)
5028 return r;
5029
bcc1c2a1 5030 evergreen_mc_program(rdev);
bcc1c2a1 5031 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5032 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5033 } else {
5034 r = evergreen_pcie_gart_enable(rdev);
5035 if (r)
5036 return r;
5037 }
bcc1c2a1 5038 evergreen_gpu_init(rdev);
bcc1c2a1 5039
d7ccd8fc 5040 r = evergreen_blit_init(rdev);
bcc1c2a1 5041 if (r) {
fb3d9e97 5042 r600_blit_fini(rdev);
27cd7769 5043 rdev->asic->copy.copy = NULL;
d7ccd8fc 5044 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
bcc1c2a1
AD		 5045 }
5046
2948f5e6
AD		 5047 /* allocate rlc buffers */
5048 if (rdev->flags & RADEON_IS_IGP) {
5049 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5050 rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
5051 rdev->rlc.cs_data = evergreen_cs_data;
5052 r = sumo_rlc_init(rdev);
5053 if (r) {
5054 DRM_ERROR("Failed to init rlc BOs!\n");
5055 return r;
5056 }
5057 }
5058
724c80e1
AD		 5059 /* allocate wb buffer */
5060 r = radeon_wb_init(rdev);
5061 if (r)
5062 return r;
5063
30eb77f4
JG		 5064 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5065 if (r) {
5066 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5067 return r;
5068 }
5069
233d1ad5
AD		 5070 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5071 if (r) {
5072 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5073 return r;
5074 }
5075
f2ba57b5
CK		 5076 r = rv770_uvd_resume(rdev);
5077 if (!r) {
5078 r = radeon_fence_driver_start_ring(rdev,
5079 R600_RING_TYPE_UVD_INDEX);
5080 if (r)
5081 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5082 }
5083
5084 if (r)
5085 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5086
bcc1c2a1 5087 /* Enable IRQ */
e49f3959
AH		 5088 if (!rdev->irq.installed) {
5089 r = radeon_irq_kms_init(rdev);
5090 if (r)
5091 return r;
5092 }
5093
bcc1c2a1
AD		 5094 r = r600_irq_init(rdev);
5095 if (r) {
5096 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5097 radeon_irq_kms_fini(rdev);
5098 return r;
5099 }
45f9a39b 5100 evergreen_irq_set(rdev);
bcc1c2a1 5101
f2ba57b5 5102 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5103 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
78c5560a
AD		 5104 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5105 0, 0xfffff, RADEON_CP_PACKET2);
bcc1c2a1
AD		 5106 if (r)
5107 return r;
233d1ad5
AD		 5108
5109 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5110 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5111 DMA_RB_RPTR, DMA_RB_WPTR,
0fcb6155 5112 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5113 if (r)
5114 return r;
5115
bcc1c2a1
AD		 5116 r = evergreen_cp_load_microcode(rdev);
5117 if (r)
5118 return r;
fe251e2f 5119 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5120 if (r)
5121 return r;
5122 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5123 if (r)
5124 return r;
fe251e2f 5125
f2ba57b5
CK		 5126 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5127 if (ring->ring_size) {
5128 r = radeon_ring_init(rdev, ring, ring->ring_size,
5129 R600_WB_UVD_RPTR_OFFSET,
5130 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5131 0, 0xfffff, RADEON_CP_PACKET2);
5132 if (!r)
5133 r = r600_uvd_init(rdev);
5134
5135 if (r)
5136 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5137 }
5138
2898c348
CK		 5139 r = radeon_ib_pool_init(rdev);
5140 if (r) {
5141 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5142 return r;
2898c348 5143 }
b15ba512 5144
69d2ae57
RM		 5145 r = r600_audio_init(rdev);
5146 if (r) {
5147 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5148 return r;
5149 }
5150
bcc1c2a1
AD		 5151 return 0;
5152}
5153
5154int evergreen_resume(struct radeon_device *rdev)
5155{
5156 int r;
5157
86f5c9ed
AD		 5158 /* reset the asic, the gfx blocks are often in a bad state
5159 * after the driver is unloaded or after a resume
5160 */
5161 if (radeon_asic_reset(rdev))
5162 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5163 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5164 * posting will perform necessary task to bring back GPU into good
5165 * shape.
5166 */
5167 /* post card */
5168 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5169
d4788db3
AD		 5170 /* init golden registers */
5171 evergreen_init_golden_registers(rdev);
5172
b15ba512 5173 rdev->accel_working = true;
bcc1c2a1
AD		 5174 r = evergreen_startup(rdev);
5175 if (r) {
755d819e 5176 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5177 rdev->accel_working = false;
bcc1c2a1
AD		 5178 return r;
5179 }
fe251e2f 5180
bcc1c2a1
AD		 5181 return r;
5182
5183}
5184
5185int evergreen_suspend(struct radeon_device *rdev)
5186{
69d2ae57 5187 r600_audio_fini(rdev);
f2ba57b5 5188 radeon_uvd_suspend(rdev);
bcc1c2a1 5189 r700_cp_stop(rdev);
233d1ad5 5190 r600_dma_stop(rdev);
f2ba57b5 5191 r600_uvd_rbc_stop(rdev);
45f9a39b 5192 evergreen_irq_suspend(rdev);
724c80e1 5193 radeon_wb_disable(rdev);
bcc1c2a1 5194 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5195
5196 return 0;
5197}
5198
bcc1c2a1
AD		 5199/* Plan is to move initialization in that function and use
5200 * helper function so that radeon_device_init pretty much
5201 * do nothing more than calling asic specific function. This
5202 * should also allow to remove a bunch of callback function
5203 * like vram_info.
5204 */
5205int evergreen_init(struct radeon_device *rdev)
5206{
5207 int r;
5208
bcc1c2a1
AD		 5209 /* Read BIOS */
5210 if (!radeon_get_bios(rdev)) {
5211 if (ASIC_IS_AVIVO(rdev))
5212 return -EINVAL;
5213 }
5214 /* Must be an ATOMBIOS */
5215 if (!rdev->is_atom_bios) {
755d819e 5216 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5217 return -EINVAL;
5218 }
5219 r = radeon_atombios_init(rdev);
5220 if (r)
5221 return r;
86f5c9ed
AD		 5222 /* reset the asic, the gfx blocks are often in a bad state
5223 * after the driver is unloaded or after a resume
5224 */
5225 if (radeon_asic_reset(rdev))
5226 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5227 /* Post card if necessary */
fd909c37 5228 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5229 if (!rdev->bios) {
5230 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5231 return -EINVAL;
5232 }
5233 DRM_INFO("GPU not posted. posting now...\n");
5234 atom_asic_init(rdev->mode_info.atom_context);
5235 }
d4788db3
AD		 5236 /* init golden registers */
5237 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5238 /* Initialize scratch registers */
5239 r600_scratch_init(rdev);
5240 /* Initialize surface registers */
5241 radeon_surface_init(rdev);
5242 /* Initialize clocks */
5243 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5244 /* Fence driver */
5245 r = radeon_fence_driver_init(rdev);
5246 if (r)
5247 return r;
d594e46a
JG		 5248 /* initialize AGP */
5249 if (rdev->flags & RADEON_IS_AGP) {
5250 r = radeon_agp_init(rdev);
5251 if (r)
5252 radeon_agp_disable(rdev);
5253 }
5254 /* initialize memory controller */
bcc1c2a1
AD		 5255 r = evergreen_mc_init(rdev);
5256 if (r)
5257 return r;
5258 /* Memory manager */
5259 r = radeon_bo_init(rdev);
5260 if (r)
5261 return r;
45f9a39b 5262
e32eb50d
CK		 5263 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5264 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5265
233d1ad5
AD		 5266 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5267 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5268
f2ba57b5
CK		 5269 r = radeon_uvd_init(rdev);
5270 if (!r) {
5271 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5272 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5273 4096);
5274 }
5275
bcc1c2a1
AD		 5276 rdev->ih.ring_obj = NULL;
5277 r600_ih_ring_init(rdev, 64 * 1024);
5278
5279 r = r600_pcie_gart_init(rdev);
5280 if (r)
5281 return r;
0fcdb61e 5282
148a03bc 5283 rdev->accel_working = true;
bcc1c2a1
AD		 5284 r = evergreen_startup(rdev);
5285 if (r) {
fe251e2f
AD		 5286 dev_err(rdev->dev, "disabling GPU acceleration\n");
5287 r700_cp_fini(rdev);
233d1ad5 5288 r600_dma_fini(rdev);
fe251e2f 5289 r600_irq_fini(rdev);
2948f5e6
AD		 5290 if (rdev->flags & RADEON_IS_IGP)
5291 sumo_rlc_fini(rdev);
724c80e1 5292 radeon_wb_fini(rdev);
2898c348 5293 radeon_ib_pool_fini(rdev);
fe251e2f 5294 radeon_irq_kms_fini(rdev);
0fcdb61e 5295 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5296 rdev->accel_working = false;
5297 }
77e00f2e
AD		 5298
5299 /* Don't start up if the MC ucode is missing on BTC parts.
5300 * The default clocks and voltages before the MC ucode
5301 * is loaded are not suffient for advanced operations.
5302 */
5303 if (ASIC_IS_DCE5(rdev)) {
5304 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5305 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5306 return -EINVAL;
5307 }
5308 }
5309
bcc1c2a1
AD		 5310 return 0;
5311}
5312
5313void evergreen_fini(struct radeon_device *rdev)
5314{
69d2ae57 5315 r600_audio_fini(rdev);
fb3d9e97 5316 r600_blit_fini(rdev);
45f9a39b 5317 r700_cp_fini(rdev);
233d1ad5 5318 r600_dma_fini(rdev);
bcc1c2a1 5319 r600_irq_fini(rdev);
2948f5e6
AD		 5320 if (rdev->flags & RADEON_IS_IGP)
5321 sumo_rlc_fini(rdev);
724c80e1 5322 radeon_wb_fini(rdev);
2898c348 5323 radeon_ib_pool_fini(rdev);
bcc1c2a1 5324 radeon_irq_kms_fini(rdev);
bcc1c2a1 5325 evergreen_pcie_gart_fini(rdev);
f2ba57b5 5326 radeon_uvd_fini(rdev);
16cdf04d 5327 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5328 radeon_gem_fini(rdev);
5329 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5330 radeon_agp_fini(rdev);
5331 radeon_bo_fini(rdev);
5332 radeon_atombios_fini(rdev);
5333 kfree(rdev->bios);
5334 rdev->bios = NULL;
bcc1c2a1 5335}
9e46a48d 5336
b07759bf 5337void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5338{
7e0e4196 5339 u32 link_width_cntl, speed_cntl;
9e46a48d 5340
d42dd579
AD		 5341 if (radeon_pcie_gen2 == 0)
5342 return;
5343
9e46a48d
AD		 5344 if (rdev->flags & RADEON_IS_IGP)
5345 return;
5346
5347 if (!(rdev->flags & RADEON_IS_PCIE))
5348 return;
5349
5350 /* x2 cards have a special sequence */
5351 if (ASIC_IS_X2(rdev))
5352 return;
5353
7e0e4196
KSS		 5354 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5355 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5356 return;
5357
492d2b61 5358 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5359 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5360 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5361 return;
5362 }
5363
197bbb3d
DA		 5364 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5365
9e46a48d
AD		 5366 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5367 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5368
492d2b61 5369 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5370 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5371 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5372
492d2b61 5373 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5374 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5375 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5376
492d2b61 5377 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5378 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5379 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5380
492d2b61 5381 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5382 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5383 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5384
492d2b61 5385 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5386 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5387 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5388
5389 } else {
492d2b61 5390 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5391 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5392 if (1)
5393 link_width_cntl |= LC_UPCONFIGURE_DIS;
5394 else
5395 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5396 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5397 }
5398}
Linux kernel - Deliverables
This page took 0.500432 seconds and 5 git commands to generate.