projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: use gart for DMA IB tests
[deliverable/linux.git] / drivers / gpu / drm / radeon / evergreen.c
CommitLineData
bcc1c2a1
AD		 1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
5a0e3ad6 25#include <linux/slab.h>
760285e7 26#include <drm/drmP.h>
bcc1c2a1 27#include "radeon.h"
e6990375 28#include "radeon_asic.h"
760285e7 29#include <drm/radeon_drm.h>
0fcdb61e 30#include "evergreend.h"
bcc1c2a1
AD		 31#include "atom.h"
32#include "avivod.h"
33#include "evergreen_reg.h"
2281a378 34#include "evergreen_blit_shaders.h"
138e4e16 35#include "radeon_ucode.h"
fe251e2f 36
4a15903d
AD		 37static const u32 crtc_offsets[6] =
38{
39 EVERGREEN_CRTC0_REGISTER_OFFSET,
40 EVERGREEN_CRTC1_REGISTER_OFFSET,
41 EVERGREEN_CRTC2_REGISTER_OFFSET,
42 EVERGREEN_CRTC3_REGISTER_OFFSET,
43 EVERGREEN_CRTC4_REGISTER_OFFSET,
44 EVERGREEN_CRTC5_REGISTER_OFFSET
45};
46
2948f5e6
AD		 47#include "clearstate_evergreen.h"
48
1fd11777 49static const u32 sumo_rlc_save_restore_register_list[] =
2948f5e6
AD		 50{
51 0x98fc,
52 0x9830,
53 0x9834,
54 0x9838,
55 0x9870,
56 0x9874,
57 0x8a14,
58 0x8b24,
59 0x8bcc,
60 0x8b10,
61 0x8d00,
62 0x8d04,
63 0x8c00,
64 0x8c04,
65 0x8c08,
66 0x8c0c,
67 0x8d8c,
68 0x8c20,
69 0x8c24,
70 0x8c28,
71 0x8c18,
72 0x8c1c,
73 0x8cf0,
74 0x8e2c,
75 0x8e38,
76 0x8c30,
77 0x9508,
78 0x9688,
79 0x9608,
80 0x960c,
81 0x9610,
82 0x9614,
83 0x88c4,
84 0x88d4,
85 0xa008,
86 0x900c,
87 0x9100,
88 0x913c,
89 0x98f8,
90 0x98f4,
91 0x9b7c,
92 0x3f8c,
93 0x8950,
94 0x8954,
95 0x8a18,
96 0x8b28,
97 0x9144,
98 0x9148,
99 0x914c,
100 0x3f90,
101 0x3f94,
102 0x915c,
103 0x9160,
104 0x9178,
105 0x917c,
106 0x9180,
107 0x918c,
108 0x9190,
109 0x9194,
110 0x9198,
111 0x919c,
112 0x91a8,
113 0x91ac,
114 0x91b0,
115 0x91b4,
116 0x91b8,
117 0x91c4,
118 0x91c8,
119 0x91cc,
120 0x91d0,
121 0x91d4,
122 0x91e0,
123 0x91e4,
124 0x91ec,
125 0x91f0,
126 0x91f4,
127 0x9200,
128 0x9204,
129 0x929c,
130 0x9150,
131 0x802c,
132};
2948f5e6 133
bcc1c2a1
AD		 134static void evergreen_gpu_init(struct radeon_device *rdev);
135void evergreen_fini(struct radeon_device *rdev);
b07759bf 136void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
f52382d7 137void evergreen_program_aspm(struct radeon_device *rdev);
1b37078b
AD		 138extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
139 int ring, u32 cp_int_cntl);
54e2e49c
AD		 140extern void cayman_vm_decode_fault(struct radeon_device *rdev,
141 u32 status, u32 addr);
22c775ce 142void cik_init_cp_pg_table(struct radeon_device *rdev);
bcc1c2a1 143
59a82d0e
AD		 144extern u32 si_get_csb_size(struct radeon_device *rdev);
145extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
a0f38609
AD		 146extern u32 cik_get_csb_size(struct radeon_device *rdev);
147extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
b5470b03 148extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
bcc1c2a1 149
d4788db3
AD		 150static const u32 evergreen_golden_registers[] =
151{
152 0x3f90, 0xffff0000, 0xff000000,
153 0x9148, 0xffff0000, 0xff000000,
154 0x3f94, 0xffff0000, 0xff000000,
155 0x914c, 0xffff0000, 0xff000000,
156 0x9b7c, 0xffffffff, 0x00000000,
157 0x8a14, 0xffffffff, 0x00000007,
158 0x8b10, 0xffffffff, 0x00000000,
159 0x960c, 0xffffffff, 0x54763210,
160 0x88c4, 0xffffffff, 0x000000c2,
161 0x88d4, 0xffffffff, 0x00000010,
162 0x8974, 0xffffffff, 0x00000000,
163 0xc78, 0x00000080, 0x00000080,
164 0x5eb4, 0xffffffff, 0x00000002,
165 0x5e78, 0xffffffff, 0x001000f0,
166 0x6104, 0x01000300, 0x00000000,
167 0x5bc0, 0x00300000, 0x00000000,
168 0x7030, 0xffffffff, 0x00000011,
169 0x7c30, 0xffffffff, 0x00000011,
170 0x10830, 0xffffffff, 0x00000011,
171 0x11430, 0xffffffff, 0x00000011,
172 0x12030, 0xffffffff, 0x00000011,
173 0x12c30, 0xffffffff, 0x00000011,
174 0xd02c, 0xffffffff, 0x08421000,
175 0x240c, 0xffffffff, 0x00000380,
176 0x8b24, 0xffffffff, 0x00ff0fff,
177 0x28a4c, 0x06000000, 0x06000000,
178 0x10c, 0x00000001, 0x00000001,
179 0x8d00, 0xffffffff, 0x100e4848,
180 0x8d04, 0xffffffff, 0x00164745,
181 0x8c00, 0xffffffff, 0xe4000003,
182 0x8c04, 0xffffffff, 0x40600060,
183 0x8c08, 0xffffffff, 0x001c001c,
184 0x8cf0, 0xffffffff, 0x08e00620,
185 0x8c20, 0xffffffff, 0x00800080,
186 0x8c24, 0xffffffff, 0x00800080,
187 0x8c18, 0xffffffff, 0x20202078,
188 0x8c1c, 0xffffffff, 0x00001010,
189 0x28350, 0xffffffff, 0x00000000,
190 0xa008, 0xffffffff, 0x00010000,
6abafb78 191 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 192 0x9508, 0xffffffff, 0x00000002,
193 0x913c, 0x0000000f, 0x0000000a
194};
195
196static const u32 evergreen_golden_registers2[] =
197{
198 0x2f4c, 0xffffffff, 0x00000000,
199 0x54f4, 0xffffffff, 0x00000000,
200 0x54f0, 0xffffffff, 0x00000000,
201 0x5498, 0xffffffff, 0x00000000,
202 0x549c, 0xffffffff, 0x00000000,
203 0x5494, 0xffffffff, 0x00000000,
204 0x53cc, 0xffffffff, 0x00000000,
205 0x53c8, 0xffffffff, 0x00000000,
206 0x53c4, 0xffffffff, 0x00000000,
207 0x53c0, 0xffffffff, 0x00000000,
208 0x53bc, 0xffffffff, 0x00000000,
209 0x53b8, 0xffffffff, 0x00000000,
210 0x53b4, 0xffffffff, 0x00000000,
211 0x53b0, 0xffffffff, 0x00000000
212};
213
214static const u32 cypress_mgcg_init[] =
215{
216 0x802c, 0xffffffff, 0xc0000000,
217 0x5448, 0xffffffff, 0x00000100,
218 0x55e4, 0xffffffff, 0x00000100,
219 0x160c, 0xffffffff, 0x00000100,
220 0x5644, 0xffffffff, 0x00000100,
221 0xc164, 0xffffffff, 0x00000100,
222 0x8a18, 0xffffffff, 0x00000100,
223 0x897c, 0xffffffff, 0x06000100,
224 0x8b28, 0xffffffff, 0x00000100,
225 0x9144, 0xffffffff, 0x00000100,
226 0x9a60, 0xffffffff, 0x00000100,
227 0x9868, 0xffffffff, 0x00000100,
228 0x8d58, 0xffffffff, 0x00000100,
229 0x9510, 0xffffffff, 0x00000100,
230 0x949c, 0xffffffff, 0x00000100,
231 0x9654, 0xffffffff, 0x00000100,
232 0x9030, 0xffffffff, 0x00000100,
233 0x9034, 0xffffffff, 0x00000100,
234 0x9038, 0xffffffff, 0x00000100,
235 0x903c, 0xffffffff, 0x00000100,
236 0x9040, 0xffffffff, 0x00000100,
237 0xa200, 0xffffffff, 0x00000100,
238 0xa204, 0xffffffff, 0x00000100,
239 0xa208, 0xffffffff, 0x00000100,
240 0xa20c, 0xffffffff, 0x00000100,
241 0x971c, 0xffffffff, 0x00000100,
242 0x977c, 0xffffffff, 0x00000100,
243 0x3f80, 0xffffffff, 0x00000100,
244 0xa210, 0xffffffff, 0x00000100,
245 0xa214, 0xffffffff, 0x00000100,
246 0x4d8, 0xffffffff, 0x00000100,
247 0x9784, 0xffffffff, 0x00000100,
248 0x9698, 0xffffffff, 0x00000100,
249 0x4d4, 0xffffffff, 0x00000200,
250 0x30cc, 0xffffffff, 0x00000100,
251 0xd0c0, 0xffffffff, 0xff000100,
252 0x802c, 0xffffffff, 0x40000000,
253 0x915c, 0xffffffff, 0x00010000,
254 0x9160, 0xffffffff, 0x00030002,
255 0x9178, 0xffffffff, 0x00070000,
256 0x917c, 0xffffffff, 0x00030002,
257 0x9180, 0xffffffff, 0x00050004,
258 0x918c, 0xffffffff, 0x00010006,
259 0x9190, 0xffffffff, 0x00090008,
260 0x9194, 0xffffffff, 0x00070000,
261 0x9198, 0xffffffff, 0x00030002,
262 0x919c, 0xffffffff, 0x00050004,
263 0x91a8, 0xffffffff, 0x00010006,
264 0x91ac, 0xffffffff, 0x00090008,
265 0x91b0, 0xffffffff, 0x00070000,
266 0x91b4, 0xffffffff, 0x00030002,
267 0x91b8, 0xffffffff, 0x00050004,
268 0x91c4, 0xffffffff, 0x00010006,
269 0x91c8, 0xffffffff, 0x00090008,
270 0x91cc, 0xffffffff, 0x00070000,
271 0x91d0, 0xffffffff, 0x00030002,
272 0x91d4, 0xffffffff, 0x00050004,
273 0x91e0, 0xffffffff, 0x00010006,
274 0x91e4, 0xffffffff, 0x00090008,
275 0x91e8, 0xffffffff, 0x00000000,
276 0x91ec, 0xffffffff, 0x00070000,
277 0x91f0, 0xffffffff, 0x00030002,
278 0x91f4, 0xffffffff, 0x00050004,
279 0x9200, 0xffffffff, 0x00010006,
280 0x9204, 0xffffffff, 0x00090008,
281 0x9208, 0xffffffff, 0x00070000,
282 0x920c, 0xffffffff, 0x00030002,
283 0x9210, 0xffffffff, 0x00050004,
284 0x921c, 0xffffffff, 0x00010006,
285 0x9220, 0xffffffff, 0x00090008,
286 0x9224, 0xffffffff, 0x00070000,
287 0x9228, 0xffffffff, 0x00030002,
288 0x922c, 0xffffffff, 0x00050004,
289 0x9238, 0xffffffff, 0x00010006,
290 0x923c, 0xffffffff, 0x00090008,
291 0x9240, 0xffffffff, 0x00070000,
292 0x9244, 0xffffffff, 0x00030002,
293 0x9248, 0xffffffff, 0x00050004,
294 0x9254, 0xffffffff, 0x00010006,
295 0x9258, 0xffffffff, 0x00090008,
296 0x925c, 0xffffffff, 0x00070000,
297 0x9260, 0xffffffff, 0x00030002,
298 0x9264, 0xffffffff, 0x00050004,
299 0x9270, 0xffffffff, 0x00010006,
300 0x9274, 0xffffffff, 0x00090008,
301 0x9278, 0xffffffff, 0x00070000,
302 0x927c, 0xffffffff, 0x00030002,
303 0x9280, 0xffffffff, 0x00050004,
304 0x928c, 0xffffffff, 0x00010006,
305 0x9290, 0xffffffff, 0x00090008,
306 0x9294, 0xffffffff, 0x00000000,
307 0x929c, 0xffffffff, 0x00000001,
308 0x802c, 0xffffffff, 0x40010000,
309 0x915c, 0xffffffff, 0x00010000,
310 0x9160, 0xffffffff, 0x00030002,
311 0x9178, 0xffffffff, 0x00070000,
312 0x917c, 0xffffffff, 0x00030002,
313 0x9180, 0xffffffff, 0x00050004,
314 0x918c, 0xffffffff, 0x00010006,
315 0x9190, 0xffffffff, 0x00090008,
316 0x9194, 0xffffffff, 0x00070000,
317 0x9198, 0xffffffff, 0x00030002,
318 0x919c, 0xffffffff, 0x00050004,
319 0x91a8, 0xffffffff, 0x00010006,
320 0x91ac, 0xffffffff, 0x00090008,
321 0x91b0, 0xffffffff, 0x00070000,
322 0x91b4, 0xffffffff, 0x00030002,
323 0x91b8, 0xffffffff, 0x00050004,
324 0x91c4, 0xffffffff, 0x00010006,
325 0x91c8, 0xffffffff, 0x00090008,
326 0x91cc, 0xffffffff, 0x00070000,
327 0x91d0, 0xffffffff, 0x00030002,
328 0x91d4, 0xffffffff, 0x00050004,
329 0x91e0, 0xffffffff, 0x00010006,
330 0x91e4, 0xffffffff, 0x00090008,
331 0x91e8, 0xffffffff, 0x00000000,
332 0x91ec, 0xffffffff, 0x00070000,
333 0x91f0, 0xffffffff, 0x00030002,
334 0x91f4, 0xffffffff, 0x00050004,
335 0x9200, 0xffffffff, 0x00010006,
336 0x9204, 0xffffffff, 0x00090008,
337 0x9208, 0xffffffff, 0x00070000,
338 0x920c, 0xffffffff, 0x00030002,
339 0x9210, 0xffffffff, 0x00050004,
340 0x921c, 0xffffffff, 0x00010006,
341 0x9220, 0xffffffff, 0x00090008,
342 0x9224, 0xffffffff, 0x00070000,
343 0x9228, 0xffffffff, 0x00030002,
344 0x922c, 0xffffffff, 0x00050004,
345 0x9238, 0xffffffff, 0x00010006,
346 0x923c, 0xffffffff, 0x00090008,
347 0x9240, 0xffffffff, 0x00070000,
348 0x9244, 0xffffffff, 0x00030002,
349 0x9248, 0xffffffff, 0x00050004,
350 0x9254, 0xffffffff, 0x00010006,
351 0x9258, 0xffffffff, 0x00090008,
352 0x925c, 0xffffffff, 0x00070000,
353 0x9260, 0xffffffff, 0x00030002,
354 0x9264, 0xffffffff, 0x00050004,
355 0x9270, 0xffffffff, 0x00010006,
356 0x9274, 0xffffffff, 0x00090008,
357 0x9278, 0xffffffff, 0x00070000,
358 0x927c, 0xffffffff, 0x00030002,
359 0x9280, 0xffffffff, 0x00050004,
360 0x928c, 0xffffffff, 0x00010006,
361 0x9290, 0xffffffff, 0x00090008,
362 0x9294, 0xffffffff, 0x00000000,
363 0x929c, 0xffffffff, 0x00000001,
364 0x802c, 0xffffffff, 0xc0000000
365};
366
367static const u32 redwood_mgcg_init[] =
368{
369 0x802c, 0xffffffff, 0xc0000000,
370 0x5448, 0xffffffff, 0x00000100,
371 0x55e4, 0xffffffff, 0x00000100,
372 0x160c, 0xffffffff, 0x00000100,
373 0x5644, 0xffffffff, 0x00000100,
374 0xc164, 0xffffffff, 0x00000100,
375 0x8a18, 0xffffffff, 0x00000100,
376 0x897c, 0xffffffff, 0x06000100,
377 0x8b28, 0xffffffff, 0x00000100,
378 0x9144, 0xffffffff, 0x00000100,
379 0x9a60, 0xffffffff, 0x00000100,
380 0x9868, 0xffffffff, 0x00000100,
381 0x8d58, 0xffffffff, 0x00000100,
382 0x9510, 0xffffffff, 0x00000100,
383 0x949c, 0xffffffff, 0x00000100,
384 0x9654, 0xffffffff, 0x00000100,
385 0x9030, 0xffffffff, 0x00000100,
386 0x9034, 0xffffffff, 0x00000100,
387 0x9038, 0xffffffff, 0x00000100,
388 0x903c, 0xffffffff, 0x00000100,
389 0x9040, 0xffffffff, 0x00000100,
390 0xa200, 0xffffffff, 0x00000100,
391 0xa204, 0xffffffff, 0x00000100,
392 0xa208, 0xffffffff, 0x00000100,
393 0xa20c, 0xffffffff, 0x00000100,
394 0x971c, 0xffffffff, 0x00000100,
395 0x977c, 0xffffffff, 0x00000100,
396 0x3f80, 0xffffffff, 0x00000100,
397 0xa210, 0xffffffff, 0x00000100,
398 0xa214, 0xffffffff, 0x00000100,
399 0x4d8, 0xffffffff, 0x00000100,
400 0x9784, 0xffffffff, 0x00000100,
401 0x9698, 0xffffffff, 0x00000100,
402 0x4d4, 0xffffffff, 0x00000200,
403 0x30cc, 0xffffffff, 0x00000100,
404 0xd0c0, 0xffffffff, 0xff000100,
405 0x802c, 0xffffffff, 0x40000000,
406 0x915c, 0xffffffff, 0x00010000,
407 0x9160, 0xffffffff, 0x00030002,
408 0x9178, 0xffffffff, 0x00070000,
409 0x917c, 0xffffffff, 0x00030002,
410 0x9180, 0xffffffff, 0x00050004,
411 0x918c, 0xffffffff, 0x00010006,
412 0x9190, 0xffffffff, 0x00090008,
413 0x9194, 0xffffffff, 0x00070000,
414 0x9198, 0xffffffff, 0x00030002,
415 0x919c, 0xffffffff, 0x00050004,
416 0x91a8, 0xffffffff, 0x00010006,
417 0x91ac, 0xffffffff, 0x00090008,
418 0x91b0, 0xffffffff, 0x00070000,
419 0x91b4, 0xffffffff, 0x00030002,
420 0x91b8, 0xffffffff, 0x00050004,
421 0x91c4, 0xffffffff, 0x00010006,
422 0x91c8, 0xffffffff, 0x00090008,
423 0x91cc, 0xffffffff, 0x00070000,
424 0x91d0, 0xffffffff, 0x00030002,
425 0x91d4, 0xffffffff, 0x00050004,
426 0x91e0, 0xffffffff, 0x00010006,
427 0x91e4, 0xffffffff, 0x00090008,
428 0x91e8, 0xffffffff, 0x00000000,
429 0x91ec, 0xffffffff, 0x00070000,
430 0x91f0, 0xffffffff, 0x00030002,
431 0x91f4, 0xffffffff, 0x00050004,
432 0x9200, 0xffffffff, 0x00010006,
433 0x9204, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437};
438
439static const u32 cedar_golden_registers[] =
440{
441 0x3f90, 0xffff0000, 0xff000000,
442 0x9148, 0xffff0000, 0xff000000,
443 0x3f94, 0xffff0000, 0xff000000,
444 0x914c, 0xffff0000, 0xff000000,
445 0x9b7c, 0xffffffff, 0x00000000,
446 0x8a14, 0xffffffff, 0x00000007,
447 0x8b10, 0xffffffff, 0x00000000,
448 0x960c, 0xffffffff, 0x54763210,
449 0x88c4, 0xffffffff, 0x000000c2,
450 0x88d4, 0xffffffff, 0x00000000,
451 0x8974, 0xffffffff, 0x00000000,
452 0xc78, 0x00000080, 0x00000080,
453 0x5eb4, 0xffffffff, 0x00000002,
454 0x5e78, 0xffffffff, 0x001000f0,
455 0x6104, 0x01000300, 0x00000000,
456 0x5bc0, 0x00300000, 0x00000000,
457 0x7030, 0xffffffff, 0x00000011,
458 0x7c30, 0xffffffff, 0x00000011,
459 0x10830, 0xffffffff, 0x00000011,
460 0x11430, 0xffffffff, 0x00000011,
461 0xd02c, 0xffffffff, 0x08421000,
462 0x240c, 0xffffffff, 0x00000380,
463 0x8b24, 0xffffffff, 0x00ff0fff,
464 0x28a4c, 0x06000000, 0x06000000,
465 0x10c, 0x00000001, 0x00000001,
466 0x8d00, 0xffffffff, 0x100e4848,
467 0x8d04, 0xffffffff, 0x00164745,
468 0x8c00, 0xffffffff, 0xe4000003,
469 0x8c04, 0xffffffff, 0x40600060,
470 0x8c08, 0xffffffff, 0x001c001c,
471 0x8cf0, 0xffffffff, 0x08e00410,
472 0x8c20, 0xffffffff, 0x00800080,
473 0x8c24, 0xffffffff, 0x00800080,
474 0x8c18, 0xffffffff, 0x20202078,
475 0x8c1c, 0xffffffff, 0x00001010,
476 0x28350, 0xffffffff, 0x00000000,
477 0xa008, 0xffffffff, 0x00010000,
6abafb78 478 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 479 0x9508, 0xffffffff, 0x00000002
480};
481
482static const u32 cedar_mgcg_init[] =
483{
484 0x802c, 0xffffffff, 0xc0000000,
485 0x5448, 0xffffffff, 0x00000100,
486 0x55e4, 0xffffffff, 0x00000100,
487 0x160c, 0xffffffff, 0x00000100,
488 0x5644, 0xffffffff, 0x00000100,
489 0xc164, 0xffffffff, 0x00000100,
490 0x8a18, 0xffffffff, 0x00000100,
491 0x897c, 0xffffffff, 0x06000100,
492 0x8b28, 0xffffffff, 0x00000100,
493 0x9144, 0xffffffff, 0x00000100,
494 0x9a60, 0xffffffff, 0x00000100,
495 0x9868, 0xffffffff, 0x00000100,
496 0x8d58, 0xffffffff, 0x00000100,
497 0x9510, 0xffffffff, 0x00000100,
498 0x949c, 0xffffffff, 0x00000100,
499 0x9654, 0xffffffff, 0x00000100,
500 0x9030, 0xffffffff, 0x00000100,
501 0x9034, 0xffffffff, 0x00000100,
502 0x9038, 0xffffffff, 0x00000100,
503 0x903c, 0xffffffff, 0x00000100,
504 0x9040, 0xffffffff, 0x00000100,
505 0xa200, 0xffffffff, 0x00000100,
506 0xa204, 0xffffffff, 0x00000100,
507 0xa208, 0xffffffff, 0x00000100,
508 0xa20c, 0xffffffff, 0x00000100,
509 0x971c, 0xffffffff, 0x00000100,
510 0x977c, 0xffffffff, 0x00000100,
511 0x3f80, 0xffffffff, 0x00000100,
512 0xa210, 0xffffffff, 0x00000100,
513 0xa214, 0xffffffff, 0x00000100,
514 0x4d8, 0xffffffff, 0x00000100,
515 0x9784, 0xffffffff, 0x00000100,
516 0x9698, 0xffffffff, 0x00000100,
517 0x4d4, 0xffffffff, 0x00000200,
518 0x30cc, 0xffffffff, 0x00000100,
519 0xd0c0, 0xffffffff, 0xff000100,
520 0x802c, 0xffffffff, 0x40000000,
521 0x915c, 0xffffffff, 0x00010000,
522 0x9178, 0xffffffff, 0x00050000,
523 0x917c, 0xffffffff, 0x00030002,
524 0x918c, 0xffffffff, 0x00010004,
525 0x9190, 0xffffffff, 0x00070006,
526 0x9194, 0xffffffff, 0x00050000,
527 0x9198, 0xffffffff, 0x00030002,
528 0x91a8, 0xffffffff, 0x00010004,
529 0x91ac, 0xffffffff, 0x00070006,
530 0x91e8, 0xffffffff, 0x00000000,
531 0x9294, 0xffffffff, 0x00000000,
532 0x929c, 0xffffffff, 0x00000001,
533 0x802c, 0xffffffff, 0xc0000000
534};
535
536static const u32 juniper_mgcg_init[] =
537{
538 0x802c, 0xffffffff, 0xc0000000,
539 0x5448, 0xffffffff, 0x00000100,
540 0x55e4, 0xffffffff, 0x00000100,
541 0x160c, 0xffffffff, 0x00000100,
542 0x5644, 0xffffffff, 0x00000100,
543 0xc164, 0xffffffff, 0x00000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x897c, 0xffffffff, 0x06000100,
546 0x8b28, 0xffffffff, 0x00000100,
547 0x9144, 0xffffffff, 0x00000100,
548 0x9a60, 0xffffffff, 0x00000100,
549 0x9868, 0xffffffff, 0x00000100,
550 0x8d58, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0x949c, 0xffffffff, 0x00000100,
553 0x9654, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0x9040, 0xffffffff, 0x00000100,
559 0xa200, 0xffffffff, 0x00000100,
560 0xa204, 0xffffffff, 0x00000100,
561 0xa208, 0xffffffff, 0x00000100,
562 0xa20c, 0xffffffff, 0x00000100,
563 0x971c, 0xffffffff, 0x00000100,
564 0xd0c0, 0xffffffff, 0xff000100,
565 0x802c, 0xffffffff, 0x40000000,
566 0x915c, 0xffffffff, 0x00010000,
567 0x9160, 0xffffffff, 0x00030002,
568 0x9178, 0xffffffff, 0x00070000,
569 0x917c, 0xffffffff, 0x00030002,
570 0x9180, 0xffffffff, 0x00050004,
571 0x918c, 0xffffffff, 0x00010006,
572 0x9190, 0xffffffff, 0x00090008,
573 0x9194, 0xffffffff, 0x00070000,
574 0x9198, 0xffffffff, 0x00030002,
575 0x919c, 0xffffffff, 0x00050004,
576 0x91a8, 0xffffffff, 0x00010006,
577 0x91ac, 0xffffffff, 0x00090008,
578 0x91b0, 0xffffffff, 0x00070000,
579 0x91b4, 0xffffffff, 0x00030002,
580 0x91b8, 0xffffffff, 0x00050004,
581 0x91c4, 0xffffffff, 0x00010006,
582 0x91c8, 0xffffffff, 0x00090008,
583 0x91cc, 0xffffffff, 0x00070000,
584 0x91d0, 0xffffffff, 0x00030002,
585 0x91d4, 0xffffffff, 0x00050004,
586 0x91e0, 0xffffffff, 0x00010006,
587 0x91e4, 0xffffffff, 0x00090008,
588 0x91e8, 0xffffffff, 0x00000000,
589 0x91ec, 0xffffffff, 0x00070000,
590 0x91f0, 0xffffffff, 0x00030002,
591 0x91f4, 0xffffffff, 0x00050004,
592 0x9200, 0xffffffff, 0x00010006,
593 0x9204, 0xffffffff, 0x00090008,
594 0x9208, 0xffffffff, 0x00070000,
595 0x920c, 0xffffffff, 0x00030002,
596 0x9210, 0xffffffff, 0x00050004,
597 0x921c, 0xffffffff, 0x00010006,
598 0x9220, 0xffffffff, 0x00090008,
599 0x9224, 0xffffffff, 0x00070000,
600 0x9228, 0xffffffff, 0x00030002,
601 0x922c, 0xffffffff, 0x00050004,
602 0x9238, 0xffffffff, 0x00010006,
603 0x923c, 0xffffffff, 0x00090008,
604 0x9240, 0xffffffff, 0x00070000,
605 0x9244, 0xffffffff, 0x00030002,
606 0x9248, 0xffffffff, 0x00050004,
607 0x9254, 0xffffffff, 0x00010006,
608 0x9258, 0xffffffff, 0x00090008,
609 0x925c, 0xffffffff, 0x00070000,
610 0x9260, 0xffffffff, 0x00030002,
611 0x9264, 0xffffffff, 0x00050004,
612 0x9270, 0xffffffff, 0x00010006,
613 0x9274, 0xffffffff, 0x00090008,
614 0x9278, 0xffffffff, 0x00070000,
615 0x927c, 0xffffffff, 0x00030002,
616 0x9280, 0xffffffff, 0x00050004,
617 0x928c, 0xffffffff, 0x00010006,
618 0x9290, 0xffffffff, 0x00090008,
619 0x9294, 0xffffffff, 0x00000000,
620 0x929c, 0xffffffff, 0x00000001,
621 0x802c, 0xffffffff, 0xc0000000,
622 0x977c, 0xffffffff, 0x00000100,
623 0x3f80, 0xffffffff, 0x00000100,
624 0xa210, 0xffffffff, 0x00000100,
625 0xa214, 0xffffffff, 0x00000100,
626 0x4d8, 0xffffffff, 0x00000100,
627 0x9784, 0xffffffff, 0x00000100,
628 0x9698, 0xffffffff, 0x00000100,
629 0x4d4, 0xffffffff, 0x00000200,
630 0x30cc, 0xffffffff, 0x00000100,
631 0x802c, 0xffffffff, 0xc0000000
632};
633
634static const u32 supersumo_golden_registers[] =
635{
636 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 637 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 638 0x7030, 0xffffffff, 0x00000011,
639 0x7c30, 0xffffffff, 0x00000011,
640 0x6104, 0x01000300, 0x00000000,
641 0x5bc0, 0x00300000, 0x00000000,
642 0x8c04, 0xffffffff, 0x40600060,
643 0x8c08, 0xffffffff, 0x001c001c,
644 0x8c20, 0xffffffff, 0x00800080,
645 0x8c24, 0xffffffff, 0x00800080,
646 0x8c18, 0xffffffff, 0x20202078,
647 0x8c1c, 0xffffffff, 0x00001010,
648 0x918c, 0xffffffff, 0x00010006,
649 0x91a8, 0xffffffff, 0x00010006,
650 0x91c4, 0xffffffff, 0x00010006,
651 0x91e0, 0xffffffff, 0x00010006,
652 0x9200, 0xffffffff, 0x00010006,
653 0x9150, 0xffffffff, 0x6e944040,
654 0x917c, 0xffffffff, 0x00030002,
655 0x9180, 0xffffffff, 0x00050004,
656 0x9198, 0xffffffff, 0x00030002,
657 0x919c, 0xffffffff, 0x00050004,
658 0x91b4, 0xffffffff, 0x00030002,
659 0x91b8, 0xffffffff, 0x00050004,
660 0x91d0, 0xffffffff, 0x00030002,
661 0x91d4, 0xffffffff, 0x00050004,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x915c, 0xffffffff, 0x00010000,
665 0x9160, 0xffffffff, 0x00030002,
666 0x3f90, 0xffff0000, 0xff000000,
667 0x9178, 0xffffffff, 0x00070000,
668 0x9194, 0xffffffff, 0x00070000,
669 0x91b0, 0xffffffff, 0x00070000,
670 0x91cc, 0xffffffff, 0x00070000,
671 0x91ec, 0xffffffff, 0x00070000,
672 0x9148, 0xffff0000, 0xff000000,
673 0x9190, 0xffffffff, 0x00090008,
674 0x91ac, 0xffffffff, 0x00090008,
675 0x91c8, 0xffffffff, 0x00090008,
676 0x91e4, 0xffffffff, 0x00090008,
677 0x9204, 0xffffffff, 0x00090008,
678 0x3f94, 0xffff0000, 0xff000000,
679 0x914c, 0xffff0000, 0xff000000,
680 0x929c, 0xffffffff, 0x00000001,
681 0x8a18, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x5644, 0xffffffff, 0x00000100,
685 0x9b7c, 0xffffffff, 0x00000000,
686 0x8030, 0xffffffff, 0x0000100a,
687 0x8a14, 0xffffffff, 0x00000007,
688 0x8b24, 0xffffffff, 0x00ff0fff,
689 0x8b10, 0xffffffff, 0x00000000,
690 0x28a4c, 0x06000000, 0x06000000,
691 0x4d8, 0xffffffff, 0x00000100,
692 0x913c, 0xffff000f, 0x0100000a,
693 0x960c, 0xffffffff, 0x54763210,
694 0x88c4, 0xffffffff, 0x000000c2,
695 0x88d4, 0xffffffff, 0x00000010,
696 0x8974, 0xffffffff, 0x00000000,
697 0xc78, 0x00000080, 0x00000080,
698 0x5e78, 0xffffffff, 0x001000f0,
699 0xd02c, 0xffffffff, 0x08421000,
700 0xa008, 0xffffffff, 0x00010000,
701 0x8d00, 0xffffffff, 0x100e4848,
702 0x8d04, 0xffffffff, 0x00164745,
703 0x8c00, 0xffffffff, 0xe4000003,
704 0x8cf0, 0x1fffffff, 0x08e00620,
705 0x28350, 0xffffffff, 0x00000000,
706 0x9508, 0xffffffff, 0x00000002
707};
708
709static const u32 sumo_golden_registers[] =
710{
711 0x900c, 0x00ffffff, 0x0017071f,
712 0x8c18, 0xffffffff, 0x10101060,
713 0x8c1c, 0xffffffff, 0x00001010,
714 0x8c30, 0x0000000f, 0x00000005,
715 0x9688, 0x0000000f, 0x00000007
716};
717
718static const u32 wrestler_golden_registers[] =
719{
720 0x5eb4, 0xffffffff, 0x00000002,
6abafb78 721 0x5c4, 0xffffffff, 0x00000001,
d4788db3
AD		 722 0x7030, 0xffffffff, 0x00000011,
723 0x7c30, 0xffffffff, 0x00000011,
724 0x6104, 0x01000300, 0x00000000,
725 0x5bc0, 0x00300000, 0x00000000,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9198, 0xffffffff, 0x00030002,
731 0x915c, 0xffffffff, 0x00010000,
732 0x3f90, 0xffff0000, 0xff000000,
733 0x9178, 0xffffffff, 0x00070000,
734 0x9194, 0xffffffff, 0x00070000,
735 0x9148, 0xffff0000, 0xff000000,
736 0x9190, 0xffffffff, 0x00090008,
737 0x91ac, 0xffffffff, 0x00090008,
738 0x3f94, 0xffff0000, 0xff000000,
739 0x914c, 0xffff0000, 0xff000000,
740 0x929c, 0xffffffff, 0x00000001,
741 0x8a18, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x9b7c, 0xffffffff, 0x00000000,
745 0x8030, 0xffffffff, 0x0000100a,
746 0x8a14, 0xffffffff, 0x00000001,
747 0x8b24, 0xffffffff, 0x00ff0fff,
748 0x8b10, 0xffffffff, 0x00000000,
749 0x28a4c, 0x06000000, 0x06000000,
750 0x4d8, 0xffffffff, 0x00000100,
751 0x913c, 0xffff000f, 0x0100000a,
752 0x960c, 0xffffffff, 0x54763210,
753 0x88c4, 0xffffffff, 0x000000c2,
754 0x88d4, 0xffffffff, 0x00000010,
755 0x8974, 0xffffffff, 0x00000000,
756 0xc78, 0x00000080, 0x00000080,
757 0x5e78, 0xffffffff, 0x001000f0,
758 0xd02c, 0xffffffff, 0x08421000,
759 0xa008, 0xffffffff, 0x00010000,
760 0x8d00, 0xffffffff, 0x100e4848,
761 0x8d04, 0xffffffff, 0x00164745,
762 0x8c00, 0xffffffff, 0xe4000003,
763 0x8cf0, 0x1fffffff, 0x08e00410,
764 0x28350, 0xffffffff, 0x00000000,
765 0x9508, 0xffffffff, 0x00000002,
766 0x900c, 0xffffffff, 0x0017071f,
767 0x8c18, 0xffffffff, 0x10101060,
768 0x8c1c, 0xffffffff, 0x00001010
769};
770
771static const u32 barts_golden_registers[] =
772{
773 0x5eb4, 0xffffffff, 0x00000002,
774 0x5e78, 0x8f311ff1, 0x001000f0,
775 0x3f90, 0xffff0000, 0xff000000,
776 0x9148, 0xffff0000, 0xff000000,
777 0x3f94, 0xffff0000, 0xff000000,
778 0x914c, 0xffff0000, 0xff000000,
779 0xc78, 0x00000080, 0x00000080,
780 0xbd4, 0x70073777, 0x00010001,
781 0xd02c, 0xbfffff1f, 0x08421000,
782 0xd0b8, 0x03773777, 0x02011003,
783 0x5bc0, 0x00200000, 0x50100000,
784 0x98f8, 0x33773777, 0x02011003,
785 0x98fc, 0xffffffff, 0x76543210,
786 0x7030, 0x31000311, 0x00000011,
787 0x2f48, 0x00000007, 0x02011003,
788 0x6b28, 0x00000010, 0x00000012,
789 0x7728, 0x00000010, 0x00000012,
790 0x10328, 0x00000010, 0x00000012,
791 0x10f28, 0x00000010, 0x00000012,
792 0x11b28, 0x00000010, 0x00000012,
793 0x12728, 0x00000010, 0x00000012,
794 0x240c, 0x000007ff, 0x00000380,
795 0x8a14, 0xf000001f, 0x00000007,
796 0x8b24, 0x3fff3fff, 0x00ff0fff,
797 0x8b10, 0x0000ff0f, 0x00000000,
798 0x28a4c, 0x07ffffff, 0x06000000,
799 0x10c, 0x00000001, 0x00010003,
800 0xa02c, 0xffffffff, 0x0000009b,
801 0x913c, 0x0000000f, 0x0100000a,
802 0x8d00, 0xffff7f7f, 0x100e4848,
803 0x8d04, 0x00ffffff, 0x00164745,
804 0x8c00, 0xfffc0003, 0xe4000003,
805 0x8c04, 0xf8ff00ff, 0x40600060,
806 0x8c08, 0x00ff00ff, 0x001c001c,
807 0x8cf0, 0x1fff1fff, 0x08e00620,
808 0x8c20, 0x0fff0fff, 0x00800080,
809 0x8c24, 0x0fff0fff, 0x00800080,
810 0x8c18, 0xffffffff, 0x20202078,
811 0x8c1c, 0x0000ffff, 0x00001010,
812 0x28350, 0x00000f01, 0x00000000,
813 0x9508, 0x3700001f, 0x00000002,
814 0x960c, 0xffffffff, 0x54763210,
815 0x88c4, 0x001f3ae3, 0x000000c2,
816 0x88d4, 0x0000001f, 0x00000010,
817 0x8974, 0xffffffff, 0x00000000
818};
819
820static const u32 turks_golden_registers[] =
821{
822 0x5eb4, 0xffffffff, 0x00000002,
823 0x5e78, 0x8f311ff1, 0x001000f0,
824 0x8c8, 0x00003000, 0x00001070,
825 0x8cc, 0x000fffff, 0x00040035,
826 0x3f90, 0xffff0000, 0xfff00000,
827 0x9148, 0xffff0000, 0xfff00000,
828 0x3f94, 0xffff0000, 0xfff00000,
829 0x914c, 0xffff0000, 0xfff00000,
830 0xc78, 0x00000080, 0x00000080,
831 0xbd4, 0x00073007, 0x00010002,
832 0xd02c, 0xbfffff1f, 0x08421000,
833 0xd0b8, 0x03773777, 0x02010002,
834 0x5bc0, 0x00200000, 0x50100000,
835 0x98f8, 0x33773777, 0x00010002,
836 0x98fc, 0xffffffff, 0x33221100,
837 0x7030, 0x31000311, 0x00000011,
838 0x2f48, 0x33773777, 0x00010002,
839 0x6b28, 0x00000010, 0x00000012,
840 0x7728, 0x00000010, 0x00000012,
841 0x10328, 0x00000010, 0x00000012,
842 0x10f28, 0x00000010, 0x00000012,
843 0x11b28, 0x00000010, 0x00000012,
844 0x12728, 0x00000010, 0x00000012,
845 0x240c, 0x000007ff, 0x00000380,
846 0x8a14, 0xf000001f, 0x00000007,
847 0x8b24, 0x3fff3fff, 0x00ff0fff,
848 0x8b10, 0x0000ff0f, 0x00000000,
849 0x28a4c, 0x07ffffff, 0x06000000,
850 0x10c, 0x00000001, 0x00010003,
851 0xa02c, 0xffffffff, 0x0000009b,
852 0x913c, 0x0000000f, 0x0100000a,
853 0x8d00, 0xffff7f7f, 0x100e4848,
854 0x8d04, 0x00ffffff, 0x00164745,
855 0x8c00, 0xfffc0003, 0xe4000003,
856 0x8c04, 0xf8ff00ff, 0x40600060,
857 0x8c08, 0x00ff00ff, 0x001c001c,
858 0x8cf0, 0x1fff1fff, 0x08e00410,
859 0x8c20, 0x0fff0fff, 0x00800080,
860 0x8c24, 0x0fff0fff, 0x00800080,
861 0x8c18, 0xffffffff, 0x20202078,
862 0x8c1c, 0x0000ffff, 0x00001010,
863 0x28350, 0x00000f01, 0x00000000,
864 0x9508, 0x3700001f, 0x00000002,
865 0x960c, 0xffffffff, 0x54763210,
866 0x88c4, 0x001f3ae3, 0x000000c2,
867 0x88d4, 0x0000001f, 0x00000010,
868 0x8974, 0xffffffff, 0x00000000
869};
870
871static const u32 caicos_golden_registers[] =
872{
873 0x5eb4, 0xffffffff, 0x00000002,
874 0x5e78, 0x8f311ff1, 0x001000f0,
875 0x8c8, 0x00003420, 0x00001450,
876 0x8cc, 0x000fffff, 0x00040035,
877 0x3f90, 0xffff0000, 0xfffc0000,
878 0x9148, 0xffff0000, 0xfffc0000,
879 0x3f94, 0xffff0000, 0xfffc0000,
880 0x914c, 0xffff0000, 0xfffc0000,
881 0xc78, 0x00000080, 0x00000080,
882 0xbd4, 0x00073007, 0x00010001,
883 0xd02c, 0xbfffff1f, 0x08421000,
884 0xd0b8, 0x03773777, 0x02010001,
885 0x5bc0, 0x00200000, 0x50100000,
886 0x98f8, 0x33773777, 0x02010001,
887 0x98fc, 0xffffffff, 0x33221100,
888 0x7030, 0x31000311, 0x00000011,
889 0x2f48, 0x33773777, 0x02010001,
890 0x6b28, 0x00000010, 0x00000012,
891 0x7728, 0x00000010, 0x00000012,
892 0x10328, 0x00000010, 0x00000012,
893 0x10f28, 0x00000010, 0x00000012,
894 0x11b28, 0x00000010, 0x00000012,
895 0x12728, 0x00000010, 0x00000012,
896 0x240c, 0x000007ff, 0x00000380,
897 0x8a14, 0xf000001f, 0x00000001,
898 0x8b24, 0x3fff3fff, 0x00ff0fff,
899 0x8b10, 0x0000ff0f, 0x00000000,
900 0x28a4c, 0x07ffffff, 0x06000000,
901 0x10c, 0x00000001, 0x00010003,
902 0xa02c, 0xffffffff, 0x0000009b,
903 0x913c, 0x0000000f, 0x0100000a,
904 0x8d00, 0xffff7f7f, 0x100e4848,
905 0x8d04, 0x00ffffff, 0x00164745,
906 0x8c00, 0xfffc0003, 0xe4000003,
907 0x8c04, 0xf8ff00ff, 0x40600060,
908 0x8c08, 0x00ff00ff, 0x001c001c,
909 0x8cf0, 0x1fff1fff, 0x08e00410,
910 0x8c20, 0x0fff0fff, 0x00800080,
911 0x8c24, 0x0fff0fff, 0x00800080,
912 0x8c18, 0xffffffff, 0x20202078,
913 0x8c1c, 0x0000ffff, 0x00001010,
914 0x28350, 0x00000f01, 0x00000000,
915 0x9508, 0x3700001f, 0x00000002,
916 0x960c, 0xffffffff, 0x54763210,
917 0x88c4, 0x001f3ae3, 0x000000c2,
918 0x88d4, 0x0000001f, 0x00000010,
919 0x8974, 0xffffffff, 0x00000000
920};
921
922static void evergreen_init_golden_registers(struct radeon_device *rdev)
923{
924 switch (rdev->family) {
925 case CHIP_CYPRESS:
926 case CHIP_HEMLOCK:
927 radeon_program_register_sequence(rdev,
928 evergreen_golden_registers,
929 (const u32)ARRAY_SIZE(evergreen_golden_registers));
930 radeon_program_register_sequence(rdev,
931 evergreen_golden_registers2,
932 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
933 radeon_program_register_sequence(rdev,
934 cypress_mgcg_init,
935 (const u32)ARRAY_SIZE(cypress_mgcg_init));
936 break;
937 case CHIP_JUNIPER:
938 radeon_program_register_sequence(rdev,
939 evergreen_golden_registers,
940 (const u32)ARRAY_SIZE(evergreen_golden_registers));
941 radeon_program_register_sequence(rdev,
942 evergreen_golden_registers2,
943 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
944 radeon_program_register_sequence(rdev,
945 juniper_mgcg_init,
946 (const u32)ARRAY_SIZE(juniper_mgcg_init));
947 break;
948 case CHIP_REDWOOD:
949 radeon_program_register_sequence(rdev,
950 evergreen_golden_registers,
951 (const u32)ARRAY_SIZE(evergreen_golden_registers));
952 radeon_program_register_sequence(rdev,
953 evergreen_golden_registers2,
954 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
955 radeon_program_register_sequence(rdev,
956 redwood_mgcg_init,
957 (const u32)ARRAY_SIZE(redwood_mgcg_init));
958 break;
959 case CHIP_CEDAR:
960 radeon_program_register_sequence(rdev,
961 cedar_golden_registers,
962 (const u32)ARRAY_SIZE(cedar_golden_registers));
963 radeon_program_register_sequence(rdev,
964 evergreen_golden_registers2,
965 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
966 radeon_program_register_sequence(rdev,
967 cedar_mgcg_init,
968 (const u32)ARRAY_SIZE(cedar_mgcg_init));
969 break;
970 case CHIP_PALM:
971 radeon_program_register_sequence(rdev,
972 wrestler_golden_registers,
973 (const u32)ARRAY_SIZE(wrestler_golden_registers));
974 break;
975 case CHIP_SUMO:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 break;
980 case CHIP_SUMO2:
981 radeon_program_register_sequence(rdev,
982 supersumo_golden_registers,
983 (const u32)ARRAY_SIZE(supersumo_golden_registers));
984 radeon_program_register_sequence(rdev,
985 sumo_golden_registers,
986 (const u32)ARRAY_SIZE(sumo_golden_registers));
987 break;
988 case CHIP_BARTS:
989 radeon_program_register_sequence(rdev,
990 barts_golden_registers,
991 (const u32)ARRAY_SIZE(barts_golden_registers));
992 break;
993 case CHIP_TURKS:
994 radeon_program_register_sequence(rdev,
995 turks_golden_registers,
996 (const u32)ARRAY_SIZE(turks_golden_registers));
997 break;
998 case CHIP_CAICOS:
999 radeon_program_register_sequence(rdev,
1000 caicos_golden_registers,
1001 (const u32)ARRAY_SIZE(caicos_golden_registers));
1002 break;
1003 default:
1004 break;
1005 }
1006}
1007
285484e2
JG		 1008void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009 unsigned *bankh, unsigned *mtaspect,
1010 unsigned *tile_split)
1011{
1012 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016 switch (*bankw) {
1017 default:
1018 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022 }
1023 switch (*bankh) {
1024 default:
1025 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029 }
1030 switch (*mtaspect) {
1031 default:
1032 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036 }
1037}
1038
23d33ba3
AD		 1039static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040 u32 cntl_reg, u32 status_reg)
1041{
1042 int r, i;
1043 struct atom_clock_dividers dividers;
1044
1045 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046 clock, false, &dividers);
1047 if (r)
1048 return r;
1049
1050 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052 for (i = 0; i < 100; i++) {
1053 if (RREG32(status_reg) & DCLK_STATUS)
1054 break;
1055 mdelay(10);
1056 }
1057 if (i == 100)
1058 return -ETIMEDOUT;
1059
1060 return 0;
1061}
1062
1063int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064{
1065 int r = 0;
1066 u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069 if (r)
1070 goto done;
1071 cg_scratch &= 0xffff0000;
1072 cg_scratch |= vclk / 100; /* Mhz */
1073
1074 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075 if (r)
1076 goto done;
1077 cg_scratch &= 0x0000ffff;
1078 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080done:
1081 WREG32(CG_SCRATCH1, cg_scratch);
1082
1083 return r;
1084}
1085
a8b4925c
AD		 1086int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087{
1088 /* start off with something large */
facd112d 1089 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
a8b4925c
AD		 1090 int r;
1091
4ed10835
CK		 1092 /* bypass vclk and dclk with bclk */
1093 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097 /* put PLL in bypass mode */
1098 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100 if (!vclk || !dclk) {
1101 /* keep the Bypass mode, put PLL to sleep */
1102 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103 return 0;
1104 }
1105
facd112d
CK		 1106 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107 16384, 0x03FFFFFF, 0, 128, 5,
1108 &fb_div, &vclk_div, &dclk_div);
1109 if (r)
1110 return r;
a8b4925c
AD		 1111
1112 /* set VCO_MODE to 1 */
1113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115 /* toggle UPLL_SLEEP to 1 then back to 0 */
1116 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119 /* deassert UPLL_RESET */
1120 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122 mdelay(1);
1123
facd112d 1124 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1125 if (r)
1126 return r;
1127
1128 /* assert UPLL_RESET again */
1129 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131 /* disable spread spectrum. */
1132 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134 /* set feedback divider */
facd112d 1135 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
a8b4925c
AD		 1136
1137 /* set ref divider to 0 */
1138 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
facd112d 1140 if (fb_div < 307200)
a8b4925c
AD		 1141 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142 else
1143 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145 /* set PDIV_A and PDIV_B */
1146 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 1147 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
a8b4925c
AD		 1148 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150 /* give the PLL some time to settle */
1151 mdelay(15);
1152
1153 /* deassert PLL_RESET */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156 mdelay(15);
1157
1158 /* switch from bypass mode to normal mode */
1159 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
facd112d 1161 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
a8b4925c
AD		 1162 if (r)
1163 return r;
1164
1165 /* switch VCLK and DCLK selection */
1166 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170 mdelay(100);
1171
1172 return 0;
1173}
1174
d054ac16
AD		 1175void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176{
c11592fe
YW		 1177 int readrq;
1178 u16 v;
d054ac16 1179
c11592fe
YW		 1180 readrq = pcie_get_readrq(rdev->pdev);
1181 v = ffs(readrq) - 8;
d054ac16
AD		 1182 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1183 * to avoid hangs or perfomance issues
1184 */
c11592fe
YW		 1185 if ((v == 0) || (v == 6) || (v == 7))
1186 pcie_set_readrq(rdev->pdev, 512);
d054ac16
AD		 1187}
1188
134b480f
AD		 1189void dce4_program_fmt(struct drm_encoder *encoder)
1190{
1191 struct drm_device *dev = encoder->dev;
1192 struct radeon_device *rdev = dev->dev_private;
1193 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1194 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1195 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1196 int bpc = 0;
1197 u32 tmp = 0;
6214bb74 1198 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 1199
6214bb74
AD		 1200 if (connector) {
1201 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 1202 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 1203 dither = radeon_connector->dither;
1204 }
134b480f
AD		 1205
1206 /* LVDS/eDP FMT is set up by atom */
1207 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1208 return;
1209
1210 /* not needed for analog */
1211 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1212 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1213 return;
1214
1215 if (bpc == 0)
1216 return;
1217
1218 switch (bpc) {
1219 case 6:
6214bb74 1220 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1221 /* XXX sort out optimal dither settings */
1222 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1223 FMT_SPATIAL_DITHER_EN);
1224 else
1225 tmp |= FMT_TRUNCATE_EN;
1226 break;
1227 case 8:
6214bb74 1228 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 1229 /* XXX sort out optimal dither settings */
1230 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1231 FMT_RGB_RANDOM_ENABLE |
1232 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1233 else
1234 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1235 break;
1236 case 10:
1237 default:
1238 /* not needed */
1239 break;
1240 }
1241
1242 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1243}
1244
10257a6d
AD		 1245static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1246{
1247 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1248 return true;
1249 else
1250 return false;
1251}
1252
1253static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1254{
1255 u32 pos1, pos2;
1256
1257 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1258 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259
1260 if (pos1 != pos2)
1261 return true;
1262 else
1263 return false;
1264}
1265
377edc8b
AD		 1266/**
1267 * dce4_wait_for_vblank - vblank wait asic callback.
1268 *
1269 * @rdev: radeon_device pointer
1270 * @crtc: crtc to wait for vblank on
1271 *
1272 * Wait for vblank on the requested crtc (evergreen+).
1273 */
3ae19b75
AD		 1274void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1275{
10257a6d 1276 unsigned i = 0;
3ae19b75 1277
4a15903d
AD		 1278 if (crtc >= rdev->num_crtc)
1279 return;
1280
10257a6d
AD		 1281 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1282 return;
1283
1284 /* depending on when we hit vblank, we may be close to active; if so,
1285 * wait for another frame.
1286 */
1287 while (dce4_is_in_vblank(rdev, crtc)) {
1288 if (i++ % 100 == 0) {
1289 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1290 break;
3ae19b75 1291 }
10257a6d
AD		 1292 }
1293
1294 while (!dce4_is_in_vblank(rdev, crtc)) {
1295 if (i++ % 100 == 0) {
1296 if (!dce4_is_counter_moving(rdev, crtc))
3ae19b75 1297 break;
3ae19b75
AD		 1298 }
1299 }
1300}
1301
377edc8b
AD		 1302/**
1303 * evergreen_page_flip - pageflip callback.
1304 *
1305 * @rdev: radeon_device pointer
1306 * @crtc_id: crtc to cleanup pageflip on
1307 * @crtc_base: new address of the crtc (GPU MC address)
1308 *
1309 * Does the actual pageflip (evergreen+).
1310 * During vblank we take the crtc lock and wait for the update_pending
1311 * bit to go high, when it does, we release the lock, and allow the
1312 * double buffered update to take place.
1313 * Returns the current update pending status.
1314 */
157fa14d 1315void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
6f34be50
AD		 1316{
1317 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1318 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
f6496479 1319 int i;
6f34be50
AD		 1320
1321 /* Lock the graphics update lock */
1322 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1323 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1324
1325 /* update the scanout addresses */
1326 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1327 upper_32_bits(crtc_base));
1328 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1329 (u32)crtc_base);
1330
1331 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1332 upper_32_bits(crtc_base));
1333 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1334 (u32)crtc_base);
1335
1336 /* Wait for update_pending to go high. */
f6496479
AD		 1337 for (i = 0; i < rdev->usec_timeout; i++) {
1338 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1339 break;
1340 udelay(1);
1341 }
6f34be50
AD		 1342 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1343
1344 /* Unlock the lock, so double-buffering can take place inside vblank */
1345 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1346 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
157fa14d
CK		 1347}
1348
1349/**
1350 * evergreen_page_flip_pending - check if page flip is still pending
1351 *
1352 * @rdev: radeon_device pointer
1353 * @crtc_id: crtc to check
1354 *
1355 * Returns the current update pending status.
1356 */
1357bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1358{
1359 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
6f34be50
AD		 1360
1361 /* Return current update_pending status: */
157fa14d
CK		 1362 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1363 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
6f34be50
AD		 1364}
1365
21a8122a 1366/* get temperature in millidegrees */
20d391d7 1367int evergreen_get_temp(struct radeon_device *rdev)
21a8122a 1368{
1c88d74f
AD		 1369 u32 temp, toffset;
1370 int actual_temp = 0;
67b3f823
AD		 1371
1372 if (rdev->family == CHIP_JUNIPER) {
1373 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1374 TOFFSET_SHIFT;
1375 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1376 TS0_ADC_DOUT_SHIFT;
1377
1378 if (toffset & 0x100)
1379 actual_temp = temp / 2 - (0x200 - toffset);
1380 else
1381 actual_temp = temp / 2 + toffset;
1382
1383 actual_temp = actual_temp * 1000;
1384
1385 } else {
1386 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1387 ASIC_T_SHIFT;
1388
1389 if (temp & 0x400)
1390 actual_temp = -256;
1391 else if (temp & 0x200)
1392 actual_temp = 255;
1393 else if (temp & 0x100) {
1394 actual_temp = temp & 0x1ff;
1395 actual_temp |= ~0x1ff;
1396 } else
1397 actual_temp = temp & 0xff;
1398
1399 actual_temp = (actual_temp * 1000) / 2;
1400 }
21a8122a 1401
67b3f823 1402 return actual_temp;
21a8122a
AD		 1403}
1404
20d391d7 1405int sumo_get_temp(struct radeon_device *rdev)
e33df25f
AD		 1406{
1407 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
20d391d7 1408 int actual_temp = temp - 49;
e33df25f
AD		 1409
1410 return actual_temp * 1000;
1411}
1412
377edc8b
AD		 1413/**
1414 * sumo_pm_init_profile - Initialize power profiles callback.
1415 *
1416 * @rdev: radeon_device pointer
1417 *
1418 * Initialize the power states used in profile mode
1419 * (sumo, trinity, SI).
1420 * Used for profile mode only.
1421 */
a4c9e2ee
AD		 1422void sumo_pm_init_profile(struct radeon_device *rdev)
1423{
1424 int idx;
1425
1426 /* default */
1427 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1428 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1429 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1430 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1431
1432 /* low,mid sh/mh */
1433 if (rdev->flags & RADEON_IS_MOBILITY)
1434 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1435 else
1436 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1437
1438 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1439 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1440 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1441 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1442
1443 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1444 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1445 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1446 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1447
1448 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1449 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1450 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1451 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1452
1453 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1454 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1455 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1456 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1457
1458 /* high sh/mh */
1459 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1460 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1463 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1464 rdev->pm.power_state[idx].num_clock_modes - 1;
1465
1466 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1469 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1470 rdev->pm.power_state[idx].num_clock_modes - 1;
1471}
1472
27810fb2
AD		 1473/**
1474 * btc_pm_init_profile - Initialize power profiles callback.
1475 *
1476 * @rdev: radeon_device pointer
1477 *
1478 * Initialize the power states used in profile mode
1479 * (BTC, cayman).
1480 * Used for profile mode only.
1481 */
1482void btc_pm_init_profile(struct radeon_device *rdev)
1483{
1484 int idx;
1485
1486 /* default */
1487 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1488 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1489 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1490 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1491 /* starting with BTC, there is one state that is used for both
1492 * MH and SH. Difference is that we always use the high clock index for
1493 * mclk.
1494 */
1495 if (rdev->flags & RADEON_IS_MOBILITY)
1496 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1497 else
1498 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1499 /* low sh */
1500 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1501 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1502 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1503 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1504 /* mid sh */
1505 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1506 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1507 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1508 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1509 /* high sh */
1510 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1511 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1512 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1513 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1514 /* low mh */
1515 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1516 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1519 /* mid mh */
1520 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1521 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1522 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1523 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1524 /* high mh */
1525 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1526 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1529}
1530
377edc8b
AD		 1531/**
1532 * evergreen_pm_misc - set additional pm hw parameters callback.
1533 *
1534 * @rdev: radeon_device pointer
1535 *
1536 * Set non-clock parameters associated with a power state
1537 * (voltage, etc.) (evergreen+).
1538 */
49e02b73
AD		 1539void evergreen_pm_misc(struct radeon_device *rdev)
1540{
a081a9d6
RM		 1541 int req_ps_idx = rdev->pm.requested_power_state_index;
1542 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1543 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1544 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
49e02b73 1545
2feea49a 1546 if (voltage->type == VOLTAGE_SW) {
c6cf7777
AD		 1547 /* 0xff0x are flags rather then an actual voltage */
1548 if ((voltage->voltage & 0xff00) == 0xff00)
a377e187 1549 return;
2feea49a 1550 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
8a83ec5e 1551 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
4d60173f 1552 rdev->pm.current_vddc = voltage->voltage;
2feea49a
AD		 1553 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1554 }
7ae764b1
AD		 1555
1556 /* starting with BTC, there is one state that is used for both
1557 * MH and SH. Difference is that we always use the high clock index for
1558 * mclk and vddci.
1559 */
1560 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1561 (rdev->family >= CHIP_BARTS) &&
1562 rdev->pm.active_crtc_count &&
1563 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1564 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1565 voltage = &rdev->pm.power_state[req_ps_idx].
1566 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1567
c6cf7777
AD		 1568 /* 0xff0x are flags rather then an actual voltage */
1569 if ((voltage->vddci & 0xff00) == 0xff00)
a377e187 1570 return;
2feea49a
AD		 1571 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1572 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1573 rdev->pm.current_vddci = voltage->vddci;
1574 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
4d60173f
AD		 1575 }
1576 }
49e02b73
AD		 1577}
1578
377edc8b
AD		 1579/**
1580 * evergreen_pm_prepare - pre-power state change callback.
1581 *
1582 * @rdev: radeon_device pointer
1583 *
1584 * Prepare for a power state change (evergreen+).
1585 */
49e02b73
AD		 1586void evergreen_pm_prepare(struct radeon_device *rdev)
1587{
1588 struct drm_device *ddev = rdev->ddev;
1589 struct drm_crtc *crtc;
1590 struct radeon_crtc *radeon_crtc;
1591 u32 tmp;
1592
1593 /* disable any active CRTCs */
1594 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1595 radeon_crtc = to_radeon_crtc(crtc);
1596 if (radeon_crtc->enabled) {
1597 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1598 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1599 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1600 }
1601 }
1602}
1603
377edc8b
AD		 1604/**
1605 * evergreen_pm_finish - post-power state change callback.
1606 *
1607 * @rdev: radeon_device pointer
1608 *
1609 * Clean up after a power state change (evergreen+).
1610 */
49e02b73
AD		 1611void evergreen_pm_finish(struct radeon_device *rdev)
1612{
1613 struct drm_device *ddev = rdev->ddev;
1614 struct drm_crtc *crtc;
1615 struct radeon_crtc *radeon_crtc;
1616 u32 tmp;
1617
1618 /* enable any active CRTCs */
1619 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1620 radeon_crtc = to_radeon_crtc(crtc);
1621 if (radeon_crtc->enabled) {
1622 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1623 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1624 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1625 }
1626 }
1627}
1628
377edc8b
AD		 1629/**
1630 * evergreen_hpd_sense - hpd sense callback.
1631 *
1632 * @rdev: radeon_device pointer
1633 * @hpd: hpd (hotplug detect) pin
1634 *
1635 * Checks if a digital monitor is connected (evergreen+).
1636 * Returns true if connected, false if not connected.
1637 */
bcc1c2a1
AD		 1638bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1639{
1640 bool connected = false;
0ca2ab52
AD		 1641
1642 switch (hpd) {
1643 case RADEON_HPD_1:
1644 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1645 connected = true;
1646 break;
1647 case RADEON_HPD_2:
1648 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1649 connected = true;
1650 break;
1651 case RADEON_HPD_3:
1652 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1653 connected = true;
1654 break;
1655 case RADEON_HPD_4:
1656 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1657 connected = true;
1658 break;
1659 case RADEON_HPD_5:
1660 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1661 connected = true;
1662 break;
1663 case RADEON_HPD_6:
1664 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1665 connected = true;
ebc54ffe 1666 break;
0ca2ab52
AD		 1667 default:
1668 break;
1669 }
1670
bcc1c2a1
AD		 1671 return connected;
1672}
1673
377edc8b
AD		 1674/**
1675 * evergreen_hpd_set_polarity - hpd set polarity callback.
1676 *
1677 * @rdev: radeon_device pointer
1678 * @hpd: hpd (hotplug detect) pin
1679 *
1680 * Set the polarity of the hpd pin (evergreen+).
1681 */
bcc1c2a1
AD		 1682void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1683 enum radeon_hpd_id hpd)
1684{
0ca2ab52
AD		 1685 u32 tmp;
1686 bool connected = evergreen_hpd_sense(rdev, hpd);
1687
1688 switch (hpd) {
1689 case RADEON_HPD_1:
1690 tmp = RREG32(DC_HPD1_INT_CONTROL);
1691 if (connected)
1692 tmp &= ~DC_HPDx_INT_POLARITY;
1693 else
1694 tmp |= DC_HPDx_INT_POLARITY;
1695 WREG32(DC_HPD1_INT_CONTROL, tmp);
1696 break;
1697 case RADEON_HPD_2:
1698 tmp = RREG32(DC_HPD2_INT_CONTROL);
1699 if (connected)
1700 tmp &= ~DC_HPDx_INT_POLARITY;
1701 else
1702 tmp |= DC_HPDx_INT_POLARITY;
1703 WREG32(DC_HPD2_INT_CONTROL, tmp);
1704 break;
1705 case RADEON_HPD_3:
1706 tmp = RREG32(DC_HPD3_INT_CONTROL);
1707 if (connected)
1708 tmp &= ~DC_HPDx_INT_POLARITY;
1709 else
1710 tmp |= DC_HPDx_INT_POLARITY;
1711 WREG32(DC_HPD3_INT_CONTROL, tmp);
1712 break;
1713 case RADEON_HPD_4:
1714 tmp = RREG32(DC_HPD4_INT_CONTROL);
1715 if (connected)
1716 tmp &= ~DC_HPDx_INT_POLARITY;
1717 else
1718 tmp |= DC_HPDx_INT_POLARITY;
1719 WREG32(DC_HPD4_INT_CONTROL, tmp);
1720 break;
1721 case RADEON_HPD_5:
1722 tmp = RREG32(DC_HPD5_INT_CONTROL);
1723 if (connected)
1724 tmp &= ~DC_HPDx_INT_POLARITY;
1725 else
1726 tmp |= DC_HPDx_INT_POLARITY;
1727 WREG32(DC_HPD5_INT_CONTROL, tmp);
1728 break;
1729 case RADEON_HPD_6:
1730 tmp = RREG32(DC_HPD6_INT_CONTROL);
1731 if (connected)
1732 tmp &= ~DC_HPDx_INT_POLARITY;
1733 else
1734 tmp |= DC_HPDx_INT_POLARITY;
1735 WREG32(DC_HPD6_INT_CONTROL, tmp);
1736 break;
1737 default:
1738 break;
1739 }
bcc1c2a1
AD		 1740}
1741
377edc8b
AD		 1742/**
1743 * evergreen_hpd_init - hpd setup callback.
1744 *
1745 * @rdev: radeon_device pointer
1746 *
1747 * Setup the hpd pins used by the card (evergreen+).
1748 * Enable the pin, set the polarity, and enable the hpd interrupts.
1749 */
bcc1c2a1
AD		 1750void evergreen_hpd_init(struct radeon_device *rdev)
1751{
0ca2ab52
AD		 1752 struct drm_device *dev = rdev->ddev;
1753 struct drm_connector *connector;
fb98257a 1754 unsigned enabled = 0;
0ca2ab52
AD		 1755 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1756 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
bcc1c2a1 1757
0ca2ab52
AD		 1758 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1759 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2e97be73
AD		 1760
1761 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1762 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1763 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1764 * aux dp channel on imac and help (but not completely fix)
1765 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1766 * also avoid interrupt storms during dpms.
1767 */
1768 continue;
1769 }
0ca2ab52
AD		 1770 switch (radeon_connector->hpd.hpd) {
1771 case RADEON_HPD_1:
1772 WREG32(DC_HPD1_CONTROL, tmp);
0ca2ab52
AD		 1773 break;
1774 case RADEON_HPD_2:
1775 WREG32(DC_HPD2_CONTROL, tmp);
0ca2ab52
AD		 1776 break;
1777 case RADEON_HPD_3:
1778 WREG32(DC_HPD3_CONTROL, tmp);
0ca2ab52
AD		 1779 break;
1780 case RADEON_HPD_4:
1781 WREG32(DC_HPD4_CONTROL, tmp);
0ca2ab52
AD		 1782 break;
1783 case RADEON_HPD_5:
1784 WREG32(DC_HPD5_CONTROL, tmp);
0ca2ab52
AD		 1785 break;
1786 case RADEON_HPD_6:
1787 WREG32(DC_HPD6_CONTROL, tmp);
0ca2ab52
AD		 1788 break;
1789 default:
1790 break;
1791 }
64912e99 1792 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
fb98257a 1793 enabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1794 }
fb98257a 1795 radeon_irq_kms_enable_hpd(rdev, enabled);
bcc1c2a1
AD		 1796}
1797
377edc8b
AD		 1798/**
1799 * evergreen_hpd_fini - hpd tear down callback.
1800 *
1801 * @rdev: radeon_device pointer
1802 *
1803 * Tear down the hpd pins used by the card (evergreen+).
1804 * Disable the hpd interrupts.
1805 */
0ca2ab52 1806void evergreen_hpd_fini(struct radeon_device *rdev)
bcc1c2a1 1807{
0ca2ab52
AD		 1808 struct drm_device *dev = rdev->ddev;
1809 struct drm_connector *connector;
fb98257a 1810 unsigned disabled = 0;
0ca2ab52
AD		 1811
1812 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1813 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1814 switch (radeon_connector->hpd.hpd) {
1815 case RADEON_HPD_1:
1816 WREG32(DC_HPD1_CONTROL, 0);
0ca2ab52
AD		 1817 break;
1818 case RADEON_HPD_2:
1819 WREG32(DC_HPD2_CONTROL, 0);
0ca2ab52
AD		 1820 break;
1821 case RADEON_HPD_3:
1822 WREG32(DC_HPD3_CONTROL, 0);
0ca2ab52
AD		 1823 break;
1824 case RADEON_HPD_4:
1825 WREG32(DC_HPD4_CONTROL, 0);
0ca2ab52
AD		 1826 break;
1827 case RADEON_HPD_5:
1828 WREG32(DC_HPD5_CONTROL, 0);
0ca2ab52
AD		 1829 break;
1830 case RADEON_HPD_6:
1831 WREG32(DC_HPD6_CONTROL, 0);
0ca2ab52
AD		 1832 break;
1833 default:
1834 break;
1835 }
fb98257a 1836 disabled |= 1 << radeon_connector->hpd.hpd;
0ca2ab52 1837 }
fb98257a 1838 radeon_irq_kms_disable_hpd(rdev, disabled);
bcc1c2a1
AD		 1839}
1840
f9d9c362
AD		 1841/* watermark setup */
1842
1843static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1844 struct radeon_crtc *radeon_crtc,
1845 struct drm_display_mode *mode,
1846 struct drm_display_mode *other_mode)
1847{
0b31e023
AD		 1848 u32 tmp, buffer_alloc, i;
1849 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
f9d9c362
AD		 1850 /*
1851 * Line Buffer Setup
1852 * There are 3 line buffers, each one shared by 2 display controllers.
1853 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1854 * the display controllers. The paritioning is done via one of four
1855 * preset allocations specified in bits 2:0:
1856 * first display controller
1857 * 0 - first half of lb (3840 * 2)
1858 * 1 - first 3/4 of lb (5760 * 2)
12dfc843 1859 * 2 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1860 * 3 - first 1/4 of lb (1920 * 2)
1861 * second display controller
1862 * 4 - second half of lb (3840 * 2)
1863 * 5 - second 3/4 of lb (5760 * 2)
12dfc843 1864 * 6 - whole lb (7680 * 2), other crtc must be disabled
f9d9c362
AD		 1865 * 7 - last 1/4 of lb (1920 * 2)
1866 */
12dfc843
AD		 1867 /* this can get tricky if we have two large displays on a paired group
1868 * of crtcs. Ideally for multiple large displays we'd assign them to
1869 * non-linked crtcs for maximum line buffer allocation.
1870 */
1871 if (radeon_crtc->base.enabled && mode) {
0b31e023 1872 if (other_mode) {
f9d9c362 1873 tmp = 0; /* 1/2 */
0b31e023
AD		 1874 buffer_alloc = 1;
1875 } else {
12dfc843 1876 tmp = 2; /* whole */
0b31e023
AD		 1877 buffer_alloc = 2;
1878 }
1879 } else {
12dfc843 1880 tmp = 0;
0b31e023
AD		 1881 buffer_alloc = 0;
1882 }
f9d9c362
AD		 1883
1884 /* second controller of the pair uses second half of the lb */
1885 if (radeon_crtc->crtc_id % 2)
1886 tmp += 4;
1887 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1888
0b31e023
AD		 1889 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1890 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1891 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1892 for (i = 0; i < rdev->usec_timeout; i++) {
1893 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1894 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1895 break;
1896 udelay(1);
1897 }
1898 }
1899
12dfc843
AD		 1900 if (radeon_crtc->base.enabled && mode) {
1901 switch (tmp) {
1902 case 0:
1903 case 4:
1904 default:
1905 if (ASIC_IS_DCE5(rdev))
1906 return 4096 * 2;
1907 else
1908 return 3840 * 2;
1909 case 1:
1910 case 5:
1911 if (ASIC_IS_DCE5(rdev))
1912 return 6144 * 2;
1913 else
1914 return 5760 * 2;
1915 case 2:
1916 case 6:
1917 if (ASIC_IS_DCE5(rdev))
1918 return 8192 * 2;
1919 else
1920 return 7680 * 2;
1921 case 3:
1922 case 7:
1923 if (ASIC_IS_DCE5(rdev))
1924 return 2048 * 2;
1925 else
1926 return 1920 * 2;
1927 }
f9d9c362 1928 }
12dfc843
AD		 1929
1930 /* controller not enabled, so no lb used */
1931 return 0;
f9d9c362
AD		 1932}
1933
ca7db22b 1934u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
f9d9c362
AD		 1935{
1936 u32 tmp = RREG32(MC_SHARED_CHMAP);
1937
1938 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1939 case 0:
1940 default:
1941 return 1;
1942 case 1:
1943 return 2;
1944 case 2:
1945 return 4;
1946 case 3:
1947 return 8;
1948 }
1949}
1950
1951struct evergreen_wm_params {
1952 u32 dram_channels; /* number of dram channels */
1953 u32 yclk; /* bandwidth per dram data pin in kHz */
1954 u32 sclk; /* engine clock in kHz */
1955 u32 disp_clk; /* display clock in kHz */
1956 u32 src_width; /* viewport width */
1957 u32 active_time; /* active display time in ns */
1958 u32 blank_time; /* blank time in ns */
1959 bool interlaced; /* mode is interlaced */
1960 fixed20_12 vsc; /* vertical scale ratio */
1961 u32 num_heads; /* number of active crtcs */
1962 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1963 u32 lb_size; /* line buffer allocated to pipe */
1964 u32 vtaps; /* vertical scaler taps */
1965};
1966
1967static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1968{
1969 /* Calculate DRAM Bandwidth and the part allocated to display. */
1970 fixed20_12 dram_efficiency; /* 0.7 */
1971 fixed20_12 yclk, dram_channels, bandwidth;
1972 fixed20_12 a;
1973
1974 a.full = dfixed_const(1000);
1975 yclk.full = dfixed_const(wm->yclk);
1976 yclk.full = dfixed_div(yclk, a);
1977 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1978 a.full = dfixed_const(10);
1979 dram_efficiency.full = dfixed_const(7);
1980 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1981 bandwidth.full = dfixed_mul(dram_channels, yclk);
1982 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1983
1984 return dfixed_trunc(bandwidth);
1985}
1986
1987static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1988{
1989 /* Calculate DRAM Bandwidth and the part allocated to display. */
1990 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1991 fixed20_12 yclk, dram_channels, bandwidth;
1992 fixed20_12 a;
1993
1994 a.full = dfixed_const(1000);
1995 yclk.full = dfixed_const(wm->yclk);
1996 yclk.full = dfixed_div(yclk, a);
1997 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1998 a.full = dfixed_const(10);
1999 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2000 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2001 bandwidth.full = dfixed_mul(dram_channels, yclk);
2002 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2003
2004 return dfixed_trunc(bandwidth);
2005}
2006
2007static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2008{
2009 /* Calculate the display Data return Bandwidth */
2010 fixed20_12 return_efficiency; /* 0.8 */
2011 fixed20_12 sclk, bandwidth;
2012 fixed20_12 a;
2013
2014 a.full = dfixed_const(1000);
2015 sclk.full = dfixed_const(wm->sclk);
2016 sclk.full = dfixed_div(sclk, a);
2017 a.full = dfixed_const(10);
2018 return_efficiency.full = dfixed_const(8);
2019 return_efficiency.full = dfixed_div(return_efficiency, a);
2020 a.full = dfixed_const(32);
2021 bandwidth.full = dfixed_mul(a, sclk);
2022 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2023
2024 return dfixed_trunc(bandwidth);
2025}
2026
2027static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2028{
2029 /* Calculate the DMIF Request Bandwidth */
2030 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2031 fixed20_12 disp_clk, bandwidth;
2032 fixed20_12 a;
2033
2034 a.full = dfixed_const(1000);
2035 disp_clk.full = dfixed_const(wm->disp_clk);
2036 disp_clk.full = dfixed_div(disp_clk, a);
2037 a.full = dfixed_const(10);
2038 disp_clk_request_efficiency.full = dfixed_const(8);
2039 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2040 a.full = dfixed_const(32);
2041 bandwidth.full = dfixed_mul(a, disp_clk);
2042 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2043
2044 return dfixed_trunc(bandwidth);
2045}
2046
2047static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2048{
2049 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2050 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2051 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2052 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2053
2054 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2055}
2056
2057static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2058{
2059 /* Calculate the display mode Average Bandwidth
2060 * DisplayMode should contain the source and destination dimensions,
2061 * timing, etc.
2062 */
2063 fixed20_12 bpp;
2064 fixed20_12 line_time;
2065 fixed20_12 src_width;
2066 fixed20_12 bandwidth;
2067 fixed20_12 a;
2068
2069 a.full = dfixed_const(1000);
2070 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2071 line_time.full = dfixed_div(line_time, a);
2072 bpp.full = dfixed_const(wm->bytes_per_pixel);
2073 src_width.full = dfixed_const(wm->src_width);
2074 bandwidth.full = dfixed_mul(src_width, bpp);
2075 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2076 bandwidth.full = dfixed_div(bandwidth, line_time);
2077
2078 return dfixed_trunc(bandwidth);
2079}
2080
2081static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2082{
2083 /* First calcualte the latency in ns */
2084 u32 mc_latency = 2000; /* 2000 ns. */
2085 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2086 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2087 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2088 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2089 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2090 (wm->num_heads * cursor_line_pair_return_time);
2091 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2092 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2093 fixed20_12 a, b, c;
2094
2095 if (wm->num_heads == 0)
2096 return 0;
2097
2098 a.full = dfixed_const(2);
2099 b.full = dfixed_const(1);
2100 if ((wm->vsc.full > a.full) ||
2101 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2102 (wm->vtaps >= 5) ||
2103 ((wm->vsc.full >= a.full) && wm->interlaced))
2104 max_src_lines_per_dst_line = 4;
2105 else
2106 max_src_lines_per_dst_line = 2;
2107
2108 a.full = dfixed_const(available_bandwidth);
2109 b.full = dfixed_const(wm->num_heads);
2110 a.full = dfixed_div(a, b);
2111
2112 b.full = dfixed_const(1000);
2113 c.full = dfixed_const(wm->disp_clk);
2114 b.full = dfixed_div(c, b);
2115 c.full = dfixed_const(wm->bytes_per_pixel);
2116 b.full = dfixed_mul(b, c);
2117
2118 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2119
2120 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2121 b.full = dfixed_const(1000);
2122 c.full = dfixed_const(lb_fill_bw);
2123 b.full = dfixed_div(c, b);
2124 a.full = dfixed_div(a, b);
2125 line_fill_time = dfixed_trunc(a);
2126
2127 if (line_fill_time < wm->active_time)
2128 return latency;
2129 else
2130 return latency + (line_fill_time - wm->active_time);
2131
2132}
2133
2134static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2135{
2136 if (evergreen_average_bandwidth(wm) <=
2137 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2138 return true;
2139 else
2140 return false;
2141};
2142
2143static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2144{
2145 if (evergreen_average_bandwidth(wm) <=
2146 (evergreen_available_bandwidth(wm) / wm->num_heads))
2147 return true;
2148 else
2149 return false;
2150};
2151
2152static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2153{
2154 u32 lb_partitions = wm->lb_size / wm->src_width;
2155 u32 line_time = wm->active_time + wm->blank_time;
2156 u32 latency_tolerant_lines;
2157 u32 latency_hiding;
2158 fixed20_12 a;
2159
2160 a.full = dfixed_const(1);
2161 if (wm->vsc.full > a.full)
2162 latency_tolerant_lines = 1;
2163 else {
2164 if (lb_partitions <= (wm->vtaps + 1))
2165 latency_tolerant_lines = 1;
2166 else
2167 latency_tolerant_lines = 2;
2168 }
2169
2170 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2171
2172 if (evergreen_latency_watermark(wm) <= latency_hiding)
2173 return true;
2174 else
2175 return false;
2176}
2177
2178static void evergreen_program_watermarks(struct radeon_device *rdev,
2179 struct radeon_crtc *radeon_crtc,
2180 u32 lb_size, u32 num_heads)
2181{
2182 struct drm_display_mode *mode = &radeon_crtc->base.mode;
cf0cfdd7
AD		 2183 struct evergreen_wm_params wm_low, wm_high;
2184 u32 dram_channels;
f9d9c362
AD		 2185 u32 pixel_period;
2186 u32 line_time = 0;
2187 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2188 u32 priority_a_mark = 0, priority_b_mark = 0;
2189 u32 priority_a_cnt = PRIORITY_OFF;
2190 u32 priority_b_cnt = PRIORITY_OFF;
2191 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2192 u32 tmp, arb_control3;
2193 fixed20_12 a, b, c;
2194
2195 if (radeon_crtc->base.enabled && num_heads && mode) {
2196 pixel_period = 1000000 / (u32)mode->clock;
2197 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2198 priority_a_cnt = 0;
2199 priority_b_cnt = 0;
cf0cfdd7
AD		 2200 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2201
2202 /* watermark for high clocks */
2203 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2204 wm_high.yclk =
2205 radeon_dpm_get_mclk(rdev, false) * 10;
2206 wm_high.sclk =
2207 radeon_dpm_get_sclk(rdev, false) * 10;
2208 } else {
2209 wm_high.yclk = rdev->pm.current_mclk * 10;
2210 wm_high.sclk = rdev->pm.current_sclk * 10;
2211 }
f9d9c362 2212
cf0cfdd7
AD		 2213 wm_high.disp_clk = mode->clock;
2214 wm_high.src_width = mode->crtc_hdisplay;
2215 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2216 wm_high.blank_time = line_time - wm_high.active_time;
2217 wm_high.interlaced = false;
f9d9c362 2218 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
cf0cfdd7
AD		 2219 wm_high.interlaced = true;
2220 wm_high.vsc = radeon_crtc->vsc;
2221 wm_high.vtaps = 1;
f9d9c362 2222 if (radeon_crtc->rmx_type != RMX_OFF)
cf0cfdd7
AD		 2223 wm_high.vtaps = 2;
2224 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2225 wm_high.lb_size = lb_size;
2226 wm_high.dram_channels = dram_channels;
2227 wm_high.num_heads = num_heads;
2228
2229 /* watermark for low clocks */
2230 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2231 wm_low.yclk =
2232 radeon_dpm_get_mclk(rdev, true) * 10;
2233 wm_low.sclk =
2234 radeon_dpm_get_sclk(rdev, true) * 10;
2235 } else {
2236 wm_low.yclk = rdev->pm.current_mclk * 10;
2237 wm_low.sclk = rdev->pm.current_sclk * 10;
2238 }
2239
2240 wm_low.disp_clk = mode->clock;
2241 wm_low.src_width = mode->crtc_hdisplay;
2242 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2243 wm_low.blank_time = line_time - wm_low.active_time;
2244 wm_low.interlaced = false;
2245 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2246 wm_low.interlaced = true;
2247 wm_low.vsc = radeon_crtc->vsc;
2248 wm_low.vtaps = 1;
2249 if (radeon_crtc->rmx_type != RMX_OFF)
2250 wm_low.vtaps = 2;
2251 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2252 wm_low.lb_size = lb_size;
2253 wm_low.dram_channels = dram_channels;
2254 wm_low.num_heads = num_heads;
f9d9c362
AD		 2255
2256 /* set for high clocks */
cf0cfdd7 2257 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
f9d9c362 2258 /* set for low clocks */
cf0cfdd7 2259 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
f9d9c362
AD		 2260
2261 /* possibly force display priority to high */
2262 /* should really do this at mode validation time... */
cf0cfdd7
AD		 2263 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2264 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2265 !evergreen_check_latency_hiding(&wm_high) ||
f9d9c362 2266 (rdev->disp_priority == 2)) {
cf0cfdd7 2267 DRM_DEBUG_KMS("force priority a to high\n");
f9d9c362 2268 priority_a_cnt |= PRIORITY_ALWAYS_ON;
cf0cfdd7
AD		 2269 }
2270 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2271 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2272 !evergreen_check_latency_hiding(&wm_low) ||
2273 (rdev->disp_priority == 2)) {
2274 DRM_DEBUG_KMS("force priority b to high\n");
f9d9c362
AD		 2275 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2276 }
2277
2278 a.full = dfixed_const(1000);
2279 b.full = dfixed_const(mode->clock);
2280 b.full = dfixed_div(b, a);
2281 c.full = dfixed_const(latency_watermark_a);
2282 c.full = dfixed_mul(c, b);
2283 c.full = dfixed_mul(c, radeon_crtc->hsc);
2284 c.full = dfixed_div(c, a);
2285 a.full = dfixed_const(16);
2286 c.full = dfixed_div(c, a);
2287 priority_a_mark = dfixed_trunc(c);
2288 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2289
2290 a.full = dfixed_const(1000);
2291 b.full = dfixed_const(mode->clock);
2292 b.full = dfixed_div(b, a);
2293 c.full = dfixed_const(latency_watermark_b);
2294 c.full = dfixed_mul(c, b);
2295 c.full = dfixed_mul(c, radeon_crtc->hsc);
2296 c.full = dfixed_div(c, a);
2297 a.full = dfixed_const(16);
2298 c.full = dfixed_div(c, a);
2299 priority_b_mark = dfixed_trunc(c);
2300 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2301 }
2302
2303 /* select wm A */
2304 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2305 tmp = arb_control3;
2306 tmp &= ~LATENCY_WATERMARK_MASK(3);
2307 tmp |= LATENCY_WATERMARK_MASK(1);
2308 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2309 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2310 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2311 LATENCY_HIGH_WATERMARK(line_time)));
2312 /* select wm B */
2313 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2314 tmp &= ~LATENCY_WATERMARK_MASK(3);
2315 tmp |= LATENCY_WATERMARK_MASK(2);
2316 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2317 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2318 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2319 LATENCY_HIGH_WATERMARK(line_time)));
2320 /* restore original selection */
2321 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2322
2323 /* write the priority marks */
2324 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2325 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2326
7178d2a6
AD		 2327 /* save values for DPM */
2328 radeon_crtc->line_time = line_time;
2329 radeon_crtc->wm_high = latency_watermark_a;
2330 radeon_crtc->wm_low = latency_watermark_b;
f9d9c362
AD		 2331}
2332
377edc8b
AD		 2333/**
2334 * evergreen_bandwidth_update - update display watermarks callback.
2335 *
2336 * @rdev: radeon_device pointer
2337 *
2338 * Update the display watermarks based on the requested mode(s)
2339 * (evergreen+).
2340 */
0ca2ab52 2341void evergreen_bandwidth_update(struct radeon_device *rdev)
bcc1c2a1 2342{
f9d9c362
AD		 2343 struct drm_display_mode *mode0 = NULL;
2344 struct drm_display_mode *mode1 = NULL;
2345 u32 num_heads = 0, lb_size;
2346 int i;
2347
8efe82ca
AD		 2348 if (!rdev->mode_info.mode_config_initialized)
2349 return;
2350
f9d9c362
AD		 2351 radeon_update_display_priority(rdev);
2352
2353 for (i = 0; i < rdev->num_crtc; i++) {
2354 if (rdev->mode_info.crtcs[i]->base.enabled)
2355 num_heads++;
2356 }
2357 for (i = 0; i < rdev->num_crtc; i += 2) {
2358 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2359 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2360 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2361 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2362 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2363 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2364 }
bcc1c2a1
AD		 2365}
2366
377edc8b
AD		 2367/**
2368 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2369 *
2370 * @rdev: radeon_device pointer
2371 *
2372 * Wait for the MC (memory controller) to be idle.
2373 * (evergreen+).
2374 * Returns 0 if the MC is idle, -1 if not.
2375 */
b9952a8a 2376int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
bcc1c2a1
AD		 2377{
2378 unsigned i;
2379 u32 tmp;
2380
2381 for (i = 0; i < rdev->usec_timeout; i++) {
2382 /* read MC_STATUS */
2383 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2384 if (!tmp)
2385 return 0;
2386 udelay(1);
2387 }
2388 return -1;
2389}
2390
2391/*
2392 * GART
2393 */
0fcdb61e
AD		 2394void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2395{
2396 unsigned i;
2397 u32 tmp;
2398
6f2f48a9
AD		 2399 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2400
0fcdb61e
AD		 2401 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2402 for (i = 0; i < rdev->usec_timeout; i++) {
2403 /* read MC_STATUS */
2404 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2405 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2406 if (tmp == 2) {
2407 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2408 return;
2409 }
2410 if (tmp) {
2411 return;
2412 }
2413 udelay(1);
2414 }
2415}
2416
1109ca09 2417static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2418{
2419 u32 tmp;
0fcdb61e 2420 int r;
bcc1c2a1 2421
c9a1be96 2422 if (rdev->gart.robj == NULL) {
bcc1c2a1
AD		 2423 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2424 return -EINVAL;
2425 }
2426 r = radeon_gart_table_vram_pin(rdev);
2427 if (r)
2428 return r;
2429 /* Setup L2 cache */
2430 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2431 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2432 EFFECTIVE_L2_QUEUE_SIZE(7));
2433 WREG32(VM_L2_CNTL2, 0);
2434 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2435 /* Setup TLB control */
2436 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2437 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2438 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2439 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
8aeb96f8
AD		 2440 if (rdev->flags & RADEON_IS_IGP) {
2441 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2442 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2443 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2444 } else {
2445 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2446 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2447 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
0b8c30bc
AD		 2448 if ((rdev->family == CHIP_JUNIPER) ||
2449 (rdev->family == CHIP_CYPRESS) ||
2450 (rdev->family == CHIP_HEMLOCK) ||
2451 (rdev->family == CHIP_BARTS))
2452 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
8aeb96f8 2453 }
bcc1c2a1
AD		 2454 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2455 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2456 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2457 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2458 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2459 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2460 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2461 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2462 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2463 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2464 (u32)(rdev->dummy_page.addr >> 12));
0fcdb61e 2465 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1 2466
0fcdb61e 2467 evergreen_pcie_gart_tlb_flush(rdev);
fcf4de5a
TV		 2468 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2469 (unsigned)(rdev->mc.gtt_size >> 20),
2470 (unsigned long long)rdev->gart.table_addr);
bcc1c2a1
AD		 2471 rdev->gart.ready = true;
2472 return 0;
2473}
2474
1109ca09 2475static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
bcc1c2a1
AD		 2476{
2477 u32 tmp;
bcc1c2a1
AD		 2478
2479 /* Disable all tables */
0fcdb61e
AD		 2480 WREG32(VM_CONTEXT0_CNTL, 0);
2481 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2482
2483 /* Setup L2 cache */
2484 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2485 EFFECTIVE_L2_QUEUE_SIZE(7));
2486 WREG32(VM_L2_CNTL2, 0);
2487 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2488 /* Setup TLB control */
2489 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2490 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2491 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2492 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2493 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2494 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2495 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2496 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
c9a1be96 2497 radeon_gart_table_vram_unpin(rdev);
bcc1c2a1
AD		 2498}
2499
1109ca09 2500static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
bcc1c2a1
AD		 2501{
2502 evergreen_pcie_gart_disable(rdev);
2503 radeon_gart_table_vram_free(rdev);
2504 radeon_gart_fini(rdev);
2505}
2506
2507
1109ca09 2508static void evergreen_agp_enable(struct radeon_device *rdev)
bcc1c2a1
AD		 2509{
2510 u32 tmp;
bcc1c2a1
AD		 2511
2512 /* Setup L2 cache */
2513 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2514 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2515 EFFECTIVE_L2_QUEUE_SIZE(7));
2516 WREG32(VM_L2_CNTL2, 0);
2517 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2518 /* Setup TLB control */
2519 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2520 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2521 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2522 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2523 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2524 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2525 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2526 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2527 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2528 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2529 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
0fcdb61e
AD		 2530 WREG32(VM_CONTEXT0_CNTL, 0);
2531 WREG32(VM_CONTEXT1_CNTL, 0);
bcc1c2a1
AD		 2532}
2533
b9952a8a 2534void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2535{
62444b74
AD		 2536 u32 crtc_enabled, tmp, frame_count, blackout;
2537 int i, j;
2538
5153550a
AD		 2539 if (!ASIC_IS_NODCE(rdev)) {
2540 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2541 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
bcc1c2a1 2542
5153550a
AD		 2543 /* disable VGA render */
2544 WREG32(VGA_RENDER_CONTROL, 0);
2545 }
62444b74
AD		 2546 /* blank the display controllers */
2547 for (i = 0; i < rdev->num_crtc; i++) {
2548 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2549 if (crtc_enabled) {
2550 save->crtc_enabled[i] = true;
2551 if (ASIC_IS_DCE6(rdev)) {
2552 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2553 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2554 radeon_wait_for_vblank(rdev, i);
abf1457b 2555 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2556 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2557 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2558 }
2559 } else {
2560 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2561 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2562 radeon_wait_for_vblank(rdev, i);
abf1457b 2563 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74
AD		 2564 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2565 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
abf1457b 2566 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2567 }
2568 }
2569 /* wait for the next frame */
2570 frame_count = radeon_get_vblank_counter(rdev, i);
2571 for (j = 0; j < rdev->usec_timeout; j++) {
2572 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2573 break;
2574 udelay(1);
2575 }
abf1457b
AD		 2576
2577 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2578 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2579 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2580 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2581 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2582 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2583 save->crtc_enabled[i] = false;
2584 /* ***** */
804cc4a0
AD		 2585 } else {
2586 save->crtc_enabled[i] = false;
62444b74 2587 }
18007401 2588 }
bcc1c2a1 2589
62444b74
AD		 2590 radeon_mc_wait_for_idle(rdev);
2591
2592 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2593 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2594 /* Block CPU access */
2595 WREG32(BIF_FB_EN, 0);
2596 /* blackout the MC */
2597 blackout &= ~BLACKOUT_MODE_MASK;
2598 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
b7eff394 2599 }
ed39fadd
AD		 2600 /* wait for the MC to settle */
2601 udelay(100);
968c0166
AD		 2602
2603 /* lock double buffered regs */
2604 for (i = 0; i < rdev->num_crtc; i++) {
2605 if (save->crtc_enabled[i]) {
2606 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2607 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2608 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2609 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2610 }
2611 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2612 if (!(tmp & 1)) {
2613 tmp |= 1;
2614 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2615 }
2616 }
2617 }
bcc1c2a1
AD		 2618}
2619
b9952a8a 2620void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
bcc1c2a1 2621{
62444b74
AD		 2622 u32 tmp, frame_count;
2623 int i, j;
18007401 2624
62444b74
AD		 2625 /* update crtc base addresses */
2626 for (i = 0; i < rdev->num_crtc; i++) {
2627 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2628 upper_32_bits(rdev->mc.vram_start));
62444b74 2629 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
18007401 2630 upper_32_bits(rdev->mc.vram_start));
62444b74 2631 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401 2632 (u32)rdev->mc.vram_start);
62444b74 2633 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
18007401
AD		 2634 (u32)rdev->mc.vram_start);
2635 }
5153550a
AD		 2636
2637 if (!ASIC_IS_NODCE(rdev)) {
2638 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2639 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2640 }
62444b74 2641
968c0166
AD		 2642 /* unlock regs and wait for update */
2643 for (i = 0; i < rdev->num_crtc; i++) {
2644 if (save->crtc_enabled[i]) {
2645 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
f53f81b2
MK		 2646 if ((tmp & 0x7) != 3) {
2647 tmp &= ~0x7;
2648 tmp |= 0x3;
968c0166
AD		 2649 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2650 }
2651 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2652 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2653 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2654 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2655 }
2656 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2657 if (tmp & 1) {
2658 tmp &= ~1;
2659 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2660 }
2661 for (j = 0; j < rdev->usec_timeout; j++) {
2662 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2663 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2664 break;
2665 udelay(1);
2666 }
2667 }
2668 }
2669
62444b74
AD		 2670 /* unblackout the MC */
2671 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2672 tmp &= ~BLACKOUT_MODE_MASK;
2673 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2674 /* allow CPU access */
2675 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2676
2677 for (i = 0; i < rdev->num_crtc; i++) {
695ddeb4 2678 if (save->crtc_enabled[i]) {
62444b74
AD		 2679 if (ASIC_IS_DCE6(rdev)) {
2680 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
3157c589 2681 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
bb588820 2682 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2683 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
bb588820 2684 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2685 } else {
2686 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2687 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
bb588820 2688 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
62444b74 2689 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
bb588820 2690 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
62444b74
AD		 2691 }
2692 /* wait for the next frame */
2693 frame_count = radeon_get_vblank_counter(rdev, i);
2694 for (j = 0; j < rdev->usec_timeout; j++) {
2695 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2696 break;
2697 udelay(1);
2698 }
2699 }
2700 }
5153550a
AD		 2701 if (!ASIC_IS_NODCE(rdev)) {
2702 /* Unlock vga access */
2703 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2704 mdelay(1);
2705 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2706 }
bcc1c2a1
AD		 2707}
2708
755d819e 2709void evergreen_mc_program(struct radeon_device *rdev)
bcc1c2a1
AD		 2710{
2711 struct evergreen_mc_save save;
2712 u32 tmp;
2713 int i, j;
2714
2715 /* Initialize HDP */
2716 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2717 WREG32((0x2c14 + j), 0x00000000);
2718 WREG32((0x2c18 + j), 0x00000000);
2719 WREG32((0x2c1c + j), 0x00000000);
2720 WREG32((0x2c20 + j), 0x00000000);
2721 WREG32((0x2c24 + j), 0x00000000);
2722 }
2723 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2724
2725 evergreen_mc_stop(rdev, &save);
2726 if (evergreen_mc_wait_for_idle(rdev)) {
2727 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2728 }
2729 /* Lockout access through VGA aperture*/
2730 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2731 /* Update configuration */
2732 if (rdev->flags & RADEON_IS_AGP) {
2733 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2734 /* VRAM before AGP */
2735 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2736 rdev->mc.vram_start >> 12);
2737 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2738 rdev->mc.gtt_end >> 12);
2739 } else {
2740 /* VRAM after AGP */
2741 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2742 rdev->mc.gtt_start >> 12);
2743 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2744 rdev->mc.vram_end >> 12);
2745 }
2746 } else {
2747 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2748 rdev->mc.vram_start >> 12);
2749 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2750 rdev->mc.vram_end >> 12);
2751 }
3b9832f6 2752 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
05b3ef69
AD		 2753 /* llano/ontario only */
2754 if ((rdev->family == CHIP_PALM) ||
2755 (rdev->family == CHIP_SUMO) ||
2756 (rdev->family == CHIP_SUMO2)) {
b4183e30
AD		 2757 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2758 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2759 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2760 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2761 }
bcc1c2a1
AD		 2762 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2763 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2764 WREG32(MC_VM_FB_LOCATION, tmp);
2765 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
c46cb4da 2766 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
46fcd2b3 2767 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
bcc1c2a1
AD		 2768 if (rdev->flags & RADEON_IS_AGP) {
2769 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2770 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2771 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2772 } else {
2773 WREG32(MC_VM_AGP_BASE, 0);
2774 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2775 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2776 }
2777 if (evergreen_mc_wait_for_idle(rdev)) {
2778 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2779 }
2780 evergreen_mc_resume(rdev, &save);
2781 /* we need to own VRAM, so turn off the VGA renderer here
2782 * to stop it overwriting our objects */
2783 rv515_vga_render_disable(rdev);
2784}
2785
bcc1c2a1
AD		 2786/*
2787 * CP.
2788 */
12920591
AD		 2789void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2790{
876dc9f3 2791 struct radeon_ring *ring = &rdev->ring[ib->ring];
89d35807 2792 u32 next_rptr;
7b1f2485 2793
12920591 2794 /* set to DX10/11 mode */
e32eb50d
CK		 2795 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2796 radeon_ring_write(ring, 1);
45df6803
CK		 2797
2798 if (ring->rptr_save_reg) {
89d35807 2799 next_rptr = ring->wptr + 3 + 4;
45df6803
CK		 2800 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2801 radeon_ring_write(ring, ((ring->rptr_save_reg -
2802 PACKET3_SET_CONFIG_REG_START) >> 2));
2803 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2804 } else if (rdev->wb.enabled) {
2805 next_rptr = ring->wptr + 5 + 4;
2806 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2807 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2808 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2809 radeon_ring_write(ring, next_rptr);
2810 radeon_ring_write(ring, 0);
45df6803
CK		 2811 }
2812
e32eb50d
CK		 2813 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2814 radeon_ring_write(ring,
0f234f5f
AD		 2815#ifdef __BIG_ENDIAN
2816 (2 << 0) |
2817#endif
2818 (ib->gpu_addr & 0xFFFFFFFC));
e32eb50d
CK		 2819 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2820 radeon_ring_write(ring, ib->length_dw);
12920591
AD		 2821}
2822
bcc1c2a1
AD		 2823
2824static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2825{
fe251e2f
AD		 2826 const __be32 *fw_data;
2827 int i;
2828
2829 if (!rdev->me_fw || !rdev->pfp_fw)
2830 return -EINVAL;
bcc1c2a1 2831
fe251e2f 2832 r700_cp_stop(rdev);
0f234f5f
AD		 2833 WREG32(CP_RB_CNTL,
2834#ifdef __BIG_ENDIAN
2835 BUF_SWAP_32BIT |
2836#endif
2837 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
fe251e2f
AD		 2838
2839 fw_data = (const __be32 *)rdev->pfp_fw->data;
2840 WREG32(CP_PFP_UCODE_ADDR, 0);
2841 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2842 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2843 WREG32(CP_PFP_UCODE_ADDR, 0);
2844
2845 fw_data = (const __be32 *)rdev->me_fw->data;
2846 WREG32(CP_ME_RAM_WADDR, 0);
2847 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2848 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2849
2850 WREG32(CP_PFP_UCODE_ADDR, 0);
2851 WREG32(CP_ME_RAM_WADDR, 0);
2852 WREG32(CP_ME_RAM_RADDR, 0);
bcc1c2a1
AD		 2853 return 0;
2854}
2855
7e7b41d2
AD		 2856static int evergreen_cp_start(struct radeon_device *rdev)
2857{
e32eb50d 2858 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2281a378 2859 int r, i;
7e7b41d2
AD		 2860 uint32_t cp_me;
2861
e32eb50d 2862 r = radeon_ring_lock(rdev, ring, 7);
7e7b41d2
AD		 2863 if (r) {
2864 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2865 return r;
2866 }
e32eb50d
CK		 2867 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2868 radeon_ring_write(ring, 0x1);
2869 radeon_ring_write(ring, 0x0);
2870 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2871 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2872 radeon_ring_write(ring, 0);
2873 radeon_ring_write(ring, 0);
1538a9e0 2874 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2875
2876 cp_me = 0xff;
2877 WREG32(CP_ME_CNTL, cp_me);
2878
e32eb50d 2879 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
7e7b41d2
AD		 2880 if (r) {
2881 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2882 return r;
2883 }
2281a378
AD		 2884
2885 /* setup clear context state */
e32eb50d
CK		 2886 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2887 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2281a378
AD		 2888
2889 for (i = 0; i < evergreen_default_size; i++)
e32eb50d 2890 radeon_ring_write(ring, evergreen_default_state[i]);
2281a378 2891
e32eb50d
CK		 2892 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2893 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2281a378
AD		 2894
2895 /* set clear context state */
e32eb50d
CK		 2896 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2897 radeon_ring_write(ring, 0);
2281a378
AD		 2898
2899 /* SQ_VTX_BASE_VTX_LOC */
e32eb50d
CK		 2900 radeon_ring_write(ring, 0xc0026f00);
2901 radeon_ring_write(ring, 0x00000000);
2902 radeon_ring_write(ring, 0x00000000);
2903 radeon_ring_write(ring, 0x00000000);
2281a378
AD		 2904
2905 /* Clear consts */
e32eb50d
CK		 2906 radeon_ring_write(ring, 0xc0036f00);
2907 radeon_ring_write(ring, 0x00000bc4);
2908 radeon_ring_write(ring, 0xffffffff);
2909 radeon_ring_write(ring, 0xffffffff);
2910 radeon_ring_write(ring, 0xffffffff);
2281a378 2911
e32eb50d
CK		 2912 radeon_ring_write(ring, 0xc0026900);
2913 radeon_ring_write(ring, 0x00000316);
2914 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2915 radeon_ring_write(ring, 0x00000010); /* */
18ff84da 2916
1538a9e0 2917 radeon_ring_unlock_commit(rdev, ring, false);
7e7b41d2
AD		 2918
2919 return 0;
2920}
2921
1109ca09 2922static int evergreen_cp_resume(struct radeon_device *rdev)
fe251e2f 2923{
e32eb50d 2924 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
fe251e2f
AD		 2925 u32 tmp;
2926 u32 rb_bufsz;
2927 int r;
2928
2929 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2930 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2931 SOFT_RESET_PA |
2932 SOFT_RESET_SH |
2933 SOFT_RESET_VGT |
a49a50da 2934 SOFT_RESET_SPI |
fe251e2f
AD		 2935 SOFT_RESET_SX));
2936 RREG32(GRBM_SOFT_RESET);
2937 mdelay(15);
2938 WREG32(GRBM_SOFT_RESET, 0);
2939 RREG32(GRBM_SOFT_RESET);
2940
2941 /* Set ring buffer size */
b72a8925
DV		 2942 rb_bufsz = order_base_2(ring->ring_size / 8);
2943 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
fe251e2f
AD		 2944#ifdef __BIG_ENDIAN
2945 tmp |= BUF_SWAP_32BIT;
32fcdbf4 2946#endif
fe251e2f 2947 WREG32(CP_RB_CNTL, tmp);
15d3332f 2948 WREG32(CP_SEM_WAIT_TIMER, 0x0);
11ef3f1f 2949 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
fe251e2f
AD		 2950
2951 /* Set the write pointer delay */
2952 WREG32(CP_RB_WPTR_DELAY, 0);
2953
2954 /* Initialize the ring buffer's read and write pointers */
2955 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2956 WREG32(CP_RB_RPTR_WR, 0);
e32eb50d
CK		 2957 ring->wptr = 0;
2958 WREG32(CP_RB_WPTR, ring->wptr);
724c80e1 2959
48fc7f7e 2960 /* set the wb address whether it's enabled or not */
0f234f5f 2961 WREG32(CP_RB_RPTR_ADDR,
0f234f5f 2962 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
724c80e1
AD		 2963 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2964 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2965
2966 if (rdev->wb.enabled)
2967 WREG32(SCRATCH_UMSK, 0xff);
2968 else {
2969 tmp |= RB_NO_UPDATE;
2970 WREG32(SCRATCH_UMSK, 0);
2971 }
2972
fe251e2f
AD		 2973 mdelay(1);
2974 WREG32(CP_RB_CNTL, tmp);
2975
e32eb50d 2976 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
fe251e2f
AD		 2977 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2978
7e7b41d2 2979 evergreen_cp_start(rdev);
e32eb50d 2980 ring->ready = true;
f712812e 2981 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
fe251e2f 2982 if (r) {
e32eb50d 2983 ring->ready = false;
fe251e2f
AD		 2984 return r;
2985 }
2986 return 0;
2987}
bcc1c2a1
AD		 2988
2989/*
2990 * Core functions
2991 */
bcc1c2a1
AD		 2992static void evergreen_gpu_init(struct radeon_device *rdev)
2993{
416a2bd2 2994 u32 gb_addr_config;
32fcdbf4 2995 u32 mc_shared_chmap, mc_arb_ramcfg;
32fcdbf4
AD		 2996 u32 sx_debug_1;
2997 u32 smx_dc_ctl0;
2998 u32 sq_config;
2999 u32 sq_lds_resource_mgmt;
3000 u32 sq_gpr_resource_mgmt_1;
3001 u32 sq_gpr_resource_mgmt_2;
3002 u32 sq_gpr_resource_mgmt_3;
3003 u32 sq_thread_resource_mgmt;
3004 u32 sq_thread_resource_mgmt_2;
3005 u32 sq_stack_resource_mgmt_1;
3006 u32 sq_stack_resource_mgmt_2;
3007 u32 sq_stack_resource_mgmt_3;
3008 u32 vgt_cache_invalidation;
f25a5c63 3009 u32 hdp_host_path_cntl, tmp;
416a2bd2 3010 u32 disabled_rb_mask;
072c44bf 3011 int i, j, ps_thread_count;
32fcdbf4
AD		 3012
3013 switch (rdev->family) {
3014 case CHIP_CYPRESS:
3015 case CHIP_HEMLOCK:
3016 rdev->config.evergreen.num_ses = 2;
3017 rdev->config.evergreen.max_pipes = 4;
3018 rdev->config.evergreen.max_tile_pipes = 8;
3019 rdev->config.evergreen.max_simds = 10;
3020 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3021 rdev->config.evergreen.max_gprs = 256;
3022 rdev->config.evergreen.max_threads = 248;
3023 rdev->config.evergreen.max_gs_threads = 32;
3024 rdev->config.evergreen.max_stack_entries = 512;
3025 rdev->config.evergreen.sx_num_of_sets = 4;
3026 rdev->config.evergreen.sx_max_export_size = 256;
3027 rdev->config.evergreen.sx_max_export_pos_size = 64;
3028 rdev->config.evergreen.sx_max_export_smx_size = 192;
3029 rdev->config.evergreen.max_hw_contexts = 8;
3030 rdev->config.evergreen.sq_num_cf_insts = 2;
3031
3032 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3033 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3034 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3035 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3036 break;
3037 case CHIP_JUNIPER:
3038 rdev->config.evergreen.num_ses = 1;
3039 rdev->config.evergreen.max_pipes = 4;
3040 rdev->config.evergreen.max_tile_pipes = 4;
3041 rdev->config.evergreen.max_simds = 10;
3042 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3043 rdev->config.evergreen.max_gprs = 256;
3044 rdev->config.evergreen.max_threads = 248;
3045 rdev->config.evergreen.max_gs_threads = 32;
3046 rdev->config.evergreen.max_stack_entries = 512;
3047 rdev->config.evergreen.sx_num_of_sets = 4;
3048 rdev->config.evergreen.sx_max_export_size = 256;
3049 rdev->config.evergreen.sx_max_export_pos_size = 64;
3050 rdev->config.evergreen.sx_max_export_smx_size = 192;
3051 rdev->config.evergreen.max_hw_contexts = 8;
3052 rdev->config.evergreen.sq_num_cf_insts = 2;
3053
3054 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3055 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3056 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3057 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3058 break;
3059 case CHIP_REDWOOD:
3060 rdev->config.evergreen.num_ses = 1;
3061 rdev->config.evergreen.max_pipes = 4;
3062 rdev->config.evergreen.max_tile_pipes = 4;
3063 rdev->config.evergreen.max_simds = 5;
3064 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3065 rdev->config.evergreen.max_gprs = 256;
3066 rdev->config.evergreen.max_threads = 248;
3067 rdev->config.evergreen.max_gs_threads = 32;
3068 rdev->config.evergreen.max_stack_entries = 256;
3069 rdev->config.evergreen.sx_num_of_sets = 4;
3070 rdev->config.evergreen.sx_max_export_size = 256;
3071 rdev->config.evergreen.sx_max_export_pos_size = 64;
3072 rdev->config.evergreen.sx_max_export_smx_size = 192;
3073 rdev->config.evergreen.max_hw_contexts = 8;
3074 rdev->config.evergreen.sq_num_cf_insts = 2;
3075
3076 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3077 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3078 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3079 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3080 break;
3081 case CHIP_CEDAR:
3082 default:
3083 rdev->config.evergreen.num_ses = 1;
3084 rdev->config.evergreen.max_pipes = 2;
3085 rdev->config.evergreen.max_tile_pipes = 2;
3086 rdev->config.evergreen.max_simds = 2;
3087 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3088 rdev->config.evergreen.max_gprs = 256;
3089 rdev->config.evergreen.max_threads = 192;
3090 rdev->config.evergreen.max_gs_threads = 16;
3091 rdev->config.evergreen.max_stack_entries = 256;
3092 rdev->config.evergreen.sx_num_of_sets = 4;
3093 rdev->config.evergreen.sx_max_export_size = 128;
3094 rdev->config.evergreen.sx_max_export_pos_size = 32;
3095 rdev->config.evergreen.sx_max_export_smx_size = 96;
3096 rdev->config.evergreen.max_hw_contexts = 4;
3097 rdev->config.evergreen.sq_num_cf_insts = 1;
3098
d5e455e4
AD		 3099 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3100 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3101 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3102 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5e455e4
AD		 3103 break;
3104 case CHIP_PALM:
3105 rdev->config.evergreen.num_ses = 1;
3106 rdev->config.evergreen.max_pipes = 2;
3107 rdev->config.evergreen.max_tile_pipes = 2;
3108 rdev->config.evergreen.max_simds = 2;
3109 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3110 rdev->config.evergreen.max_gprs = 256;
3111 rdev->config.evergreen.max_threads = 192;
3112 rdev->config.evergreen.max_gs_threads = 16;
3113 rdev->config.evergreen.max_stack_entries = 256;
3114 rdev->config.evergreen.sx_num_of_sets = 4;
3115 rdev->config.evergreen.sx_max_export_size = 128;
3116 rdev->config.evergreen.sx_max_export_pos_size = 32;
3117 rdev->config.evergreen.sx_max_export_smx_size = 96;
3118 rdev->config.evergreen.max_hw_contexts = 4;
3119 rdev->config.evergreen.sq_num_cf_insts = 1;
3120
d5c5a72f
AD		 3121 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3122 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3123 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3124 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3125 break;
3126 case CHIP_SUMO:
3127 rdev->config.evergreen.num_ses = 1;
3128 rdev->config.evergreen.max_pipes = 4;
bd25f078 3129 rdev->config.evergreen.max_tile_pipes = 4;
d5c5a72f
AD		 3130 if (rdev->pdev->device == 0x9648)
3131 rdev->config.evergreen.max_simds = 3;
3132 else if ((rdev->pdev->device == 0x9647) ||
3133 (rdev->pdev->device == 0x964a))
3134 rdev->config.evergreen.max_simds = 4;
3135 else
3136 rdev->config.evergreen.max_simds = 5;
3137 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3138 rdev->config.evergreen.max_gprs = 256;
3139 rdev->config.evergreen.max_threads = 248;
3140 rdev->config.evergreen.max_gs_threads = 32;
3141 rdev->config.evergreen.max_stack_entries = 256;
3142 rdev->config.evergreen.sx_num_of_sets = 4;
3143 rdev->config.evergreen.sx_max_export_size = 256;
3144 rdev->config.evergreen.sx_max_export_pos_size = 64;
3145 rdev->config.evergreen.sx_max_export_smx_size = 192;
3146 rdev->config.evergreen.max_hw_contexts = 8;
3147 rdev->config.evergreen.sq_num_cf_insts = 2;
3148
3149 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3150 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3151 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3152 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
d5c5a72f
AD		 3153 break;
3154 case CHIP_SUMO2:
3155 rdev->config.evergreen.num_ses = 1;
3156 rdev->config.evergreen.max_pipes = 4;
3157 rdev->config.evergreen.max_tile_pipes = 4;
3158 rdev->config.evergreen.max_simds = 2;
3159 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3160 rdev->config.evergreen.max_gprs = 256;
3161 rdev->config.evergreen.max_threads = 248;
3162 rdev->config.evergreen.max_gs_threads = 32;
3163 rdev->config.evergreen.max_stack_entries = 512;
3164 rdev->config.evergreen.sx_num_of_sets = 4;
3165 rdev->config.evergreen.sx_max_export_size = 256;
3166 rdev->config.evergreen.sx_max_export_pos_size = 64;
3167 rdev->config.evergreen.sx_max_export_smx_size = 192;
50b8f5ae 3168 rdev->config.evergreen.max_hw_contexts = 4;
d5c5a72f
AD		 3169 rdev->config.evergreen.sq_num_cf_insts = 2;
3170
adb68fa2
AD		 3171 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3172 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3173 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
bd25f078 3174 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3175 break;
3176 case CHIP_BARTS:
3177 rdev->config.evergreen.num_ses = 2;
3178 rdev->config.evergreen.max_pipes = 4;
3179 rdev->config.evergreen.max_tile_pipes = 8;
3180 rdev->config.evergreen.max_simds = 7;
3181 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3182 rdev->config.evergreen.max_gprs = 256;
3183 rdev->config.evergreen.max_threads = 248;
3184 rdev->config.evergreen.max_gs_threads = 32;
3185 rdev->config.evergreen.max_stack_entries = 512;
3186 rdev->config.evergreen.sx_num_of_sets = 4;
3187 rdev->config.evergreen.sx_max_export_size = 256;
3188 rdev->config.evergreen.sx_max_export_pos_size = 64;
3189 rdev->config.evergreen.sx_max_export_smx_size = 192;
3190 rdev->config.evergreen.max_hw_contexts = 8;
3191 rdev->config.evergreen.sq_num_cf_insts = 2;
3192
3193 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3194 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3195 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3196 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3197 break;
3198 case CHIP_TURKS:
3199 rdev->config.evergreen.num_ses = 1;
3200 rdev->config.evergreen.max_pipes = 4;
3201 rdev->config.evergreen.max_tile_pipes = 4;
3202 rdev->config.evergreen.max_simds = 6;
3203 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3204 rdev->config.evergreen.max_gprs = 256;
3205 rdev->config.evergreen.max_threads = 248;
3206 rdev->config.evergreen.max_gs_threads = 32;
3207 rdev->config.evergreen.max_stack_entries = 256;
3208 rdev->config.evergreen.sx_num_of_sets = 4;
3209 rdev->config.evergreen.sx_max_export_size = 256;
3210 rdev->config.evergreen.sx_max_export_pos_size = 64;
3211 rdev->config.evergreen.sx_max_export_smx_size = 192;
3212 rdev->config.evergreen.max_hw_contexts = 8;
3213 rdev->config.evergreen.sq_num_cf_insts = 2;
3214
3215 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3216 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3217 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3218 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
adb68fa2
AD		 3219 break;
3220 case CHIP_CAICOS:
3221 rdev->config.evergreen.num_ses = 1;
bd25f078 3222 rdev->config.evergreen.max_pipes = 2;
adb68fa2
AD		 3223 rdev->config.evergreen.max_tile_pipes = 2;
3224 rdev->config.evergreen.max_simds = 2;
3225 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3226 rdev->config.evergreen.max_gprs = 256;
3227 rdev->config.evergreen.max_threads = 192;
3228 rdev->config.evergreen.max_gs_threads = 16;
3229 rdev->config.evergreen.max_stack_entries = 256;
3230 rdev->config.evergreen.sx_num_of_sets = 4;
3231 rdev->config.evergreen.sx_max_export_size = 128;
3232 rdev->config.evergreen.sx_max_export_pos_size = 32;
3233 rdev->config.evergreen.sx_max_export_smx_size = 96;
3234 rdev->config.evergreen.max_hw_contexts = 4;
3235 rdev->config.evergreen.sq_num_cf_insts = 1;
3236
32fcdbf4
AD		 3237 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3238 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3239 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
416a2bd2 3240 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
32fcdbf4
AD		 3241 break;
3242 }
3243
3244 /* Initialize HDP */
3245 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3246 WREG32((0x2c14 + j), 0x00000000);
3247 WREG32((0x2c18 + j), 0x00000000);
3248 WREG32((0x2c1c + j), 0x00000000);
3249 WREG32((0x2c20 + j), 0x00000000);
3250 WREG32((0x2c24 + j), 0x00000000);
3251 }
3252
3253 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3254
d054ac16
AD		 3255 evergreen_fix_pci_max_read_req_size(rdev);
3256
32fcdbf4 3257 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
05b3ef69
AD		 3258 if ((rdev->family == CHIP_PALM) ||
3259 (rdev->family == CHIP_SUMO) ||
3260 (rdev->family == CHIP_SUMO2))
d9282fca
AD		 3261 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3262 else
3263 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
32fcdbf4 3264
1aa52bd3
AD		 3265 /* setup tiling info dword. gb_addr_config is not adequate since it does
3266 * not have bank info, so create a custom tiling dword.
3267 * bits 3:0 num_pipes
3268 * bits 7:4 num_banks
3269 * bits 11:8 group_size
3270 * bits 15:12 row_size
3271 */
3272 rdev->config.evergreen.tile_config = 0;
3273 switch (rdev->config.evergreen.max_tile_pipes) {
3274 case 1:
3275 default:
3276 rdev->config.evergreen.tile_config |= (0 << 0);
3277 break;
3278 case 2:
3279 rdev->config.evergreen.tile_config |= (1 << 0);
3280 break;
3281 case 4:
3282 rdev->config.evergreen.tile_config |= (2 << 0);
3283 break;
3284 case 8:
3285 rdev->config.evergreen.tile_config |= (3 << 0);
3286 break;
3287 }
d698a34d 3288 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
5bfa4879 3289 if (rdev->flags & RADEON_IS_IGP)
d698a34d 3290 rdev->config.evergreen.tile_config |= 1 << 4;
29d65406 3291 else {
c8d15edc
AD		 3292 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3293 case 0: /* four banks */
29d65406 3294 rdev->config.evergreen.tile_config |= 0 << 4;
c8d15edc
AD		 3295 break;
3296 case 1: /* eight banks */
3297 rdev->config.evergreen.tile_config |= 1 << 4;
3298 break;
3299 case 2: /* sixteen banks */
3300 default:
3301 rdev->config.evergreen.tile_config |= 2 << 4;
3302 break;
3303 }
29d65406 3304 }
416a2bd2 3305 rdev->config.evergreen.tile_config |= 0 << 8;
1aa52bd3
AD		 3306 rdev->config.evergreen.tile_config |=
3307 ((gb_addr_config & 0x30000000) >> 28) << 12;
3308
416a2bd2
AD		 3309 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3310 u32 efuse_straps_4;
3311 u32 efuse_straps_3;
32fcdbf4 3312
ff82bbc4
AD		 3313 efuse_straps_4 = RREG32_RCU(0x204);
3314 efuse_straps_3 = RREG32_RCU(0x203);
416a2bd2
AD		 3315 tmp = (((efuse_straps_4 & 0xf) << 4) |
3316 ((efuse_straps_3 & 0xf0000000) >> 28));
3317 } else {
3318 tmp = 0;
3319 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3320 u32 rb_disable_bitmap;
3321
3322 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3323 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3324 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3325 tmp <<= 4;
3326 tmp |= rb_disable_bitmap;
32fcdbf4 3327 }
416a2bd2
AD		 3328 }
3329 /* enabled rb are just the one not disabled :) */
3330 disabled_rb_mask = tmp;
cedb655a
AD		 3331 tmp = 0;
3332 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3333 tmp |= (1 << i);
3334 /* if all the backends are disabled, fix it up here */
3335 if ((disabled_rb_mask & tmp) == tmp) {
3336 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3337 disabled_rb_mask &= ~(1 << i);
3338 }
32fcdbf4 3339
65fcf668
AD		 3340 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3341 u32 simd_disable_bitmap;
3342
3343 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3344 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3345 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3346 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3347 tmp <<= 16;
3348 tmp |= simd_disable_bitmap;
3349 }
3350 rdev->config.evergreen.active_simds = hweight32(~tmp);
3351
416a2bd2
AD		 3352 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3353 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
32fcdbf4 3354
416a2bd2
AD		 3355 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3356 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3357 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
233d1ad5 3358 WREG32(DMA_TILING_CONFIG, gb_addr_config);
9a21059d
CK		 3359 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3360 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3361 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
32fcdbf4 3362
f7eb9730
AD		 3363 if ((rdev->config.evergreen.max_backends == 1) &&
3364 (rdev->flags & RADEON_IS_IGP)) {
3365 if ((disabled_rb_mask & 3) == 1) {
3366 /* RB0 disabled, RB1 enabled */
3367 tmp = 0x11111111;
3368 } else {
3369 /* RB1 disabled, RB0 enabled */
3370 tmp = 0x00000000;
3371 }
3372 } else {
3373 tmp = gb_addr_config & NUM_PIPES_MASK;
3374 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3375 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3376 }
416a2bd2 3377 WREG32(GB_BACKEND_MAP, tmp);
32fcdbf4
AD		 3378
3379 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3380 WREG32(CGTS_TCC_DISABLE, 0);
3381 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3382 WREG32(CGTS_USER_TCC_DISABLE, 0);
3383
3384 /* set HW defaults for 3D engine */
3385 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3386 ROQ_IB2_START(0x2b)));
3387
3388 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3389
3390 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3391 SYNC_GRADIENT |
3392 SYNC_WALKER |
3393 SYNC_ALIGNER));
3394
3395 sx_debug_1 = RREG32(SX_DEBUG_1);
3396 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3397 WREG32(SX_DEBUG_1, sx_debug_1);
3398
3399
3400 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3401 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3402 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3403 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3404
b866d133
AD		 3405 if (rdev->family <= CHIP_SUMO2)
3406 WREG32(SMX_SAR_CTL0, 0x00010000);
3407
32fcdbf4
AD		 3408 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3409 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3410 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3411
3412 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3413 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3414 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3415
3416 WREG32(VGT_NUM_INSTANCES, 1);
3417 WREG32(SPI_CONFIG_CNTL, 0);
3418 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3419 WREG32(CP_PERFMON_CNTL, 0);
3420
3421 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3422 FETCH_FIFO_HIWATER(0x4) |
3423 DONE_FIFO_HIWATER(0xe0) |
3424 ALU_UPDATE_FIFO_HIWATER(0x8)));
3425
3426 sq_config = RREG32(SQ_CONFIG);
3427 sq_config &= ~(PS_PRIO(3) |
3428 VS_PRIO(3) |
3429 GS_PRIO(3) |
3430 ES_PRIO(3));
3431 sq_config |= (VC_ENABLE |
3432 EXPORT_SRC_C |
3433 PS_PRIO(0) |
3434 VS_PRIO(1) |
3435 GS_PRIO(2) |
3436 ES_PRIO(3));
3437
d5e455e4
AD		 3438 switch (rdev->family) {
3439 case CHIP_CEDAR:
3440 case CHIP_PALM:
d5c5a72f
AD		 3441 case CHIP_SUMO:
3442 case CHIP_SUMO2:
adb68fa2 3443 case CHIP_CAICOS:
32fcdbf4
AD		 3444 /* no vertex cache */
3445 sq_config &= ~VC_ENABLE;
d5e455e4
AD		 3446 break;
3447 default:
3448 break;
3449 }
32fcdbf4
AD		 3450
3451 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3452
3453 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3454 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3455 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3456 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3457 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3458 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3459 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3460
d5e455e4
AD		 3461 switch (rdev->family) {
3462 case CHIP_CEDAR:
3463 case CHIP_PALM:
d5c5a72f
AD		 3464 case CHIP_SUMO:
3465 case CHIP_SUMO2:
32fcdbf4 3466 ps_thread_count = 96;
d5e455e4
AD		 3467 break;
3468 default:
32fcdbf4 3469 ps_thread_count = 128;
d5e455e4
AD		 3470 break;
3471 }
32fcdbf4
AD		 3472
3473 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
f96b35cd
AD		 3474 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3475 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3476 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3477 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3478 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
32fcdbf4
AD		 3479
3480 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3481 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3482 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3483 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3484 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3485 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3486
3487 WREG32(SQ_CONFIG, sq_config);
3488 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3489 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3490 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3491 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3492 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3493 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3494 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3495 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3496 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3497 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3498
3499 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3500 FORCE_EOV_MAX_REZ_CNT(255)));
3501
d5e455e4
AD		 3502 switch (rdev->family) {
3503 case CHIP_CEDAR:
3504 case CHIP_PALM:
d5c5a72f
AD		 3505 case CHIP_SUMO:
3506 case CHIP_SUMO2:
adb68fa2 3507 case CHIP_CAICOS:
32fcdbf4 3508 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
d5e455e4
AD		 3509 break;
3510 default:
32fcdbf4 3511 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
d5e455e4
AD		 3512 break;
3513 }
32fcdbf4
AD		 3514 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3515 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3516
3517 WREG32(VGT_GS_VERTEX_REUSE, 16);
12920591 3518 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
32fcdbf4
AD		 3519 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3520
60a4a3e0
AD		 3521 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3522 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3523
32fcdbf4
AD		 3524 WREG32(CB_PERF_CTR0_SEL_0, 0);
3525 WREG32(CB_PERF_CTR0_SEL_1, 0);
3526 WREG32(CB_PERF_CTR1_SEL_0, 0);
3527 WREG32(CB_PERF_CTR1_SEL_1, 0);
3528 WREG32(CB_PERF_CTR2_SEL_0, 0);
3529 WREG32(CB_PERF_CTR2_SEL_1, 0);
3530 WREG32(CB_PERF_CTR3_SEL_0, 0);
3531 WREG32(CB_PERF_CTR3_SEL_1, 0);
3532
60a4a3e0
AD		 3533 /* clear render buffer base addresses */
3534 WREG32(CB_COLOR0_BASE, 0);
3535 WREG32(CB_COLOR1_BASE, 0);
3536 WREG32(CB_COLOR2_BASE, 0);
3537 WREG32(CB_COLOR3_BASE, 0);
3538 WREG32(CB_COLOR4_BASE, 0);
3539 WREG32(CB_COLOR5_BASE, 0);
3540 WREG32(CB_COLOR6_BASE, 0);
3541 WREG32(CB_COLOR7_BASE, 0);
3542 WREG32(CB_COLOR8_BASE, 0);
3543 WREG32(CB_COLOR9_BASE, 0);
3544 WREG32(CB_COLOR10_BASE, 0);
3545 WREG32(CB_COLOR11_BASE, 0);
3546
3547 /* set the shader const cache sizes to 0 */
3548 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3549 WREG32(i, 0);
3550 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3551 WREG32(i, 0);
3552
f25a5c63
AD		 3553 tmp = RREG32(HDP_MISC_CNTL);
3554 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3555 WREG32(HDP_MISC_CNTL, tmp);
3556
32fcdbf4
AD		 3557 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3558 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3559
3560 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3561
3562 udelay(50);
3563
bcc1c2a1
AD		 3564}
3565
3566int evergreen_mc_init(struct radeon_device *rdev)
3567{
bcc1c2a1
AD		 3568 u32 tmp;
3569 int chansize, numchan;
bcc1c2a1
AD		 3570
3571 /* Get VRAM informations */
3572 rdev->mc.vram_is_ddr = true;
05b3ef69
AD		 3573 if ((rdev->family == CHIP_PALM) ||
3574 (rdev->family == CHIP_SUMO) ||
3575 (rdev->family == CHIP_SUMO2))
8208441b
AD		 3576 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3577 else
3578 tmp = RREG32(MC_ARB_RAMCFG);
bcc1c2a1
AD		 3579 if (tmp & CHANSIZE_OVERRIDE) {
3580 chansize = 16;
3581 } else if (tmp & CHANSIZE_MASK) {
3582 chansize = 64;
3583 } else {
3584 chansize = 32;
3585 }
3586 tmp = RREG32(MC_SHARED_CHMAP);
3587 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3588 case 0:
3589 default:
3590 numchan = 1;
3591 break;
3592 case 1:
3593 numchan = 2;
3594 break;
3595 case 2:
3596 numchan = 4;
3597 break;
3598 case 3:
3599 numchan = 8;
3600 break;
3601 }
3602 rdev->mc.vram_width = numchan * chansize;
3603 /* Could aper size report 0 ? */
01d73a69
JC		 3604 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3605 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
bcc1c2a1 3606 /* Setup GPU memory space */
05b3ef69
AD		 3607 if ((rdev->family == CHIP_PALM) ||
3608 (rdev->family == CHIP_SUMO) ||
3609 (rdev->family == CHIP_SUMO2)) {
6eb18f8b
AD		 3610 /* size in bytes on fusion */
3611 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3612 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3613 } else {
05b3ef69 3614 /* size in MB on evergreen/cayman/tn */
fc986034
NOS		 3615 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3616 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
6eb18f8b 3617 }
51e5fcd3 3618 rdev->mc.visible_vram_size = rdev->mc.aper_size;
0ef0c1f7 3619 r700_vram_gtt_location(rdev, &rdev->mc);
f47299c5
AD		 3620 radeon_update_bandwidth_info(rdev);
3621
bcc1c2a1
AD		 3622 return 0;
3623}
d594e46a 3624
187e3593 3625void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
bcc1c2a1 3626{
64c56e8c 3627 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
747943ea 3628 RREG32(GRBM_STATUS));
64c56e8c 3629 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
747943ea 3630 RREG32(GRBM_STATUS_SE0));
64c56e8c 3631 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
747943ea 3632 RREG32(GRBM_STATUS_SE1));
64c56e8c 3633 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
747943ea 3634 RREG32(SRBM_STATUS));
a65a4369
AD		 3635 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3636 RREG32(SRBM_STATUS2));
440a7cd8
JG		 3637 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3638 RREG32(CP_STALLED_STAT1));
3639 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3640 RREG32(CP_STALLED_STAT2));
3641 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3642 RREG32(CP_BUSY_STAT));
3643 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3644 RREG32(CP_STAT));
eaaa6983
JG		 3645 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3646 RREG32(DMA_STATUS_REG));
168757ea
AD		 3647 if (rdev->family >= CHIP_CAYMAN) {
3648 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3649 RREG32(DMA_STATUS_REG + 0x800));
3650 }
0ecebb9e
AD		 3651}
3652
168757ea 3653bool evergreen_is_display_hung(struct radeon_device *rdev)
0ecebb9e 3654{
a65a4369
AD		 3655 u32 crtc_hung = 0;
3656 u32 crtc_status[6];
3657 u32 i, j, tmp;
3658
3659 for (i = 0; i < rdev->num_crtc; i++) {
3660 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3661 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3662 crtc_hung |= (1 << i);
3663 }
3664 }
3665
3666 for (j = 0; j < 10; j++) {
3667 for (i = 0; i < rdev->num_crtc; i++) {
3668 if (crtc_hung & (1 << i)) {
3669 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3670 if (tmp != crtc_status[i])
3671 crtc_hung &= ~(1 << i);
3672 }
3673 }
3674 if (crtc_hung == 0)
3675 return false;
3676 udelay(100);
3677 }
3678
3679 return true;
3680}
3681
2483b4ea 3682u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
a65a4369
AD		 3683{
3684 u32 reset_mask = 0;
b7630473 3685 u32 tmp;
0ecebb9e 3686
a65a4369
AD		 3687 /* GRBM_STATUS */
3688 tmp = RREG32(GRBM_STATUS);
3689 if (tmp & (PA_BUSY | SC_BUSY |
3690 SH_BUSY | SX_BUSY |
3691 TA_BUSY | VGT_BUSY |
3692 DB_BUSY | CB_BUSY |
3693 SPI_BUSY | VGT_BUSY_NO_DMA))
3694 reset_mask |= RADEON_RESET_GFX;
3695
3696 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3697 CP_BUSY | CP_COHERENCY_BUSY))
3698 reset_mask |= RADEON_RESET_CP;
3699
3700 if (tmp & GRBM_EE_BUSY)
3701 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
19fc42ed 3702
a65a4369
AD		 3703 /* DMA_STATUS_REG */
3704 tmp = RREG32(DMA_STATUS_REG);
3705 if (!(tmp & DMA_IDLE))
3706 reset_mask |= RADEON_RESET_DMA;
3707
3708 /* SRBM_STATUS2 */
3709 tmp = RREG32(SRBM_STATUS2);
3710 if (tmp & DMA_BUSY)
3711 reset_mask |= RADEON_RESET_DMA;
3712
3713 /* SRBM_STATUS */
3714 tmp = RREG32(SRBM_STATUS);
3715 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3716 reset_mask |= RADEON_RESET_RLC;
3717
3718 if (tmp & IH_BUSY)
3719 reset_mask |= RADEON_RESET_IH;
3720
3721 if (tmp & SEM_BUSY)
3722 reset_mask |= RADEON_RESET_SEM;
3723
3724 if (tmp & GRBM_RQ_PENDING)
3725 reset_mask |= RADEON_RESET_GRBM;
3726
3727 if (tmp & VMC_BUSY)
3728 reset_mask |= RADEON_RESET_VMC;
3729
3730 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3731 MCC_BUSY | MCD_BUSY))
3732 reset_mask |= RADEON_RESET_MC;
3733
3734 if (evergreen_is_display_hung(rdev))
3735 reset_mask |= RADEON_RESET_DISPLAY;
3736
3737 /* VM_L2_STATUS */
3738 tmp = RREG32(VM_L2_STATUS);
3739 if (tmp & L2_BUSY)
3740 reset_mask |= RADEON_RESET_VMC;
3741
d808fc88
AD		 3742 /* Skip MC reset as it's mostly likely not hung, just busy */
3743 if (reset_mask & RADEON_RESET_MC) {
3744 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3745 reset_mask &= ~RADEON_RESET_MC;
3746 }
3747
a65a4369
AD		 3748 return reset_mask;
3749}
3750
3751static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3752{
3753 struct evergreen_mc_save save;
3754 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3755 u32 tmp;
19fc42ed 3756
0ecebb9e 3757 if (reset_mask == 0)
a65a4369 3758 return;
0ecebb9e
AD		 3759
3760 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3761
b7630473
AD		 3762 evergreen_print_gpu_status_regs(rdev);
3763
b7630473
AD		 3764 /* Disable CP parsing/prefetching */
3765 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3766
3767 if (reset_mask & RADEON_RESET_DMA) {
3768 /* Disable DMA */
3769 tmp = RREG32(DMA_RB_CNTL);
3770 tmp &= ~DMA_RB_ENABLE;
3771 WREG32(DMA_RB_CNTL, tmp);
3772 }
3773
b21b6e7a
AD		 3774 udelay(50);
3775
3776 evergreen_mc_stop(rdev, &save);
3777 if (evergreen_mc_wait_for_idle(rdev)) {
3778 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3779 }
3780
b7630473
AD		 3781 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3782 grbm_soft_reset |= SOFT_RESET_DB |
3783 SOFT_RESET_CB |
3784 SOFT_RESET_PA |
3785 SOFT_RESET_SC |
3786 SOFT_RESET_SPI |
3787 SOFT_RESET_SX |
3788 SOFT_RESET_SH |
3789 SOFT_RESET_TC |
3790 SOFT_RESET_TA |
3791 SOFT_RESET_VC |
3792 SOFT_RESET_VGT;
3793 }
3794
3795 if (reset_mask & RADEON_RESET_CP) {
3796 grbm_soft_reset |= SOFT_RESET_CP |
3797 SOFT_RESET_VGT;
3798
3799 srbm_soft_reset |= SOFT_RESET_GRBM;
3800 }
0ecebb9e
AD		 3801
3802 if (reset_mask & RADEON_RESET_DMA)
b7630473
AD		 3803 srbm_soft_reset |= SOFT_RESET_DMA;
3804
a65a4369
AD		 3805 if (reset_mask & RADEON_RESET_DISPLAY)
3806 srbm_soft_reset |= SOFT_RESET_DC;
3807
3808 if (reset_mask & RADEON_RESET_RLC)
3809 srbm_soft_reset |= SOFT_RESET_RLC;
3810
3811 if (reset_mask & RADEON_RESET_SEM)
3812 srbm_soft_reset |= SOFT_RESET_SEM;
3813
3814 if (reset_mask & RADEON_RESET_IH)
3815 srbm_soft_reset |= SOFT_RESET_IH;
3816
3817 if (reset_mask & RADEON_RESET_GRBM)
3818 srbm_soft_reset |= SOFT_RESET_GRBM;
3819
3820 if (reset_mask & RADEON_RESET_VMC)
3821 srbm_soft_reset |= SOFT_RESET_VMC;
3822
24178ec4
AD		 3823 if (!(rdev->flags & RADEON_IS_IGP)) {
3824 if (reset_mask & RADEON_RESET_MC)
3825 srbm_soft_reset |= SOFT_RESET_MC;
3826 }
a65a4369 3827
b7630473
AD		 3828 if (grbm_soft_reset) {
3829 tmp = RREG32(GRBM_SOFT_RESET);
3830 tmp |= grbm_soft_reset;
3831 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3832 WREG32(GRBM_SOFT_RESET, tmp);
3833 tmp = RREG32(GRBM_SOFT_RESET);
3834
3835 udelay(50);
3836
3837 tmp &= ~grbm_soft_reset;
3838 WREG32(GRBM_SOFT_RESET, tmp);
3839 tmp = RREG32(GRBM_SOFT_RESET);
3840 }
3841
3842 if (srbm_soft_reset) {
3843 tmp = RREG32(SRBM_SOFT_RESET);
3844 tmp |= srbm_soft_reset;
3845 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3846 WREG32(SRBM_SOFT_RESET, tmp);
3847 tmp = RREG32(SRBM_SOFT_RESET);
3848
3849 udelay(50);
3850
3851 tmp &= ~srbm_soft_reset;
3852 WREG32(SRBM_SOFT_RESET, tmp);
3853 tmp = RREG32(SRBM_SOFT_RESET);
3854 }
0ecebb9e
AD		 3855
3856 /* Wait a little for things to settle down */
3857 udelay(50);
3858
747943ea 3859 evergreen_mc_resume(rdev, &save);
b7630473
AD		 3860 udelay(50);
3861
b7630473 3862 evergreen_print_gpu_status_regs(rdev);
bcc1c2a1
AD		 3863}
3864
b5470b03
AD		 3865void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3866{
3867 struct evergreen_mc_save save;
3868 u32 tmp, i;
3869
3870 dev_info(rdev->dev, "GPU pci config reset\n");
3871
3872 /* disable dpm? */
3873
3874 /* Disable CP parsing/prefetching */
3875 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3876 udelay(50);
3877 /* Disable DMA */
3878 tmp = RREG32(DMA_RB_CNTL);
3879 tmp &= ~DMA_RB_ENABLE;
3880 WREG32(DMA_RB_CNTL, tmp);
3881 /* XXX other engines? */
3882
3883 /* halt the rlc */
3884 r600_rlc_stop(rdev);
3885
3886 udelay(50);
3887
3888 /* set mclk/sclk to bypass */
3889 rv770_set_clk_bypass_mode(rdev);
3890 /* disable BM */
3891 pci_clear_master(rdev->pdev);
3892 /* disable mem access */
3893 evergreen_mc_stop(rdev, &save);
3894 if (evergreen_mc_wait_for_idle(rdev)) {
3895 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3896 }
3897 /* reset */
3898 radeon_pci_config_reset(rdev);
3899 /* wait for asic to come out of reset */
3900 for (i = 0; i < rdev->usec_timeout; i++) {
3901 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3902 break;
3903 udelay(1);
3904 }
3905}
3906
a2d07b74 3907int evergreen_asic_reset(struct radeon_device *rdev)
bcc1c2a1 3908{
a65a4369
AD		 3909 u32 reset_mask;
3910
3911 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3912
3913 if (reset_mask)
3914 r600_set_bios_scratch_engine_hung(rdev, true);
3915
b5470b03 3916 /* try soft reset */
a65a4369
AD		 3917 evergreen_gpu_soft_reset(rdev, reset_mask);
3918
3919 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3920
b5470b03
AD		 3921 /* try pci config reset */
3922 if (reset_mask && radeon_hard_reset)
3923 evergreen_gpu_pci_config_reset(rdev);
3924
3925 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3926
a65a4369
AD		 3927 if (!reset_mask)
3928 r600_set_bios_scratch_engine_hung(rdev, false);
3929
3930 return 0;
747943ea
AD		 3931}
3932
123bc183
AD		 3933/**
3934 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3935 *
3936 * @rdev: radeon_device pointer
3937 * @ring: radeon_ring structure holding ring information
3938 *
3939 * Check if the GFX engine is locked up.
3940 * Returns true if the engine appears to be locked up, false if not.
3941 */
3942bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3943{
3944 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3945
3946 if (!(reset_mask & (RADEON_RESET_GFX |
3947 RADEON_RESET_COMPUTE |
3948 RADEON_RESET_CP))) {
ff212f25 3949 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 3950 return false;
3951 }
123bc183
AD		 3952 return radeon_ring_test_lockup(rdev, ring);
3953}
3954
2948f5e6
AD		 3955/*
3956 * RLC
3957 */
3958#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3959#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3960
3961void sumo_rlc_fini(struct radeon_device *rdev)
3962{
3963 int r;
3964
3965 /* save restore block */
3966 if (rdev->rlc.save_restore_obj) {
3967 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3968 if (unlikely(r != 0))
3969 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3970 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3971 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3972
3973 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3974 rdev->rlc.save_restore_obj = NULL;
3975 }
3976
3977 /* clear state block */
3978 if (rdev->rlc.clear_state_obj) {
3979 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3980 if (unlikely(r != 0))
3981 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3982 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3983 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3984
3985 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3986 rdev->rlc.clear_state_obj = NULL;
3987 }
22c775ce
AD		 3988
3989 /* clear state block */
3990 if (rdev->rlc.cp_table_obj) {
3991 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3992 if (unlikely(r != 0))
3993 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3994 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3995 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3996
3997 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3998 rdev->rlc.cp_table_obj = NULL;
3999 }
2948f5e6
AD		 4000}
4001
22c775ce
AD		 4002#define CP_ME_TABLE_SIZE 96
4003
2948f5e6
AD		 4004int sumo_rlc_init(struct radeon_device *rdev)
4005{
1fd11777 4006 const u32 *src_ptr;
2948f5e6
AD		 4007 volatile u32 *dst_ptr;
4008 u32 dws, data, i, j, k, reg_num;
59a82d0e 4009 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
2948f5e6 4010 u64 reg_list_mc_addr;
1fd11777 4011 const struct cs_section_def *cs_data;
2948f5e6
AD		 4012 int r;
4013
4014 src_ptr = rdev->rlc.reg_list;
4015 dws = rdev->rlc.reg_list_size;
a0f38609
AD		 4016 if (rdev->family >= CHIP_BONAIRE) {
4017 dws += (5 * 16) + 48 + 48 + 64;
4018 }
2948f5e6
AD		 4019 cs_data = rdev->rlc.cs_data;
4020
10b7ca7e
AD		 4021 if (src_ptr) {
4022 /* save restore block */
4023 if (rdev->rlc.save_restore_obj == NULL) {
4024 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4025 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4026 NULL, &rdev->rlc.save_restore_obj);
10b7ca7e
AD		 4027 if (r) {
4028 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4029 return r;
4030 }
4031 }
4032
4033 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4034 if (unlikely(r != 0)) {
4035 sumo_rlc_fini(rdev);
4036 return r;
4037 }
4038 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4039 &rdev->rlc.save_restore_gpu_addr);
2948f5e6 4040 if (r) {
10b7ca7e
AD		 4041 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4042 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4043 sumo_rlc_fini(rdev);
2948f5e6
AD		 4044 return r;
4045 }
2948f5e6 4046
10b7ca7e
AD		 4047 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4048 if (r) {
4049 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4050 sumo_rlc_fini(rdev);
4051 return r;
4052 }
4053 /* write the sr buffer */
4054 dst_ptr = rdev->rlc.sr_ptr;
1fd11777
AD		 4055 if (rdev->family >= CHIP_TAHITI) {
4056 /* SI */
59a82d0e 4057 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6ba81e53 4058 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
1fd11777
AD		 4059 } else {
4060 /* ON/LN/TN */
4061 /* format:
4062 * dw0: (reg2 << 16) | reg1
4063 * dw1: reg1 save space
4064 * dw2: reg2 save space
4065 */
4066 for (i = 0; i < dws; i++) {
4067 data = src_ptr[i] >> 2;
4068 i++;
4069 if (i < dws)
4070 data |= (src_ptr[i] >> 2) << 16;
4071 j = (((i - 1) * 3) / 2);
6ba81e53 4072 dst_ptr[j] = cpu_to_le32(data);
1fd11777
AD		 4073 }
4074 j = ((i * 3) / 2);
6ba81e53 4075 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
10b7ca7e 4076 }
10b7ca7e 4077 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
2948f5e6 4078 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
2948f5e6 4079 }
2948f5e6 4080
10b7ca7e
AD		 4081 if (cs_data) {
4082 /* clear state block */
a0f38609
AD		 4083 if (rdev->family >= CHIP_BONAIRE) {
4084 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4085 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e
AD		 4086 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4087 dws = rdev->rlc.clear_state_size + (256 / 4);
4088 } else {
4089 reg_list_num = 0;
4090 dws = 0;
4091 for (i = 0; cs_data[i].section != NULL; i++) {
4092 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4093 reg_list_num++;
4094 dws += cs_data[i].section[j].reg_count;
4095 }
10b7ca7e 4096 }
59a82d0e
AD		 4097 reg_list_blk_index = (3 * reg_list_num + 2);
4098 dws += reg_list_blk_index;
4099 rdev->rlc.clear_state_size = dws;
2948f5e6 4100 }
2948f5e6 4101
10b7ca7e 4102 if (rdev->rlc.clear_state_obj == NULL) {
59a82d0e 4103 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
02376d82 4104 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4105 NULL, &rdev->rlc.clear_state_obj);
10b7ca7e
AD		 4106 if (r) {
4107 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4108 sumo_rlc_fini(rdev);
4109 return r;
4110 }
4111 }
4112 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4113 if (unlikely(r != 0)) {
4114 sumo_rlc_fini(rdev);
4115 return r;
4116 }
4117 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4118 &rdev->rlc.clear_state_gpu_addr);
2948f5e6 4119 if (r) {
10b7ca7e
AD		 4120 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4121 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
2948f5e6
AD		 4122 sumo_rlc_fini(rdev);
4123 return r;
4124 }
2948f5e6 4125
10b7ca7e
AD		 4126 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4127 if (r) {
4128 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4129 sumo_rlc_fini(rdev);
4130 return r;
4131 }
4132 /* set up the cs buffer */
4133 dst_ptr = rdev->rlc.cs_ptr;
a0f38609
AD		 4134 if (rdev->family >= CHIP_BONAIRE) {
4135 cik_get_csb_buffer(rdev, dst_ptr);
4136 } else if (rdev->family >= CHIP_TAHITI) {
59a82d0e 4137 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
6ba81e53
AD		 4138 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4139 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4140 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
59a82d0e
AD		 4141 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4142 } else {
4143 reg_list_hdr_blk_index = 0;
4144 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4145 data = upper_32_bits(reg_list_mc_addr);
6ba81e53 4146 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
2948f5e6 4147 reg_list_hdr_blk_index++;
59a82d0e
AD		 4148 for (i = 0; cs_data[i].section != NULL; i++) {
4149 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4150 reg_num = cs_data[i].section[j].reg_count;
4151 data = reg_list_mc_addr & 0xffffffff;
6ba81e53 4152 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4153 reg_list_hdr_blk_index++;
4154
4155 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
6ba81e53 4156 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4157 reg_list_hdr_blk_index++;
4158
4159 data = 0x08000000 | (reg_num * 4);
6ba81e53 4160 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
59a82d0e
AD		 4161 reg_list_hdr_blk_index++;
4162
4163 for (k = 0; k < reg_num; k++) {
4164 data = cs_data[i].section[j].extent[k];
6ba81e53 4165 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
59a82d0e
AD		 4166 }
4167 reg_list_mc_addr += reg_num * 4;
4168 reg_list_blk_index += reg_num;
10b7ca7e 4169 }
2948f5e6 4170 }
6ba81e53 4171 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
2948f5e6 4172 }
10b7ca7e
AD		 4173 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4174 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
2948f5e6 4175 }
2948f5e6 4176
22c775ce
AD		 4177 if (rdev->rlc.cp_table_size) {
4178 if (rdev->rlc.cp_table_obj == NULL) {
02376d82
MD		 4179 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4180 PAGE_SIZE, true,
4181 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
831b6966 4182 NULL, &rdev->rlc.cp_table_obj);
22c775ce
AD		 4183 if (r) {
4184 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4185 sumo_rlc_fini(rdev);
4186 return r;
4187 }
4188 }
4189
4190 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4191 if (unlikely(r != 0)) {
4192 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4193 sumo_rlc_fini(rdev);
4194 return r;
4195 }
4196 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4197 &rdev->rlc.cp_table_gpu_addr);
4198 if (r) {
4199 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4200 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4201 sumo_rlc_fini(rdev);
4202 return r;
4203 }
4204 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4205 if (r) {
4206 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4207 sumo_rlc_fini(rdev);
4208 return r;
4209 }
4210
4211 cik_init_cp_pg_table(rdev);
4212
4213 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4214 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4215
4216 }
2948f5e6
AD		 4217
4218 return 0;
4219}
4220
4221static void evergreen_rlc_start(struct radeon_device *rdev)
4222{
8ba10463
AD		 4223 u32 mask = RLC_ENABLE;
4224
4225 if (rdev->flags & RADEON_IS_IGP) {
4226 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
8ba10463
AD		 4227 }
4228
4229 WREG32(RLC_CNTL, mask);
2948f5e6
AD		 4230}
4231
4232int evergreen_rlc_resume(struct radeon_device *rdev)
4233{
4234 u32 i;
4235 const __be32 *fw_data;
4236
4237 if (!rdev->rlc_fw)
4238 return -EINVAL;
4239
4240 r600_rlc_stop(rdev);
4241
4242 WREG32(RLC_HB_CNTL, 0);
4243
4244 if (rdev->flags & RADEON_IS_IGP) {
8ba10463
AD		 4245 if (rdev->family == CHIP_ARUBA) {
4246 u32 always_on_bitmap =
4247 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4248 /* find out the number of active simds */
4249 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4250 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4251 tmp = hweight32(~tmp);
4252 if (tmp == rdev->config.cayman.max_simds_per_se) {
4253 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4254 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4255 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4256 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4257 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4258 }
4259 } else {
4260 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4261 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4262 }
2948f5e6
AD		 4263 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4264 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4265 } else {
4266 WREG32(RLC_HB_BASE, 0);
4267 WREG32(RLC_HB_RPTR, 0);
4268 WREG32(RLC_HB_WPTR, 0);
8ba10463
AD		 4269 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4270 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
2948f5e6 4271 }
2948f5e6
AD		 4272 WREG32(RLC_MC_CNTL, 0);
4273 WREG32(RLC_UCODE_CNTL, 0);
4274
4275 fw_data = (const __be32 *)rdev->rlc_fw->data;
4276 if (rdev->family >= CHIP_ARUBA) {
4277 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4278 WREG32(RLC_UCODE_ADDR, i);
4279 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4280 }
4281 } else if (rdev->family >= CHIP_CAYMAN) {
4282 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4283 WREG32(RLC_UCODE_ADDR, i);
4284 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4285 }
4286 } else {
4287 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4288 WREG32(RLC_UCODE_ADDR, i);
4289 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4290 }
4291 }
4292 WREG32(RLC_UCODE_ADDR, 0);
4293
4294 evergreen_rlc_start(rdev);
4295
4296 return 0;
4297}
4298
45f9a39b
AD		 4299/* Interrupts */
4300
4301u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4302{
46437057 4303 if (crtc >= rdev->num_crtc)
45f9a39b 4304 return 0;
46437057
AD		 4305 else
4306 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
45f9a39b
AD		 4307}
4308
4309void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4310{
4311 u32 tmp;
4312
1b37078b
AD		 4313 if (rdev->family >= CHIP_CAYMAN) {
4314 cayman_cp_int_cntl_setup(rdev, 0,
4315 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4316 cayman_cp_int_cntl_setup(rdev, 1, 0);
4317 cayman_cp_int_cntl_setup(rdev, 2, 0);
f60cbd11
AD		 4318 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4319 WREG32(CAYMAN_DMA1_CNTL, tmp);
1b37078b
AD		 4320 } else
4321 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
233d1ad5
AD		 4322 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4323 WREG32(DMA_CNTL, tmp);
45f9a39b
AD		 4324 WREG32(GRBM_INT_CNTL, 0);
4325 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4326 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4327 if (rdev->num_crtc >= 4) {
18007401
AD		 4328 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4329 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4330 }
4331 if (rdev->num_crtc >= 6) {
18007401
AD		 4332 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4333 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4334 }
45f9a39b
AD		 4335
4336 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4337 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
b7eff394 4338 if (rdev->num_crtc >= 4) {
18007401
AD		 4339 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4340 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
b7eff394
AD		 4341 }
4342 if (rdev->num_crtc >= 6) {
18007401
AD		 4343 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4344 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4345 }
45f9a39b 4346
e9a321c6
AD		 4347 /* only one DAC on DCE5 */
4348 if (!ASIC_IS_DCE5(rdev))
05b3ef69 4349 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
45f9a39b
AD		 4350 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4351
4352 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4353 WREG32(DC_HPD1_INT_CONTROL, tmp);
4354 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4355 WREG32(DC_HPD2_INT_CONTROL, tmp);
4356 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4357 WREG32(DC_HPD3_INT_CONTROL, tmp);
4358 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4359 WREG32(DC_HPD4_INT_CONTROL, tmp);
4360 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4361 WREG32(DC_HPD5_INT_CONTROL, tmp);
4362 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4363 WREG32(DC_HPD6_INT_CONTROL, tmp);
4364
4365}
4366
4367int evergreen_irq_set(struct radeon_device *rdev)
4368{
4369 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
1b37078b 4370 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
45f9a39b
AD		 4371 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4372 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2031f77c 4373 u32 grbm_int_cntl = 0;
f122c610 4374 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
f60cbd11 4375 u32 dma_cntl, dma_cntl1 = 0;
dc50ba7f 4376 u32 thermal_int = 0;
45f9a39b
AD		 4377
4378 if (!rdev->irq.installed) {
fce7d61b 4379 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
45f9a39b
AD		 4380 return -EINVAL;
4381 }
4382 /* don't enable anything if the ih is disabled */
4383 if (!rdev->ih.enabled) {
4384 r600_disable_interrupts(rdev);
4385 /* force the active interrupt state to all disabled */
4386 evergreen_disable_interrupt_state(rdev);
4387 return 0;
4388 }
4389
4390 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4391 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4392 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4393 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4394 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4395 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
d70229f7
AD		 4396 if (rdev->family == CHIP_ARUBA)
4397 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4398 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4399 else
4400 thermal_int = RREG32(CG_THERMAL_INT) &
4401 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
45f9a39b 4402
f122c610
AD		 4403 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4404 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4405 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4406 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4407 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4408 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4409
233d1ad5
AD		 4410 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4411
1b37078b
AD		 4412 if (rdev->family >= CHIP_CAYMAN) {
4413 /* enable CP interrupts on all rings */
736fc37f 4414 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4415 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4416 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4417 }
736fc37f 4418 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
1b37078b
AD		 4419 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4420 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4421 }
736fc37f 4422 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
1b37078b
AD		 4423 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4424 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4425 }
4426 } else {
736fc37f 4427 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
1b37078b
AD		 4428 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4429 cp_int_cntl |= RB_INT_ENABLE;
4430 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4431 }
45f9a39b 4432 }
1b37078b 4433
233d1ad5
AD		 4434 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4435 DRM_DEBUG("r600_irq_set: sw int dma\n");
4436 dma_cntl |= TRAP_ENABLE;
4437 }
4438
f60cbd11
AD		 4439 if (rdev->family >= CHIP_CAYMAN) {
4440 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4441 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4442 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4443 dma_cntl1 |= TRAP_ENABLE;
4444 }
4445 }
4446
dc50ba7f
AD		 4447 if (rdev->irq.dpm_thermal) {
4448 DRM_DEBUG("dpm thermal\n");
4449 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4450 }
4451
6f34be50 4452 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4453 atomic_read(&rdev->irq.pflip[0])) {
45f9a39b
AD		 4454 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4455 crtc1 |= VBLANK_INT_MASK;
4456 }
6f34be50 4457 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4458 atomic_read(&rdev->irq.pflip[1])) {
45f9a39b
AD		 4459 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4460 crtc2 |= VBLANK_INT_MASK;
4461 }
6f34be50 4462 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4463 atomic_read(&rdev->irq.pflip[2])) {
45f9a39b
AD		 4464 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4465 crtc3 |= VBLANK_INT_MASK;
4466 }
6f34be50 4467 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4468 atomic_read(&rdev->irq.pflip[3])) {
45f9a39b
AD		 4469 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4470 crtc4 |= VBLANK_INT_MASK;
4471 }
6f34be50 4472 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4473 atomic_read(&rdev->irq.pflip[4])) {
45f9a39b
AD		 4474 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4475 crtc5 |= VBLANK_INT_MASK;
4476 }
6f34be50 4477 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4478 atomic_read(&rdev->irq.pflip[5])) {
45f9a39b
AD		 4479 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4480 crtc6 |= VBLANK_INT_MASK;
4481 }
4482 if (rdev->irq.hpd[0]) {
4483 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4484 hpd1 |= DC_HPDx_INT_EN;
4485 }
4486 if (rdev->irq.hpd[1]) {
4487 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4488 hpd2 |= DC_HPDx_INT_EN;
4489 }
4490 if (rdev->irq.hpd[2]) {
4491 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4492 hpd3 |= DC_HPDx_INT_EN;
4493 }
4494 if (rdev->irq.hpd[3]) {
4495 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4496 hpd4 |= DC_HPDx_INT_EN;
4497 }
4498 if (rdev->irq.hpd[4]) {
4499 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4500 hpd5 |= DC_HPDx_INT_EN;
4501 }
4502 if (rdev->irq.hpd[5]) {
4503 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4504 hpd6 |= DC_HPDx_INT_EN;
4505 }
f122c610
AD		 4506 if (rdev->irq.afmt[0]) {
4507 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4508 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4509 }
4510 if (rdev->irq.afmt[1]) {
4511 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4512 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4513 }
4514 if (rdev->irq.afmt[2]) {
4515 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4516 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4517 }
4518 if (rdev->irq.afmt[3]) {
4519 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4520 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4521 }
4522 if (rdev->irq.afmt[4]) {
4523 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4524 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4525 }
4526 if (rdev->irq.afmt[5]) {
4527 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4528 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4529 }
45f9a39b 4530
1b37078b
AD		 4531 if (rdev->family >= CHIP_CAYMAN) {
4532 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4533 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4534 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4535 } else
4536 WREG32(CP_INT_CNTL, cp_int_cntl);
233d1ad5
AD		 4537
4538 WREG32(DMA_CNTL, dma_cntl);
4539
f60cbd11
AD		 4540 if (rdev->family >= CHIP_CAYMAN)
4541 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4542
2031f77c 4543 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
45f9a39b
AD		 4544
4545 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4546 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
b7eff394 4547 if (rdev->num_crtc >= 4) {
18007401
AD		 4548 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4549 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
b7eff394
AD		 4550 }
4551 if (rdev->num_crtc >= 6) {
18007401
AD		 4552 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4553 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4554 }
45f9a39b 4555
f5d636d2
CK		 4556 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4557 GRPH_PFLIP_INT_MASK);
4558 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4559 GRPH_PFLIP_INT_MASK);
b7eff394 4560 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 4561 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4562 GRPH_PFLIP_INT_MASK);
4563 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4564 GRPH_PFLIP_INT_MASK);
b7eff394
AD		 4565 }
4566 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 4567 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4568 GRPH_PFLIP_INT_MASK);
4569 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4570 GRPH_PFLIP_INT_MASK);
b7eff394 4571 }
6f34be50 4572
45f9a39b
AD		 4573 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4574 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4575 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4576 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4577 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4578 WREG32(DC_HPD6_INT_CONTROL, hpd6);
d70229f7
AD		 4579 if (rdev->family == CHIP_ARUBA)
4580 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4581 else
4582 WREG32(CG_THERMAL_INT, thermal_int);
45f9a39b 4583
f122c610
AD		 4584 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4585 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4586 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4587 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4588 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4589 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4590
bcc1c2a1
AD		 4591 return 0;
4592}
4593
cbdd4501 4594static void evergreen_irq_ack(struct radeon_device *rdev)
45f9a39b
AD		 4595{
4596 u32 tmp;
4597
6f34be50
AD		 4598 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4599 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4600 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4601 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4602 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4603 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4604 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4605 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
b7eff394
AD		 4606 if (rdev->num_crtc >= 4) {
4607 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4608 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4609 }
4610 if (rdev->num_crtc >= 6) {
4611 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4612 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4613 }
6f34be50 4614
f122c610
AD		 4615 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4616 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4617 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4618 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4619 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4620 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4621
6f34be50
AD		 4622 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4623 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4624 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4625 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6f34be50 4626 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
45f9a39b 4627 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4628 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
45f9a39b 4629 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6f34be50 4630 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
45f9a39b 4631 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6f34be50 4632 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
45f9a39b
AD		 4633 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4634
b7eff394
AD		 4635 if (rdev->num_crtc >= 4) {
4636 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4637 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4638 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4639 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4640 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4641 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4642 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4643 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4644 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4645 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4646 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4647 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4648 }
4649
4650 if (rdev->num_crtc >= 6) {
4651 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4652 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4653 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4654 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4655 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4656 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4657 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4658 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4659 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4660 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4661 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4662 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4663 }
45f9a39b 4664
6f34be50 4665 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
45f9a39b
AD		 4666 tmp = RREG32(DC_HPD1_INT_CONTROL);
4667 tmp |= DC_HPDx_INT_ACK;
4668 WREG32(DC_HPD1_INT_CONTROL, tmp);
4669 }
6f34be50 4670 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
45f9a39b
AD		 4671 tmp = RREG32(DC_HPD2_INT_CONTROL);
4672 tmp |= DC_HPDx_INT_ACK;
4673 WREG32(DC_HPD2_INT_CONTROL, tmp);
4674 }
6f34be50 4675 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
45f9a39b
AD		 4676 tmp = RREG32(DC_HPD3_INT_CONTROL);
4677 tmp |= DC_HPDx_INT_ACK;
4678 WREG32(DC_HPD3_INT_CONTROL, tmp);
4679 }
6f34be50 4680 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
45f9a39b
AD		 4681 tmp = RREG32(DC_HPD4_INT_CONTROL);
4682 tmp |= DC_HPDx_INT_ACK;
4683 WREG32(DC_HPD4_INT_CONTROL, tmp);
4684 }
6f34be50 4685 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
45f9a39b
AD		 4686 tmp = RREG32(DC_HPD5_INT_CONTROL);
4687 tmp |= DC_HPDx_INT_ACK;
4688 WREG32(DC_HPD5_INT_CONTROL, tmp);
4689 }
6f34be50 4690 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
45f9a39b
AD		 4691 tmp = RREG32(DC_HPD5_INT_CONTROL);
4692 tmp |= DC_HPDx_INT_ACK;
4693 WREG32(DC_HPD6_INT_CONTROL, tmp);
4694 }
f122c610
AD		 4695 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4696 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4697 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4698 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4699 }
4700 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4701 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4702 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4703 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4704 }
4705 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4706 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4707 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4708 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4709 }
4710 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4711 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4712 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4713 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4714 }
4715 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4716 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4717 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4718 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4719 }
4720 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4721 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4722 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4723 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4724 }
45f9a39b
AD		 4725}
4726
1109ca09 4727static void evergreen_irq_disable(struct radeon_device *rdev)
45f9a39b 4728{
45f9a39b
AD		 4729 r600_disable_interrupts(rdev);
4730 /* Wait and acknowledge irq */
4731 mdelay(1);
6f34be50 4732 evergreen_irq_ack(rdev);
45f9a39b
AD		 4733 evergreen_disable_interrupt_state(rdev);
4734}
4735
755d819e 4736void evergreen_irq_suspend(struct radeon_device *rdev)
45f9a39b
AD		 4737{
4738 evergreen_irq_disable(rdev);
4739 r600_rlc_stop(rdev);
4740}
4741
cbdd4501 4742static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
45f9a39b
AD		 4743{
4744 u32 wptr, tmp;
4745
724c80e1 4746 if (rdev->wb.enabled)
204ae24d 4747 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
724c80e1
AD		 4748 else
4749 wptr = RREG32(IH_RB_WPTR);
45f9a39b
AD		 4750
4751 if (wptr & RB_OVERFLOW) {
11bab0ae 4752 wptr &= ~RB_OVERFLOW;
45f9a39b
AD		 4753 /* When a ring buffer overflow happen start parsing interrupt
4754 * from the last not overwritten vector (wptr + 16). Hopefully
4755 * this should allow us to catchup.
4756 */
6cc2fda2
MD		 4757 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4758 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
45f9a39b
AD		 4759 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4760 tmp = RREG32(IH_RB_CNTL);
4761 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4762 WREG32(IH_RB_CNTL, tmp);
4763 }
4764 return (wptr & rdev->ih.ptr_mask);
4765}
4766
4767int evergreen_irq_process(struct radeon_device *rdev)
4768{
682f1a54
DA		 4769 u32 wptr;
4770 u32 rptr;
45f9a39b
AD		 4771 u32 src_id, src_data;
4772 u32 ring_index;
45f9a39b 4773 bool queue_hotplug = false;
f122c610 4774 bool queue_hdmi = false;
dc50ba7f 4775 bool queue_thermal = false;
54e2e49c 4776 u32 status, addr;
45f9a39b 4777
682f1a54 4778 if (!rdev->ih.enabled || rdev->shutdown)
45f9a39b
AD		 4779 return IRQ_NONE;
4780
682f1a54 4781 wptr = evergreen_get_ih_wptr(rdev);
c20dc369
CK		 4782
4783restart_ih:
4784 /* is somebody else already processing irqs? */
4785 if (atomic_xchg(&rdev->ih.lock, 1))
4786 return IRQ_NONE;
4787
682f1a54
DA		 4788 rptr = rdev->ih.rptr;
4789 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
45f9a39b 4790
964f6645
BH		 4791 /* Order reading of wptr vs. reading of IH ring data */
4792 rmb();
4793
45f9a39b 4794 /* display interrupts */
6f34be50 4795 evergreen_irq_ack(rdev);
45f9a39b 4796
45f9a39b
AD		 4797 while (rptr != wptr) {
4798 /* wptr/rptr are in bytes! */
4799 ring_index = rptr / 4;
0f234f5f
AD		 4800 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4801 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
45f9a39b
AD		 4802
4803 switch (src_id) {
4804 case 1: /* D1 vblank/vline */
4805 switch (src_data) {
4806 case 0: /* D1 vblank */
6f34be50 4807 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6f34be50
AD		 4808 if (rdev->irq.crtc_vblank_int[0]) {
4809 drm_handle_vblank(rdev->ddev, 0);
4810 rdev->pm.vblank_sync = true;
4811 wake_up(&rdev->irq.vblank_queue);
4812 }
736fc37f 4813 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 4814 radeon_crtc_handle_vblank(rdev, 0);
6f34be50 4815 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
45f9a39b
AD		 4816 DRM_DEBUG("IH: D1 vblank\n");
4817 }
4818 break;
4819 case 1: /* D1 vline */
6f34be50
AD		 4820 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4821 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
45f9a39b
AD		 4822 DRM_DEBUG("IH: D1 vline\n");
4823 }
4824 break;
4825 default:
4826 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4827 break;
4828 }
4829 break;
4830 case 2: /* D2 vblank/vline */
4831 switch (src_data) {
4832 case 0: /* D2 vblank */
6f34be50 4833 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6f34be50
AD		 4834 if (rdev->irq.crtc_vblank_int[1]) {
4835 drm_handle_vblank(rdev->ddev, 1);
4836 rdev->pm.vblank_sync = true;
4837 wake_up(&rdev->irq.vblank_queue);
4838 }
736fc37f 4839 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 4840 radeon_crtc_handle_vblank(rdev, 1);
6f34be50 4841 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
45f9a39b
AD		 4842 DRM_DEBUG("IH: D2 vblank\n");
4843 }
4844 break;
4845 case 1: /* D2 vline */
6f34be50
AD		 4846 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4847 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
45f9a39b
AD		 4848 DRM_DEBUG("IH: D2 vline\n");
4849 }
4850 break;
4851 default:
4852 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4853 break;
4854 }
4855 break;
4856 case 3: /* D3 vblank/vline */
4857 switch (src_data) {
4858 case 0: /* D3 vblank */
6f34be50
AD		 4859 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4860 if (rdev->irq.crtc_vblank_int[2]) {
4861 drm_handle_vblank(rdev->ddev, 2);
4862 rdev->pm.vblank_sync = true;
4863 wake_up(&rdev->irq.vblank_queue);
4864 }
736fc37f 4865 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 4866 radeon_crtc_handle_vblank(rdev, 2);
6f34be50 4867 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
45f9a39b
AD		 4868 DRM_DEBUG("IH: D3 vblank\n");
4869 }
4870 break;
4871 case 1: /* D3 vline */
6f34be50
AD		 4872 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4873 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
45f9a39b
AD		 4874 DRM_DEBUG("IH: D3 vline\n");
4875 }
4876 break;
4877 default:
4878 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4879 break;
4880 }
4881 break;
4882 case 4: /* D4 vblank/vline */
4883 switch (src_data) {
4884 case 0: /* D4 vblank */
6f34be50
AD		 4885 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4886 if (rdev->irq.crtc_vblank_int[3]) {
4887 drm_handle_vblank(rdev->ddev, 3);
4888 rdev->pm.vblank_sync = true;
4889 wake_up(&rdev->irq.vblank_queue);
4890 }
736fc37f 4891 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 4892 radeon_crtc_handle_vblank(rdev, 3);
6f34be50 4893 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
45f9a39b
AD		 4894 DRM_DEBUG("IH: D4 vblank\n");
4895 }
4896 break;
4897 case 1: /* D4 vline */
6f34be50
AD		 4898 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4899 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
45f9a39b
AD		 4900 DRM_DEBUG("IH: D4 vline\n");
4901 }
4902 break;
4903 default:
4904 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905 break;
4906 }
4907 break;
4908 case 5: /* D5 vblank/vline */
4909 switch (src_data) {
4910 case 0: /* D5 vblank */
6f34be50
AD		 4911 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4912 if (rdev->irq.crtc_vblank_int[4]) {
4913 drm_handle_vblank(rdev->ddev, 4);
4914 rdev->pm.vblank_sync = true;
4915 wake_up(&rdev->irq.vblank_queue);
4916 }
736fc37f 4917 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 4918 radeon_crtc_handle_vblank(rdev, 4);
6f34be50 4919 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
45f9a39b
AD		 4920 DRM_DEBUG("IH: D5 vblank\n");
4921 }
4922 break;
4923 case 1: /* D5 vline */
6f34be50
AD		 4924 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4925 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
45f9a39b
AD		 4926 DRM_DEBUG("IH: D5 vline\n");
4927 }
4928 break;
4929 default:
4930 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4931 break;
4932 }
4933 break;
4934 case 6: /* D6 vblank/vline */
4935 switch (src_data) {
4936 case 0: /* D6 vblank */
6f34be50
AD		 4937 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4938 if (rdev->irq.crtc_vblank_int[5]) {
4939 drm_handle_vblank(rdev->ddev, 5);
4940 rdev->pm.vblank_sync = true;
4941 wake_up(&rdev->irq.vblank_queue);
4942 }
736fc37f 4943 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 4944 radeon_crtc_handle_vblank(rdev, 5);
6f34be50 4945 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
45f9a39b
AD		 4946 DRM_DEBUG("IH: D6 vblank\n");
4947 }
4948 break;
4949 case 1: /* D6 vline */
6f34be50
AD		 4950 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4951 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
45f9a39b
AD		 4952 DRM_DEBUG("IH: D6 vline\n");
4953 }
4954 break;
4955 default:
4956 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4957 break;
4958 }
4959 break;
f5d636d2
CK		 4960 case 8: /* D1 page flip */
4961 case 10: /* D2 page flip */
4962 case 12: /* D3 page flip */
4963 case 14: /* D4 page flip */
4964 case 16: /* D5 page flip */
4965 case 18: /* D6 page flip */
4966 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 4967 if (radeon_use_pflipirq > 0)
4968 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 4969 break;
45f9a39b
AD		 4970 case 42: /* HPD hotplug */
4971 switch (src_data) {
4972 case 0:
6f34be50
AD		 4973 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4974 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
45f9a39b
AD		 4975 queue_hotplug = true;
4976 DRM_DEBUG("IH: HPD1\n");
4977 }
4978 break;
4979 case 1:
6f34be50
AD		 4980 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4981 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
45f9a39b
AD		 4982 queue_hotplug = true;
4983 DRM_DEBUG("IH: HPD2\n");
4984 }
4985 break;
4986 case 2:
6f34be50
AD		 4987 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4988 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
45f9a39b
AD		 4989 queue_hotplug = true;
4990 DRM_DEBUG("IH: HPD3\n");
4991 }
4992 break;
4993 case 3:
6f34be50
AD		 4994 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4995 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
45f9a39b
AD		 4996 queue_hotplug = true;
4997 DRM_DEBUG("IH: HPD4\n");
4998 }
4999 break;
5000 case 4:
6f34be50
AD		 5001 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5002 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
45f9a39b
AD		 5003 queue_hotplug = true;
5004 DRM_DEBUG("IH: HPD5\n");
5005 }
5006 break;
5007 case 5:
6f34be50
AD		 5008 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5009 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
45f9a39b
AD		 5010 queue_hotplug = true;
5011 DRM_DEBUG("IH: HPD6\n");
5012 }
5013 break;
5014 default:
5015 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5016 break;
5017 }
5018 break;
f122c610
AD		 5019 case 44: /* hdmi */
5020 switch (src_data) {
5021 case 0:
5022 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5023 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5024 queue_hdmi = true;
5025 DRM_DEBUG("IH: HDMI0\n");
5026 }
5027 break;
5028 case 1:
5029 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5030 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5031 queue_hdmi = true;
5032 DRM_DEBUG("IH: HDMI1\n");
5033 }
5034 break;
5035 case 2:
5036 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5037 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5038 queue_hdmi = true;
5039 DRM_DEBUG("IH: HDMI2\n");
5040 }
5041 break;
5042 case 3:
5043 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5044 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5045 queue_hdmi = true;
5046 DRM_DEBUG("IH: HDMI3\n");
5047 }
5048 break;
5049 case 4:
5050 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5051 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5052 queue_hdmi = true;
5053 DRM_DEBUG("IH: HDMI4\n");
5054 }
5055 break;
5056 case 5:
5057 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5058 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5059 queue_hdmi = true;
5060 DRM_DEBUG("IH: HDMI5\n");
5061 }
5062 break;
5063 default:
5064 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5065 break;
5066 }
f2ba57b5
CK		 5067 case 124: /* UVD */
5068 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5069 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
f122c610 5070 break;
ae133a11
CK		 5071 case 146:
5072 case 147:
54e2e49c
AD		 5073 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5074 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
9b7d786b
CK		 5075 /* reset addr and status */
5076 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5077 if (addr == 0x0 && status == 0x0)
5078 break;
ae133a11
CK		 5079 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5080 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
54e2e49c 5081 addr);
ae133a11 5082 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
54e2e49c
AD		 5083 status);
5084 cayman_vm_decode_fault(rdev, status, addr);
ae133a11 5085 break;
45f9a39b
AD		 5086 case 176: /* CP_INT in ring buffer */
5087 case 177: /* CP_INT in IB1 */
5088 case 178: /* CP_INT in IB2 */
5089 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
7465280c 5090 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b
AD		 5091 break;
5092 case 181: /* CP EOP event */
5093 DRM_DEBUG("IH: CP EOP\n");
1b37078b
AD		 5094 if (rdev->family >= CHIP_CAYMAN) {
5095 switch (src_data) {
5096 case 0:
5097 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5098 break;
5099 case 1:
5100 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5101 break;
5102 case 2:
5103 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5104 break;
5105 }
5106 } else
5107 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
45f9a39b 5108 break;
233d1ad5
AD		 5109 case 224: /* DMA trap event */
5110 DRM_DEBUG("IH: DMA trap\n");
5111 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5112 break;
dc50ba7f
AD		 5113 case 230: /* thermal low to high */
5114 DRM_DEBUG("IH: thermal low to high\n");
5115 rdev->pm.dpm.thermal.high_to_low = false;
5116 queue_thermal = true;
5117 break;
5118 case 231: /* thermal high to low */
5119 DRM_DEBUG("IH: thermal high to low\n");
5120 rdev->pm.dpm.thermal.high_to_low = true;
5121 queue_thermal = true;
5122 break;
2031f77c 5123 case 233: /* GUI IDLE */
303c805c 5124 DRM_DEBUG("IH: GUI idle\n");
2031f77c 5125 break;
f60cbd11
AD		 5126 case 244: /* DMA trap event */
5127 if (rdev->family >= CHIP_CAYMAN) {
5128 DRM_DEBUG("IH: DMA1 trap\n");
5129 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5130 }
5131 break;
45f9a39b
AD		 5132 default:
5133 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5134 break;
5135 }
5136
5137 /* wptr/rptr are in bytes! */
5138 rptr += 16;
5139 rptr &= rdev->ih.ptr_mask;
f55e03b9 5140 WREG32(IH_RB_RPTR, rptr);
45f9a39b 5141 }
45f9a39b 5142 if (queue_hotplug)
32c87fca 5143 schedule_work(&rdev->hotplug_work);
f122c610
AD		 5144 if (queue_hdmi)
5145 schedule_work(&rdev->audio_work);
dc50ba7f
AD		 5146 if (queue_thermal && rdev->pm.dpm_enabled)
5147 schedule_work(&rdev->pm.dpm.thermal.work);
45f9a39b 5148 rdev->ih.rptr = rptr;
c20dc369
CK		 5149 atomic_set(&rdev->ih.lock, 0);
5150
5151 /* make sure wptr hasn't changed while processing */
5152 wptr = evergreen_get_ih_wptr(rdev);
5153 if (wptr != rptr)
5154 goto restart_ih;
5155
45f9a39b
AD		 5156 return IRQ_HANDLED;
5157}
5158
bcc1c2a1
AD		 5159static int evergreen_startup(struct radeon_device *rdev)
5160{
f2ba57b5 5161 struct radeon_ring *ring;
bcc1c2a1
AD		 5162 int r;
5163
9e46a48d 5164 /* enable pcie gen2 link */
cd54033a 5165 evergreen_pcie_gen2_enable(rdev);
f52382d7
AD		 5166 /* enable aspm */
5167 evergreen_program_aspm(rdev);
9e46a48d 5168
e5903d39
AD		 5169 /* scratch needs to be initialized before MC */
5170 r = r600_vram_scratch_init(rdev);
5171 if (r)
5172 return r;
5173
6fab3feb
AD		 5174 evergreen_mc_program(rdev);
5175
6c7bccea 5176 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
755d819e 5177 r = ni_mc_load_microcode(rdev);
bcc1c2a1 5178 if (r) {
0af62b01 5179 DRM_ERROR("Failed to load MC firmware!\n");
bcc1c2a1
AD		 5180 return r;
5181 }
5182 }
fe251e2f 5183
bcc1c2a1 5184 if (rdev->flags & RADEON_IS_AGP) {
0fcdb61e 5185 evergreen_agp_enable(rdev);
bcc1c2a1
AD		 5186 } else {
5187 r = evergreen_pcie_gart_enable(rdev);
5188 if (r)
5189 return r;
5190 }
bcc1c2a1 5191 evergreen_gpu_init(rdev);
bcc1c2a1 5192
2948f5e6
AD		 5193 /* allocate rlc buffers */
5194 if (rdev->flags & RADEON_IS_IGP) {
5195 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
1fd11777
AD		 5196 rdev->rlc.reg_list_size =
5197 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
2948f5e6
AD		 5198 rdev->rlc.cs_data = evergreen_cs_data;
5199 r = sumo_rlc_init(rdev);
5200 if (r) {
5201 DRM_ERROR("Failed to init rlc BOs!\n");
5202 return r;
5203 }
5204 }
5205
724c80e1
AD		 5206 /* allocate wb buffer */
5207 r = radeon_wb_init(rdev);
5208 if (r)
5209 return r;
5210
30eb77f4
JG		 5211 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5212 if (r) {
5213 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5214 return r;
5215 }
5216
233d1ad5
AD		 5217 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5218 if (r) {
5219 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5220 return r;
5221 }
5222
e409b128 5223 r = uvd_v2_2_resume(rdev);
f2ba57b5
CK		 5224 if (!r) {
5225 r = radeon_fence_driver_start_ring(rdev,
5226 R600_RING_TYPE_UVD_INDEX);
5227 if (r)
5228 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5229 }
5230
5231 if (r)
5232 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5233
bcc1c2a1 5234 /* Enable IRQ */
e49f3959
AH		 5235 if (!rdev->irq.installed) {
5236 r = radeon_irq_kms_init(rdev);
5237 if (r)
5238 return r;
5239 }
5240
bcc1c2a1
AD		 5241 r = r600_irq_init(rdev);
5242 if (r) {
5243 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5244 radeon_irq_kms_fini(rdev);
5245 return r;
5246 }
45f9a39b 5247 evergreen_irq_set(rdev);
bcc1c2a1 5248
f2ba57b5 5249 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
e32eb50d 5250 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 5251 RADEON_CP_PACKET2);
bcc1c2a1
AD		 5252 if (r)
5253 return r;
233d1ad5
AD		 5254
5255 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5256 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 5257 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
233d1ad5
AD		 5258 if (r)
5259 return r;
5260
bcc1c2a1
AD		 5261 r = evergreen_cp_load_microcode(rdev);
5262 if (r)
5263 return r;
fe251e2f 5264 r = evergreen_cp_resume(rdev);
233d1ad5
AD		 5265 if (r)
5266 return r;
5267 r = r600_dma_resume(rdev);
bcc1c2a1
AD		 5268 if (r)
5269 return r;
fe251e2f 5270
f2ba57b5
CK		 5271 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5272 if (ring->ring_size) {
02c9f7fa 5273 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 5274 RADEON_CP_PACKET2);
f2ba57b5 5275 if (!r)
e409b128 5276 r = uvd_v1_0_init(rdev);
f2ba57b5
CK		 5277
5278 if (r)
5279 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5280 }
5281
2898c348
CK		 5282 r = radeon_ib_pool_init(rdev);
5283 if (r) {
5284 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
b15ba512 5285 return r;
2898c348 5286 }
b15ba512 5287
69d2ae57
RM		 5288 r = r600_audio_init(rdev);
5289 if (r) {
5290 DRM_ERROR("radeon: audio init failed\n");
b15ba512
JG		 5291 return r;
5292 }
5293
bcc1c2a1
AD		 5294 return 0;
5295}
5296
5297int evergreen_resume(struct radeon_device *rdev)
5298{
5299 int r;
5300
86f5c9ed
AD		 5301 /* reset the asic, the gfx blocks are often in a bad state
5302 * after the driver is unloaded or after a resume
5303 */
5304 if (radeon_asic_reset(rdev))
5305 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1
AD		 5306 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5307 * posting will perform necessary task to bring back GPU into good
5308 * shape.
5309 */
5310 /* post card */
5311 atom_asic_init(rdev->mode_info.atom_context);
bcc1c2a1 5312
d4788db3
AD		 5313 /* init golden registers */
5314 evergreen_init_golden_registers(rdev);
5315
bc6a6295
AD		 5316 if (rdev->pm.pm_method == PM_METHOD_DPM)
5317 radeon_pm_resume(rdev);
6c7bccea 5318
b15ba512 5319 rdev->accel_working = true;
bcc1c2a1
AD		 5320 r = evergreen_startup(rdev);
5321 if (r) {
755d819e 5322 DRM_ERROR("evergreen startup failed on resume\n");
6b7746e8 5323 rdev->accel_working = false;
bcc1c2a1
AD		 5324 return r;
5325 }
fe251e2f 5326
bcc1c2a1
AD		 5327 return r;
5328
5329}
5330
5331int evergreen_suspend(struct radeon_device *rdev)
5332{
6c7bccea 5333 radeon_pm_suspend(rdev);
69d2ae57 5334 r600_audio_fini(rdev);
e409b128 5335 uvd_v1_0_fini(rdev);
f2ba57b5 5336 radeon_uvd_suspend(rdev);
bcc1c2a1 5337 r700_cp_stop(rdev);
233d1ad5 5338 r600_dma_stop(rdev);
45f9a39b 5339 evergreen_irq_suspend(rdev);
724c80e1 5340 radeon_wb_disable(rdev);
bcc1c2a1 5341 evergreen_pcie_gart_disable(rdev);
d7ccd8fc
AD		 5342
5343 return 0;
5344}
5345
bcc1c2a1
AD		 5346/* Plan is to move initialization in that function and use
5347 * helper function so that radeon_device_init pretty much
5348 * do nothing more than calling asic specific function. This
5349 * should also allow to remove a bunch of callback function
5350 * like vram_info.
5351 */
5352int evergreen_init(struct radeon_device *rdev)
5353{
5354 int r;
5355
bcc1c2a1
AD		 5356 /* Read BIOS */
5357 if (!radeon_get_bios(rdev)) {
5358 if (ASIC_IS_AVIVO(rdev))
5359 return -EINVAL;
5360 }
5361 /* Must be an ATOMBIOS */
5362 if (!rdev->is_atom_bios) {
755d819e 5363 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
bcc1c2a1
AD		 5364 return -EINVAL;
5365 }
5366 r = radeon_atombios_init(rdev);
5367 if (r)
5368 return r;
86f5c9ed
AD		 5369 /* reset the asic, the gfx blocks are often in a bad state
5370 * after the driver is unloaded or after a resume
5371 */
5372 if (radeon_asic_reset(rdev))
5373 dev_warn(rdev->dev, "GPU reset failed !\n");
bcc1c2a1 5374 /* Post card if necessary */
fd909c37 5375 if (!radeon_card_posted(rdev)) {
bcc1c2a1
AD		 5376 if (!rdev->bios) {
5377 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5378 return -EINVAL;
5379 }
5380 DRM_INFO("GPU not posted. posting now...\n");
5381 atom_asic_init(rdev->mode_info.atom_context);
5382 }
d4788db3
AD		 5383 /* init golden registers */
5384 evergreen_init_golden_registers(rdev);
bcc1c2a1
AD		 5385 /* Initialize scratch registers */
5386 r600_scratch_init(rdev);
5387 /* Initialize surface registers */
5388 radeon_surface_init(rdev);
5389 /* Initialize clocks */
5390 radeon_get_clock_info(rdev->ddev);
bcc1c2a1
AD		 5391 /* Fence driver */
5392 r = radeon_fence_driver_init(rdev);
5393 if (r)
5394 return r;
d594e46a
JG		 5395 /* initialize AGP */
5396 if (rdev->flags & RADEON_IS_AGP) {
5397 r = radeon_agp_init(rdev);
5398 if (r)
5399 radeon_agp_disable(rdev);
5400 }
5401 /* initialize memory controller */
bcc1c2a1
AD		 5402 r = evergreen_mc_init(rdev);
5403 if (r)
5404 return r;
5405 /* Memory manager */
5406 r = radeon_bo_init(rdev);
5407 if (r)
5408 return r;
45f9a39b 5409
01ac8794
AD		 5410 if (ASIC_IS_DCE5(rdev)) {
5411 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5412 r = ni_init_microcode(rdev);
5413 if (r) {
5414 DRM_ERROR("Failed to load firmware!\n");
5415 return r;
5416 }
5417 }
5418 } else {
5419 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5420 r = r600_init_microcode(rdev);
5421 if (r) {
5422 DRM_ERROR("Failed to load firmware!\n");
5423 return r;
5424 }
5425 }
5426 }
5427
6c7bccea
AD		 5428 /* Initialize power management */
5429 radeon_pm_init(rdev);
5430
e32eb50d
CK		 5431 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5432 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
bcc1c2a1 5433
233d1ad5
AD		 5434 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5435 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5436
f2ba57b5
CK		 5437 r = radeon_uvd_init(rdev);
5438 if (!r) {
5439 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5440 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5441 4096);
5442 }
5443
bcc1c2a1
AD		 5444 rdev->ih.ring_obj = NULL;
5445 r600_ih_ring_init(rdev, 64 * 1024);
5446
5447 r = r600_pcie_gart_init(rdev);
5448 if (r)
5449 return r;
0fcdb61e 5450
148a03bc 5451 rdev->accel_working = true;
bcc1c2a1
AD		 5452 r = evergreen_startup(rdev);
5453 if (r) {
fe251e2f
AD		 5454 dev_err(rdev->dev, "disabling GPU acceleration\n");
5455 r700_cp_fini(rdev);
233d1ad5 5456 r600_dma_fini(rdev);
fe251e2f 5457 r600_irq_fini(rdev);
2948f5e6
AD		 5458 if (rdev->flags & RADEON_IS_IGP)
5459 sumo_rlc_fini(rdev);
724c80e1 5460 radeon_wb_fini(rdev);
2898c348 5461 radeon_ib_pool_fini(rdev);
fe251e2f 5462 radeon_irq_kms_fini(rdev);
0fcdb61e 5463 evergreen_pcie_gart_fini(rdev);
bcc1c2a1
AD		 5464 rdev->accel_working = false;
5465 }
77e00f2e
AD		 5466
5467 /* Don't start up if the MC ucode is missing on BTC parts.
5468 * The default clocks and voltages before the MC ucode
5469 * is loaded are not suffient for advanced operations.
5470 */
5471 if (ASIC_IS_DCE5(rdev)) {
5472 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5473 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5474 return -EINVAL;
5475 }
5476 }
5477
bcc1c2a1
AD		 5478 return 0;
5479}
5480
5481void evergreen_fini(struct radeon_device *rdev)
5482{
6c7bccea 5483 radeon_pm_fini(rdev);
69d2ae57 5484 r600_audio_fini(rdev);
45f9a39b 5485 r700_cp_fini(rdev);
233d1ad5 5486 r600_dma_fini(rdev);
bcc1c2a1 5487 r600_irq_fini(rdev);
2948f5e6
AD		 5488 if (rdev->flags & RADEON_IS_IGP)
5489 sumo_rlc_fini(rdev);
724c80e1 5490 radeon_wb_fini(rdev);
2898c348 5491 radeon_ib_pool_fini(rdev);
bcc1c2a1 5492 radeon_irq_kms_fini(rdev);
e409b128 5493 uvd_v1_0_fini(rdev);
f2ba57b5 5494 radeon_uvd_fini(rdev);
d9654413 5495 evergreen_pcie_gart_fini(rdev);
16cdf04d 5496 r600_vram_scratch_fini(rdev);
bcc1c2a1
AD		 5497 radeon_gem_fini(rdev);
5498 radeon_fence_driver_fini(rdev);
bcc1c2a1
AD		 5499 radeon_agp_fini(rdev);
5500 radeon_bo_fini(rdev);
5501 radeon_atombios_fini(rdev);
5502 kfree(rdev->bios);
5503 rdev->bios = NULL;
bcc1c2a1 5504}
9e46a48d 5505
b07759bf 5506void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
9e46a48d 5507{
7e0e4196 5508 u32 link_width_cntl, speed_cntl;
9e46a48d 5509
d42dd579
AD		 5510 if (radeon_pcie_gen2 == 0)
5511 return;
5512
9e46a48d
AD		 5513 if (rdev->flags & RADEON_IS_IGP)
5514 return;
5515
5516 if (!(rdev->flags & RADEON_IS_PCIE))
5517 return;
5518
5519 /* x2 cards have a special sequence */
5520 if (ASIC_IS_X2(rdev))
5521 return;
5522
7e0e4196
KSS		 5523 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5524 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
197bbb3d
DA		 5525 return;
5526
492d2b61 5527 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3691feea
AD		 5528 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5529 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5530 return;
5531 }
5532
197bbb3d
DA		 5533 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5534
9e46a48d
AD		 5535 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5536 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5537
492d2b61 5538 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d 5539 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5540 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d 5541
492d2b61 5542 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5543 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
492d2b61 5544 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5545
492d2b61 5546 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5547 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5548 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5549
492d2b61 5550 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5551 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
492d2b61 5552 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d 5553
492d2b61 5554 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9e46a48d 5555 speed_cntl |= LC_GEN2_EN_STRAP;
492d2b61 5556 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9e46a48d
AD		 5557
5558 } else {
492d2b61 5559 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9e46a48d
AD		 5560 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5561 if (1)
5562 link_width_cntl |= LC_UPCONFIGURE_DIS;
5563 else
5564 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
492d2b61 5565 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
9e46a48d
AD		 5566 }
5567}
f52382d7
AD		 5568
5569void evergreen_program_aspm(struct radeon_device *rdev)
5570{
5571 u32 data, orig;
5572 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5573 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5574 /* fusion_platform = true
5575 * if the system is a fusion system
5576 * (APU or DGPU in a fusion system).
5577 * todo: check if the system is a fusion platform.
5578 */
5579 bool fusion_platform = false;
5580
1294d4a3
AD		 5581 if (radeon_aspm == 0)
5582 return;
5583
f52382d7
AD		 5584 if (!(rdev->flags & RADEON_IS_PCIE))
5585 return;
5586
5587 switch (rdev->family) {
5588 case CHIP_CYPRESS:
5589 case CHIP_HEMLOCK:
5590 case CHIP_JUNIPER:
5591 case CHIP_REDWOOD:
5592 case CHIP_CEDAR:
5593 case CHIP_SUMO:
5594 case CHIP_SUMO2:
5595 case CHIP_PALM:
5596 case CHIP_ARUBA:
5597 disable_l0s = true;
5598 break;
5599 default:
5600 disable_l0s = false;
5601 break;
5602 }
5603
5604 if (rdev->flags & RADEON_IS_IGP)
5605 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5606
5607 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5608 if (fusion_platform)
5609 data &= ~MULTI_PIF;
5610 else
5611 data |= MULTI_PIF;
5612 if (data != orig)
5613 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5614
5615 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5616 if (fusion_platform)
5617 data &= ~MULTI_PIF;
5618 else
5619 data |= MULTI_PIF;
5620 if (data != orig)
5621 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5622
5623 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5624 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5625 if (!disable_l0s) {
5626 if (rdev->family >= CHIP_BARTS)
5627 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5628 else
5629 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5630 }
5631
5632 if (!disable_l1) {
5633 if (rdev->family >= CHIP_BARTS)
5634 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5635 else
5636 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5637
5638 if (!disable_plloff_in_l1) {
5639 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5640 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5641 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5642 if (data != orig)
5643 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5644
5645 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5646 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5647 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5648 if (data != orig)
5649 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5650
5651 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5652 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5653 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5654 if (data != orig)
5655 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5656
5657 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5658 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5659 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5660 if (data != orig)
5661 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5662
5663 if (rdev->family >= CHIP_BARTS) {
5664 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5665 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5666 data |= PLL_RAMP_UP_TIME_0(4);
5667 if (data != orig)
5668 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5669
5670 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5671 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5672 data |= PLL_RAMP_UP_TIME_1(4);
5673 if (data != orig)
5674 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5675
5676 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5677 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5678 data |= PLL_RAMP_UP_TIME_0(4);
5679 if (data != orig)
5680 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5681
5682 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5683 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5684 data |= PLL_RAMP_UP_TIME_1(4);
5685 if (data != orig)
5686 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5687 }
5688
5689 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5690 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5691 data |= LC_DYN_LANES_PWR_STATE(3);
5692 if (data != orig)
5693 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5694
5695 if (rdev->family >= CHIP_BARTS) {
5696 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5697 data &= ~LS2_EXIT_TIME_MASK;
5698 data |= LS2_EXIT_TIME(1);
5699 if (data != orig)
5700 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5701
5702 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5703 data &= ~LS2_EXIT_TIME_MASK;
5704 data |= LS2_EXIT_TIME(1);
5705 if (data != orig)
5706 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5707 }
5708 }
5709 }
5710
5711 /* evergreen parts only */
5712 if (rdev->family < CHIP_BARTS)
5713 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5714
5715 if (pcie_lc_cntl != pcie_lc_cntl_old)
5716 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5717}
Linux kernel - Deliverables
This page took 0.662423 seconds and 5 git commands to generate.