2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
33 #include "si_blit_shaders.h"
34 #include "clearstate_si.h"
35 #include "radeon_ucode.h"
38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51 MODULE_FIRMWARE("radeon/VERDE_me.bin");
52 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
55 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57 MODULE_FIRMWARE("radeon/OLAND_me.bin");
58 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
61 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
69 static void si_pcie_gen3_enable(struct radeon_device
*rdev
);
70 static void si_program_aspm(struct radeon_device
*rdev
);
71 extern int r600_ih_ring_alloc(struct radeon_device
*rdev
);
72 extern void r600_ih_ring_fini(struct radeon_device
*rdev
);
73 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device
*rdev
);
74 extern void evergreen_mc_stop(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
75 extern void evergreen_mc_resume(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
76 extern u32
evergreen_get_number_of_dram_channels(struct radeon_device
*rdev
);
77 extern void evergreen_print_gpu_status_regs(struct radeon_device
*rdev
);
78 extern bool evergreen_is_display_hung(struct radeon_device
*rdev
);
80 static const u32 verde_rlc_save_restore_register_list
[] =
82 (0x8000 << 16) | (0x98f4 >> 2),
84 (0x8040 << 16) | (0x98f4 >> 2),
86 (0x8000 << 16) | (0xe80 >> 2),
88 (0x8040 << 16) | (0xe80 >> 2),
90 (0x8000 << 16) | (0x89bc >> 2),
92 (0x8040 << 16) | (0x89bc >> 2),
94 (0x8000 << 16) | (0x8c1c >> 2),
96 (0x8040 << 16) | (0x8c1c >> 2),
98 (0x9c00 << 16) | (0x98f0 >> 2),
100 (0x9c00 << 16) | (0xe7c >> 2),
102 (0x8000 << 16) | (0x9148 >> 2),
104 (0x8040 << 16) | (0x9148 >> 2),
106 (0x9c00 << 16) | (0x9150 >> 2),
108 (0x9c00 << 16) | (0x897c >> 2),
110 (0x9c00 << 16) | (0x8d8c >> 2),
112 (0x9c00 << 16) | (0xac54 >> 2),
115 (0x9c00 << 16) | (0x98f8 >> 2),
117 (0x9c00 << 16) | (0x9910 >> 2),
119 (0x9c00 << 16) | (0x9914 >> 2),
121 (0x9c00 << 16) | (0x9918 >> 2),
123 (0x9c00 << 16) | (0x991c >> 2),
125 (0x9c00 << 16) | (0x9920 >> 2),
127 (0x9c00 << 16) | (0x9924 >> 2),
129 (0x9c00 << 16) | (0x9928 >> 2),
131 (0x9c00 << 16) | (0x992c >> 2),
133 (0x9c00 << 16) | (0x9930 >> 2),
135 (0x9c00 << 16) | (0x9934 >> 2),
137 (0x9c00 << 16) | (0x9938 >> 2),
139 (0x9c00 << 16) | (0x993c >> 2),
141 (0x9c00 << 16) | (0x9940 >> 2),
143 (0x9c00 << 16) | (0x9944 >> 2),
145 (0x9c00 << 16) | (0x9948 >> 2),
147 (0x9c00 << 16) | (0x994c >> 2),
149 (0x9c00 << 16) | (0x9950 >> 2),
151 (0x9c00 << 16) | (0x9954 >> 2),
153 (0x9c00 << 16) | (0x9958 >> 2),
155 (0x9c00 << 16) | (0x995c >> 2),
157 (0x9c00 << 16) | (0x9960 >> 2),
159 (0x9c00 << 16) | (0x9964 >> 2),
161 (0x9c00 << 16) | (0x9968 >> 2),
163 (0x9c00 << 16) | (0x996c >> 2),
165 (0x9c00 << 16) | (0x9970 >> 2),
167 (0x9c00 << 16) | (0x9974 >> 2),
169 (0x9c00 << 16) | (0x9978 >> 2),
171 (0x9c00 << 16) | (0x997c >> 2),
173 (0x9c00 << 16) | (0x9980 >> 2),
175 (0x9c00 << 16) | (0x9984 >> 2),
177 (0x9c00 << 16) | (0x9988 >> 2),
179 (0x9c00 << 16) | (0x998c >> 2),
181 (0x9c00 << 16) | (0x8c00 >> 2),
183 (0x9c00 << 16) | (0x8c14 >> 2),
185 (0x9c00 << 16) | (0x8c04 >> 2),
187 (0x9c00 << 16) | (0x8c08 >> 2),
189 (0x8000 << 16) | (0x9b7c >> 2),
191 (0x8040 << 16) | (0x9b7c >> 2),
193 (0x8000 << 16) | (0xe84 >> 2),
195 (0x8040 << 16) | (0xe84 >> 2),
197 (0x8000 << 16) | (0x89c0 >> 2),
199 (0x8040 << 16) | (0x89c0 >> 2),
201 (0x8000 << 16) | (0x914c >> 2),
203 (0x8040 << 16) | (0x914c >> 2),
205 (0x8000 << 16) | (0x8c20 >> 2),
207 (0x8040 << 16) | (0x8c20 >> 2),
209 (0x8000 << 16) | (0x9354 >> 2),
211 (0x8040 << 16) | (0x9354 >> 2),
213 (0x9c00 << 16) | (0x9060 >> 2),
215 (0x9c00 << 16) | (0x9364 >> 2),
217 (0x9c00 << 16) | (0x9100 >> 2),
219 (0x9c00 << 16) | (0x913c >> 2),
221 (0x8000 << 16) | (0x90e0 >> 2),
223 (0x8000 << 16) | (0x90e4 >> 2),
225 (0x8000 << 16) | (0x90e8 >> 2),
227 (0x8040 << 16) | (0x90e0 >> 2),
229 (0x8040 << 16) | (0x90e4 >> 2),
231 (0x8040 << 16) | (0x90e8 >> 2),
233 (0x9c00 << 16) | (0x8bcc >> 2),
235 (0x9c00 << 16) | (0x8b24 >> 2),
237 (0x9c00 << 16) | (0x88c4 >> 2),
239 (0x9c00 << 16) | (0x8e50 >> 2),
241 (0x9c00 << 16) | (0x8c0c >> 2),
243 (0x9c00 << 16) | (0x8e58 >> 2),
245 (0x9c00 << 16) | (0x8e5c >> 2),
247 (0x9c00 << 16) | (0x9508 >> 2),
249 (0x9c00 << 16) | (0x950c >> 2),
251 (0x9c00 << 16) | (0x9494 >> 2),
253 (0x9c00 << 16) | (0xac0c >> 2),
255 (0x9c00 << 16) | (0xac10 >> 2),
257 (0x9c00 << 16) | (0xac14 >> 2),
259 (0x9c00 << 16) | (0xae00 >> 2),
261 (0x9c00 << 16) | (0xac08 >> 2),
263 (0x9c00 << 16) | (0x88d4 >> 2),
265 (0x9c00 << 16) | (0x88c8 >> 2),
267 (0x9c00 << 16) | (0x88cc >> 2),
269 (0x9c00 << 16) | (0x89b0 >> 2),
271 (0x9c00 << 16) | (0x8b10 >> 2),
273 (0x9c00 << 16) | (0x8a14 >> 2),
275 (0x9c00 << 16) | (0x9830 >> 2),
277 (0x9c00 << 16) | (0x9834 >> 2),
279 (0x9c00 << 16) | (0x9838 >> 2),
281 (0x9c00 << 16) | (0x9a10 >> 2),
283 (0x8000 << 16) | (0x9870 >> 2),
285 (0x8000 << 16) | (0x9874 >> 2),
287 (0x8001 << 16) | (0x9870 >> 2),
289 (0x8001 << 16) | (0x9874 >> 2),
291 (0x8040 << 16) | (0x9870 >> 2),
293 (0x8040 << 16) | (0x9874 >> 2),
295 (0x8041 << 16) | (0x9870 >> 2),
297 (0x8041 << 16) | (0x9874 >> 2),
302 static const u32 tahiti_golden_rlc_registers
[] =
304 0xc424, 0xffffffff, 0x00601005,
305 0xc47c, 0xffffffff, 0x10104040,
306 0xc488, 0xffffffff, 0x0100000a,
307 0xc314, 0xffffffff, 0x00000800,
308 0xc30c, 0xffffffff, 0x800000f4,
309 0xf4a8, 0xffffffff, 0x00000000
312 static const u32 tahiti_golden_registers
[] =
314 0x9a10, 0x00010000, 0x00018208,
315 0x9830, 0xffffffff, 0x00000000,
316 0x9834, 0xf00fffff, 0x00000400,
317 0x9838, 0x0002021c, 0x00020200,
318 0xc78, 0x00000080, 0x00000000,
319 0xd030, 0x000300c0, 0x00800040,
320 0xd830, 0x000300c0, 0x00800040,
321 0x5bb0, 0x000000f0, 0x00000070,
322 0x5bc0, 0x00200000, 0x50100000,
323 0x7030, 0x31000311, 0x00000011,
324 0x277c, 0x00000003, 0x000007ff,
325 0x240c, 0x000007ff, 0x00000000,
326 0x8a14, 0xf000001f, 0x00000007,
327 0x8b24, 0xffffffff, 0x00ffffff,
328 0x8b10, 0x0000ff0f, 0x00000000,
329 0x28a4c, 0x07ffffff, 0x4e000000,
330 0x28350, 0x3f3f3fff, 0x2a00126a,
331 0x30, 0x000000ff, 0x0040,
332 0x34, 0x00000040, 0x00004040,
333 0x9100, 0x07ffffff, 0x03000000,
334 0x8e88, 0x01ff1f3f, 0x00000000,
335 0x8e84, 0x01ff1f3f, 0x00000000,
336 0x9060, 0x0000007f, 0x00000020,
337 0x9508, 0x00010000, 0x00010000,
338 0xac14, 0x00000200, 0x000002fb,
339 0xac10, 0xffffffff, 0x0000543b,
340 0xac0c, 0xffffffff, 0xa9210876,
341 0x88d0, 0xffffffff, 0x000fff40,
342 0x88d4, 0x0000001f, 0x00000010,
343 0x1410, 0x20000000, 0x20fffed8,
344 0x15c0, 0x000c0fc0, 0x000c0400
347 static const u32 tahiti_golden_registers2
[] =
349 0xc64, 0x00000001, 0x00000001
352 static const u32 pitcairn_golden_rlc_registers
[] =
354 0xc424, 0xffffffff, 0x00601004,
355 0xc47c, 0xffffffff, 0x10102020,
356 0xc488, 0xffffffff, 0x01000020,
357 0xc314, 0xffffffff, 0x00000800,
358 0xc30c, 0xffffffff, 0x800000a4
361 static const u32 pitcairn_golden_registers
[] =
363 0x9a10, 0x00010000, 0x00018208,
364 0x9830, 0xffffffff, 0x00000000,
365 0x9834, 0xf00fffff, 0x00000400,
366 0x9838, 0x0002021c, 0x00020200,
367 0xc78, 0x00000080, 0x00000000,
368 0xd030, 0x000300c0, 0x00800040,
369 0xd830, 0x000300c0, 0x00800040,
370 0x5bb0, 0x000000f0, 0x00000070,
371 0x5bc0, 0x00200000, 0x50100000,
372 0x7030, 0x31000311, 0x00000011,
373 0x2ae4, 0x00073ffe, 0x000022a2,
374 0x240c, 0x000007ff, 0x00000000,
375 0x8a14, 0xf000001f, 0x00000007,
376 0x8b24, 0xffffffff, 0x00ffffff,
377 0x8b10, 0x0000ff0f, 0x00000000,
378 0x28a4c, 0x07ffffff, 0x4e000000,
379 0x28350, 0x3f3f3fff, 0x2a00126a,
380 0x30, 0x000000ff, 0x0040,
381 0x34, 0x00000040, 0x00004040,
382 0x9100, 0x07ffffff, 0x03000000,
383 0x9060, 0x0000007f, 0x00000020,
384 0x9508, 0x00010000, 0x00010000,
385 0xac14, 0x000003ff, 0x000000f7,
386 0xac10, 0xffffffff, 0x00000000,
387 0xac0c, 0xffffffff, 0x32761054,
388 0x88d4, 0x0000001f, 0x00000010,
389 0x15c0, 0x000c0fc0, 0x000c0400
392 static const u32 verde_golden_rlc_registers
[] =
394 0xc424, 0xffffffff, 0x033f1005,
395 0xc47c, 0xffffffff, 0x10808020,
396 0xc488, 0xffffffff, 0x00800008,
397 0xc314, 0xffffffff, 0x00001000,
398 0xc30c, 0xffffffff, 0x80010014
401 static const u32 verde_golden_registers
[] =
403 0x9a10, 0x00010000, 0x00018208,
404 0x9830, 0xffffffff, 0x00000000,
405 0x9834, 0xf00fffff, 0x00000400,
406 0x9838, 0x0002021c, 0x00020200,
407 0xc78, 0x00000080, 0x00000000,
408 0xd030, 0x000300c0, 0x00800040,
409 0xd030, 0x000300c0, 0x00800040,
410 0xd830, 0x000300c0, 0x00800040,
411 0xd830, 0x000300c0, 0x00800040,
412 0x5bb0, 0x000000f0, 0x00000070,
413 0x5bc0, 0x00200000, 0x50100000,
414 0x7030, 0x31000311, 0x00000011,
415 0x2ae4, 0x00073ffe, 0x000022a2,
416 0x2ae4, 0x00073ffe, 0x000022a2,
417 0x2ae4, 0x00073ffe, 0x000022a2,
418 0x240c, 0x000007ff, 0x00000000,
419 0x240c, 0x000007ff, 0x00000000,
420 0x240c, 0x000007ff, 0x00000000,
421 0x8a14, 0xf000001f, 0x00000007,
422 0x8a14, 0xf000001f, 0x00000007,
423 0x8a14, 0xf000001f, 0x00000007,
424 0x8b24, 0xffffffff, 0x00ffffff,
425 0x8b10, 0x0000ff0f, 0x00000000,
426 0x28a4c, 0x07ffffff, 0x4e000000,
427 0x28350, 0x3f3f3fff, 0x0000124a,
428 0x28350, 0x3f3f3fff, 0x0000124a,
429 0x28350, 0x3f3f3fff, 0x0000124a,
430 0x30, 0x000000ff, 0x0040,
431 0x34, 0x00000040, 0x00004040,
432 0x9100, 0x07ffffff, 0x03000000,
433 0x9100, 0x07ffffff, 0x03000000,
434 0x8e88, 0x01ff1f3f, 0x00000000,
435 0x8e88, 0x01ff1f3f, 0x00000000,
436 0x8e88, 0x01ff1f3f, 0x00000000,
437 0x8e84, 0x01ff1f3f, 0x00000000,
438 0x8e84, 0x01ff1f3f, 0x00000000,
439 0x8e84, 0x01ff1f3f, 0x00000000,
440 0x9060, 0x0000007f, 0x00000020,
441 0x9508, 0x00010000, 0x00010000,
442 0xac14, 0x000003ff, 0x00000003,
443 0xac14, 0x000003ff, 0x00000003,
444 0xac14, 0x000003ff, 0x00000003,
445 0xac10, 0xffffffff, 0x00000000,
446 0xac10, 0xffffffff, 0x00000000,
447 0xac10, 0xffffffff, 0x00000000,
448 0xac0c, 0xffffffff, 0x00001032,
449 0xac0c, 0xffffffff, 0x00001032,
450 0xac0c, 0xffffffff, 0x00001032,
451 0x88d4, 0x0000001f, 0x00000010,
452 0x88d4, 0x0000001f, 0x00000010,
453 0x88d4, 0x0000001f, 0x00000010,
454 0x15c0, 0x000c0fc0, 0x000c0400
457 static const u32 oland_golden_rlc_registers
[] =
459 0xc424, 0xffffffff, 0x00601005,
460 0xc47c, 0xffffffff, 0x10104040,
461 0xc488, 0xffffffff, 0x0100000a,
462 0xc314, 0xffffffff, 0x00000800,
463 0xc30c, 0xffffffff, 0x800000f4
466 static const u32 oland_golden_registers
[] =
468 0x9a10, 0x00010000, 0x00018208,
469 0x9830, 0xffffffff, 0x00000000,
470 0x9834, 0xf00fffff, 0x00000400,
471 0x9838, 0x0002021c, 0x00020200,
472 0xc78, 0x00000080, 0x00000000,
473 0xd030, 0x000300c0, 0x00800040,
474 0xd830, 0x000300c0, 0x00800040,
475 0x5bb0, 0x000000f0, 0x00000070,
476 0x5bc0, 0x00200000, 0x50100000,
477 0x7030, 0x31000311, 0x00000011,
478 0x2ae4, 0x00073ffe, 0x000022a2,
479 0x240c, 0x000007ff, 0x00000000,
480 0x8a14, 0xf000001f, 0x00000007,
481 0x8b24, 0xffffffff, 0x00ffffff,
482 0x8b10, 0x0000ff0f, 0x00000000,
483 0x28a4c, 0x07ffffff, 0x4e000000,
484 0x28350, 0x3f3f3fff, 0x00000082,
485 0x30, 0x000000ff, 0x0040,
486 0x34, 0x00000040, 0x00004040,
487 0x9100, 0x07ffffff, 0x03000000,
488 0x9060, 0x0000007f, 0x00000020,
489 0x9508, 0x00010000, 0x00010000,
490 0xac14, 0x000003ff, 0x000000f3,
491 0xac10, 0xffffffff, 0x00000000,
492 0xac0c, 0xffffffff, 0x00003210,
493 0x88d4, 0x0000001f, 0x00000010,
494 0x15c0, 0x000c0fc0, 0x000c0400
497 static const u32 hainan_golden_registers
[] =
499 0x9a10, 0x00010000, 0x00018208,
500 0x9830, 0xffffffff, 0x00000000,
501 0x9834, 0xf00fffff, 0x00000400,
502 0x9838, 0x0002021c, 0x00020200,
503 0xd0c0, 0xff000fff, 0x00000100,
504 0xd030, 0x000300c0, 0x00800040,
505 0xd8c0, 0xff000fff, 0x00000100,
506 0xd830, 0x000300c0, 0x00800040,
507 0x2ae4, 0x00073ffe, 0x000022a2,
508 0x240c, 0x000007ff, 0x00000000,
509 0x8a14, 0xf000001f, 0x00000007,
510 0x8b24, 0xffffffff, 0x00ffffff,
511 0x8b10, 0x0000ff0f, 0x00000000,
512 0x28a4c, 0x07ffffff, 0x4e000000,
513 0x28350, 0x3f3f3fff, 0x00000000,
514 0x30, 0x000000ff, 0x0040,
515 0x34, 0x00000040, 0x00004040,
516 0x9100, 0x03e00000, 0x03600000,
517 0x9060, 0x0000007f, 0x00000020,
518 0x9508, 0x00010000, 0x00010000,
519 0xac14, 0x000003ff, 0x000000f1,
520 0xac10, 0xffffffff, 0x00000000,
521 0xac0c, 0xffffffff, 0x00003210,
522 0x88d4, 0x0000001f, 0x00000010,
523 0x15c0, 0x000c0fc0, 0x000c0400
526 static const u32 hainan_golden_registers2
[] =
528 0x98f8, 0xffffffff, 0x02010001
531 static const u32 tahiti_mgcg_cgcg_init
[] =
533 0xc400, 0xffffffff, 0xfffffffc,
534 0x802c, 0xffffffff, 0xe0000000,
535 0x9a60, 0xffffffff, 0x00000100,
536 0x92a4, 0xffffffff, 0x00000100,
537 0xc164, 0xffffffff, 0x00000100,
538 0x9774, 0xffffffff, 0x00000100,
539 0x8984, 0xffffffff, 0x06000100,
540 0x8a18, 0xffffffff, 0x00000100,
541 0x92a0, 0xffffffff, 0x00000100,
542 0xc380, 0xffffffff, 0x00000100,
543 0x8b28, 0xffffffff, 0x00000100,
544 0x9144, 0xffffffff, 0x00000100,
545 0x8d88, 0xffffffff, 0x00000100,
546 0x8d8c, 0xffffffff, 0x00000100,
547 0x9030, 0xffffffff, 0x00000100,
548 0x9034, 0xffffffff, 0x00000100,
549 0x9038, 0xffffffff, 0x00000100,
550 0x903c, 0xffffffff, 0x00000100,
551 0xad80, 0xffffffff, 0x00000100,
552 0xac54, 0xffffffff, 0x00000100,
553 0x897c, 0xffffffff, 0x06000100,
554 0x9868, 0xffffffff, 0x00000100,
555 0x9510, 0xffffffff, 0x00000100,
556 0xaf04, 0xffffffff, 0x00000100,
557 0xae04, 0xffffffff, 0x00000100,
558 0x949c, 0xffffffff, 0x00000100,
559 0x802c, 0xffffffff, 0xe0000000,
560 0x9160, 0xffffffff, 0x00010000,
561 0x9164, 0xffffffff, 0x00030002,
562 0x9168, 0xffffffff, 0x00040007,
563 0x916c, 0xffffffff, 0x00060005,
564 0x9170, 0xffffffff, 0x00090008,
565 0x9174, 0xffffffff, 0x00020001,
566 0x9178, 0xffffffff, 0x00040003,
567 0x917c, 0xffffffff, 0x00000007,
568 0x9180, 0xffffffff, 0x00060005,
569 0x9184, 0xffffffff, 0x00090008,
570 0x9188, 0xffffffff, 0x00030002,
571 0x918c, 0xffffffff, 0x00050004,
572 0x9190, 0xffffffff, 0x00000008,
573 0x9194, 0xffffffff, 0x00070006,
574 0x9198, 0xffffffff, 0x000a0009,
575 0x919c, 0xffffffff, 0x00040003,
576 0x91a0, 0xffffffff, 0x00060005,
577 0x91a4, 0xffffffff, 0x00000009,
578 0x91a8, 0xffffffff, 0x00080007,
579 0x91ac, 0xffffffff, 0x000b000a,
580 0x91b0, 0xffffffff, 0x00050004,
581 0x91b4, 0xffffffff, 0x00070006,
582 0x91b8, 0xffffffff, 0x0008000b,
583 0x91bc, 0xffffffff, 0x000a0009,
584 0x91c0, 0xffffffff, 0x000d000c,
585 0x91c4, 0xffffffff, 0x00060005,
586 0x91c8, 0xffffffff, 0x00080007,
587 0x91cc, 0xffffffff, 0x0000000b,
588 0x91d0, 0xffffffff, 0x000a0009,
589 0x91d4, 0xffffffff, 0x000d000c,
590 0x91d8, 0xffffffff, 0x00070006,
591 0x91dc, 0xffffffff, 0x00090008,
592 0x91e0, 0xffffffff, 0x0000000c,
593 0x91e4, 0xffffffff, 0x000b000a,
594 0x91e8, 0xffffffff, 0x000e000d,
595 0x91ec, 0xffffffff, 0x00080007,
596 0x91f0, 0xffffffff, 0x000a0009,
597 0x91f4, 0xffffffff, 0x0000000d,
598 0x91f8, 0xffffffff, 0x000c000b,
599 0x91fc, 0xffffffff, 0x000f000e,
600 0x9200, 0xffffffff, 0x00090008,
601 0x9204, 0xffffffff, 0x000b000a,
602 0x9208, 0xffffffff, 0x000c000f,
603 0x920c, 0xffffffff, 0x000e000d,
604 0x9210, 0xffffffff, 0x00110010,
605 0x9214, 0xffffffff, 0x000a0009,
606 0x9218, 0xffffffff, 0x000c000b,
607 0x921c, 0xffffffff, 0x0000000f,
608 0x9220, 0xffffffff, 0x000e000d,
609 0x9224, 0xffffffff, 0x00110010,
610 0x9228, 0xffffffff, 0x000b000a,
611 0x922c, 0xffffffff, 0x000d000c,
612 0x9230, 0xffffffff, 0x00000010,
613 0x9234, 0xffffffff, 0x000f000e,
614 0x9238, 0xffffffff, 0x00120011,
615 0x923c, 0xffffffff, 0x000c000b,
616 0x9240, 0xffffffff, 0x000e000d,
617 0x9244, 0xffffffff, 0x00000011,
618 0x9248, 0xffffffff, 0x0010000f,
619 0x924c, 0xffffffff, 0x00130012,
620 0x9250, 0xffffffff, 0x000d000c,
621 0x9254, 0xffffffff, 0x000f000e,
622 0x9258, 0xffffffff, 0x00100013,
623 0x925c, 0xffffffff, 0x00120011,
624 0x9260, 0xffffffff, 0x00150014,
625 0x9264, 0xffffffff, 0x000e000d,
626 0x9268, 0xffffffff, 0x0010000f,
627 0x926c, 0xffffffff, 0x00000013,
628 0x9270, 0xffffffff, 0x00120011,
629 0x9274, 0xffffffff, 0x00150014,
630 0x9278, 0xffffffff, 0x000f000e,
631 0x927c, 0xffffffff, 0x00110010,
632 0x9280, 0xffffffff, 0x00000014,
633 0x9284, 0xffffffff, 0x00130012,
634 0x9288, 0xffffffff, 0x00160015,
635 0x928c, 0xffffffff, 0x0010000f,
636 0x9290, 0xffffffff, 0x00120011,
637 0x9294, 0xffffffff, 0x00000015,
638 0x9298, 0xffffffff, 0x00140013,
639 0x929c, 0xffffffff, 0x00170016,
640 0x9150, 0xffffffff, 0x96940200,
641 0x8708, 0xffffffff, 0x00900100,
642 0xc478, 0xffffffff, 0x00000080,
643 0xc404, 0xffffffff, 0x0020003f,
644 0x30, 0xffffffff, 0x0000001c,
645 0x34, 0x000f0000, 0x000f0000,
646 0x160c, 0xffffffff, 0x00000100,
647 0x1024, 0xffffffff, 0x00000100,
648 0x102c, 0x00000101, 0x00000000,
649 0x20a8, 0xffffffff, 0x00000104,
650 0x264c, 0x000c0000, 0x000c0000,
651 0x2648, 0x000c0000, 0x000c0000,
652 0x55e4, 0xff000fff, 0x00000100,
653 0x55e8, 0x00000001, 0x00000001,
654 0x2f50, 0x00000001, 0x00000001,
655 0x30cc, 0xc0000fff, 0x00000104,
656 0xc1e4, 0x00000001, 0x00000001,
657 0xd0c0, 0xfffffff0, 0x00000100,
658 0xd8c0, 0xfffffff0, 0x00000100
661 static const u32 pitcairn_mgcg_cgcg_init
[] =
663 0xc400, 0xffffffff, 0xfffffffc,
664 0x802c, 0xffffffff, 0xe0000000,
665 0x9a60, 0xffffffff, 0x00000100,
666 0x92a4, 0xffffffff, 0x00000100,
667 0xc164, 0xffffffff, 0x00000100,
668 0x9774, 0xffffffff, 0x00000100,
669 0x8984, 0xffffffff, 0x06000100,
670 0x8a18, 0xffffffff, 0x00000100,
671 0x92a0, 0xffffffff, 0x00000100,
672 0xc380, 0xffffffff, 0x00000100,
673 0x8b28, 0xffffffff, 0x00000100,
674 0x9144, 0xffffffff, 0x00000100,
675 0x8d88, 0xffffffff, 0x00000100,
676 0x8d8c, 0xffffffff, 0x00000100,
677 0x9030, 0xffffffff, 0x00000100,
678 0x9034, 0xffffffff, 0x00000100,
679 0x9038, 0xffffffff, 0x00000100,
680 0x903c, 0xffffffff, 0x00000100,
681 0xad80, 0xffffffff, 0x00000100,
682 0xac54, 0xffffffff, 0x00000100,
683 0x897c, 0xffffffff, 0x06000100,
684 0x9868, 0xffffffff, 0x00000100,
685 0x9510, 0xffffffff, 0x00000100,
686 0xaf04, 0xffffffff, 0x00000100,
687 0xae04, 0xffffffff, 0x00000100,
688 0x949c, 0xffffffff, 0x00000100,
689 0x802c, 0xffffffff, 0xe0000000,
690 0x9160, 0xffffffff, 0x00010000,
691 0x9164, 0xffffffff, 0x00030002,
692 0x9168, 0xffffffff, 0x00040007,
693 0x916c, 0xffffffff, 0x00060005,
694 0x9170, 0xffffffff, 0x00090008,
695 0x9174, 0xffffffff, 0x00020001,
696 0x9178, 0xffffffff, 0x00040003,
697 0x917c, 0xffffffff, 0x00000007,
698 0x9180, 0xffffffff, 0x00060005,
699 0x9184, 0xffffffff, 0x00090008,
700 0x9188, 0xffffffff, 0x00030002,
701 0x918c, 0xffffffff, 0x00050004,
702 0x9190, 0xffffffff, 0x00000008,
703 0x9194, 0xffffffff, 0x00070006,
704 0x9198, 0xffffffff, 0x000a0009,
705 0x919c, 0xffffffff, 0x00040003,
706 0x91a0, 0xffffffff, 0x00060005,
707 0x91a4, 0xffffffff, 0x00000009,
708 0x91a8, 0xffffffff, 0x00080007,
709 0x91ac, 0xffffffff, 0x000b000a,
710 0x91b0, 0xffffffff, 0x00050004,
711 0x91b4, 0xffffffff, 0x00070006,
712 0x91b8, 0xffffffff, 0x0008000b,
713 0x91bc, 0xffffffff, 0x000a0009,
714 0x91c0, 0xffffffff, 0x000d000c,
715 0x9200, 0xffffffff, 0x00090008,
716 0x9204, 0xffffffff, 0x000b000a,
717 0x9208, 0xffffffff, 0x000c000f,
718 0x920c, 0xffffffff, 0x000e000d,
719 0x9210, 0xffffffff, 0x00110010,
720 0x9214, 0xffffffff, 0x000a0009,
721 0x9218, 0xffffffff, 0x000c000b,
722 0x921c, 0xffffffff, 0x0000000f,
723 0x9220, 0xffffffff, 0x000e000d,
724 0x9224, 0xffffffff, 0x00110010,
725 0x9228, 0xffffffff, 0x000b000a,
726 0x922c, 0xffffffff, 0x000d000c,
727 0x9230, 0xffffffff, 0x00000010,
728 0x9234, 0xffffffff, 0x000f000e,
729 0x9238, 0xffffffff, 0x00120011,
730 0x923c, 0xffffffff, 0x000c000b,
731 0x9240, 0xffffffff, 0x000e000d,
732 0x9244, 0xffffffff, 0x00000011,
733 0x9248, 0xffffffff, 0x0010000f,
734 0x924c, 0xffffffff, 0x00130012,
735 0x9250, 0xffffffff, 0x000d000c,
736 0x9254, 0xffffffff, 0x000f000e,
737 0x9258, 0xffffffff, 0x00100013,
738 0x925c, 0xffffffff, 0x00120011,
739 0x9260, 0xffffffff, 0x00150014,
740 0x9150, 0xffffffff, 0x96940200,
741 0x8708, 0xffffffff, 0x00900100,
742 0xc478, 0xffffffff, 0x00000080,
743 0xc404, 0xffffffff, 0x0020003f,
744 0x30, 0xffffffff, 0x0000001c,
745 0x34, 0x000f0000, 0x000f0000,
746 0x160c, 0xffffffff, 0x00000100,
747 0x1024, 0xffffffff, 0x00000100,
748 0x102c, 0x00000101, 0x00000000,
749 0x20a8, 0xffffffff, 0x00000104,
750 0x55e4, 0xff000fff, 0x00000100,
751 0x55e8, 0x00000001, 0x00000001,
752 0x2f50, 0x00000001, 0x00000001,
753 0x30cc, 0xc0000fff, 0x00000104,
754 0xc1e4, 0x00000001, 0x00000001,
755 0xd0c0, 0xfffffff0, 0x00000100,
756 0xd8c0, 0xfffffff0, 0x00000100
759 static const u32 verde_mgcg_cgcg_init
[] =
761 0xc400, 0xffffffff, 0xfffffffc,
762 0x802c, 0xffffffff, 0xe0000000,
763 0x9a60, 0xffffffff, 0x00000100,
764 0x92a4, 0xffffffff, 0x00000100,
765 0xc164, 0xffffffff, 0x00000100,
766 0x9774, 0xffffffff, 0x00000100,
767 0x8984, 0xffffffff, 0x06000100,
768 0x8a18, 0xffffffff, 0x00000100,
769 0x92a0, 0xffffffff, 0x00000100,
770 0xc380, 0xffffffff, 0x00000100,
771 0x8b28, 0xffffffff, 0x00000100,
772 0x9144, 0xffffffff, 0x00000100,
773 0x8d88, 0xffffffff, 0x00000100,
774 0x8d8c, 0xffffffff, 0x00000100,
775 0x9030, 0xffffffff, 0x00000100,
776 0x9034, 0xffffffff, 0x00000100,
777 0x9038, 0xffffffff, 0x00000100,
778 0x903c, 0xffffffff, 0x00000100,
779 0xad80, 0xffffffff, 0x00000100,
780 0xac54, 0xffffffff, 0x00000100,
781 0x897c, 0xffffffff, 0x06000100,
782 0x9868, 0xffffffff, 0x00000100,
783 0x9510, 0xffffffff, 0x00000100,
784 0xaf04, 0xffffffff, 0x00000100,
785 0xae04, 0xffffffff, 0x00000100,
786 0x949c, 0xffffffff, 0x00000100,
787 0x802c, 0xffffffff, 0xe0000000,
788 0x9160, 0xffffffff, 0x00010000,
789 0x9164, 0xffffffff, 0x00030002,
790 0x9168, 0xffffffff, 0x00040007,
791 0x916c, 0xffffffff, 0x00060005,
792 0x9170, 0xffffffff, 0x00090008,
793 0x9174, 0xffffffff, 0x00020001,
794 0x9178, 0xffffffff, 0x00040003,
795 0x917c, 0xffffffff, 0x00000007,
796 0x9180, 0xffffffff, 0x00060005,
797 0x9184, 0xffffffff, 0x00090008,
798 0x9188, 0xffffffff, 0x00030002,
799 0x918c, 0xffffffff, 0x00050004,
800 0x9190, 0xffffffff, 0x00000008,
801 0x9194, 0xffffffff, 0x00070006,
802 0x9198, 0xffffffff, 0x000a0009,
803 0x919c, 0xffffffff, 0x00040003,
804 0x91a0, 0xffffffff, 0x00060005,
805 0x91a4, 0xffffffff, 0x00000009,
806 0x91a8, 0xffffffff, 0x00080007,
807 0x91ac, 0xffffffff, 0x000b000a,
808 0x91b0, 0xffffffff, 0x00050004,
809 0x91b4, 0xffffffff, 0x00070006,
810 0x91b8, 0xffffffff, 0x0008000b,
811 0x91bc, 0xffffffff, 0x000a0009,
812 0x91c0, 0xffffffff, 0x000d000c,
813 0x9200, 0xffffffff, 0x00090008,
814 0x9204, 0xffffffff, 0x000b000a,
815 0x9208, 0xffffffff, 0x000c000f,
816 0x920c, 0xffffffff, 0x000e000d,
817 0x9210, 0xffffffff, 0x00110010,
818 0x9214, 0xffffffff, 0x000a0009,
819 0x9218, 0xffffffff, 0x000c000b,
820 0x921c, 0xffffffff, 0x0000000f,
821 0x9220, 0xffffffff, 0x000e000d,
822 0x9224, 0xffffffff, 0x00110010,
823 0x9228, 0xffffffff, 0x000b000a,
824 0x922c, 0xffffffff, 0x000d000c,
825 0x9230, 0xffffffff, 0x00000010,
826 0x9234, 0xffffffff, 0x000f000e,
827 0x9238, 0xffffffff, 0x00120011,
828 0x923c, 0xffffffff, 0x000c000b,
829 0x9240, 0xffffffff, 0x000e000d,
830 0x9244, 0xffffffff, 0x00000011,
831 0x9248, 0xffffffff, 0x0010000f,
832 0x924c, 0xffffffff, 0x00130012,
833 0x9250, 0xffffffff, 0x000d000c,
834 0x9254, 0xffffffff, 0x000f000e,
835 0x9258, 0xffffffff, 0x00100013,
836 0x925c, 0xffffffff, 0x00120011,
837 0x9260, 0xffffffff, 0x00150014,
838 0x9150, 0xffffffff, 0x96940200,
839 0x8708, 0xffffffff, 0x00900100,
840 0xc478, 0xffffffff, 0x00000080,
841 0xc404, 0xffffffff, 0x0020003f,
842 0x30, 0xffffffff, 0x0000001c,
843 0x34, 0x000f0000, 0x000f0000,
844 0x160c, 0xffffffff, 0x00000100,
845 0x1024, 0xffffffff, 0x00000100,
846 0x102c, 0x00000101, 0x00000000,
847 0x20a8, 0xffffffff, 0x00000104,
848 0x264c, 0x000c0000, 0x000c0000,
849 0x2648, 0x000c0000, 0x000c0000,
850 0x55e4, 0xff000fff, 0x00000100,
851 0x55e8, 0x00000001, 0x00000001,
852 0x2f50, 0x00000001, 0x00000001,
853 0x30cc, 0xc0000fff, 0x00000104,
854 0xc1e4, 0x00000001, 0x00000001,
855 0xd0c0, 0xfffffff0, 0x00000100,
856 0xd8c0, 0xfffffff0, 0x00000100
859 static const u32 oland_mgcg_cgcg_init
[] =
861 0xc400, 0xffffffff, 0xfffffffc,
862 0x802c, 0xffffffff, 0xe0000000,
863 0x9a60, 0xffffffff, 0x00000100,
864 0x92a4, 0xffffffff, 0x00000100,
865 0xc164, 0xffffffff, 0x00000100,
866 0x9774, 0xffffffff, 0x00000100,
867 0x8984, 0xffffffff, 0x06000100,
868 0x8a18, 0xffffffff, 0x00000100,
869 0x92a0, 0xffffffff, 0x00000100,
870 0xc380, 0xffffffff, 0x00000100,
871 0x8b28, 0xffffffff, 0x00000100,
872 0x9144, 0xffffffff, 0x00000100,
873 0x8d88, 0xffffffff, 0x00000100,
874 0x8d8c, 0xffffffff, 0x00000100,
875 0x9030, 0xffffffff, 0x00000100,
876 0x9034, 0xffffffff, 0x00000100,
877 0x9038, 0xffffffff, 0x00000100,
878 0x903c, 0xffffffff, 0x00000100,
879 0xad80, 0xffffffff, 0x00000100,
880 0xac54, 0xffffffff, 0x00000100,
881 0x897c, 0xffffffff, 0x06000100,
882 0x9868, 0xffffffff, 0x00000100,
883 0x9510, 0xffffffff, 0x00000100,
884 0xaf04, 0xffffffff, 0x00000100,
885 0xae04, 0xffffffff, 0x00000100,
886 0x949c, 0xffffffff, 0x00000100,
887 0x802c, 0xffffffff, 0xe0000000,
888 0x9160, 0xffffffff, 0x00010000,
889 0x9164, 0xffffffff, 0x00030002,
890 0x9168, 0xffffffff, 0x00040007,
891 0x916c, 0xffffffff, 0x00060005,
892 0x9170, 0xffffffff, 0x00090008,
893 0x9174, 0xffffffff, 0x00020001,
894 0x9178, 0xffffffff, 0x00040003,
895 0x917c, 0xffffffff, 0x00000007,
896 0x9180, 0xffffffff, 0x00060005,
897 0x9184, 0xffffffff, 0x00090008,
898 0x9188, 0xffffffff, 0x00030002,
899 0x918c, 0xffffffff, 0x00050004,
900 0x9190, 0xffffffff, 0x00000008,
901 0x9194, 0xffffffff, 0x00070006,
902 0x9198, 0xffffffff, 0x000a0009,
903 0x919c, 0xffffffff, 0x00040003,
904 0x91a0, 0xffffffff, 0x00060005,
905 0x91a4, 0xffffffff, 0x00000009,
906 0x91a8, 0xffffffff, 0x00080007,
907 0x91ac, 0xffffffff, 0x000b000a,
908 0x91b0, 0xffffffff, 0x00050004,
909 0x91b4, 0xffffffff, 0x00070006,
910 0x91b8, 0xffffffff, 0x0008000b,
911 0x91bc, 0xffffffff, 0x000a0009,
912 0x91c0, 0xffffffff, 0x000d000c,
913 0x91c4, 0xffffffff, 0x00060005,
914 0x91c8, 0xffffffff, 0x00080007,
915 0x91cc, 0xffffffff, 0x0000000b,
916 0x91d0, 0xffffffff, 0x000a0009,
917 0x91d4, 0xffffffff, 0x000d000c,
918 0x9150, 0xffffffff, 0x96940200,
919 0x8708, 0xffffffff, 0x00900100,
920 0xc478, 0xffffffff, 0x00000080,
921 0xc404, 0xffffffff, 0x0020003f,
922 0x30, 0xffffffff, 0x0000001c,
923 0x34, 0x000f0000, 0x000f0000,
924 0x160c, 0xffffffff, 0x00000100,
925 0x1024, 0xffffffff, 0x00000100,
926 0x102c, 0x00000101, 0x00000000,
927 0x20a8, 0xffffffff, 0x00000104,
928 0x264c, 0x000c0000, 0x000c0000,
929 0x2648, 0x000c0000, 0x000c0000,
930 0x55e4, 0xff000fff, 0x00000100,
931 0x55e8, 0x00000001, 0x00000001,
932 0x2f50, 0x00000001, 0x00000001,
933 0x30cc, 0xc0000fff, 0x00000104,
934 0xc1e4, 0x00000001, 0x00000001,
935 0xd0c0, 0xfffffff0, 0x00000100,
936 0xd8c0, 0xfffffff0, 0x00000100
939 static const u32 hainan_mgcg_cgcg_init
[] =
941 0xc400, 0xffffffff, 0xfffffffc,
942 0x802c, 0xffffffff, 0xe0000000,
943 0x9a60, 0xffffffff, 0x00000100,
944 0x92a4, 0xffffffff, 0x00000100,
945 0xc164, 0xffffffff, 0x00000100,
946 0x9774, 0xffffffff, 0x00000100,
947 0x8984, 0xffffffff, 0x06000100,
948 0x8a18, 0xffffffff, 0x00000100,
949 0x92a0, 0xffffffff, 0x00000100,
950 0xc380, 0xffffffff, 0x00000100,
951 0x8b28, 0xffffffff, 0x00000100,
952 0x9144, 0xffffffff, 0x00000100,
953 0x8d88, 0xffffffff, 0x00000100,
954 0x8d8c, 0xffffffff, 0x00000100,
955 0x9030, 0xffffffff, 0x00000100,
956 0x9034, 0xffffffff, 0x00000100,
957 0x9038, 0xffffffff, 0x00000100,
958 0x903c, 0xffffffff, 0x00000100,
959 0xad80, 0xffffffff, 0x00000100,
960 0xac54, 0xffffffff, 0x00000100,
961 0x897c, 0xffffffff, 0x06000100,
962 0x9868, 0xffffffff, 0x00000100,
963 0x9510, 0xffffffff, 0x00000100,
964 0xaf04, 0xffffffff, 0x00000100,
965 0xae04, 0xffffffff, 0x00000100,
966 0x949c, 0xffffffff, 0x00000100,
967 0x802c, 0xffffffff, 0xe0000000,
968 0x9160, 0xffffffff, 0x00010000,
969 0x9164, 0xffffffff, 0x00030002,
970 0x9168, 0xffffffff, 0x00040007,
971 0x916c, 0xffffffff, 0x00060005,
972 0x9170, 0xffffffff, 0x00090008,
973 0x9174, 0xffffffff, 0x00020001,
974 0x9178, 0xffffffff, 0x00040003,
975 0x917c, 0xffffffff, 0x00000007,
976 0x9180, 0xffffffff, 0x00060005,
977 0x9184, 0xffffffff, 0x00090008,
978 0x9188, 0xffffffff, 0x00030002,
979 0x918c, 0xffffffff, 0x00050004,
980 0x9190, 0xffffffff, 0x00000008,
981 0x9194, 0xffffffff, 0x00070006,
982 0x9198, 0xffffffff, 0x000a0009,
983 0x919c, 0xffffffff, 0x00040003,
984 0x91a0, 0xffffffff, 0x00060005,
985 0x91a4, 0xffffffff, 0x00000009,
986 0x91a8, 0xffffffff, 0x00080007,
987 0x91ac, 0xffffffff, 0x000b000a,
988 0x91b0, 0xffffffff, 0x00050004,
989 0x91b4, 0xffffffff, 0x00070006,
990 0x91b8, 0xffffffff, 0x0008000b,
991 0x91bc, 0xffffffff, 0x000a0009,
992 0x91c0, 0xffffffff, 0x000d000c,
993 0x91c4, 0xffffffff, 0x00060005,
994 0x91c8, 0xffffffff, 0x00080007,
995 0x91cc, 0xffffffff, 0x0000000b,
996 0x91d0, 0xffffffff, 0x000a0009,
997 0x91d4, 0xffffffff, 0x000d000c,
998 0x9150, 0xffffffff, 0x96940200,
999 0x8708, 0xffffffff, 0x00900100,
1000 0xc478, 0xffffffff, 0x00000080,
1001 0xc404, 0xffffffff, 0x0020003f,
1002 0x30, 0xffffffff, 0x0000001c,
1003 0x34, 0x000f0000, 0x000f0000,
1004 0x160c, 0xffffffff, 0x00000100,
1005 0x1024, 0xffffffff, 0x00000100,
1006 0x20a8, 0xffffffff, 0x00000104,
1007 0x264c, 0x000c0000, 0x000c0000,
1008 0x2648, 0x000c0000, 0x000c0000,
1009 0x2f50, 0x00000001, 0x00000001,
1010 0x30cc, 0xc0000fff, 0x00000104,
1011 0xc1e4, 0x00000001, 0x00000001,
1012 0xd0c0, 0xfffffff0, 0x00000100,
1013 0xd8c0, 0xfffffff0, 0x00000100
1016 static u32 verde_pg_init
[] =
1018 0x353c, 0xffffffff, 0x40000,
1019 0x3538, 0xffffffff, 0x200010ff,
1020 0x353c, 0xffffffff, 0x0,
1021 0x353c, 0xffffffff, 0x0,
1022 0x353c, 0xffffffff, 0x0,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x7007,
1026 0x3538, 0xffffffff, 0x300010ff,
1027 0x353c, 0xffffffff, 0x0,
1028 0x353c, 0xffffffff, 0x0,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x400000,
1033 0x3538, 0xffffffff, 0x100010ff,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x120200,
1040 0x3538, 0xffffffff, 0x500010ff,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x1e1e16,
1047 0x3538, 0xffffffff, 0x600010ff,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x171f1e,
1054 0x3538, 0xffffffff, 0x700010ff,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x3538, 0xffffffff, 0x9ff,
1062 0x3500, 0xffffffff, 0x0,
1063 0x3504, 0xffffffff, 0x10000800,
1064 0x3504, 0xffffffff, 0xf,
1065 0x3504, 0xffffffff, 0xf,
1066 0x3500, 0xffffffff, 0x4,
1067 0x3504, 0xffffffff, 0x1000051e,
1068 0x3504, 0xffffffff, 0xffff,
1069 0x3504, 0xffffffff, 0xffff,
1070 0x3500, 0xffffffff, 0x8,
1071 0x3504, 0xffffffff, 0x80500,
1072 0x3500, 0xffffffff, 0x12,
1073 0x3504, 0xffffffff, 0x9050c,
1074 0x3500, 0xffffffff, 0x1d,
1075 0x3504, 0xffffffff, 0xb052c,
1076 0x3500, 0xffffffff, 0x2a,
1077 0x3504, 0xffffffff, 0x1053e,
1078 0x3500, 0xffffffff, 0x2d,
1079 0x3504, 0xffffffff, 0x10546,
1080 0x3500, 0xffffffff, 0x30,
1081 0x3504, 0xffffffff, 0xa054e,
1082 0x3500, 0xffffffff, 0x3c,
1083 0x3504, 0xffffffff, 0x1055f,
1084 0x3500, 0xffffffff, 0x3f,
1085 0x3504, 0xffffffff, 0x10567,
1086 0x3500, 0xffffffff, 0x42,
1087 0x3504, 0xffffffff, 0x1056f,
1088 0x3500, 0xffffffff, 0x45,
1089 0x3504, 0xffffffff, 0x10572,
1090 0x3500, 0xffffffff, 0x48,
1091 0x3504, 0xffffffff, 0x20575,
1092 0x3500, 0xffffffff, 0x4c,
1093 0x3504, 0xffffffff, 0x190801,
1094 0x3500, 0xffffffff, 0x67,
1095 0x3504, 0xffffffff, 0x1082a,
1096 0x3500, 0xffffffff, 0x6a,
1097 0x3504, 0xffffffff, 0x1b082d,
1098 0x3500, 0xffffffff, 0x87,
1099 0x3504, 0xffffffff, 0x310851,
1100 0x3500, 0xffffffff, 0xba,
1101 0x3504, 0xffffffff, 0x891,
1102 0x3500, 0xffffffff, 0xbc,
1103 0x3504, 0xffffffff, 0x893,
1104 0x3500, 0xffffffff, 0xbe,
1105 0x3504, 0xffffffff, 0x20895,
1106 0x3500, 0xffffffff, 0xc2,
1107 0x3504, 0xffffffff, 0x20899,
1108 0x3500, 0xffffffff, 0xc6,
1109 0x3504, 0xffffffff, 0x2089d,
1110 0x3500, 0xffffffff, 0xca,
1111 0x3504, 0xffffffff, 0x8a1,
1112 0x3500, 0xffffffff, 0xcc,
1113 0x3504, 0xffffffff, 0x8a3,
1114 0x3500, 0xffffffff, 0xce,
1115 0x3504, 0xffffffff, 0x308a5,
1116 0x3500, 0xffffffff, 0xd3,
1117 0x3504, 0xffffffff, 0x6d08cd,
1118 0x3500, 0xffffffff, 0x142,
1119 0x3504, 0xffffffff, 0x2000095a,
1120 0x3504, 0xffffffff, 0x1,
1121 0x3500, 0xffffffff, 0x144,
1122 0x3504, 0xffffffff, 0x301f095b,
1123 0x3500, 0xffffffff, 0x165,
1124 0x3504, 0xffffffff, 0xc094d,
1125 0x3500, 0xffffffff, 0x173,
1126 0x3504, 0xffffffff, 0xf096d,
1127 0x3500, 0xffffffff, 0x184,
1128 0x3504, 0xffffffff, 0x15097f,
1129 0x3500, 0xffffffff, 0x19b,
1130 0x3504, 0xffffffff, 0xc0998,
1131 0x3500, 0xffffffff, 0x1a9,
1132 0x3504, 0xffffffff, 0x409a7,
1133 0x3500, 0xffffffff, 0x1af,
1134 0x3504, 0xffffffff, 0xcdc,
1135 0x3500, 0xffffffff, 0x1b1,
1136 0x3504, 0xffffffff, 0x800,
1137 0x3508, 0xffffffff, 0x6c9b2000,
1138 0x3510, 0xfc00, 0x2000,
1139 0x3544, 0xffffffff, 0xfc0,
1140 0x28d4, 0x00000100, 0x100
1143 static void si_init_golden_registers(struct radeon_device
*rdev
)
1145 switch (rdev
->family
) {
1147 radeon_program_register_sequence(rdev
,
1148 tahiti_golden_registers
,
1149 (const u32
)ARRAY_SIZE(tahiti_golden_registers
));
1150 radeon_program_register_sequence(rdev
,
1151 tahiti_golden_rlc_registers
,
1152 (const u32
)ARRAY_SIZE(tahiti_golden_rlc_registers
));
1153 radeon_program_register_sequence(rdev
,
1154 tahiti_mgcg_cgcg_init
,
1155 (const u32
)ARRAY_SIZE(tahiti_mgcg_cgcg_init
));
1156 radeon_program_register_sequence(rdev
,
1157 tahiti_golden_registers2
,
1158 (const u32
)ARRAY_SIZE(tahiti_golden_registers2
));
1161 radeon_program_register_sequence(rdev
,
1162 pitcairn_golden_registers
,
1163 (const u32
)ARRAY_SIZE(pitcairn_golden_registers
));
1164 radeon_program_register_sequence(rdev
,
1165 pitcairn_golden_rlc_registers
,
1166 (const u32
)ARRAY_SIZE(pitcairn_golden_rlc_registers
));
1167 radeon_program_register_sequence(rdev
,
1168 pitcairn_mgcg_cgcg_init
,
1169 (const u32
)ARRAY_SIZE(pitcairn_mgcg_cgcg_init
));
1172 radeon_program_register_sequence(rdev
,
1173 verde_golden_registers
,
1174 (const u32
)ARRAY_SIZE(verde_golden_registers
));
1175 radeon_program_register_sequence(rdev
,
1176 verde_golden_rlc_registers
,
1177 (const u32
)ARRAY_SIZE(verde_golden_rlc_registers
));
1178 radeon_program_register_sequence(rdev
,
1179 verde_mgcg_cgcg_init
,
1180 (const u32
)ARRAY_SIZE(verde_mgcg_cgcg_init
));
1181 radeon_program_register_sequence(rdev
,
1183 (const u32
)ARRAY_SIZE(verde_pg_init
));
1186 radeon_program_register_sequence(rdev
,
1187 oland_golden_registers
,
1188 (const u32
)ARRAY_SIZE(oland_golden_registers
));
1189 radeon_program_register_sequence(rdev
,
1190 oland_golden_rlc_registers
,
1191 (const u32
)ARRAY_SIZE(oland_golden_rlc_registers
));
1192 radeon_program_register_sequence(rdev
,
1193 oland_mgcg_cgcg_init
,
1194 (const u32
)ARRAY_SIZE(oland_mgcg_cgcg_init
));
1197 radeon_program_register_sequence(rdev
,
1198 hainan_golden_registers
,
1199 (const u32
)ARRAY_SIZE(hainan_golden_registers
));
1200 radeon_program_register_sequence(rdev
,
1201 hainan_golden_registers2
,
1202 (const u32
)ARRAY_SIZE(hainan_golden_registers2
));
1203 radeon_program_register_sequence(rdev
,
1204 hainan_mgcg_cgcg_init
,
1205 (const u32
)ARRAY_SIZE(hainan_mgcg_cgcg_init
));
1212 #define PCIE_BUS_CLK 10000
1213 #define TCLK (PCIE_BUS_CLK / 10)
1216 * si_get_xclk - get the xclk
1218 * @rdev: radeon_device pointer
1220 * Returns the reference clock used by the gfx engine
1223 u32
si_get_xclk(struct radeon_device
*rdev
)
1225 u32 reference_clock
= rdev
->clock
.spll
.reference_freq
;
1228 tmp
= RREG32(CG_CLKPIN_CNTL_2
);
1229 if (tmp
& MUX_TCLK_TO_XCLK
)
1232 tmp
= RREG32(CG_CLKPIN_CNTL
);
1233 if (tmp
& XTALIN_DIVIDE
)
1234 return reference_clock
/ 4;
1236 return reference_clock
;
1239 /* get temperature in millidegrees */
1240 int si_get_temp(struct radeon_device
*rdev
)
1243 int actual_temp
= 0;
1245 temp
= (RREG32(CG_MULT_THERMAL_STATUS
) & CTF_TEMP_MASK
) >>
1251 actual_temp
= temp
& 0x1ff;
1253 actual_temp
= (actual_temp
* 1000);
1258 #define TAHITI_IO_MC_REGS_SIZE 36
1260 static const u32 tahiti_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1261 {0x0000006f, 0x03044000},
1262 {0x00000070, 0x0480c018},
1263 {0x00000071, 0x00000040},
1264 {0x00000072, 0x01000000},
1265 {0x00000074, 0x000000ff},
1266 {0x00000075, 0x00143400},
1267 {0x00000076, 0x08ec0800},
1268 {0x00000077, 0x040000cc},
1269 {0x00000079, 0x00000000},
1270 {0x0000007a, 0x21000409},
1271 {0x0000007c, 0x00000000},
1272 {0x0000007d, 0xe8000000},
1273 {0x0000007e, 0x044408a8},
1274 {0x0000007f, 0x00000003},
1275 {0x00000080, 0x00000000},
1276 {0x00000081, 0x01000000},
1277 {0x00000082, 0x02000000},
1278 {0x00000083, 0x00000000},
1279 {0x00000084, 0xe3f3e4f4},
1280 {0x00000085, 0x00052024},
1281 {0x00000087, 0x00000000},
1282 {0x00000088, 0x66036603},
1283 {0x00000089, 0x01000000},
1284 {0x0000008b, 0x1c0a0000},
1285 {0x0000008c, 0xff010000},
1286 {0x0000008e, 0xffffefff},
1287 {0x0000008f, 0xfff3efff},
1288 {0x00000090, 0xfff3efbf},
1289 {0x00000094, 0x00101101},
1290 {0x00000095, 0x00000fff},
1291 {0x00000096, 0x00116fff},
1292 {0x00000097, 0x60010000},
1293 {0x00000098, 0x10010000},
1294 {0x00000099, 0x00006000},
1295 {0x0000009a, 0x00001000},
1296 {0x0000009f, 0x00a77400}
1299 static const u32 pitcairn_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1300 {0x0000006f, 0x03044000},
1301 {0x00000070, 0x0480c018},
1302 {0x00000071, 0x00000040},
1303 {0x00000072, 0x01000000},
1304 {0x00000074, 0x000000ff},
1305 {0x00000075, 0x00143400},
1306 {0x00000076, 0x08ec0800},
1307 {0x00000077, 0x040000cc},
1308 {0x00000079, 0x00000000},
1309 {0x0000007a, 0x21000409},
1310 {0x0000007c, 0x00000000},
1311 {0x0000007d, 0xe8000000},
1312 {0x0000007e, 0x044408a8},
1313 {0x0000007f, 0x00000003},
1314 {0x00000080, 0x00000000},
1315 {0x00000081, 0x01000000},
1316 {0x00000082, 0x02000000},
1317 {0x00000083, 0x00000000},
1318 {0x00000084, 0xe3f3e4f4},
1319 {0x00000085, 0x00052024},
1320 {0x00000087, 0x00000000},
1321 {0x00000088, 0x66036603},
1322 {0x00000089, 0x01000000},
1323 {0x0000008b, 0x1c0a0000},
1324 {0x0000008c, 0xff010000},
1325 {0x0000008e, 0xffffefff},
1326 {0x0000008f, 0xfff3efff},
1327 {0x00000090, 0xfff3efbf},
1328 {0x00000094, 0x00101101},
1329 {0x00000095, 0x00000fff},
1330 {0x00000096, 0x00116fff},
1331 {0x00000097, 0x60010000},
1332 {0x00000098, 0x10010000},
1333 {0x00000099, 0x00006000},
1334 {0x0000009a, 0x00001000},
1335 {0x0000009f, 0x00a47400}
1338 static const u32 verde_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1339 {0x0000006f, 0x03044000},
1340 {0x00000070, 0x0480c018},
1341 {0x00000071, 0x00000040},
1342 {0x00000072, 0x01000000},
1343 {0x00000074, 0x000000ff},
1344 {0x00000075, 0x00143400},
1345 {0x00000076, 0x08ec0800},
1346 {0x00000077, 0x040000cc},
1347 {0x00000079, 0x00000000},
1348 {0x0000007a, 0x21000409},
1349 {0x0000007c, 0x00000000},
1350 {0x0000007d, 0xe8000000},
1351 {0x0000007e, 0x044408a8},
1352 {0x0000007f, 0x00000003},
1353 {0x00000080, 0x00000000},
1354 {0x00000081, 0x01000000},
1355 {0x00000082, 0x02000000},
1356 {0x00000083, 0x00000000},
1357 {0x00000084, 0xe3f3e4f4},
1358 {0x00000085, 0x00052024},
1359 {0x00000087, 0x00000000},
1360 {0x00000088, 0x66036603},
1361 {0x00000089, 0x01000000},
1362 {0x0000008b, 0x1c0a0000},
1363 {0x0000008c, 0xff010000},
1364 {0x0000008e, 0xffffefff},
1365 {0x0000008f, 0xfff3efff},
1366 {0x00000090, 0xfff3efbf},
1367 {0x00000094, 0x00101101},
1368 {0x00000095, 0x00000fff},
1369 {0x00000096, 0x00116fff},
1370 {0x00000097, 0x60010000},
1371 {0x00000098, 0x10010000},
1372 {0x00000099, 0x00006000},
1373 {0x0000009a, 0x00001000},
1374 {0x0000009f, 0x00a37400}
1377 static const u32 oland_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1378 {0x0000006f, 0x03044000},
1379 {0x00000070, 0x0480c018},
1380 {0x00000071, 0x00000040},
1381 {0x00000072, 0x01000000},
1382 {0x00000074, 0x000000ff},
1383 {0x00000075, 0x00143400},
1384 {0x00000076, 0x08ec0800},
1385 {0x00000077, 0x040000cc},
1386 {0x00000079, 0x00000000},
1387 {0x0000007a, 0x21000409},
1388 {0x0000007c, 0x00000000},
1389 {0x0000007d, 0xe8000000},
1390 {0x0000007e, 0x044408a8},
1391 {0x0000007f, 0x00000003},
1392 {0x00000080, 0x00000000},
1393 {0x00000081, 0x01000000},
1394 {0x00000082, 0x02000000},
1395 {0x00000083, 0x00000000},
1396 {0x00000084, 0xe3f3e4f4},
1397 {0x00000085, 0x00052024},
1398 {0x00000087, 0x00000000},
1399 {0x00000088, 0x66036603},
1400 {0x00000089, 0x01000000},
1401 {0x0000008b, 0x1c0a0000},
1402 {0x0000008c, 0xff010000},
1403 {0x0000008e, 0xffffefff},
1404 {0x0000008f, 0xfff3efff},
1405 {0x00000090, 0xfff3efbf},
1406 {0x00000094, 0x00101101},
1407 {0x00000095, 0x00000fff},
1408 {0x00000096, 0x00116fff},
1409 {0x00000097, 0x60010000},
1410 {0x00000098, 0x10010000},
1411 {0x00000099, 0x00006000},
1412 {0x0000009a, 0x00001000},
1413 {0x0000009f, 0x00a17730}
1416 static const u32 hainan_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1417 {0x0000006f, 0x03044000},
1418 {0x00000070, 0x0480c018},
1419 {0x00000071, 0x00000040},
1420 {0x00000072, 0x01000000},
1421 {0x00000074, 0x000000ff},
1422 {0x00000075, 0x00143400},
1423 {0x00000076, 0x08ec0800},
1424 {0x00000077, 0x040000cc},
1425 {0x00000079, 0x00000000},
1426 {0x0000007a, 0x21000409},
1427 {0x0000007c, 0x00000000},
1428 {0x0000007d, 0xe8000000},
1429 {0x0000007e, 0x044408a8},
1430 {0x0000007f, 0x00000003},
1431 {0x00000080, 0x00000000},
1432 {0x00000081, 0x01000000},
1433 {0x00000082, 0x02000000},
1434 {0x00000083, 0x00000000},
1435 {0x00000084, 0xe3f3e4f4},
1436 {0x00000085, 0x00052024},
1437 {0x00000087, 0x00000000},
1438 {0x00000088, 0x66036603},
1439 {0x00000089, 0x01000000},
1440 {0x0000008b, 0x1c0a0000},
1441 {0x0000008c, 0xff010000},
1442 {0x0000008e, 0xffffefff},
1443 {0x0000008f, 0xfff3efff},
1444 {0x00000090, 0xfff3efbf},
1445 {0x00000094, 0x00101101},
1446 {0x00000095, 0x00000fff},
1447 {0x00000096, 0x00116fff},
1448 {0x00000097, 0x60010000},
1449 {0x00000098, 0x10010000},
1450 {0x00000099, 0x00006000},
1451 {0x0000009a, 0x00001000},
1452 {0x0000009f, 0x00a07730}
1456 static int si_mc_load_microcode(struct radeon_device
*rdev
)
1458 const __be32
*fw_data
;
1459 u32 running
, blackout
= 0;
1461 int i
, ucode_size
, regs_size
;
1466 switch (rdev
->family
) {
1468 io_mc_regs
= (u32
*)&tahiti_io_mc_regs
;
1469 ucode_size
= SI_MC_UCODE_SIZE
;
1470 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1473 io_mc_regs
= (u32
*)&pitcairn_io_mc_regs
;
1474 ucode_size
= SI_MC_UCODE_SIZE
;
1475 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1479 io_mc_regs
= (u32
*)&verde_io_mc_regs
;
1480 ucode_size
= SI_MC_UCODE_SIZE
;
1481 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1484 io_mc_regs
= (u32
*)&oland_io_mc_regs
;
1485 ucode_size
= OLAND_MC_UCODE_SIZE
;
1486 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1489 io_mc_regs
= (u32
*)&hainan_io_mc_regs
;
1490 ucode_size
= OLAND_MC_UCODE_SIZE
;
1491 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1495 running
= RREG32(MC_SEQ_SUP_CNTL
) & RUN_MASK
;
1499 blackout
= RREG32(MC_SHARED_BLACKOUT_CNTL
);
1500 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
| 1);
1503 /* reset the engine and set to writable */
1504 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1505 WREG32(MC_SEQ_SUP_CNTL
, 0x00000010);
1507 /* load mc io regs */
1508 for (i
= 0; i
< regs_size
; i
++) {
1509 WREG32(MC_SEQ_IO_DEBUG_INDEX
, io_mc_regs
[(i
<< 1)]);
1510 WREG32(MC_SEQ_IO_DEBUG_DATA
, io_mc_regs
[(i
<< 1) + 1]);
1512 /* load the MC ucode */
1513 fw_data
= (const __be32
*)rdev
->mc_fw
->data
;
1514 for (i
= 0; i
< ucode_size
; i
++)
1515 WREG32(MC_SEQ_SUP_PGM
, be32_to_cpup(fw_data
++));
1517 /* put the engine back into the active state */
1518 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1519 WREG32(MC_SEQ_SUP_CNTL
, 0x00000004);
1520 WREG32(MC_SEQ_SUP_CNTL
, 0x00000001);
1522 /* wait for training to complete */
1523 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1524 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D0
)
1528 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1529 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D1
)
1535 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
);
1541 static int si_init_microcode(struct radeon_device
*rdev
)
1543 const char *chip_name
;
1544 const char *rlc_chip_name
;
1545 size_t pfp_req_size
, me_req_size
, ce_req_size
, rlc_req_size
, mc_req_size
;
1546 size_t smc_req_size
;
1552 switch (rdev
->family
) {
1554 chip_name
= "TAHITI";
1555 rlc_chip_name
= "TAHITI";
1556 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1557 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1558 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1559 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1560 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1561 smc_req_size
= ALIGN(TAHITI_SMC_UCODE_SIZE
, 4);
1564 chip_name
= "PITCAIRN";
1565 rlc_chip_name
= "PITCAIRN";
1566 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1567 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1568 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1569 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1570 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1571 smc_req_size
= ALIGN(PITCAIRN_SMC_UCODE_SIZE
, 4);
1574 chip_name
= "VERDE";
1575 rlc_chip_name
= "VERDE";
1576 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1577 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1578 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1579 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1580 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1581 smc_req_size
= ALIGN(VERDE_SMC_UCODE_SIZE
, 4);
1584 chip_name
= "OLAND";
1585 rlc_chip_name
= "OLAND";
1586 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1587 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1588 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1589 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1590 mc_req_size
= OLAND_MC_UCODE_SIZE
* 4;
1591 smc_req_size
= ALIGN(OLAND_SMC_UCODE_SIZE
, 4);
1594 chip_name
= "HAINAN";
1595 rlc_chip_name
= "HAINAN";
1596 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1597 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1598 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1599 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1600 mc_req_size
= OLAND_MC_UCODE_SIZE
* 4;
1601 smc_req_size
= ALIGN(HAINAN_SMC_UCODE_SIZE
, 4);
1606 DRM_INFO("Loading %s Microcode\n", chip_name
);
1608 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_pfp.bin", chip_name
);
1609 err
= request_firmware(&rdev
->pfp_fw
, fw_name
, rdev
->dev
);
1612 if (rdev
->pfp_fw
->size
!= pfp_req_size
) {
1614 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1615 rdev
->pfp_fw
->size
, fw_name
);
1620 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_me.bin", chip_name
);
1621 err
= request_firmware(&rdev
->me_fw
, fw_name
, rdev
->dev
);
1624 if (rdev
->me_fw
->size
!= me_req_size
) {
1626 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1627 rdev
->me_fw
->size
, fw_name
);
1631 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_ce.bin", chip_name
);
1632 err
= request_firmware(&rdev
->ce_fw
, fw_name
, rdev
->dev
);
1635 if (rdev
->ce_fw
->size
!= ce_req_size
) {
1637 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1638 rdev
->ce_fw
->size
, fw_name
);
1642 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_rlc.bin", rlc_chip_name
);
1643 err
= request_firmware(&rdev
->rlc_fw
, fw_name
, rdev
->dev
);
1646 if (rdev
->rlc_fw
->size
!= rlc_req_size
) {
1648 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1649 rdev
->rlc_fw
->size
, fw_name
);
1653 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mc.bin", chip_name
);
1654 err
= request_firmware(&rdev
->mc_fw
, fw_name
, rdev
->dev
);
1657 if (rdev
->mc_fw
->size
!= mc_req_size
) {
1659 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1660 rdev
->mc_fw
->size
, fw_name
);
1664 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_smc.bin", chip_name
);
1665 err
= request_firmware(&rdev
->smc_fw
, fw_name
, rdev
->dev
);
1668 "smc: error loading firmware \"%s\"\n",
1670 release_firmware(rdev
->smc_fw
);
1671 rdev
->smc_fw
= NULL
;
1672 } else if (rdev
->smc_fw
->size
!= smc_req_size
) {
1674 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1675 rdev
->smc_fw
->size
, fw_name
);
1683 "si_cp: Failed to load firmware \"%s\"\n",
1685 release_firmware(rdev
->pfp_fw
);
1686 rdev
->pfp_fw
= NULL
;
1687 release_firmware(rdev
->me_fw
);
1689 release_firmware(rdev
->ce_fw
);
1691 release_firmware(rdev
->rlc_fw
);
1692 rdev
->rlc_fw
= NULL
;
1693 release_firmware(rdev
->mc_fw
);
1695 release_firmware(rdev
->smc_fw
);
1696 rdev
->smc_fw
= NULL
;
1701 /* watermark setup */
1702 static u32
dce6_line_buffer_adjust(struct radeon_device
*rdev
,
1703 struct radeon_crtc
*radeon_crtc
,
1704 struct drm_display_mode
*mode
,
1705 struct drm_display_mode
*other_mode
)
1710 * There are 3 line buffers, each one shared by 2 display controllers.
1711 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1712 * the display controllers. The paritioning is done via one of four
1713 * preset allocations specified in bits 21:20:
1715 * 2 - whole lb, other crtc must be disabled
1717 /* this can get tricky if we have two large displays on a paired group
1718 * of crtcs. Ideally for multiple large displays we'd assign them to
1719 * non-linked crtcs for maximum line buffer allocation.
1721 if (radeon_crtc
->base
.enabled
&& mode
) {
1725 tmp
= 2; /* whole */
1729 WREG32(DC_LB_MEMORY_SPLIT
+ radeon_crtc
->crtc_offset
,
1730 DC_LB_MEMORY_CONFIG(tmp
));
1732 if (radeon_crtc
->base
.enabled
&& mode
) {
1742 /* controller not enabled, so no lb used */
1746 static u32
si_get_number_of_dram_channels(struct radeon_device
*rdev
)
1748 u32 tmp
= RREG32(MC_SHARED_CHMAP
);
1750 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
1773 struct dce6_wm_params
{
1774 u32 dram_channels
; /* number of dram channels */
1775 u32 yclk
; /* bandwidth per dram data pin in kHz */
1776 u32 sclk
; /* engine clock in kHz */
1777 u32 disp_clk
; /* display clock in kHz */
1778 u32 src_width
; /* viewport width */
1779 u32 active_time
; /* active display time in ns */
1780 u32 blank_time
; /* blank time in ns */
1781 bool interlaced
; /* mode is interlaced */
1782 fixed20_12 vsc
; /* vertical scale ratio */
1783 u32 num_heads
; /* number of active crtcs */
1784 u32 bytes_per_pixel
; /* bytes per pixel display + overlay */
1785 u32 lb_size
; /* line buffer allocated to pipe */
1786 u32 vtaps
; /* vertical scaler taps */
1789 static u32
dce6_dram_bandwidth(struct dce6_wm_params
*wm
)
1791 /* Calculate raw DRAM Bandwidth */
1792 fixed20_12 dram_efficiency
; /* 0.7 */
1793 fixed20_12 yclk
, dram_channels
, bandwidth
;
1796 a
.full
= dfixed_const(1000);
1797 yclk
.full
= dfixed_const(wm
->yclk
);
1798 yclk
.full
= dfixed_div(yclk
, a
);
1799 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
1800 a
.full
= dfixed_const(10);
1801 dram_efficiency
.full
= dfixed_const(7);
1802 dram_efficiency
.full
= dfixed_div(dram_efficiency
, a
);
1803 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
1804 bandwidth
.full
= dfixed_mul(bandwidth
, dram_efficiency
);
1806 return dfixed_trunc(bandwidth
);
1809 static u32
dce6_dram_bandwidth_for_display(struct dce6_wm_params
*wm
)
1811 /* Calculate DRAM Bandwidth and the part allocated to display. */
1812 fixed20_12 disp_dram_allocation
; /* 0.3 to 0.7 */
1813 fixed20_12 yclk
, dram_channels
, bandwidth
;
1816 a
.full
= dfixed_const(1000);
1817 yclk
.full
= dfixed_const(wm
->yclk
);
1818 yclk
.full
= dfixed_div(yclk
, a
);
1819 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
1820 a
.full
= dfixed_const(10);
1821 disp_dram_allocation
.full
= dfixed_const(3); /* XXX worse case value 0.3 */
1822 disp_dram_allocation
.full
= dfixed_div(disp_dram_allocation
, a
);
1823 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
1824 bandwidth
.full
= dfixed_mul(bandwidth
, disp_dram_allocation
);
1826 return dfixed_trunc(bandwidth
);
1829 static u32
dce6_data_return_bandwidth(struct dce6_wm_params
*wm
)
1831 /* Calculate the display Data return Bandwidth */
1832 fixed20_12 return_efficiency
; /* 0.8 */
1833 fixed20_12 sclk
, bandwidth
;
1836 a
.full
= dfixed_const(1000);
1837 sclk
.full
= dfixed_const(wm
->sclk
);
1838 sclk
.full
= dfixed_div(sclk
, a
);
1839 a
.full
= dfixed_const(10);
1840 return_efficiency
.full
= dfixed_const(8);
1841 return_efficiency
.full
= dfixed_div(return_efficiency
, a
);
1842 a
.full
= dfixed_const(32);
1843 bandwidth
.full
= dfixed_mul(a
, sclk
);
1844 bandwidth
.full
= dfixed_mul(bandwidth
, return_efficiency
);
1846 return dfixed_trunc(bandwidth
);
1849 static u32
dce6_get_dmif_bytes_per_request(struct dce6_wm_params
*wm
)
1854 static u32
dce6_dmif_request_bandwidth(struct dce6_wm_params
*wm
)
1856 /* Calculate the DMIF Request Bandwidth */
1857 fixed20_12 disp_clk_request_efficiency
; /* 0.8 */
1858 fixed20_12 disp_clk
, sclk
, bandwidth
;
1859 fixed20_12 a
, b1
, b2
;
1862 a
.full
= dfixed_const(1000);
1863 disp_clk
.full
= dfixed_const(wm
->disp_clk
);
1864 disp_clk
.full
= dfixed_div(disp_clk
, a
);
1865 a
.full
= dfixed_const(dce6_get_dmif_bytes_per_request(wm
) / 2);
1866 b1
.full
= dfixed_mul(a
, disp_clk
);
1868 a
.full
= dfixed_const(1000);
1869 sclk
.full
= dfixed_const(wm
->sclk
);
1870 sclk
.full
= dfixed_div(sclk
, a
);
1871 a
.full
= dfixed_const(dce6_get_dmif_bytes_per_request(wm
));
1872 b2
.full
= dfixed_mul(a
, sclk
);
1874 a
.full
= dfixed_const(10);
1875 disp_clk_request_efficiency
.full
= dfixed_const(8);
1876 disp_clk_request_efficiency
.full
= dfixed_div(disp_clk_request_efficiency
, a
);
1878 min_bandwidth
= min(dfixed_trunc(b1
), dfixed_trunc(b2
));
1880 a
.full
= dfixed_const(min_bandwidth
);
1881 bandwidth
.full
= dfixed_mul(a
, disp_clk_request_efficiency
);
1883 return dfixed_trunc(bandwidth
);
1886 static u32
dce6_available_bandwidth(struct dce6_wm_params
*wm
)
1888 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1889 u32 dram_bandwidth
= dce6_dram_bandwidth(wm
);
1890 u32 data_return_bandwidth
= dce6_data_return_bandwidth(wm
);
1891 u32 dmif_req_bandwidth
= dce6_dmif_request_bandwidth(wm
);
1893 return min(dram_bandwidth
, min(data_return_bandwidth
, dmif_req_bandwidth
));
1896 static u32
dce6_average_bandwidth(struct dce6_wm_params
*wm
)
1898 /* Calculate the display mode Average Bandwidth
1899 * DisplayMode should contain the source and destination dimensions,
1903 fixed20_12 line_time
;
1904 fixed20_12 src_width
;
1905 fixed20_12 bandwidth
;
1908 a
.full
= dfixed_const(1000);
1909 line_time
.full
= dfixed_const(wm
->active_time
+ wm
->blank_time
);
1910 line_time
.full
= dfixed_div(line_time
, a
);
1911 bpp
.full
= dfixed_const(wm
->bytes_per_pixel
);
1912 src_width
.full
= dfixed_const(wm
->src_width
);
1913 bandwidth
.full
= dfixed_mul(src_width
, bpp
);
1914 bandwidth
.full
= dfixed_mul(bandwidth
, wm
->vsc
);
1915 bandwidth
.full
= dfixed_div(bandwidth
, line_time
);
1917 return dfixed_trunc(bandwidth
);
1920 static u32
dce6_latency_watermark(struct dce6_wm_params
*wm
)
1922 /* First calcualte the latency in ns */
1923 u32 mc_latency
= 2000; /* 2000 ns. */
1924 u32 available_bandwidth
= dce6_available_bandwidth(wm
);
1925 u32 worst_chunk_return_time
= (512 * 8 * 1000) / available_bandwidth
;
1926 u32 cursor_line_pair_return_time
= (128 * 4 * 1000) / available_bandwidth
;
1927 u32 dc_latency
= 40000000 / wm
->disp_clk
; /* dc pipe latency */
1928 u32 other_heads_data_return_time
= ((wm
->num_heads
+ 1) * worst_chunk_return_time
) +
1929 (wm
->num_heads
* cursor_line_pair_return_time
);
1930 u32 latency
= mc_latency
+ other_heads_data_return_time
+ dc_latency
;
1931 u32 max_src_lines_per_dst_line
, lb_fill_bw
, line_fill_time
;
1932 u32 tmp
, dmif_size
= 12288;
1935 if (wm
->num_heads
== 0)
1938 a
.full
= dfixed_const(2);
1939 b
.full
= dfixed_const(1);
1940 if ((wm
->vsc
.full
> a
.full
) ||
1941 ((wm
->vsc
.full
> b
.full
) && (wm
->vtaps
>= 3)) ||
1943 ((wm
->vsc
.full
>= a
.full
) && wm
->interlaced
))
1944 max_src_lines_per_dst_line
= 4;
1946 max_src_lines_per_dst_line
= 2;
1948 a
.full
= dfixed_const(available_bandwidth
);
1949 b
.full
= dfixed_const(wm
->num_heads
);
1950 a
.full
= dfixed_div(a
, b
);
1952 b
.full
= dfixed_const(mc_latency
+ 512);
1953 c
.full
= dfixed_const(wm
->disp_clk
);
1954 b
.full
= dfixed_div(b
, c
);
1956 c
.full
= dfixed_const(dmif_size
);
1957 b
.full
= dfixed_div(c
, b
);
1959 tmp
= min(dfixed_trunc(a
), dfixed_trunc(b
));
1961 b
.full
= dfixed_const(1000);
1962 c
.full
= dfixed_const(wm
->disp_clk
);
1963 b
.full
= dfixed_div(c
, b
);
1964 c
.full
= dfixed_const(wm
->bytes_per_pixel
);
1965 b
.full
= dfixed_mul(b
, c
);
1967 lb_fill_bw
= min(tmp
, dfixed_trunc(b
));
1969 a
.full
= dfixed_const(max_src_lines_per_dst_line
* wm
->src_width
* wm
->bytes_per_pixel
);
1970 b
.full
= dfixed_const(1000);
1971 c
.full
= dfixed_const(lb_fill_bw
);
1972 b
.full
= dfixed_div(c
, b
);
1973 a
.full
= dfixed_div(a
, b
);
1974 line_fill_time
= dfixed_trunc(a
);
1976 if (line_fill_time
< wm
->active_time
)
1979 return latency
+ (line_fill_time
- wm
->active_time
);
1983 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params
*wm
)
1985 if (dce6_average_bandwidth(wm
) <=
1986 (dce6_dram_bandwidth_for_display(wm
) / wm
->num_heads
))
1992 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params
*wm
)
1994 if (dce6_average_bandwidth(wm
) <=
1995 (dce6_available_bandwidth(wm
) / wm
->num_heads
))
2001 static bool dce6_check_latency_hiding(struct dce6_wm_params
*wm
)
2003 u32 lb_partitions
= wm
->lb_size
/ wm
->src_width
;
2004 u32 line_time
= wm
->active_time
+ wm
->blank_time
;
2005 u32 latency_tolerant_lines
;
2009 a
.full
= dfixed_const(1);
2010 if (wm
->vsc
.full
> a
.full
)
2011 latency_tolerant_lines
= 1;
2013 if (lb_partitions
<= (wm
->vtaps
+ 1))
2014 latency_tolerant_lines
= 1;
2016 latency_tolerant_lines
= 2;
2019 latency_hiding
= (latency_tolerant_lines
* line_time
+ wm
->blank_time
);
2021 if (dce6_latency_watermark(wm
) <= latency_hiding
)
2027 static void dce6_program_watermarks(struct radeon_device
*rdev
,
2028 struct radeon_crtc
*radeon_crtc
,
2029 u32 lb_size
, u32 num_heads
)
2031 struct drm_display_mode
*mode
= &radeon_crtc
->base
.mode
;
2032 struct dce6_wm_params wm_low
, wm_high
;
2036 u32 latency_watermark_a
= 0, latency_watermark_b
= 0;
2037 u32 priority_a_mark
= 0, priority_b_mark
= 0;
2038 u32 priority_a_cnt
= PRIORITY_OFF
;
2039 u32 priority_b_cnt
= PRIORITY_OFF
;
2040 u32 tmp
, arb_control3
;
2043 if (radeon_crtc
->base
.enabled
&& num_heads
&& mode
) {
2044 pixel_period
= 1000000 / (u32
)mode
->clock
;
2045 line_time
= min((u32
)mode
->crtc_htotal
* pixel_period
, (u32
)65535);
2049 if (rdev
->family
== CHIP_ARUBA
)
2050 dram_channels
= evergreen_get_number_of_dram_channels(rdev
);
2052 dram_channels
= si_get_number_of_dram_channels(rdev
);
2054 /* watermark for high clocks */
2055 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) && rdev
->pm
.dpm_enabled
) {
2057 radeon_dpm_get_mclk(rdev
, false) * 10;
2059 radeon_dpm_get_sclk(rdev
, false) * 10;
2061 wm_high
.yclk
= rdev
->pm
.current_mclk
* 10;
2062 wm_high
.sclk
= rdev
->pm
.current_sclk
* 10;
2065 wm_high
.disp_clk
= mode
->clock
;
2066 wm_high
.src_width
= mode
->crtc_hdisplay
;
2067 wm_high
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
2068 wm_high
.blank_time
= line_time
- wm_high
.active_time
;
2069 wm_high
.interlaced
= false;
2070 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
2071 wm_high
.interlaced
= true;
2072 wm_high
.vsc
= radeon_crtc
->vsc
;
2074 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
2076 wm_high
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
2077 wm_high
.lb_size
= lb_size
;
2078 wm_high
.dram_channels
= dram_channels
;
2079 wm_high
.num_heads
= num_heads
;
2081 /* watermark for low clocks */
2082 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) && rdev
->pm
.dpm_enabled
) {
2084 radeon_dpm_get_mclk(rdev
, true) * 10;
2086 radeon_dpm_get_sclk(rdev
, true) * 10;
2088 wm_low
.yclk
= rdev
->pm
.current_mclk
* 10;
2089 wm_low
.sclk
= rdev
->pm
.current_sclk
* 10;
2092 wm_low
.disp_clk
= mode
->clock
;
2093 wm_low
.src_width
= mode
->crtc_hdisplay
;
2094 wm_low
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
2095 wm_low
.blank_time
= line_time
- wm_low
.active_time
;
2096 wm_low
.interlaced
= false;
2097 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
2098 wm_low
.interlaced
= true;
2099 wm_low
.vsc
= radeon_crtc
->vsc
;
2101 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
2103 wm_low
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
2104 wm_low
.lb_size
= lb_size
;
2105 wm_low
.dram_channels
= dram_channels
;
2106 wm_low
.num_heads
= num_heads
;
2108 /* set for high clocks */
2109 latency_watermark_a
= min(dce6_latency_watermark(&wm_high
), (u32
)65535);
2110 /* set for low clocks */
2111 latency_watermark_b
= min(dce6_latency_watermark(&wm_low
), (u32
)65535);
2113 /* possibly force display priority to high */
2114 /* should really do this at mode validation time... */
2115 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high
) ||
2116 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high
) ||
2117 !dce6_check_latency_hiding(&wm_high
) ||
2118 (rdev
->disp_priority
== 2)) {
2119 DRM_DEBUG_KMS("force priority to high\n");
2120 priority_a_cnt
|= PRIORITY_ALWAYS_ON
;
2121 priority_b_cnt
|= PRIORITY_ALWAYS_ON
;
2123 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low
) ||
2124 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low
) ||
2125 !dce6_check_latency_hiding(&wm_low
) ||
2126 (rdev
->disp_priority
== 2)) {
2127 DRM_DEBUG_KMS("force priority to high\n");
2128 priority_a_cnt
|= PRIORITY_ALWAYS_ON
;
2129 priority_b_cnt
|= PRIORITY_ALWAYS_ON
;
2132 a
.full
= dfixed_const(1000);
2133 b
.full
= dfixed_const(mode
->clock
);
2134 b
.full
= dfixed_div(b
, a
);
2135 c
.full
= dfixed_const(latency_watermark_a
);
2136 c
.full
= dfixed_mul(c
, b
);
2137 c
.full
= dfixed_mul(c
, radeon_crtc
->hsc
);
2138 c
.full
= dfixed_div(c
, a
);
2139 a
.full
= dfixed_const(16);
2140 c
.full
= dfixed_div(c
, a
);
2141 priority_a_mark
= dfixed_trunc(c
);
2142 priority_a_cnt
|= priority_a_mark
& PRIORITY_MARK_MASK
;
2144 a
.full
= dfixed_const(1000);
2145 b
.full
= dfixed_const(mode
->clock
);
2146 b
.full
= dfixed_div(b
, a
);
2147 c
.full
= dfixed_const(latency_watermark_b
);
2148 c
.full
= dfixed_mul(c
, b
);
2149 c
.full
= dfixed_mul(c
, radeon_crtc
->hsc
);
2150 c
.full
= dfixed_div(c
, a
);
2151 a
.full
= dfixed_const(16);
2152 c
.full
= dfixed_div(c
, a
);
2153 priority_b_mark
= dfixed_trunc(c
);
2154 priority_b_cnt
|= priority_b_mark
& PRIORITY_MARK_MASK
;
2158 arb_control3
= RREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
);
2160 tmp
&= ~LATENCY_WATERMARK_MASK(3);
2161 tmp
|= LATENCY_WATERMARK_MASK(1);
2162 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, tmp
);
2163 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
2164 (LATENCY_LOW_WATERMARK(latency_watermark_a
) |
2165 LATENCY_HIGH_WATERMARK(line_time
)));
2167 tmp
= RREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
);
2168 tmp
&= ~LATENCY_WATERMARK_MASK(3);
2169 tmp
|= LATENCY_WATERMARK_MASK(2);
2170 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, tmp
);
2171 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
2172 (LATENCY_LOW_WATERMARK(latency_watermark_b
) |
2173 LATENCY_HIGH_WATERMARK(line_time
)));
2174 /* restore original selection */
2175 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, arb_control3
);
2177 /* write the priority marks */
2178 WREG32(PRIORITY_A_CNT
+ radeon_crtc
->crtc_offset
, priority_a_cnt
);
2179 WREG32(PRIORITY_B_CNT
+ radeon_crtc
->crtc_offset
, priority_b_cnt
);
2181 /* save values for DPM */
2182 radeon_crtc
->line_time
= line_time
;
2183 radeon_crtc
->wm_high
= latency_watermark_a
;
2184 radeon_crtc
->wm_low
= latency_watermark_b
;
2187 void dce6_bandwidth_update(struct radeon_device
*rdev
)
2189 struct drm_display_mode
*mode0
= NULL
;
2190 struct drm_display_mode
*mode1
= NULL
;
2191 u32 num_heads
= 0, lb_size
;
2194 radeon_update_display_priority(rdev
);
2196 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
2197 if (rdev
->mode_info
.crtcs
[i
]->base
.enabled
)
2200 for (i
= 0; i
< rdev
->num_crtc
; i
+= 2) {
2201 mode0
= &rdev
->mode_info
.crtcs
[i
]->base
.mode
;
2202 mode1
= &rdev
->mode_info
.crtcs
[i
+1]->base
.mode
;
2203 lb_size
= dce6_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
], mode0
, mode1
);
2204 dce6_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
], lb_size
, num_heads
);
2205 lb_size
= dce6_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
+1], mode1
, mode0
);
2206 dce6_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
+1], lb_size
, num_heads
);
2213 static void si_tiling_mode_table_init(struct radeon_device
*rdev
)
2215 const u32 num_tile_mode_states
= 32;
2216 u32 reg_offset
, gb_tile_moden
, split_equal_to_row_size
;
2218 switch (rdev
->config
.si
.mem_row_size_in_kb
) {
2220 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_1KB
;
2224 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_2KB
;
2227 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_4KB
;
2231 if ((rdev
->family
== CHIP_TAHITI
) ||
2232 (rdev
->family
== CHIP_PITCAIRN
)) {
2233 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2234 switch (reg_offset
) {
2235 case 0: /* non-AA compressed depth or any compressed stencil */
2236 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2237 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2240 NUM_BANKS(ADDR_SURF_16_BANK
) |
2241 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2242 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2243 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2245 case 1: /* 2xAA/4xAA compressed depth only */
2246 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2247 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2248 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2249 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2250 NUM_BANKS(ADDR_SURF_16_BANK
) |
2251 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2252 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2253 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2255 case 2: /* 8xAA compressed depth only */
2256 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2257 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2258 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2259 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2260 NUM_BANKS(ADDR_SURF_16_BANK
) |
2261 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2262 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2263 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2265 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2266 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2267 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2268 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2269 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2270 NUM_BANKS(ADDR_SURF_16_BANK
) |
2271 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2272 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2273 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2275 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2276 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2277 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2278 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2279 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2280 NUM_BANKS(ADDR_SURF_16_BANK
) |
2281 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2282 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2283 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2285 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2286 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2287 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2288 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2289 TILE_SPLIT(split_equal_to_row_size
) |
2290 NUM_BANKS(ADDR_SURF_16_BANK
) |
2291 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2292 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2293 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2295 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2296 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2297 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2298 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2299 TILE_SPLIT(split_equal_to_row_size
) |
2300 NUM_BANKS(ADDR_SURF_16_BANK
) |
2301 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2302 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2303 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2305 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2306 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2307 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2309 TILE_SPLIT(split_equal_to_row_size
) |
2310 NUM_BANKS(ADDR_SURF_16_BANK
) |
2311 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2312 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2313 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2315 case 8: /* 1D and 1D Array Surfaces */
2316 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2317 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2320 NUM_BANKS(ADDR_SURF_16_BANK
) |
2321 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2322 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2323 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2325 case 9: /* Displayable maps. */
2326 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2327 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2328 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2329 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2330 NUM_BANKS(ADDR_SURF_16_BANK
) |
2331 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2332 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2333 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2335 case 10: /* Display 8bpp. */
2336 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2337 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2338 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2339 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2340 NUM_BANKS(ADDR_SURF_16_BANK
) |
2341 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2342 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2343 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2345 case 11: /* Display 16bpp. */
2346 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2347 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2348 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2349 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2350 NUM_BANKS(ADDR_SURF_16_BANK
) |
2351 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2352 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2353 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2355 case 12: /* Display 32bpp. */
2356 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2357 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2358 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2359 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2360 NUM_BANKS(ADDR_SURF_16_BANK
) |
2361 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2365 case 13: /* Thin. */
2366 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2367 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2368 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2369 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2370 NUM_BANKS(ADDR_SURF_16_BANK
) |
2371 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2372 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2373 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2375 case 14: /* Thin 8 bpp. */
2376 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2377 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2378 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2379 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2380 NUM_BANKS(ADDR_SURF_16_BANK
) |
2381 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2382 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2383 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2385 case 15: /* Thin 16 bpp. */
2386 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2387 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2388 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2389 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2390 NUM_BANKS(ADDR_SURF_16_BANK
) |
2391 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2395 case 16: /* Thin 32 bpp. */
2396 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2397 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2398 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2399 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2400 NUM_BANKS(ADDR_SURF_16_BANK
) |
2401 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2402 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2403 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2405 case 17: /* Thin 64 bpp. */
2406 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2407 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2408 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2409 TILE_SPLIT(split_equal_to_row_size
) |
2410 NUM_BANKS(ADDR_SURF_16_BANK
) |
2411 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2412 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2413 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2415 case 21: /* 8 bpp PRT. */
2416 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2417 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2418 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2419 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2420 NUM_BANKS(ADDR_SURF_16_BANK
) |
2421 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2425 case 22: /* 16 bpp PRT */
2426 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2427 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2428 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2429 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2430 NUM_BANKS(ADDR_SURF_16_BANK
) |
2431 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2432 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2433 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2435 case 23: /* 32 bpp PRT */
2436 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2437 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2438 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2439 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2440 NUM_BANKS(ADDR_SURF_16_BANK
) |
2441 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2442 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2443 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2445 case 24: /* 64 bpp PRT */
2446 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2447 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2448 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2449 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2450 NUM_BANKS(ADDR_SURF_16_BANK
) |
2451 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2452 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2453 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2455 case 25: /* 128 bpp PRT */
2456 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2457 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2458 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2459 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB
) |
2460 NUM_BANKS(ADDR_SURF_8_BANK
) |
2461 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2469 rdev
->config
.si
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2470 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2472 } else if ((rdev
->family
== CHIP_VERDE
) ||
2473 (rdev
->family
== CHIP_OLAND
) ||
2474 (rdev
->family
== CHIP_HAINAN
)) {
2475 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2476 switch (reg_offset
) {
2477 case 0: /* non-AA compressed depth or any compressed stencil */
2478 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2479 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2480 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2481 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2482 NUM_BANKS(ADDR_SURF_16_BANK
) |
2483 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2484 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2485 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2487 case 1: /* 2xAA/4xAA compressed depth only */
2488 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2489 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2490 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2491 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2492 NUM_BANKS(ADDR_SURF_16_BANK
) |
2493 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2497 case 2: /* 8xAA compressed depth only */
2498 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2499 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2500 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2501 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2502 NUM_BANKS(ADDR_SURF_16_BANK
) |
2503 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2507 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2508 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2509 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2510 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2511 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2512 NUM_BANKS(ADDR_SURF_16_BANK
) |
2513 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2514 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2515 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2517 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2518 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2519 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2520 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2521 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2522 NUM_BANKS(ADDR_SURF_16_BANK
) |
2523 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2527 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2528 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2529 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2530 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2531 TILE_SPLIT(split_equal_to_row_size
) |
2532 NUM_BANKS(ADDR_SURF_16_BANK
) |
2533 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2534 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2535 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2537 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2538 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2539 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2540 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2541 TILE_SPLIT(split_equal_to_row_size
) |
2542 NUM_BANKS(ADDR_SURF_16_BANK
) |
2543 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2544 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2545 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2547 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2548 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2549 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2550 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2551 TILE_SPLIT(split_equal_to_row_size
) |
2552 NUM_BANKS(ADDR_SURF_16_BANK
) |
2553 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2554 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2555 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2557 case 8: /* 1D and 1D Array Surfaces */
2558 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2559 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2560 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2561 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2562 NUM_BANKS(ADDR_SURF_16_BANK
) |
2563 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2564 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2565 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2567 case 9: /* Displayable maps. */
2568 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2569 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2570 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2571 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2572 NUM_BANKS(ADDR_SURF_16_BANK
) |
2573 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2574 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2575 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2577 case 10: /* Display 8bpp. */
2578 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2579 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2580 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2581 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2582 NUM_BANKS(ADDR_SURF_16_BANK
) |
2583 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2584 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2585 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2587 case 11: /* Display 16bpp. */
2588 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2589 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2590 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2591 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2592 NUM_BANKS(ADDR_SURF_16_BANK
) |
2593 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2594 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2595 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2597 case 12: /* Display 32bpp. */
2598 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2599 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2600 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2601 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2602 NUM_BANKS(ADDR_SURF_16_BANK
) |
2603 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2604 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2605 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2607 case 13: /* Thin. */
2608 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2609 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2610 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2611 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2612 NUM_BANKS(ADDR_SURF_16_BANK
) |
2613 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2614 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2615 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2617 case 14: /* Thin 8 bpp. */
2618 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2619 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2621 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2622 NUM_BANKS(ADDR_SURF_16_BANK
) |
2623 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2624 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2625 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2627 case 15: /* Thin 16 bpp. */
2628 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2629 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2630 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2631 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2632 NUM_BANKS(ADDR_SURF_16_BANK
) |
2633 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2634 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2635 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2637 case 16: /* Thin 32 bpp. */
2638 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2639 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2640 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2641 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2642 NUM_BANKS(ADDR_SURF_16_BANK
) |
2643 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2644 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2645 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2647 case 17: /* Thin 64 bpp. */
2648 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2649 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2650 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2651 TILE_SPLIT(split_equal_to_row_size
) |
2652 NUM_BANKS(ADDR_SURF_16_BANK
) |
2653 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2654 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2655 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2657 case 21: /* 8 bpp PRT. */
2658 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2659 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2660 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2661 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2662 NUM_BANKS(ADDR_SURF_16_BANK
) |
2663 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2664 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2665 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2667 case 22: /* 16 bpp PRT */
2668 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2669 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2670 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2671 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2672 NUM_BANKS(ADDR_SURF_16_BANK
) |
2673 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2674 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2675 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2677 case 23: /* 32 bpp PRT */
2678 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2679 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2680 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2681 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2682 NUM_BANKS(ADDR_SURF_16_BANK
) |
2683 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2684 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2685 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2687 case 24: /* 64 bpp PRT */
2688 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2689 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2690 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2691 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2692 NUM_BANKS(ADDR_SURF_16_BANK
) |
2693 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2694 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2695 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2697 case 25: /* 128 bpp PRT */
2698 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2699 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2700 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2701 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB
) |
2702 NUM_BANKS(ADDR_SURF_8_BANK
) |
2703 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2704 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2705 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2711 rdev
->config
.si
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2712 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2715 DRM_ERROR("unknown asic: 0x%x\n", rdev
->family
);
2718 static void si_select_se_sh(struct radeon_device
*rdev
,
2719 u32 se_num
, u32 sh_num
)
2721 u32 data
= INSTANCE_BROADCAST_WRITES
;
2723 if ((se_num
== 0xffffffff) && (sh_num
== 0xffffffff))
2724 data
|= SH_BROADCAST_WRITES
| SE_BROADCAST_WRITES
;
2725 else if (se_num
== 0xffffffff)
2726 data
|= SE_BROADCAST_WRITES
| SH_INDEX(sh_num
);
2727 else if (sh_num
== 0xffffffff)
2728 data
|= SH_BROADCAST_WRITES
| SE_INDEX(se_num
);
2730 data
|= SH_INDEX(sh_num
) | SE_INDEX(se_num
);
2731 WREG32(GRBM_GFX_INDEX
, data
);
2734 static u32
si_create_bitmask(u32 bit_width
)
2738 for (i
= 0; i
< bit_width
; i
++) {
2745 static u32
si_get_cu_enabled(struct radeon_device
*rdev
, u32 cu_per_sh
)
2749 data
= RREG32(CC_GC_SHADER_ARRAY_CONFIG
);
2751 data
&= INACTIVE_CUS_MASK
;
2754 data
|= RREG32(GC_USER_SHADER_ARRAY_CONFIG
);
2756 data
>>= INACTIVE_CUS_SHIFT
;
2758 mask
= si_create_bitmask(cu_per_sh
);
2760 return ~data
& mask
;
2763 static void si_setup_spi(struct radeon_device
*rdev
,
2764 u32 se_num
, u32 sh_per_se
,
2768 u32 data
, mask
, active_cu
;
2770 for (i
= 0; i
< se_num
; i
++) {
2771 for (j
= 0; j
< sh_per_se
; j
++) {
2772 si_select_se_sh(rdev
, i
, j
);
2773 data
= RREG32(SPI_STATIC_THREAD_MGMT_3
);
2774 active_cu
= si_get_cu_enabled(rdev
, cu_per_sh
);
2777 for (k
= 0; k
< 16; k
++) {
2779 if (active_cu
& mask
) {
2781 WREG32(SPI_STATIC_THREAD_MGMT_3
, data
);
2787 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2790 static u32
si_get_rb_disabled(struct radeon_device
*rdev
,
2791 u32 max_rb_num
, u32 se_num
,
2796 data
= RREG32(CC_RB_BACKEND_DISABLE
);
2798 data
&= BACKEND_DISABLE_MASK
;
2801 data
|= RREG32(GC_USER_RB_BACKEND_DISABLE
);
2803 data
>>= BACKEND_DISABLE_SHIFT
;
2805 mask
= si_create_bitmask(max_rb_num
/ se_num
/ sh_per_se
);
2810 static void si_setup_rb(struct radeon_device
*rdev
,
2811 u32 se_num
, u32 sh_per_se
,
2816 u32 disabled_rbs
= 0;
2817 u32 enabled_rbs
= 0;
2819 for (i
= 0; i
< se_num
; i
++) {
2820 for (j
= 0; j
< sh_per_se
; j
++) {
2821 si_select_se_sh(rdev
, i
, j
);
2822 data
= si_get_rb_disabled(rdev
, max_rb_num
, se_num
, sh_per_se
);
2823 disabled_rbs
|= data
<< ((i
* sh_per_se
+ j
) * TAHITI_RB_BITMAP_WIDTH_PER_SH
);
2826 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2829 for (i
= 0; i
< max_rb_num
; i
++) {
2830 if (!(disabled_rbs
& mask
))
2831 enabled_rbs
|= mask
;
2835 for (i
= 0; i
< se_num
; i
++) {
2836 si_select_se_sh(rdev
, i
, 0xffffffff);
2838 for (j
= 0; j
< sh_per_se
; j
++) {
2839 switch (enabled_rbs
& 3) {
2841 data
|= (RASTER_CONFIG_RB_MAP_0
<< (i
* sh_per_se
+ j
) * 2);
2844 data
|= (RASTER_CONFIG_RB_MAP_3
<< (i
* sh_per_se
+ j
) * 2);
2848 data
|= (RASTER_CONFIG_RB_MAP_2
<< (i
* sh_per_se
+ j
) * 2);
2853 WREG32(PA_SC_RASTER_CONFIG
, data
);
2855 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2858 static void si_gpu_init(struct radeon_device
*rdev
)
2860 u32 gb_addr_config
= 0;
2861 u32 mc_shared_chmap
, mc_arb_ramcfg
;
2863 u32 hdp_host_path_cntl
;
2867 switch (rdev
->family
) {
2869 rdev
->config
.si
.max_shader_engines
= 2;
2870 rdev
->config
.si
.max_tile_pipes
= 12;
2871 rdev
->config
.si
.max_cu_per_sh
= 8;
2872 rdev
->config
.si
.max_sh_per_se
= 2;
2873 rdev
->config
.si
.max_backends_per_se
= 4;
2874 rdev
->config
.si
.max_texture_channel_caches
= 12;
2875 rdev
->config
.si
.max_gprs
= 256;
2876 rdev
->config
.si
.max_gs_threads
= 32;
2877 rdev
->config
.si
.max_hw_contexts
= 8;
2879 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2880 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x100;
2881 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2882 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2883 gb_addr_config
= TAHITI_GB_ADDR_CONFIG_GOLDEN
;
2886 rdev
->config
.si
.max_shader_engines
= 2;
2887 rdev
->config
.si
.max_tile_pipes
= 8;
2888 rdev
->config
.si
.max_cu_per_sh
= 5;
2889 rdev
->config
.si
.max_sh_per_se
= 2;
2890 rdev
->config
.si
.max_backends_per_se
= 4;
2891 rdev
->config
.si
.max_texture_channel_caches
= 8;
2892 rdev
->config
.si
.max_gprs
= 256;
2893 rdev
->config
.si
.max_gs_threads
= 32;
2894 rdev
->config
.si
.max_hw_contexts
= 8;
2896 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2897 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x100;
2898 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2899 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2900 gb_addr_config
= TAHITI_GB_ADDR_CONFIG_GOLDEN
;
2904 rdev
->config
.si
.max_shader_engines
= 1;
2905 rdev
->config
.si
.max_tile_pipes
= 4;
2906 rdev
->config
.si
.max_cu_per_sh
= 5;
2907 rdev
->config
.si
.max_sh_per_se
= 2;
2908 rdev
->config
.si
.max_backends_per_se
= 4;
2909 rdev
->config
.si
.max_texture_channel_caches
= 4;
2910 rdev
->config
.si
.max_gprs
= 256;
2911 rdev
->config
.si
.max_gs_threads
= 32;
2912 rdev
->config
.si
.max_hw_contexts
= 8;
2914 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2915 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2916 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2917 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2918 gb_addr_config
= VERDE_GB_ADDR_CONFIG_GOLDEN
;
2921 rdev
->config
.si
.max_shader_engines
= 1;
2922 rdev
->config
.si
.max_tile_pipes
= 4;
2923 rdev
->config
.si
.max_cu_per_sh
= 6;
2924 rdev
->config
.si
.max_sh_per_se
= 1;
2925 rdev
->config
.si
.max_backends_per_se
= 2;
2926 rdev
->config
.si
.max_texture_channel_caches
= 4;
2927 rdev
->config
.si
.max_gprs
= 256;
2928 rdev
->config
.si
.max_gs_threads
= 16;
2929 rdev
->config
.si
.max_hw_contexts
= 8;
2931 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2932 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2933 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2934 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2935 gb_addr_config
= VERDE_GB_ADDR_CONFIG_GOLDEN
;
2938 rdev
->config
.si
.max_shader_engines
= 1;
2939 rdev
->config
.si
.max_tile_pipes
= 4;
2940 rdev
->config
.si
.max_cu_per_sh
= 5;
2941 rdev
->config
.si
.max_sh_per_se
= 1;
2942 rdev
->config
.si
.max_backends_per_se
= 1;
2943 rdev
->config
.si
.max_texture_channel_caches
= 2;
2944 rdev
->config
.si
.max_gprs
= 256;
2945 rdev
->config
.si
.max_gs_threads
= 16;
2946 rdev
->config
.si
.max_hw_contexts
= 8;
2948 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2949 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2950 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2951 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2952 gb_addr_config
= HAINAN_GB_ADDR_CONFIG_GOLDEN
;
2956 /* Initialize HDP */
2957 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
2958 WREG32((0x2c14 + j
), 0x00000000);
2959 WREG32((0x2c18 + j
), 0x00000000);
2960 WREG32((0x2c1c + j
), 0x00000000);
2961 WREG32((0x2c20 + j
), 0x00000000);
2962 WREG32((0x2c24 + j
), 0x00000000);
2965 WREG32(GRBM_CNTL
, GRBM_READ_TIMEOUT(0xff));
2967 evergreen_fix_pci_max_read_req_size(rdev
);
2969 WREG32(BIF_FB_EN
, FB_READ_EN
| FB_WRITE_EN
);
2971 mc_shared_chmap
= RREG32(MC_SHARED_CHMAP
);
2972 mc_arb_ramcfg
= RREG32(MC_ARB_RAMCFG
);
2974 rdev
->config
.si
.num_tile_pipes
= rdev
->config
.si
.max_tile_pipes
;
2975 rdev
->config
.si
.mem_max_burst_length_bytes
= 256;
2976 tmp
= (mc_arb_ramcfg
& NOOFCOLS_MASK
) >> NOOFCOLS_SHIFT
;
2977 rdev
->config
.si
.mem_row_size_in_kb
= (4 * (1 << (8 + tmp
))) / 1024;
2978 if (rdev
->config
.si
.mem_row_size_in_kb
> 4)
2979 rdev
->config
.si
.mem_row_size_in_kb
= 4;
2980 /* XXX use MC settings? */
2981 rdev
->config
.si
.shader_engine_tile_size
= 32;
2982 rdev
->config
.si
.num_gpus
= 1;
2983 rdev
->config
.si
.multi_gpu_tile_size
= 64;
2985 /* fix up row size */
2986 gb_addr_config
&= ~ROW_SIZE_MASK
;
2987 switch (rdev
->config
.si
.mem_row_size_in_kb
) {
2990 gb_addr_config
|= ROW_SIZE(0);
2993 gb_addr_config
|= ROW_SIZE(1);
2996 gb_addr_config
|= ROW_SIZE(2);
3000 /* setup tiling info dword. gb_addr_config is not adequate since it does
3001 * not have bank info, so create a custom tiling dword.
3002 * bits 3:0 num_pipes
3003 * bits 7:4 num_banks
3004 * bits 11:8 group_size
3005 * bits 15:12 row_size
3007 rdev
->config
.si
.tile_config
= 0;
3008 switch (rdev
->config
.si
.num_tile_pipes
) {
3010 rdev
->config
.si
.tile_config
|= (0 << 0);
3013 rdev
->config
.si
.tile_config
|= (1 << 0);
3016 rdev
->config
.si
.tile_config
|= (2 << 0);
3020 /* XXX what about 12? */
3021 rdev
->config
.si
.tile_config
|= (3 << 0);
3024 switch ((mc_arb_ramcfg
& NOOFBANK_MASK
) >> NOOFBANK_SHIFT
) {
3025 case 0: /* four banks */
3026 rdev
->config
.si
.tile_config
|= 0 << 4;
3028 case 1: /* eight banks */
3029 rdev
->config
.si
.tile_config
|= 1 << 4;
3031 case 2: /* sixteen banks */
3033 rdev
->config
.si
.tile_config
|= 2 << 4;
3036 rdev
->config
.si
.tile_config
|=
3037 ((gb_addr_config
& PIPE_INTERLEAVE_SIZE_MASK
) >> PIPE_INTERLEAVE_SIZE_SHIFT
) << 8;
3038 rdev
->config
.si
.tile_config
|=
3039 ((gb_addr_config
& ROW_SIZE_MASK
) >> ROW_SIZE_SHIFT
) << 12;
3041 WREG32(GB_ADDR_CONFIG
, gb_addr_config
);
3042 WREG32(DMIF_ADDR_CONFIG
, gb_addr_config
);
3043 WREG32(DMIF_ADDR_CALC
, gb_addr_config
);
3044 WREG32(HDP_ADDR_CONFIG
, gb_addr_config
);
3045 WREG32(DMA_TILING_CONFIG
+ DMA0_REGISTER_OFFSET
, gb_addr_config
);
3046 WREG32(DMA_TILING_CONFIG
+ DMA1_REGISTER_OFFSET
, gb_addr_config
);
3047 if (rdev
->has_uvd
) {
3048 WREG32(UVD_UDEC_ADDR_CONFIG
, gb_addr_config
);
3049 WREG32(UVD_UDEC_DB_ADDR_CONFIG
, gb_addr_config
);
3050 WREG32(UVD_UDEC_DBW_ADDR_CONFIG
, gb_addr_config
);
3053 si_tiling_mode_table_init(rdev
);
3055 si_setup_rb(rdev
, rdev
->config
.si
.max_shader_engines
,
3056 rdev
->config
.si
.max_sh_per_se
,
3057 rdev
->config
.si
.max_backends_per_se
);
3059 si_setup_spi(rdev
, rdev
->config
.si
.max_shader_engines
,
3060 rdev
->config
.si
.max_sh_per_se
,
3061 rdev
->config
.si
.max_cu_per_sh
);
3064 /* set HW defaults for 3D engine */
3065 WREG32(CP_QUEUE_THRESHOLDS
, (ROQ_IB1_START(0x16) |
3066 ROQ_IB2_START(0x2b)));
3067 WREG32(CP_MEQ_THRESHOLDS
, MEQ1_START(0x30) | MEQ2_START(0x60));
3069 sx_debug_1
= RREG32(SX_DEBUG_1
);
3070 WREG32(SX_DEBUG_1
, sx_debug_1
);
3072 WREG32(SPI_CONFIG_CNTL_1
, VTX_DONE_DELAY(4));
3074 WREG32(PA_SC_FIFO_SIZE
, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev
->config
.si
.sc_prim_fifo_size_frontend
) |
3075 SC_BACKEND_PRIM_FIFO_SIZE(rdev
->config
.si
.sc_prim_fifo_size_backend
) |
3076 SC_HIZ_TILE_FIFO_SIZE(rdev
->config
.si
.sc_hiz_tile_fifo_size
) |
3077 SC_EARLYZ_TILE_FIFO_SIZE(rdev
->config
.si
.sc_earlyz_tile_fifo_size
)));
3079 WREG32(VGT_NUM_INSTANCES
, 1);
3081 WREG32(CP_PERFMON_CNTL
, 0);
3083 WREG32(SQ_CONFIG
, 0);
3085 WREG32(PA_SC_FORCE_EOV_MAX_CNTS
, (FORCE_EOV_MAX_CLK_CNT(4095) |
3086 FORCE_EOV_MAX_REZ_CNT(255)));
3088 WREG32(VGT_CACHE_INVALIDATION
, CACHE_INVALIDATION(VC_AND_TC
) |
3089 AUTO_INVLD_EN(ES_AND_GS_AUTO
));
3091 WREG32(VGT_GS_VERTEX_REUSE
, 16);
3092 WREG32(PA_SC_LINE_STIPPLE_STATE
, 0);
3094 WREG32(CB_PERFCOUNTER0_SELECT0
, 0);
3095 WREG32(CB_PERFCOUNTER0_SELECT1
, 0);
3096 WREG32(CB_PERFCOUNTER1_SELECT0
, 0);
3097 WREG32(CB_PERFCOUNTER1_SELECT1
, 0);
3098 WREG32(CB_PERFCOUNTER2_SELECT0
, 0);
3099 WREG32(CB_PERFCOUNTER2_SELECT1
, 0);
3100 WREG32(CB_PERFCOUNTER3_SELECT0
, 0);
3101 WREG32(CB_PERFCOUNTER3_SELECT1
, 0);
3103 tmp
= RREG32(HDP_MISC_CNTL
);
3104 tmp
|= HDP_FLUSH_INVALIDATE_CACHE
;
3105 WREG32(HDP_MISC_CNTL
, tmp
);
3107 hdp_host_path_cntl
= RREG32(HDP_HOST_PATH_CNTL
);
3108 WREG32(HDP_HOST_PATH_CNTL
, hdp_host_path_cntl
);
3110 WREG32(PA_CL_ENHANCE
, CLIP_VTX_REORDER_ENA
| NUM_CLIP_SEQ(3));
3116 * GPU scratch registers helpers function.
3118 static void si_scratch_init(struct radeon_device
*rdev
)
3122 rdev
->scratch
.num_reg
= 7;
3123 rdev
->scratch
.reg_base
= SCRATCH_REG0
;
3124 for (i
= 0; i
< rdev
->scratch
.num_reg
; i
++) {
3125 rdev
->scratch
.free
[i
] = true;
3126 rdev
->scratch
.reg
[i
] = rdev
->scratch
.reg_base
+ (i
* 4);
3130 void si_fence_ring_emit(struct radeon_device
*rdev
,
3131 struct radeon_fence
*fence
)
3133 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
3134 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
3136 /* flush read cache over gart */
3137 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
3138 radeon_ring_write(ring
, (CP_COHER_CNTL2
- PACKET3_SET_CONFIG_REG_START
) >> 2);
3139 radeon_ring_write(ring
, 0);
3140 radeon_ring_write(ring
, PACKET3(PACKET3_SURFACE_SYNC
, 3));
3141 radeon_ring_write(ring
, PACKET3_TCL1_ACTION_ENA
|
3142 PACKET3_TC_ACTION_ENA
|
3143 PACKET3_SH_KCACHE_ACTION_ENA
|
3144 PACKET3_SH_ICACHE_ACTION_ENA
);
3145 radeon_ring_write(ring
, 0xFFFFFFFF);
3146 radeon_ring_write(ring
, 0);
3147 radeon_ring_write(ring
, 10); /* poll interval */
3148 /* EVENT_WRITE_EOP - flush caches, send int */
3149 radeon_ring_write(ring
, PACKET3(PACKET3_EVENT_WRITE_EOP
, 4));
3150 radeon_ring_write(ring
, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) | EVENT_INDEX(5));
3151 radeon_ring_write(ring
, addr
& 0xffffffff);
3152 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3153 radeon_ring_write(ring
, fence
->seq
);
3154 radeon_ring_write(ring
, 0);
3160 void si_ring_ib_execute(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
3162 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
3165 if (ib
->is_const_ib
) {
3166 /* set switch buffer packet before const IB */
3167 radeon_ring_write(ring
, PACKET3(PACKET3_SWITCH_BUFFER
, 0));
3168 radeon_ring_write(ring
, 0);
3170 header
= PACKET3(PACKET3_INDIRECT_BUFFER_CONST
, 2);
3173 if (ring
->rptr_save_reg
) {
3174 next_rptr
= ring
->wptr
+ 3 + 4 + 8;
3175 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
3176 radeon_ring_write(ring
, ((ring
->rptr_save_reg
-
3177 PACKET3_SET_CONFIG_REG_START
) >> 2));
3178 radeon_ring_write(ring
, next_rptr
);
3179 } else if (rdev
->wb
.enabled
) {
3180 next_rptr
= ring
->wptr
+ 5 + 4 + 8;
3181 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
3182 radeon_ring_write(ring
, (1 << 8));
3183 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
3184 radeon_ring_write(ring
, upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xffffffff);
3185 radeon_ring_write(ring
, next_rptr
);
3188 header
= PACKET3(PACKET3_INDIRECT_BUFFER
, 2);
3191 radeon_ring_write(ring
, header
);
3192 radeon_ring_write(ring
,
3196 (ib
->gpu_addr
& 0xFFFFFFFC));
3197 radeon_ring_write(ring
, upper_32_bits(ib
->gpu_addr
) & 0xFFFF);
3198 radeon_ring_write(ring
, ib
->length_dw
|
3199 (ib
->vm
? (ib
->vm
->id
<< 24) : 0));
3201 if (!ib
->is_const_ib
) {
3202 /* flush read cache over gart for this vmid */
3203 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
3204 radeon_ring_write(ring
, (CP_COHER_CNTL2
- PACKET3_SET_CONFIG_REG_START
) >> 2);
3205 radeon_ring_write(ring
, ib
->vm
? ib
->vm
->id
: 0);
3206 radeon_ring_write(ring
, PACKET3(PACKET3_SURFACE_SYNC
, 3));
3207 radeon_ring_write(ring
, PACKET3_TCL1_ACTION_ENA
|
3208 PACKET3_TC_ACTION_ENA
|
3209 PACKET3_SH_KCACHE_ACTION_ENA
|
3210 PACKET3_SH_ICACHE_ACTION_ENA
);
3211 radeon_ring_write(ring
, 0xFFFFFFFF);
3212 radeon_ring_write(ring
, 0);
3213 radeon_ring_write(ring
, 10); /* poll interval */
3220 static void si_cp_enable(struct radeon_device
*rdev
, bool enable
)
3223 WREG32(CP_ME_CNTL
, 0);
3225 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.visible_vram_size
);
3226 WREG32(CP_ME_CNTL
, (CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
));
3227 WREG32(SCRATCH_UMSK
, 0);
3228 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3229 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
3230 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
3235 static int si_cp_load_microcode(struct radeon_device
*rdev
)
3237 const __be32
*fw_data
;
3240 if (!rdev
->me_fw
|| !rdev
->pfp_fw
)
3243 si_cp_enable(rdev
, false);
3246 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
3247 WREG32(CP_PFP_UCODE_ADDR
, 0);
3248 for (i
= 0; i
< SI_PFP_UCODE_SIZE
; i
++)
3249 WREG32(CP_PFP_UCODE_DATA
, be32_to_cpup(fw_data
++));
3250 WREG32(CP_PFP_UCODE_ADDR
, 0);
3253 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
3254 WREG32(CP_CE_UCODE_ADDR
, 0);
3255 for (i
= 0; i
< SI_CE_UCODE_SIZE
; i
++)
3256 WREG32(CP_CE_UCODE_DATA
, be32_to_cpup(fw_data
++));
3257 WREG32(CP_CE_UCODE_ADDR
, 0);
3260 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
3261 WREG32(CP_ME_RAM_WADDR
, 0);
3262 for (i
= 0; i
< SI_PM4_UCODE_SIZE
; i
++)
3263 WREG32(CP_ME_RAM_DATA
, be32_to_cpup(fw_data
++));
3264 WREG32(CP_ME_RAM_WADDR
, 0);
3266 WREG32(CP_PFP_UCODE_ADDR
, 0);
3267 WREG32(CP_CE_UCODE_ADDR
, 0);
3268 WREG32(CP_ME_RAM_WADDR
, 0);
3269 WREG32(CP_ME_RAM_RADDR
, 0);
3273 static int si_cp_start(struct radeon_device
*rdev
)
3275 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3278 r
= radeon_ring_lock(rdev
, ring
, 7 + 4);
3280 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
3284 radeon_ring_write(ring
, PACKET3(PACKET3_ME_INITIALIZE
, 5));
3285 radeon_ring_write(ring
, 0x1);
3286 radeon_ring_write(ring
, 0x0);
3287 radeon_ring_write(ring
, rdev
->config
.si
.max_hw_contexts
- 1);
3288 radeon_ring_write(ring
, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3289 radeon_ring_write(ring
, 0);
3290 radeon_ring_write(ring
, 0);
3292 /* init the CE partitions */
3293 radeon_ring_write(ring
, PACKET3(PACKET3_SET_BASE
, 2));
3294 radeon_ring_write(ring
, PACKET3_BASE_INDEX(CE_PARTITION_BASE
));
3295 radeon_ring_write(ring
, 0xc000);
3296 radeon_ring_write(ring
, 0xe000);
3297 radeon_ring_unlock_commit(rdev
, ring
);
3299 si_cp_enable(rdev
, true);
3301 r
= radeon_ring_lock(rdev
, ring
, si_default_size
+ 10);
3303 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
3307 /* setup clear context state */
3308 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3309 radeon_ring_write(ring
, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
3311 for (i
= 0; i
< si_default_size
; i
++)
3312 radeon_ring_write(ring
, si_default_state
[i
]);
3314 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3315 radeon_ring_write(ring
, PACKET3_PREAMBLE_END_CLEAR_STATE
);
3317 /* set clear context state */
3318 radeon_ring_write(ring
, PACKET3(PACKET3_CLEAR_STATE
, 0));
3319 radeon_ring_write(ring
, 0);
3321 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONTEXT_REG
, 2));
3322 radeon_ring_write(ring
, 0x00000316);
3323 radeon_ring_write(ring
, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3324 radeon_ring_write(ring
, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3326 radeon_ring_unlock_commit(rdev
, ring
);
3328 for (i
= RADEON_RING_TYPE_GFX_INDEX
; i
<= CAYMAN_RING_TYPE_CP2_INDEX
; ++i
) {
3329 ring
= &rdev
->ring
[i
];
3330 r
= radeon_ring_lock(rdev
, ring
, 2);
3332 /* clear the compute context state */
3333 radeon_ring_write(ring
, PACKET3_COMPUTE(PACKET3_CLEAR_STATE
, 0));
3334 radeon_ring_write(ring
, 0);
3336 radeon_ring_unlock_commit(rdev
, ring
);
3342 static void si_cp_fini(struct radeon_device
*rdev
)
3344 struct radeon_ring
*ring
;
3345 si_cp_enable(rdev
, false);
3347 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3348 radeon_ring_fini(rdev
, ring
);
3349 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3351 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
3352 radeon_ring_fini(rdev
, ring
);
3353 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3355 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
3356 radeon_ring_fini(rdev
, ring
);
3357 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3360 static int si_cp_resume(struct radeon_device
*rdev
)
3362 struct radeon_ring
*ring
;
3367 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3368 WREG32(GRBM_SOFT_RESET
, (SOFT_RESET_CP
|
3373 RREG32(GRBM_SOFT_RESET
);
3375 WREG32(GRBM_SOFT_RESET
, 0);
3376 RREG32(GRBM_SOFT_RESET
);
3378 WREG32(CP_SEM_WAIT_TIMER
, 0x0);
3379 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL
, 0x0);
3381 /* Set the write pointer delay */
3382 WREG32(CP_RB_WPTR_DELAY
, 0);
3384 WREG32(CP_DEBUG
, 0);
3385 WREG32(SCRATCH_ADDR
, ((rdev
->wb
.gpu_addr
+ RADEON_WB_SCRATCH_OFFSET
) >> 8) & 0xFFFFFFFF);
3387 /* ring 0 - compute and gfx */
3388 /* Set ring buffer size */
3389 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3390 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3391 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3393 tmp
|= BUF_SWAP_32BIT
;
3395 WREG32(CP_RB0_CNTL
, tmp
);
3397 /* Initialize the ring buffer's read and write pointers */
3398 WREG32(CP_RB0_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3400 WREG32(CP_RB0_WPTR
, ring
->wptr
);
3402 /* set the wb address whether it's enabled or not */
3403 WREG32(CP_RB0_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFFFFFFFC);
3404 WREG32(CP_RB0_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFF);
3406 if (rdev
->wb
.enabled
)
3407 WREG32(SCRATCH_UMSK
, 0xff);
3409 tmp
|= RB_NO_UPDATE
;
3410 WREG32(SCRATCH_UMSK
, 0);
3414 WREG32(CP_RB0_CNTL
, tmp
);
3416 WREG32(CP_RB0_BASE
, ring
->gpu_addr
>> 8);
3418 ring
->rptr
= RREG32(CP_RB0_RPTR
);
3420 /* ring1 - compute only */
3421 /* Set ring buffer size */
3422 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
3423 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3424 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3426 tmp
|= BUF_SWAP_32BIT
;
3428 WREG32(CP_RB1_CNTL
, tmp
);
3430 /* Initialize the ring buffer's read and write pointers */
3431 WREG32(CP_RB1_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3433 WREG32(CP_RB1_WPTR
, ring
->wptr
);
3435 /* set the wb address whether it's enabled or not */
3436 WREG32(CP_RB1_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
) & 0xFFFFFFFC);
3437 WREG32(CP_RB1_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
) & 0xFF);
3440 WREG32(CP_RB1_CNTL
, tmp
);
3442 WREG32(CP_RB1_BASE
, ring
->gpu_addr
>> 8);
3444 ring
->rptr
= RREG32(CP_RB1_RPTR
);
3446 /* ring2 - compute only */
3447 /* Set ring buffer size */
3448 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
3449 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3450 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3452 tmp
|= BUF_SWAP_32BIT
;
3454 WREG32(CP_RB2_CNTL
, tmp
);
3456 /* Initialize the ring buffer's read and write pointers */
3457 WREG32(CP_RB2_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3459 WREG32(CP_RB2_WPTR
, ring
->wptr
);
3461 /* set the wb address whether it's enabled or not */
3462 WREG32(CP_RB2_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
) & 0xFFFFFFFC);
3463 WREG32(CP_RB2_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
) & 0xFF);
3466 WREG32(CP_RB2_CNTL
, tmp
);
3468 WREG32(CP_RB2_BASE
, ring
->gpu_addr
>> 8);
3470 ring
->rptr
= RREG32(CP_RB2_RPTR
);
3472 /* start the rings */
3474 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= true;
3475 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= true;
3476 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= true;
3477 r
= radeon_ring_test(rdev
, RADEON_RING_TYPE_GFX_INDEX
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
3479 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3480 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
3481 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
3484 r
= radeon_ring_test(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
, &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
]);
3486 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
3488 r
= radeon_ring_test(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
, &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
]);
3490 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
3496 static u32
si_gpu_check_soft_reset(struct radeon_device
*rdev
)
3502 tmp
= RREG32(GRBM_STATUS
);
3503 if (tmp
& (PA_BUSY
| SC_BUSY
|
3504 BCI_BUSY
| SX_BUSY
|
3505 TA_BUSY
| VGT_BUSY
|
3507 GDS_BUSY
| SPI_BUSY
|
3508 IA_BUSY
| IA_BUSY_NO_DMA
))
3509 reset_mask
|= RADEON_RESET_GFX
;
3511 if (tmp
& (CF_RQ_PENDING
| PF_RQ_PENDING
|
3512 CP_BUSY
| CP_COHERENCY_BUSY
))
3513 reset_mask
|= RADEON_RESET_CP
;
3515 if (tmp
& GRBM_EE_BUSY
)
3516 reset_mask
|= RADEON_RESET_GRBM
| RADEON_RESET_GFX
| RADEON_RESET_CP
;
3519 tmp
= RREG32(GRBM_STATUS2
);
3520 if (tmp
& (RLC_RQ_PENDING
| RLC_BUSY
))
3521 reset_mask
|= RADEON_RESET_RLC
;
3523 /* DMA_STATUS_REG 0 */
3524 tmp
= RREG32(DMA_STATUS_REG
+ DMA0_REGISTER_OFFSET
);
3525 if (!(tmp
& DMA_IDLE
))
3526 reset_mask
|= RADEON_RESET_DMA
;
3528 /* DMA_STATUS_REG 1 */
3529 tmp
= RREG32(DMA_STATUS_REG
+ DMA1_REGISTER_OFFSET
);
3530 if (!(tmp
& DMA_IDLE
))
3531 reset_mask
|= RADEON_RESET_DMA1
;
3534 tmp
= RREG32(SRBM_STATUS2
);
3536 reset_mask
|= RADEON_RESET_DMA
;
3538 if (tmp
& DMA1_BUSY
)
3539 reset_mask
|= RADEON_RESET_DMA1
;
3542 tmp
= RREG32(SRBM_STATUS
);
3545 reset_mask
|= RADEON_RESET_IH
;
3548 reset_mask
|= RADEON_RESET_SEM
;
3550 if (tmp
& GRBM_RQ_PENDING
)
3551 reset_mask
|= RADEON_RESET_GRBM
;
3554 reset_mask
|= RADEON_RESET_VMC
;
3556 if (tmp
& (MCB_BUSY
| MCB_NON_DISPLAY_BUSY
|
3557 MCC_BUSY
| MCD_BUSY
))
3558 reset_mask
|= RADEON_RESET_MC
;
3560 if (evergreen_is_display_hung(rdev
))
3561 reset_mask
|= RADEON_RESET_DISPLAY
;
3564 tmp
= RREG32(VM_L2_STATUS
);
3566 reset_mask
|= RADEON_RESET_VMC
;
3568 /* Skip MC reset as it's mostly likely not hung, just busy */
3569 if (reset_mask
& RADEON_RESET_MC
) {
3570 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask
);
3571 reset_mask
&= ~RADEON_RESET_MC
;
3577 static void si_gpu_soft_reset(struct radeon_device
*rdev
, u32 reset_mask
)
3579 struct evergreen_mc_save save
;
3580 u32 grbm_soft_reset
= 0, srbm_soft_reset
= 0;
3583 if (reset_mask
== 0)
3586 dev_info(rdev
->dev
, "GPU softreset: 0x%08X\n", reset_mask
);
3588 evergreen_print_gpu_status_regs(rdev
);
3589 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3590 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
));
3591 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3592 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
));
3594 /* Disable CP parsing/prefetching */
3595 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
);
3597 if (reset_mask
& RADEON_RESET_DMA
) {
3599 tmp
= RREG32(DMA_RB_CNTL
+ DMA0_REGISTER_OFFSET
);
3600 tmp
&= ~DMA_RB_ENABLE
;
3601 WREG32(DMA_RB_CNTL
+ DMA0_REGISTER_OFFSET
, tmp
);
3603 if (reset_mask
& RADEON_RESET_DMA1
) {
3605 tmp
= RREG32(DMA_RB_CNTL
+ DMA1_REGISTER_OFFSET
);
3606 tmp
&= ~DMA_RB_ENABLE
;
3607 WREG32(DMA_RB_CNTL
+ DMA1_REGISTER_OFFSET
, tmp
);
3612 evergreen_mc_stop(rdev
, &save
);
3613 if (evergreen_mc_wait_for_idle(rdev
)) {
3614 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3617 if (reset_mask
& (RADEON_RESET_GFX
| RADEON_RESET_COMPUTE
| RADEON_RESET_CP
)) {
3618 grbm_soft_reset
= SOFT_RESET_CB
|
3632 if (reset_mask
& RADEON_RESET_CP
) {
3633 grbm_soft_reset
|= SOFT_RESET_CP
| SOFT_RESET_VGT
;
3635 srbm_soft_reset
|= SOFT_RESET_GRBM
;
3638 if (reset_mask
& RADEON_RESET_DMA
)
3639 srbm_soft_reset
|= SOFT_RESET_DMA
;
3641 if (reset_mask
& RADEON_RESET_DMA1
)
3642 srbm_soft_reset
|= SOFT_RESET_DMA1
;
3644 if (reset_mask
& RADEON_RESET_DISPLAY
)
3645 srbm_soft_reset
|= SOFT_RESET_DC
;
3647 if (reset_mask
& RADEON_RESET_RLC
)
3648 grbm_soft_reset
|= SOFT_RESET_RLC
;
3650 if (reset_mask
& RADEON_RESET_SEM
)
3651 srbm_soft_reset
|= SOFT_RESET_SEM
;
3653 if (reset_mask
& RADEON_RESET_IH
)
3654 srbm_soft_reset
|= SOFT_RESET_IH
;
3656 if (reset_mask
& RADEON_RESET_GRBM
)
3657 srbm_soft_reset
|= SOFT_RESET_GRBM
;
3659 if (reset_mask
& RADEON_RESET_VMC
)
3660 srbm_soft_reset
|= SOFT_RESET_VMC
;
3662 if (reset_mask
& RADEON_RESET_MC
)
3663 srbm_soft_reset
|= SOFT_RESET_MC
;
3665 if (grbm_soft_reset
) {
3666 tmp
= RREG32(GRBM_SOFT_RESET
);
3667 tmp
|= grbm_soft_reset
;
3668 dev_info(rdev
->dev
, "GRBM_SOFT_RESET=0x%08X\n", tmp
);
3669 WREG32(GRBM_SOFT_RESET
, tmp
);
3670 tmp
= RREG32(GRBM_SOFT_RESET
);
3674 tmp
&= ~grbm_soft_reset
;
3675 WREG32(GRBM_SOFT_RESET
, tmp
);
3676 tmp
= RREG32(GRBM_SOFT_RESET
);
3679 if (srbm_soft_reset
) {
3680 tmp
= RREG32(SRBM_SOFT_RESET
);
3681 tmp
|= srbm_soft_reset
;
3682 dev_info(rdev
->dev
, "SRBM_SOFT_RESET=0x%08X\n", tmp
);
3683 WREG32(SRBM_SOFT_RESET
, tmp
);
3684 tmp
= RREG32(SRBM_SOFT_RESET
);
3688 tmp
&= ~srbm_soft_reset
;
3689 WREG32(SRBM_SOFT_RESET
, tmp
);
3690 tmp
= RREG32(SRBM_SOFT_RESET
);
3693 /* Wait a little for things to settle down */
3696 evergreen_mc_resume(rdev
, &save
);
3699 evergreen_print_gpu_status_regs(rdev
);
3702 int si_asic_reset(struct radeon_device
*rdev
)
3706 reset_mask
= si_gpu_check_soft_reset(rdev
);
3709 r600_set_bios_scratch_engine_hung(rdev
, true);
3711 si_gpu_soft_reset(rdev
, reset_mask
);
3713 reset_mask
= si_gpu_check_soft_reset(rdev
);
3716 r600_set_bios_scratch_engine_hung(rdev
, false);
3722 * si_gfx_is_lockup - Check if the GFX engine is locked up
3724 * @rdev: radeon_device pointer
3725 * @ring: radeon_ring structure holding ring information
3727 * Check if the GFX engine is locked up.
3728 * Returns true if the engine appears to be locked up, false if not.
3730 bool si_gfx_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3732 u32 reset_mask
= si_gpu_check_soft_reset(rdev
);
3734 if (!(reset_mask
& (RADEON_RESET_GFX
|
3735 RADEON_RESET_COMPUTE
|
3736 RADEON_RESET_CP
))) {
3737 radeon_ring_lockup_update(ring
);
3740 /* force CP activities */
3741 radeon_ring_force_activity(rdev
, ring
);
3742 return radeon_ring_test_lockup(rdev
, ring
);
3746 * si_dma_is_lockup - Check if the DMA engine is locked up
3748 * @rdev: radeon_device pointer
3749 * @ring: radeon_ring structure holding ring information
3751 * Check if the async DMA engine is locked up.
3752 * Returns true if the engine appears to be locked up, false if not.
3754 bool si_dma_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3756 u32 reset_mask
= si_gpu_check_soft_reset(rdev
);
3759 if (ring
->idx
== R600_RING_TYPE_DMA_INDEX
)
3760 mask
= RADEON_RESET_DMA
;
3762 mask
= RADEON_RESET_DMA1
;
3764 if (!(reset_mask
& mask
)) {
3765 radeon_ring_lockup_update(ring
);
3768 /* force ring activities */
3769 radeon_ring_force_activity(rdev
, ring
);
3770 return radeon_ring_test_lockup(rdev
, ring
);
3774 static void si_mc_program(struct radeon_device
*rdev
)
3776 struct evergreen_mc_save save
;
3780 /* Initialize HDP */
3781 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
3782 WREG32((0x2c14 + j
), 0x00000000);
3783 WREG32((0x2c18 + j
), 0x00000000);
3784 WREG32((0x2c1c + j
), 0x00000000);
3785 WREG32((0x2c20 + j
), 0x00000000);
3786 WREG32((0x2c24 + j
), 0x00000000);
3788 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL
, 0);
3790 evergreen_mc_stop(rdev
, &save
);
3791 if (radeon_mc_wait_for_idle(rdev
)) {
3792 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3794 if (!ASIC_IS_NODCE(rdev
))
3795 /* Lockout access through VGA aperture*/
3796 WREG32(VGA_HDP_CONTROL
, VGA_MEMORY_DISABLE
);
3797 /* Update configuration */
3798 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
3799 rdev
->mc
.vram_start
>> 12);
3800 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
3801 rdev
->mc
.vram_end
>> 12);
3802 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR
,
3803 rdev
->vram_scratch
.gpu_addr
>> 12);
3804 tmp
= ((rdev
->mc
.vram_end
>> 24) & 0xFFFF) << 16;
3805 tmp
|= ((rdev
->mc
.vram_start
>> 24) & 0xFFFF);
3806 WREG32(MC_VM_FB_LOCATION
, tmp
);
3807 /* XXX double check these! */
3808 WREG32(HDP_NONSURFACE_BASE
, (rdev
->mc
.vram_start
>> 8));
3809 WREG32(HDP_NONSURFACE_INFO
, (2 << 7) | (1 << 30));
3810 WREG32(HDP_NONSURFACE_SIZE
, 0x3FFFFFFF);
3811 WREG32(MC_VM_AGP_BASE
, 0);
3812 WREG32(MC_VM_AGP_TOP
, 0x0FFFFFFF);
3813 WREG32(MC_VM_AGP_BOT
, 0x0FFFFFFF);
3814 if (radeon_mc_wait_for_idle(rdev
)) {
3815 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3817 evergreen_mc_resume(rdev
, &save
);
3818 if (!ASIC_IS_NODCE(rdev
)) {
3819 /* we need to own VRAM, so turn off the VGA renderer here
3820 * to stop it overwriting our objects */
3821 rv515_vga_render_disable(rdev
);
3825 void si_vram_gtt_location(struct radeon_device
*rdev
,
3826 struct radeon_mc
*mc
)
3828 if (mc
->mc_vram_size
> 0xFFC0000000ULL
) {
3829 /* leave room for at least 1024M GTT */
3830 dev_warn(rdev
->dev
, "limiting VRAM\n");
3831 mc
->real_vram_size
= 0xFFC0000000ULL
;
3832 mc
->mc_vram_size
= 0xFFC0000000ULL
;
3834 radeon_vram_location(rdev
, &rdev
->mc
, 0);
3835 rdev
->mc
.gtt_base_align
= 0;
3836 radeon_gtt_location(rdev
, mc
);
3839 static int si_mc_init(struct radeon_device
*rdev
)
3842 int chansize
, numchan
;
3844 /* Get VRAM informations */
3845 rdev
->mc
.vram_is_ddr
= true;
3846 tmp
= RREG32(MC_ARB_RAMCFG
);
3847 if (tmp
& CHANSIZE_OVERRIDE
) {
3849 } else if (tmp
& CHANSIZE_MASK
) {
3854 tmp
= RREG32(MC_SHARED_CHMAP
);
3855 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
3885 rdev
->mc
.vram_width
= numchan
* chansize
;
3886 /* Could aper size report 0 ? */
3887 rdev
->mc
.aper_base
= pci_resource_start(rdev
->pdev
, 0);
3888 rdev
->mc
.aper_size
= pci_resource_len(rdev
->pdev
, 0);
3889 /* size in MB on si */
3890 rdev
->mc
.mc_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
3891 rdev
->mc
.real_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
3892 rdev
->mc
.visible_vram_size
= rdev
->mc
.aper_size
;
3893 si_vram_gtt_location(rdev
, &rdev
->mc
);
3894 radeon_update_bandwidth_info(rdev
);
3902 void si_pcie_gart_tlb_flush(struct radeon_device
*rdev
)
3904 /* flush hdp cache */
3905 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL
, 0x1);
3907 /* bits 0-15 are the VM contexts0-15 */
3908 WREG32(VM_INVALIDATE_REQUEST
, 1);
3911 static int si_pcie_gart_enable(struct radeon_device
*rdev
)
3915 if (rdev
->gart
.robj
== NULL
) {
3916 dev_err(rdev
->dev
, "No VRAM object for PCIE GART.\n");
3919 r
= radeon_gart_table_vram_pin(rdev
);
3922 radeon_gart_restore(rdev
);
3923 /* Setup TLB control */
3924 WREG32(MC_VM_MX_L1_TLB_CNTL
,
3927 SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
3928 ENABLE_ADVANCED_DRIVER_MODEL
|
3929 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
3930 /* Setup L2 cache */
3931 WREG32(VM_L2_CNTL
, ENABLE_L2_CACHE
|
3932 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
3933 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
3934 EFFECTIVE_L2_QUEUE_SIZE(7) |
3935 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3936 WREG32(VM_L2_CNTL2
, INVALIDATE_ALL_L1_TLBS
| INVALIDATE_L2_CACHE
);
3937 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
3938 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3939 /* setup context0 */
3940 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR
, rdev
->mc
.gtt_start
>> 12);
3941 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR
, rdev
->mc
.gtt_end
>> 12);
3942 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
, rdev
->gart
.table_addr
>> 12);
3943 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR
,
3944 (u32
)(rdev
->dummy_page
.addr
>> 12));
3945 WREG32(VM_CONTEXT0_CNTL2
, 0);
3946 WREG32(VM_CONTEXT0_CNTL
, (ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(0) |
3947 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
));
3953 /* empty context1-15 */
3954 /* set vm size, must be a multiple of 4 */
3955 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR
, 0);
3956 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR
, rdev
->vm_manager
.max_pfn
);
3957 /* Assign the pt base to something valid for now; the pts used for
3958 * the VMs are determined by the application and setup and assigned
3959 * on the fly in the vm part of radeon_gart.c
3961 for (i
= 1; i
< 16; i
++) {
3963 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (i
<< 2),
3964 rdev
->gart
.table_addr
>> 12);
3966 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((i
- 8) << 2),
3967 rdev
->gart
.table_addr
>> 12);
3970 /* enable context1-15 */
3971 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR
,
3972 (u32
)(rdev
->dummy_page
.addr
>> 12));
3973 WREG32(VM_CONTEXT1_CNTL2
, 4);
3974 WREG32(VM_CONTEXT1_CNTL
, ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(1) |
3975 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3976 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
3977 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3978 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
3979 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3980 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT
|
3981 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3982 VALID_PROTECTION_FAULT_ENABLE_DEFAULT
|
3983 READ_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3984 READ_PROTECTION_FAULT_ENABLE_DEFAULT
|
3985 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3986 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT
);
3988 si_pcie_gart_tlb_flush(rdev
);
3989 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3990 (unsigned)(rdev
->mc
.gtt_size
>> 20),
3991 (unsigned long long)rdev
->gart
.table_addr
);
3992 rdev
->gart
.ready
= true;
3996 static void si_pcie_gart_disable(struct radeon_device
*rdev
)
3998 /* Disable all tables */
3999 WREG32(VM_CONTEXT0_CNTL
, 0);
4000 WREG32(VM_CONTEXT1_CNTL
, 0);
4001 /* Setup TLB control */
4002 WREG32(MC_VM_MX_L1_TLB_CNTL
, SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
4003 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
4004 /* Setup L2 cache */
4005 WREG32(VM_L2_CNTL
, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
4006 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
4007 EFFECTIVE_L2_QUEUE_SIZE(7) |
4008 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4009 WREG32(VM_L2_CNTL2
, 0);
4010 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
4011 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4012 radeon_gart_table_vram_unpin(rdev
);
4015 static void si_pcie_gart_fini(struct radeon_device
*rdev
)
4017 si_pcie_gart_disable(rdev
);
4018 radeon_gart_table_vram_free(rdev
);
4019 radeon_gart_fini(rdev
);
4023 static bool si_vm_reg_valid(u32 reg
)
4025 /* context regs are fine */
4029 /* check config regs */
4031 case GRBM_GFX_INDEX
:
4032 case CP_STRMOUT_CNTL
:
4033 case VGT_VTX_VECT_EJECT_REG
:
4034 case VGT_CACHE_INVALIDATION
:
4035 case VGT_ESGS_RING_SIZE
:
4036 case VGT_GSVS_RING_SIZE
:
4037 case VGT_GS_VERTEX_REUSE
:
4038 case VGT_PRIMITIVE_TYPE
:
4039 case VGT_INDEX_TYPE
:
4040 case VGT_NUM_INDICES
:
4041 case VGT_NUM_INSTANCES
:
4042 case VGT_TF_RING_SIZE
:
4043 case VGT_HS_OFFCHIP_PARAM
:
4044 case VGT_TF_MEMORY_BASE
:
4046 case PA_SU_LINE_STIPPLE_VALUE
:
4047 case PA_SC_LINE_STIPPLE_STATE
:
4050 case SPI_STATIC_THREAD_MGMT_1
:
4051 case SPI_STATIC_THREAD_MGMT_2
:
4052 case SPI_STATIC_THREAD_MGMT_3
:
4053 case SPI_PS_MAX_WAVE_ID
:
4054 case SPI_CONFIG_CNTL
:
4055 case SPI_CONFIG_CNTL_1
:
4059 DRM_ERROR("Invalid register 0x%x in CS\n", reg
);
4064 static int si_vm_packet3_ce_check(struct radeon_device
*rdev
,
4065 u32
*ib
, struct radeon_cs_packet
*pkt
)
4067 switch (pkt
->opcode
) {
4069 case PACKET3_SET_BASE
:
4070 case PACKET3_SET_CE_DE_COUNTERS
:
4071 case PACKET3_LOAD_CONST_RAM
:
4072 case PACKET3_WRITE_CONST_RAM
:
4073 case PACKET3_WRITE_CONST_RAM_OFFSET
:
4074 case PACKET3_DUMP_CONST_RAM
:
4075 case PACKET3_INCREMENT_CE_COUNTER
:
4076 case PACKET3_WAIT_ON_DE_COUNTER
:
4077 case PACKET3_CE_WRITE
:
4080 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt
->opcode
);
4086 static int si_vm_packet3_gfx_check(struct radeon_device
*rdev
,
4087 u32
*ib
, struct radeon_cs_packet
*pkt
)
4089 u32 idx
= pkt
->idx
+ 1;
4090 u32 idx_value
= ib
[idx
];
4091 u32 start_reg
, end_reg
, reg
, i
;
4094 switch (pkt
->opcode
) {
4096 case PACKET3_SET_BASE
:
4097 case PACKET3_CLEAR_STATE
:
4098 case PACKET3_INDEX_BUFFER_SIZE
:
4099 case PACKET3_DISPATCH_DIRECT
:
4100 case PACKET3_DISPATCH_INDIRECT
:
4101 case PACKET3_ALLOC_GDS
:
4102 case PACKET3_WRITE_GDS_RAM
:
4103 case PACKET3_ATOMIC_GDS
:
4104 case PACKET3_ATOMIC
:
4105 case PACKET3_OCCLUSION_QUERY
:
4106 case PACKET3_SET_PREDICATION
:
4107 case PACKET3_COND_EXEC
:
4108 case PACKET3_PRED_EXEC
:
4109 case PACKET3_DRAW_INDIRECT
:
4110 case PACKET3_DRAW_INDEX_INDIRECT
:
4111 case PACKET3_INDEX_BASE
:
4112 case PACKET3_DRAW_INDEX_2
:
4113 case PACKET3_CONTEXT_CONTROL
:
4114 case PACKET3_INDEX_TYPE
:
4115 case PACKET3_DRAW_INDIRECT_MULTI
:
4116 case PACKET3_DRAW_INDEX_AUTO
:
4117 case PACKET3_DRAW_INDEX_IMMD
:
4118 case PACKET3_NUM_INSTANCES
:
4119 case PACKET3_DRAW_INDEX_MULTI_AUTO
:
4120 case PACKET3_STRMOUT_BUFFER_UPDATE
:
4121 case PACKET3_DRAW_INDEX_OFFSET_2
:
4122 case PACKET3_DRAW_INDEX_MULTI_ELEMENT
:
4123 case PACKET3_DRAW_INDEX_INDIRECT_MULTI
:
4124 case PACKET3_MPEG_INDEX
:
4125 case PACKET3_WAIT_REG_MEM
:
4126 case PACKET3_MEM_WRITE
:
4127 case PACKET3_PFP_SYNC_ME
:
4128 case PACKET3_SURFACE_SYNC
:
4129 case PACKET3_EVENT_WRITE
:
4130 case PACKET3_EVENT_WRITE_EOP
:
4131 case PACKET3_EVENT_WRITE_EOS
:
4132 case PACKET3_SET_CONTEXT_REG
:
4133 case PACKET3_SET_CONTEXT_REG_INDIRECT
:
4134 case PACKET3_SET_SH_REG
:
4135 case PACKET3_SET_SH_REG_OFFSET
:
4136 case PACKET3_INCREMENT_DE_COUNTER
:
4137 case PACKET3_WAIT_ON_CE_COUNTER
:
4138 case PACKET3_WAIT_ON_AVAIL_BUFFER
:
4139 case PACKET3_ME_WRITE
:
4141 case PACKET3_COPY_DATA
:
4142 if ((idx_value
& 0xf00) == 0) {
4143 reg
= ib
[idx
+ 3] * 4;
4144 if (!si_vm_reg_valid(reg
))
4148 case PACKET3_WRITE_DATA
:
4149 if ((idx_value
& 0xf00) == 0) {
4150 start_reg
= ib
[idx
+ 1] * 4;
4151 if (idx_value
& 0x10000) {
4152 if (!si_vm_reg_valid(start_reg
))
4155 for (i
= 0; i
< (pkt
->count
- 2); i
++) {
4156 reg
= start_reg
+ (4 * i
);
4157 if (!si_vm_reg_valid(reg
))
4163 case PACKET3_COND_WRITE
:
4164 if (idx_value
& 0x100) {
4165 reg
= ib
[idx
+ 5] * 4;
4166 if (!si_vm_reg_valid(reg
))
4170 case PACKET3_COPY_DW
:
4171 if (idx_value
& 0x2) {
4172 reg
= ib
[idx
+ 3] * 4;
4173 if (!si_vm_reg_valid(reg
))
4177 case PACKET3_SET_CONFIG_REG
:
4178 start_reg
= (idx_value
<< 2) + PACKET3_SET_CONFIG_REG_START
;
4179 end_reg
= 4 * pkt
->count
+ start_reg
- 4;
4180 if ((start_reg
< PACKET3_SET_CONFIG_REG_START
) ||
4181 (start_reg
>= PACKET3_SET_CONFIG_REG_END
) ||
4182 (end_reg
>= PACKET3_SET_CONFIG_REG_END
)) {
4183 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4186 for (i
= 0; i
< pkt
->count
; i
++) {
4187 reg
= start_reg
+ (4 * i
);
4188 if (!si_vm_reg_valid(reg
))
4192 case PACKET3_CP_DMA
:
4193 command
= ib
[idx
+ 4];
4195 if (command
& PACKET3_CP_DMA_CMD_SAS
) {
4196 /* src address space is register */
4197 if (((info
& 0x60000000) >> 29) == 0) {
4198 start_reg
= idx_value
<< 2;
4199 if (command
& PACKET3_CP_DMA_CMD_SAIC
) {
4201 if (!si_vm_reg_valid(reg
)) {
4202 DRM_ERROR("CP DMA Bad SRC register\n");
4206 for (i
= 0; i
< (command
& 0x1fffff); i
++) {
4207 reg
= start_reg
+ (4 * i
);
4208 if (!si_vm_reg_valid(reg
)) {
4209 DRM_ERROR("CP DMA Bad SRC register\n");
4216 if (command
& PACKET3_CP_DMA_CMD_DAS
) {
4217 /* dst address space is register */
4218 if (((info
& 0x00300000) >> 20) == 0) {
4219 start_reg
= ib
[idx
+ 2];
4220 if (command
& PACKET3_CP_DMA_CMD_DAIC
) {
4222 if (!si_vm_reg_valid(reg
)) {
4223 DRM_ERROR("CP DMA Bad DST register\n");
4227 for (i
= 0; i
< (command
& 0x1fffff); i
++) {
4228 reg
= start_reg
+ (4 * i
);
4229 if (!si_vm_reg_valid(reg
)) {
4230 DRM_ERROR("CP DMA Bad DST register\n");
4239 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt
->opcode
);
4245 static int si_vm_packet3_compute_check(struct radeon_device
*rdev
,
4246 u32
*ib
, struct radeon_cs_packet
*pkt
)
4248 u32 idx
= pkt
->idx
+ 1;
4249 u32 idx_value
= ib
[idx
];
4250 u32 start_reg
, reg
, i
;
4252 switch (pkt
->opcode
) {
4254 case PACKET3_SET_BASE
:
4255 case PACKET3_CLEAR_STATE
:
4256 case PACKET3_DISPATCH_DIRECT
:
4257 case PACKET3_DISPATCH_INDIRECT
:
4258 case PACKET3_ALLOC_GDS
:
4259 case PACKET3_WRITE_GDS_RAM
:
4260 case PACKET3_ATOMIC_GDS
:
4261 case PACKET3_ATOMIC
:
4262 case PACKET3_OCCLUSION_QUERY
:
4263 case PACKET3_SET_PREDICATION
:
4264 case PACKET3_COND_EXEC
:
4265 case PACKET3_PRED_EXEC
:
4266 case PACKET3_CONTEXT_CONTROL
:
4267 case PACKET3_STRMOUT_BUFFER_UPDATE
:
4268 case PACKET3_WAIT_REG_MEM
:
4269 case PACKET3_MEM_WRITE
:
4270 case PACKET3_PFP_SYNC_ME
:
4271 case PACKET3_SURFACE_SYNC
:
4272 case PACKET3_EVENT_WRITE
:
4273 case PACKET3_EVENT_WRITE_EOP
:
4274 case PACKET3_EVENT_WRITE_EOS
:
4275 case PACKET3_SET_CONTEXT_REG
:
4276 case PACKET3_SET_CONTEXT_REG_INDIRECT
:
4277 case PACKET3_SET_SH_REG
:
4278 case PACKET3_SET_SH_REG_OFFSET
:
4279 case PACKET3_INCREMENT_DE_COUNTER
:
4280 case PACKET3_WAIT_ON_CE_COUNTER
:
4281 case PACKET3_WAIT_ON_AVAIL_BUFFER
:
4282 case PACKET3_ME_WRITE
:
4284 case PACKET3_COPY_DATA
:
4285 if ((idx_value
& 0xf00) == 0) {
4286 reg
= ib
[idx
+ 3] * 4;
4287 if (!si_vm_reg_valid(reg
))
4291 case PACKET3_WRITE_DATA
:
4292 if ((idx_value
& 0xf00) == 0) {
4293 start_reg
= ib
[idx
+ 1] * 4;
4294 if (idx_value
& 0x10000) {
4295 if (!si_vm_reg_valid(start_reg
))
4298 for (i
= 0; i
< (pkt
->count
- 2); i
++) {
4299 reg
= start_reg
+ (4 * i
);
4300 if (!si_vm_reg_valid(reg
))
4306 case PACKET3_COND_WRITE
:
4307 if (idx_value
& 0x100) {
4308 reg
= ib
[idx
+ 5] * 4;
4309 if (!si_vm_reg_valid(reg
))
4313 case PACKET3_COPY_DW
:
4314 if (idx_value
& 0x2) {
4315 reg
= ib
[idx
+ 3] * 4;
4316 if (!si_vm_reg_valid(reg
))
4321 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt
->opcode
);
4327 int si_ib_parse(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
4331 struct radeon_cs_packet pkt
;
4335 pkt
.type
= RADEON_CP_PACKET_GET_TYPE(ib
->ptr
[idx
]);
4336 pkt
.count
= RADEON_CP_PACKET_GET_COUNT(ib
->ptr
[idx
]);
4339 case RADEON_PACKET_TYPE0
:
4340 dev_err(rdev
->dev
, "Packet0 not allowed!\n");
4343 case RADEON_PACKET_TYPE2
:
4346 case RADEON_PACKET_TYPE3
:
4347 pkt
.opcode
= RADEON_CP_PACKET3_GET_OPCODE(ib
->ptr
[idx
]);
4348 if (ib
->is_const_ib
)
4349 ret
= si_vm_packet3_ce_check(rdev
, ib
->ptr
, &pkt
);
4352 case RADEON_RING_TYPE_GFX_INDEX
:
4353 ret
= si_vm_packet3_gfx_check(rdev
, ib
->ptr
, &pkt
);
4355 case CAYMAN_RING_TYPE_CP1_INDEX
:
4356 case CAYMAN_RING_TYPE_CP2_INDEX
:
4357 ret
= si_vm_packet3_compute_check(rdev
, ib
->ptr
, &pkt
);
4360 dev_err(rdev
->dev
, "Non-PM4 ring %d !\n", ib
->ring
);
4365 idx
+= pkt
.count
+ 2;
4368 dev_err(rdev
->dev
, "Unknown packet type %d !\n", pkt
.type
);
4374 } while (idx
< ib
->length_dw
);
4382 int si_vm_init(struct radeon_device
*rdev
)
4385 rdev
->vm_manager
.nvm
= 16;
4386 /* base offset of vram pages */
4387 rdev
->vm_manager
.vram_base_offset
= 0;
4392 void si_vm_fini(struct radeon_device
*rdev
)
4397 * si_vm_decode_fault - print human readable fault info
4399 * @rdev: radeon_device pointer
4400 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4401 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4403 * Print human readable fault information (SI).
4405 static void si_vm_decode_fault(struct radeon_device
*rdev
,
4406 u32 status
, u32 addr
)
4408 u32 mc_id
= (status
& MEMORY_CLIENT_ID_MASK
) >> MEMORY_CLIENT_ID_SHIFT
;
4409 u32 vmid
= (status
& FAULT_VMID_MASK
) >> FAULT_VMID_SHIFT
;
4410 u32 protections
= (status
& PROTECTIONS_MASK
) >> PROTECTIONS_SHIFT
;
4413 if (rdev
->family
== CHIP_TAHITI
) {
4654 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4655 protections
, vmid
, addr
,
4656 (status
& MEMORY_CLIENT_RW_MASK
) ? "write" : "read",
4661 * si_vm_set_page - update the page tables using the CP
4663 * @rdev: radeon_device pointer
4664 * @ib: indirect buffer to fill with commands
4665 * @pe: addr of the page entry
4666 * @addr: dst addr to write into pe
4667 * @count: number of page entries to update
4668 * @incr: increase next addr by incr bytes
4669 * @flags: access flags
4671 * Update the page tables using the CP (SI).
4673 void si_vm_set_page(struct radeon_device
*rdev
,
4674 struct radeon_ib
*ib
,
4676 uint64_t addr
, unsigned count
,
4677 uint32_t incr
, uint32_t flags
)
4679 uint32_t r600_flags
= cayman_vm_page_flags(rdev
, flags
);
4683 if (rdev
->asic
->vm
.pt_ring_index
== RADEON_RING_TYPE_GFX_INDEX
) {
4685 ndw
= 2 + count
* 2;
4689 ib
->ptr
[ib
->length_dw
++] = PACKET3(PACKET3_WRITE_DATA
, ndw
);
4690 ib
->ptr
[ib
->length_dw
++] = (WRITE_DATA_ENGINE_SEL(0) |
4691 WRITE_DATA_DST_SEL(1));
4692 ib
->ptr
[ib
->length_dw
++] = pe
;
4693 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
);
4694 for (; ndw
> 2; ndw
-= 2, --count
, pe
+= 8) {
4695 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4696 value
= radeon_vm_map_gart(rdev
, addr
);
4697 value
&= 0xFFFFFFFFFFFFF000ULL
;
4698 } else if (flags
& RADEON_VM_PAGE_VALID
) {
4704 value
|= r600_flags
;
4705 ib
->ptr
[ib
->length_dw
++] = value
;
4706 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4711 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4717 /* for non-physically contiguous pages (system) */
4718 ib
->ptr
[ib
->length_dw
++] = DMA_PACKET(DMA_PACKET_WRITE
, 0, 0, 0, ndw
);
4719 ib
->ptr
[ib
->length_dw
++] = pe
;
4720 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
4721 for (; ndw
> 0; ndw
-= 2, --count
, pe
+= 8) {
4722 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4723 value
= radeon_vm_map_gart(rdev
, addr
);
4724 value
&= 0xFFFFFFFFFFFFF000ULL
;
4725 } else if (flags
& RADEON_VM_PAGE_VALID
) {
4731 value
|= r600_flags
;
4732 ib
->ptr
[ib
->length_dw
++] = value
;
4733 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4742 if (flags
& RADEON_VM_PAGE_VALID
)
4746 /* for physically contiguous pages (vram) */
4747 ib
->ptr
[ib
->length_dw
++] = DMA_PTE_PDE_PACKET(ndw
);
4748 ib
->ptr
[ib
->length_dw
++] = pe
; /* dst addr */
4749 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
4750 ib
->ptr
[ib
->length_dw
++] = r600_flags
; /* mask */
4751 ib
->ptr
[ib
->length_dw
++] = 0;
4752 ib
->ptr
[ib
->length_dw
++] = value
; /* value */
4753 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4754 ib
->ptr
[ib
->length_dw
++] = incr
; /* increment size */
4755 ib
->ptr
[ib
->length_dw
++] = 0;
4757 addr
+= (ndw
/ 2) * incr
;
4761 while (ib
->length_dw
& 0x7)
4762 ib
->ptr
[ib
->length_dw
++] = DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0);
4766 void si_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
4768 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
4773 /* write new base address */
4774 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4775 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4776 WRITE_DATA_DST_SEL(0)));
4779 radeon_ring_write(ring
,
4780 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2);
4782 radeon_ring_write(ring
,
4783 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2);
4785 radeon_ring_write(ring
, 0);
4786 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
4788 /* flush hdp cache */
4789 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4790 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4791 WRITE_DATA_DST_SEL(0)));
4792 radeon_ring_write(ring
, HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2);
4793 radeon_ring_write(ring
, 0);
4794 radeon_ring_write(ring
, 0x1);
4796 /* bits 0-15 are the VM contexts0-15 */
4797 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4798 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4799 WRITE_DATA_DST_SEL(0)));
4800 radeon_ring_write(ring
, VM_INVALIDATE_REQUEST
>> 2);
4801 radeon_ring_write(ring
, 0);
4802 radeon_ring_write(ring
, 1 << vm
->id
);
4804 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4805 radeon_ring_write(ring
, PACKET3(PACKET3_PFP_SYNC_ME
, 0));
4806 radeon_ring_write(ring
, 0x0);
4809 void si_dma_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
4811 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
4816 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4818 radeon_ring_write(ring
, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2));
4820 radeon_ring_write(ring
, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2));
4822 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
4824 /* flush hdp cache */
4825 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4826 radeon_ring_write(ring
, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2));
4827 radeon_ring_write(ring
, 1);
4829 /* bits 0-7 are the VM contexts0-7 */
4830 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4831 radeon_ring_write(ring
, (0xf << 16) | (VM_INVALIDATE_REQUEST
>> 2));
4832 radeon_ring_write(ring
, 1 << vm
->id
);
4836 * Power and clock gating
4838 static void si_wait_for_rlc_serdes(struct radeon_device
*rdev
)
4842 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4843 if (RREG32(RLC_SERDES_MASTER_BUSY_0
) == 0)
4848 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4849 if (RREG32(RLC_SERDES_MASTER_BUSY_1
) == 0)
4855 static void si_enable_gui_idle_interrupt(struct radeon_device
*rdev
,
4858 u32 tmp
= RREG32(CP_INT_CNTL_RING0
);
4863 tmp
|= (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
4865 tmp
&= ~(CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
4866 WREG32(CP_INT_CNTL_RING0
, tmp
);
4869 /* read a gfx register */
4870 tmp
= RREG32(DB_DEPTH_INFO
);
4872 mask
= RLC_BUSY_STATUS
| GFX_POWER_STATUS
| GFX_CLOCK_STATUS
| GFX_LS_STATUS
;
4873 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4874 if ((RREG32(RLC_STAT
) & mask
) == (GFX_CLOCK_STATUS
| GFX_POWER_STATUS
))
4881 static void si_set_uvd_dcm(struct radeon_device
*rdev
,
4886 tmp
= RREG32(UVD_CGC_CTRL
);
4887 tmp
&= ~(CLK_OD_MASK
| CG_DT_MASK
);
4888 tmp
|= DCM
| CG_DT(1) | CLK_OD(4);
4892 tmp2
= DYN_OR_EN
| DYN_RR_EN
| G_DIV_ID(7);
4898 WREG32(UVD_CGC_CTRL
, tmp
);
4899 WREG32_UVD_CTX(UVD_CGC_CTRL2
, tmp2
);
4902 static void si_init_uvd_internal_cg(struct radeon_device
*rdev
)
4904 bool hw_mode
= true;
4907 si_set_uvd_dcm(rdev
, false);
4909 u32 tmp
= RREG32(UVD_CGC_CTRL
);
4911 WREG32(UVD_CGC_CTRL
, tmp
);
4915 static u32
si_halt_rlc(struct radeon_device
*rdev
)
4919 orig
= data
= RREG32(RLC_CNTL
);
4921 if (data
& RLC_ENABLE
) {
4922 data
&= ~RLC_ENABLE
;
4923 WREG32(RLC_CNTL
, data
);
4925 si_wait_for_rlc_serdes(rdev
);
4931 static void si_update_rlc(struct radeon_device
*rdev
, u32 rlc
)
4935 tmp
= RREG32(RLC_CNTL
);
4937 WREG32(RLC_CNTL
, rlc
);
4940 static void si_enable_dma_pg(struct radeon_device
*rdev
, bool enable
)
4944 orig
= data
= RREG32(DMA_PG
);
4946 data
|= PG_CNTL_ENABLE
;
4948 data
&= ~PG_CNTL_ENABLE
;
4950 WREG32(DMA_PG
, data
);
4953 static void si_init_dma_pg(struct radeon_device
*rdev
)
4957 WREG32(DMA_PGFSM_WRITE
, 0x00002000);
4958 WREG32(DMA_PGFSM_CONFIG
, 0x100010ff);
4960 for (tmp
= 0; tmp
< 5; tmp
++)
4961 WREG32(DMA_PGFSM_WRITE
, 0);
4964 static void si_enable_gfx_cgpg(struct radeon_device
*rdev
,
4970 tmp
= RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4971 WREG32(RLC_TTOP_D
, tmp
);
4973 tmp
= RREG32(RLC_PG_CNTL
);
4974 tmp
|= GFX_PG_ENABLE
;
4975 WREG32(RLC_PG_CNTL
, tmp
);
4977 tmp
= RREG32(RLC_AUTO_PG_CTRL
);
4979 WREG32(RLC_AUTO_PG_CTRL
, tmp
);
4981 tmp
= RREG32(RLC_AUTO_PG_CTRL
);
4983 WREG32(RLC_AUTO_PG_CTRL
, tmp
);
4985 tmp
= RREG32(DB_RENDER_CONTROL
);
4989 static void si_init_gfx_cgpg(struct radeon_device
*rdev
)
4993 WREG32(RLC_SAVE_AND_RESTORE_BASE
, rdev
->rlc
.save_restore_gpu_addr
>> 8);
4995 tmp
= RREG32(RLC_PG_CNTL
);
4997 WREG32(RLC_PG_CNTL
, tmp
);
4999 WREG32(RLC_CLEAR_STATE_RESTORE_BASE
, rdev
->rlc
.clear_state_gpu_addr
>> 8);
5001 tmp
= RREG32(RLC_AUTO_PG_CTRL
);
5003 tmp
&= ~GRBM_REG_SGIT_MASK
;
5004 tmp
|= GRBM_REG_SGIT(0x700);
5005 tmp
&= ~PG_AFTER_GRBM_REG_ST_MASK
;
5006 WREG32(RLC_AUTO_PG_CTRL
, tmp
);
5009 static u32
si_get_cu_active_bitmap(struct radeon_device
*rdev
, u32 se
, u32 sh
)
5011 u32 mask
= 0, tmp
, tmp1
;
5014 si_select_se_sh(rdev
, se
, sh
);
5015 tmp
= RREG32(CC_GC_SHADER_ARRAY_CONFIG
);
5016 tmp1
= RREG32(GC_USER_SHADER_ARRAY_CONFIG
);
5017 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5024 for (i
= 0; i
< rdev
->config
.si
.max_cu_per_sh
; i
++) {
5029 return (~tmp
) & mask
;
5032 static void si_init_ao_cu_mask(struct radeon_device
*rdev
)
5034 u32 i
, j
, k
, active_cu_number
= 0;
5035 u32 mask
, counter
, cu_bitmap
;
5038 for (i
= 0; i
< rdev
->config
.si
.max_shader_engines
; i
++) {
5039 for (j
= 0; j
< rdev
->config
.si
.max_sh_per_se
; j
++) {
5043 for (k
= 0; k
< rdev
->config
.si
.max_cu_per_sh
; k
++) {
5044 if (si_get_cu_active_bitmap(rdev
, i
, j
) & mask
) {
5052 active_cu_number
+= counter
;
5053 tmp
|= (cu_bitmap
<< (i
* 16 + j
* 8));
5057 WREG32(RLC_PG_AO_CU_MASK
, tmp
);
5059 tmp
= RREG32(RLC_MAX_PG_CU
);
5060 tmp
&= ~MAX_PU_CU_MASK
;
5061 tmp
|= MAX_PU_CU(active_cu_number
);
5062 WREG32(RLC_MAX_PG_CU
, tmp
);
5065 static void si_enable_cgcg(struct radeon_device
*rdev
,
5068 u32 data
, orig
, tmp
;
5070 orig
= data
= RREG32(RLC_CGCG_CGLS_CTRL
);
5072 si_enable_gui_idle_interrupt(rdev
, enable
);
5075 WREG32(RLC_GCPM_GENERAL_3
, 0x00000080);
5077 tmp
= si_halt_rlc(rdev
);
5079 WREG32(RLC_SERDES_WR_MASTER_MASK_0
, 0xffffffff);
5080 WREG32(RLC_SERDES_WR_MASTER_MASK_1
, 0xffffffff);
5081 WREG32(RLC_SERDES_WR_CTRL
, 0x00b000ff);
5083 si_wait_for_rlc_serdes(rdev
);
5085 si_update_rlc(rdev
, tmp
);
5087 WREG32(RLC_SERDES_WR_CTRL
, 0x007000ff);
5089 data
|= CGCG_EN
| CGLS_EN
;
5091 RREG32(CB_CGTT_SCLK_CTRL
);
5092 RREG32(CB_CGTT_SCLK_CTRL
);
5093 RREG32(CB_CGTT_SCLK_CTRL
);
5094 RREG32(CB_CGTT_SCLK_CTRL
);
5096 data
&= ~(CGCG_EN
| CGLS_EN
);
5100 WREG32(RLC_CGCG_CGLS_CTRL
, data
);
5103 static void si_enable_mgcg(struct radeon_device
*rdev
,
5106 u32 data
, orig
, tmp
= 0;
5109 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5112 WREG32(CGTS_SM_CTRL_REG
, data
);
5114 orig
= data
= RREG32(CP_MEM_SLP_CNTL
);
5115 data
|= CP_MEM_LS_EN
;
5117 WREG32(CP_MEM_SLP_CNTL
, data
);
5119 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5122 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5124 tmp
= si_halt_rlc(rdev
);
5126 WREG32(RLC_SERDES_WR_MASTER_MASK_0
, 0xffffffff);
5127 WREG32(RLC_SERDES_WR_MASTER_MASK_1
, 0xffffffff);
5128 WREG32(RLC_SERDES_WR_CTRL
, 0x00d000ff);
5130 si_update_rlc(rdev
, tmp
);
5132 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5135 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5137 data
= RREG32(CP_MEM_SLP_CNTL
);
5138 if (data
& CP_MEM_LS_EN
) {
5139 data
&= ~CP_MEM_LS_EN
;
5140 WREG32(CP_MEM_SLP_CNTL
, data
);
5142 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5143 data
|= LS_OVERRIDE
| OVERRIDE
;
5145 WREG32(CGTS_SM_CTRL_REG
, data
);
5147 tmp
= si_halt_rlc(rdev
);
5149 WREG32(RLC_SERDES_WR_MASTER_MASK_0
, 0xffffffff);
5150 WREG32(RLC_SERDES_WR_MASTER_MASK_1
, 0xffffffff);
5151 WREG32(RLC_SERDES_WR_CTRL
, 0x00e000ff);
5153 si_update_rlc(rdev
, tmp
);
5157 static void si_enable_uvd_mgcg(struct radeon_device
*rdev
,
5160 u32 orig
, data
, tmp
;
5163 tmp
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
5165 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, tmp
);
5167 orig
= data
= RREG32(UVD_CGC_CTRL
);
5170 WREG32(UVD_CGC_CTRL
, data
);
5172 WREG32_SMC(SMC_CG_IND_START
+ CG_CGTT_LOCAL_0
, 0);
5173 WREG32_SMC(SMC_CG_IND_START
+ CG_CGTT_LOCAL_1
, 0);
5175 tmp
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
5177 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, tmp
);
5179 orig
= data
= RREG32(UVD_CGC_CTRL
);
5182 WREG32(UVD_CGC_CTRL
, data
);
5184 WREG32_SMC(SMC_CG_IND_START
+ CG_CGTT_LOCAL_0
, 0xffffffff);
5185 WREG32_SMC(SMC_CG_IND_START
+ CG_CGTT_LOCAL_1
, 0xffffffff);
5189 static const u32 mc_cg_registers
[] =
5202 static void si_enable_mc_ls(struct radeon_device
*rdev
,
5208 for (i
= 0; i
< ARRAY_SIZE(mc_cg_registers
); i
++) {
5209 orig
= data
= RREG32(mc_cg_registers
[i
]);
5211 data
|= MC_LS_ENABLE
;
5213 data
&= ~MC_LS_ENABLE
;
5215 WREG32(mc_cg_registers
[i
], data
);
5220 static void si_init_cg(struct radeon_device
*rdev
)
5222 si_enable_mgcg(rdev
, true);
5223 si_enable_cgcg(rdev
, false);
5224 /* disable MC LS on Tahiti */
5225 if (rdev
->family
== CHIP_TAHITI
)
5226 si_enable_mc_ls(rdev
, false);
5227 if (rdev
->has_uvd
) {
5228 si_enable_uvd_mgcg(rdev
, true);
5229 si_init_uvd_internal_cg(rdev
);
5233 static void si_fini_cg(struct radeon_device
*rdev
)
5236 si_enable_uvd_mgcg(rdev
, false);
5237 si_enable_cgcg(rdev
, false);
5238 si_enable_mgcg(rdev
, false);
5241 static void si_init_pg(struct radeon_device
*rdev
)
5243 bool has_pg
= false;
5245 /* only cape verde supports PG */
5246 if (rdev
->family
== CHIP_VERDE
)
5250 si_init_ao_cu_mask(rdev
);
5251 si_init_dma_pg(rdev
);
5252 si_enable_dma_pg(rdev
, true);
5253 si_init_gfx_cgpg(rdev
);
5254 si_enable_gfx_cgpg(rdev
, true);
5256 WREG32(RLC_SAVE_AND_RESTORE_BASE
, rdev
->rlc
.save_restore_gpu_addr
>> 8);
5257 WREG32(RLC_CLEAR_STATE_RESTORE_BASE
, rdev
->rlc
.clear_state_gpu_addr
>> 8);
5261 static void si_fini_pg(struct radeon_device
*rdev
)
5263 bool has_pg
= false;
5265 /* only cape verde supports PG */
5266 if (rdev
->family
== CHIP_VERDE
)
5270 si_enable_dma_pg(rdev
, false);
5271 si_enable_gfx_cgpg(rdev
, false);
5278 void si_rlc_fini(struct radeon_device
*rdev
)
5282 /* save restore block */
5283 if (rdev
->rlc
.save_restore_obj
) {
5284 r
= radeon_bo_reserve(rdev
->rlc
.save_restore_obj
, false);
5285 if (unlikely(r
!= 0))
5286 dev_warn(rdev
->dev
, "(%d) reserve RLC sr bo failed\n", r
);
5287 radeon_bo_unpin(rdev
->rlc
.save_restore_obj
);
5288 radeon_bo_unreserve(rdev
->rlc
.save_restore_obj
);
5290 radeon_bo_unref(&rdev
->rlc
.save_restore_obj
);
5291 rdev
->rlc
.save_restore_obj
= NULL
;
5294 /* clear state block */
5295 if (rdev
->rlc
.clear_state_obj
) {
5296 r
= radeon_bo_reserve(rdev
->rlc
.clear_state_obj
, false);
5297 if (unlikely(r
!= 0))
5298 dev_warn(rdev
->dev
, "(%d) reserve RLC c bo failed\n", r
);
5299 radeon_bo_unpin(rdev
->rlc
.clear_state_obj
);
5300 radeon_bo_unreserve(rdev
->rlc
.clear_state_obj
);
5302 radeon_bo_unref(&rdev
->rlc
.clear_state_obj
);
5303 rdev
->rlc
.clear_state_obj
= NULL
;
5307 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
5309 int si_rlc_init(struct radeon_device
*rdev
)
5311 volatile u32
*dst_ptr
;
5312 u32 dws
, data
, i
, j
, k
, reg_num
;
5313 u32 reg_list_num
, reg_list_hdr_blk_index
, reg_list_blk_index
;
5314 u64 reg_list_mc_addr
;
5315 const struct cs_section_def
*cs_data
= si_cs_data
;
5318 /* save restore block */
5319 if (rdev
->rlc
.save_restore_obj
== NULL
) {
5320 r
= radeon_bo_create(rdev
, RADEON_GPU_PAGE_SIZE
, PAGE_SIZE
, true,
5321 RADEON_GEM_DOMAIN_VRAM
, NULL
,
5322 &rdev
->rlc
.save_restore_obj
);
5324 dev_warn(rdev
->dev
, "(%d) create RLC sr bo failed\n", r
);
5329 r
= radeon_bo_reserve(rdev
->rlc
.save_restore_obj
, false);
5330 if (unlikely(r
!= 0)) {
5334 r
= radeon_bo_pin(rdev
->rlc
.save_restore_obj
, RADEON_GEM_DOMAIN_VRAM
,
5335 &rdev
->rlc
.save_restore_gpu_addr
);
5337 radeon_bo_unreserve(rdev
->rlc
.save_restore_obj
);
5338 dev_warn(rdev
->dev
, "(%d) pin RLC sr bo failed\n", r
);
5343 if (rdev
->family
== CHIP_VERDE
) {
5344 r
= radeon_bo_kmap(rdev
->rlc
.save_restore_obj
, (void **)&rdev
->rlc
.sr_ptr
);
5346 dev_warn(rdev
->dev
, "(%d) map RLC sr bo failed\n", r
);
5350 /* write the sr buffer */
5351 dst_ptr
= rdev
->rlc
.sr_ptr
;
5352 for (i
= 0; i
< ARRAY_SIZE(verde_rlc_save_restore_register_list
); i
++) {
5353 dst_ptr
[i
] = verde_rlc_save_restore_register_list
[i
];
5355 radeon_bo_kunmap(rdev
->rlc
.save_restore_obj
);
5357 radeon_bo_unreserve(rdev
->rlc
.save_restore_obj
);
5359 /* clear state block */
5362 for (i
= 0; cs_data
[i
].section
!= NULL
; i
++) {
5363 for (j
= 0; cs_data
[i
].section
[j
].extent
!= NULL
; j
++) {
5365 dws
+= cs_data
[i
].section
[j
].reg_count
;
5368 reg_list_blk_index
= (3 * reg_list_num
+ 2);
5369 dws
+= reg_list_blk_index
;
5371 if (rdev
->rlc
.clear_state_obj
== NULL
) {
5372 r
= radeon_bo_create(rdev
, dws
* 4, PAGE_SIZE
, true,
5373 RADEON_GEM_DOMAIN_VRAM
, NULL
, &rdev
->rlc
.clear_state_obj
);
5375 dev_warn(rdev
->dev
, "(%d) create RLC c bo failed\n", r
);
5380 r
= radeon_bo_reserve(rdev
->rlc
.clear_state_obj
, false);
5381 if (unlikely(r
!= 0)) {
5385 r
= radeon_bo_pin(rdev
->rlc
.clear_state_obj
, RADEON_GEM_DOMAIN_VRAM
,
5386 &rdev
->rlc
.clear_state_gpu_addr
);
5389 radeon_bo_unreserve(rdev
->rlc
.clear_state_obj
);
5390 dev_warn(rdev
->dev
, "(%d) pin RLC c bo failed\n", r
);
5394 r
= radeon_bo_kmap(rdev
->rlc
.clear_state_obj
, (void **)&rdev
->rlc
.cs_ptr
);
5396 dev_warn(rdev
->dev
, "(%d) map RLC c bo failed\n", r
);
5400 /* set up the cs buffer */
5401 dst_ptr
= rdev
->rlc
.cs_ptr
;
5402 reg_list_hdr_blk_index
= 0;
5403 reg_list_mc_addr
= rdev
->rlc
.clear_state_gpu_addr
+ (reg_list_blk_index
* 4);
5404 data
= upper_32_bits(reg_list_mc_addr
);
5405 dst_ptr
[reg_list_hdr_blk_index
] = data
;
5406 reg_list_hdr_blk_index
++;
5407 for (i
= 0; cs_data
[i
].section
!= NULL
; i
++) {
5408 for (j
= 0; cs_data
[i
].section
[j
].extent
!= NULL
; j
++) {
5409 reg_num
= cs_data
[i
].section
[j
].reg_count
;
5410 data
= reg_list_mc_addr
& 0xffffffff;
5411 dst_ptr
[reg_list_hdr_blk_index
] = data
;
5412 reg_list_hdr_blk_index
++;
5414 data
= (cs_data
[i
].section
[j
].reg_index
* 4) & 0xffffffff;
5415 dst_ptr
[reg_list_hdr_blk_index
] = data
;
5416 reg_list_hdr_blk_index
++;
5418 data
= 0x08000000 | (reg_num
* 4);
5419 dst_ptr
[reg_list_hdr_blk_index
] = data
;
5420 reg_list_hdr_blk_index
++;
5422 for (k
= 0; k
< reg_num
; k
++) {
5423 data
= cs_data
[i
].section
[j
].extent
[k
];
5424 dst_ptr
[reg_list_blk_index
+ k
] = data
;
5426 reg_list_mc_addr
+= reg_num
* 4;
5427 reg_list_blk_index
+= reg_num
;
5430 dst_ptr
[reg_list_hdr_blk_index
] = RLC_CLEAR_STATE_END_MARKER
;
5432 radeon_bo_kunmap(rdev
->rlc
.clear_state_obj
);
5433 radeon_bo_unreserve(rdev
->rlc
.clear_state_obj
);
5438 static void si_rlc_reset(struct radeon_device
*rdev
)
5440 u32 tmp
= RREG32(GRBM_SOFT_RESET
);
5442 tmp
|= SOFT_RESET_RLC
;
5443 WREG32(GRBM_SOFT_RESET
, tmp
);
5445 tmp
&= ~SOFT_RESET_RLC
;
5446 WREG32(GRBM_SOFT_RESET
, tmp
);
5450 static void si_rlc_stop(struct radeon_device
*rdev
)
5452 WREG32(RLC_CNTL
, 0);
5454 si_enable_gui_idle_interrupt(rdev
, false);
5456 si_wait_for_rlc_serdes(rdev
);
5459 static void si_rlc_start(struct radeon_device
*rdev
)
5461 WREG32(RLC_CNTL
, RLC_ENABLE
);
5463 si_enable_gui_idle_interrupt(rdev
, true);
5468 static bool si_lbpw_supported(struct radeon_device
*rdev
)
5472 /* Enable LBPW only for DDR3 */
5473 tmp
= RREG32(MC_SEQ_MISC0
);
5474 if ((tmp
& 0xF0000000) == 0xB0000000)
5479 static void si_enable_lbpw(struct radeon_device
*rdev
, bool enable
)
5483 tmp
= RREG32(RLC_LB_CNTL
);
5485 tmp
|= LOAD_BALANCE_ENABLE
;
5487 tmp
&= ~LOAD_BALANCE_ENABLE
;
5488 WREG32(RLC_LB_CNTL
, tmp
);
5491 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5492 WREG32(SPI_LB_CU_MASK
, 0x00ff);
5496 static int si_rlc_resume(struct radeon_device
*rdev
)
5499 const __be32
*fw_data
;
5512 WREG32(RLC_RL_BASE
, 0);
5513 WREG32(RLC_RL_SIZE
, 0);
5514 WREG32(RLC_LB_CNTL
, 0);
5515 WREG32(RLC_LB_CNTR_MAX
, 0xffffffff);
5516 WREG32(RLC_LB_CNTR_INIT
, 0);
5517 WREG32(RLC_LB_INIT_CU_MASK
, 0xffffffff);
5519 WREG32(RLC_MC_CNTL
, 0);
5520 WREG32(RLC_UCODE_CNTL
, 0);
5522 fw_data
= (const __be32
*)rdev
->rlc_fw
->data
;
5523 for (i
= 0; i
< SI_RLC_UCODE_SIZE
; i
++) {
5524 WREG32(RLC_UCODE_ADDR
, i
);
5525 WREG32(RLC_UCODE_DATA
, be32_to_cpup(fw_data
++));
5527 WREG32(RLC_UCODE_ADDR
, 0);
5529 si_enable_lbpw(rdev
, si_lbpw_supported(rdev
));
5536 static void si_enable_interrupts(struct radeon_device
*rdev
)
5538 u32 ih_cntl
= RREG32(IH_CNTL
);
5539 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
5541 ih_cntl
|= ENABLE_INTR
;
5542 ih_rb_cntl
|= IH_RB_ENABLE
;
5543 WREG32(IH_CNTL
, ih_cntl
);
5544 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
5545 rdev
->ih
.enabled
= true;
5548 static void si_disable_interrupts(struct radeon_device
*rdev
)
5550 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
5551 u32 ih_cntl
= RREG32(IH_CNTL
);
5553 ih_rb_cntl
&= ~IH_RB_ENABLE
;
5554 ih_cntl
&= ~ENABLE_INTR
;
5555 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
5556 WREG32(IH_CNTL
, ih_cntl
);
5557 /* set rptr, wptr to 0 */
5558 WREG32(IH_RB_RPTR
, 0);
5559 WREG32(IH_RB_WPTR
, 0);
5560 rdev
->ih
.enabled
= false;
5564 static void si_disable_interrupt_state(struct radeon_device
*rdev
)
5568 WREG32(CP_INT_CNTL_RING0
, CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
5569 WREG32(CP_INT_CNTL_RING1
, 0);
5570 WREG32(CP_INT_CNTL_RING2
, 0);
5571 tmp
= RREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
5572 WREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
, tmp
);
5573 tmp
= RREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
5574 WREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
, tmp
);
5575 WREG32(GRBM_INT_CNTL
, 0);
5576 if (rdev
->num_crtc
>= 2) {
5577 WREG32(INT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
5578 WREG32(INT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
5580 if (rdev
->num_crtc
>= 4) {
5581 WREG32(INT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
5582 WREG32(INT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
5584 if (rdev
->num_crtc
>= 6) {
5585 WREG32(INT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
5586 WREG32(INT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
5589 if (rdev
->num_crtc
>= 2) {
5590 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
5591 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
5593 if (rdev
->num_crtc
>= 4) {
5594 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
5595 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
5597 if (rdev
->num_crtc
>= 6) {
5598 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
5599 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
5602 if (!ASIC_IS_NODCE(rdev
)) {
5603 WREG32(DACA_AUTODETECT_INT_CONTROL
, 0);
5605 tmp
= RREG32(DC_HPD1_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5606 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
5607 tmp
= RREG32(DC_HPD2_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5608 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
5609 tmp
= RREG32(DC_HPD3_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5610 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
5611 tmp
= RREG32(DC_HPD4_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5612 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
5613 tmp
= RREG32(DC_HPD5_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5614 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
5615 tmp
= RREG32(DC_HPD6_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5616 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
5620 static int si_irq_init(struct radeon_device
*rdev
)
5624 u32 interrupt_cntl
, ih_cntl
, ih_rb_cntl
;
5627 ret
= r600_ih_ring_alloc(rdev
);
5632 si_disable_interrupts(rdev
);
5635 ret
= si_rlc_resume(rdev
);
5637 r600_ih_ring_fini(rdev
);
5641 /* setup interrupt control */
5642 /* set dummy read address to ring address */
5643 WREG32(INTERRUPT_CNTL2
, rdev
->ih
.gpu_addr
>> 8);
5644 interrupt_cntl
= RREG32(INTERRUPT_CNTL
);
5645 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5646 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5648 interrupt_cntl
&= ~IH_DUMMY_RD_OVERRIDE
;
5649 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5650 interrupt_cntl
&= ~IH_REQ_NONSNOOP_EN
;
5651 WREG32(INTERRUPT_CNTL
, interrupt_cntl
);
5653 WREG32(IH_RB_BASE
, rdev
->ih
.gpu_addr
>> 8);
5654 rb_bufsz
= drm_order(rdev
->ih
.ring_size
/ 4);
5656 ih_rb_cntl
= (IH_WPTR_OVERFLOW_ENABLE
|
5657 IH_WPTR_OVERFLOW_CLEAR
|
5660 if (rdev
->wb
.enabled
)
5661 ih_rb_cntl
|= IH_WPTR_WRITEBACK_ENABLE
;
5663 /* set the writeback address whether it's enabled or not */
5664 WREG32(IH_RB_WPTR_ADDR_LO
, (rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFFFFFFFC);
5665 WREG32(IH_RB_WPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFF);
5667 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
5669 /* set rptr, wptr to 0 */
5670 WREG32(IH_RB_RPTR
, 0);
5671 WREG32(IH_RB_WPTR
, 0);
5673 /* Default settings for IH_CNTL (disabled at first) */
5674 ih_cntl
= MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5675 /* RPTR_REARM only works if msi's are enabled */
5676 if (rdev
->msi_enabled
)
5677 ih_cntl
|= RPTR_REARM
;
5678 WREG32(IH_CNTL
, ih_cntl
);
5680 /* force the active interrupt state to all disabled */
5681 si_disable_interrupt_state(rdev
);
5683 pci_set_master(rdev
->pdev
);
5686 si_enable_interrupts(rdev
);
5691 int si_irq_set(struct radeon_device
*rdev
)
5693 u32 cp_int_cntl
= CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
;
5694 u32 cp_int_cntl1
= 0, cp_int_cntl2
= 0;
5695 u32 crtc1
= 0, crtc2
= 0, crtc3
= 0, crtc4
= 0, crtc5
= 0, crtc6
= 0;
5696 u32 hpd1
= 0, hpd2
= 0, hpd3
= 0, hpd4
= 0, hpd5
= 0, hpd6
= 0;
5697 u32 grbm_int_cntl
= 0;
5698 u32 grph1
= 0, grph2
= 0, grph3
= 0, grph4
= 0, grph5
= 0, grph6
= 0;
5699 u32 dma_cntl
, dma_cntl1
;
5700 u32 thermal_int
= 0;
5702 if (!rdev
->irq
.installed
) {
5703 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5706 /* don't enable anything if the ih is disabled */
5707 if (!rdev
->ih
.enabled
) {
5708 si_disable_interrupts(rdev
);
5709 /* force the active interrupt state to all disabled */
5710 si_disable_interrupt_state(rdev
);
5714 if (!ASIC_IS_NODCE(rdev
)) {
5715 hpd1
= RREG32(DC_HPD1_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5716 hpd2
= RREG32(DC_HPD2_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5717 hpd3
= RREG32(DC_HPD3_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5718 hpd4
= RREG32(DC_HPD4_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5719 hpd5
= RREG32(DC_HPD5_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5720 hpd6
= RREG32(DC_HPD6_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5723 dma_cntl
= RREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
5724 dma_cntl1
= RREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
5726 thermal_int
= RREG32(CG_THERMAL_INT
) &
5727 ~(THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
);
5729 /* enable CP interrupts on all rings */
5730 if (atomic_read(&rdev
->irq
.ring_int
[RADEON_RING_TYPE_GFX_INDEX
])) {
5731 DRM_DEBUG("si_irq_set: sw int gfx\n");
5732 cp_int_cntl
|= TIME_STAMP_INT_ENABLE
;
5734 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP1_INDEX
])) {
5735 DRM_DEBUG("si_irq_set: sw int cp1\n");
5736 cp_int_cntl1
|= TIME_STAMP_INT_ENABLE
;
5738 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP2_INDEX
])) {
5739 DRM_DEBUG("si_irq_set: sw int cp2\n");
5740 cp_int_cntl2
|= TIME_STAMP_INT_ENABLE
;
5742 if (atomic_read(&rdev
->irq
.ring_int
[R600_RING_TYPE_DMA_INDEX
])) {
5743 DRM_DEBUG("si_irq_set: sw int dma\n");
5744 dma_cntl
|= TRAP_ENABLE
;
5747 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_DMA1_INDEX
])) {
5748 DRM_DEBUG("si_irq_set: sw int dma1\n");
5749 dma_cntl1
|= TRAP_ENABLE
;
5751 if (rdev
->irq
.crtc_vblank_int
[0] ||
5752 atomic_read(&rdev
->irq
.pflip
[0])) {
5753 DRM_DEBUG("si_irq_set: vblank 0\n");
5754 crtc1
|= VBLANK_INT_MASK
;
5756 if (rdev
->irq
.crtc_vblank_int
[1] ||
5757 atomic_read(&rdev
->irq
.pflip
[1])) {
5758 DRM_DEBUG("si_irq_set: vblank 1\n");
5759 crtc2
|= VBLANK_INT_MASK
;
5761 if (rdev
->irq
.crtc_vblank_int
[2] ||
5762 atomic_read(&rdev
->irq
.pflip
[2])) {
5763 DRM_DEBUG("si_irq_set: vblank 2\n");
5764 crtc3
|= VBLANK_INT_MASK
;
5766 if (rdev
->irq
.crtc_vblank_int
[3] ||
5767 atomic_read(&rdev
->irq
.pflip
[3])) {
5768 DRM_DEBUG("si_irq_set: vblank 3\n");
5769 crtc4
|= VBLANK_INT_MASK
;
5771 if (rdev
->irq
.crtc_vblank_int
[4] ||
5772 atomic_read(&rdev
->irq
.pflip
[4])) {
5773 DRM_DEBUG("si_irq_set: vblank 4\n");
5774 crtc5
|= VBLANK_INT_MASK
;
5776 if (rdev
->irq
.crtc_vblank_int
[5] ||
5777 atomic_read(&rdev
->irq
.pflip
[5])) {
5778 DRM_DEBUG("si_irq_set: vblank 5\n");
5779 crtc6
|= VBLANK_INT_MASK
;
5781 if (rdev
->irq
.hpd
[0]) {
5782 DRM_DEBUG("si_irq_set: hpd 1\n");
5783 hpd1
|= DC_HPDx_INT_EN
;
5785 if (rdev
->irq
.hpd
[1]) {
5786 DRM_DEBUG("si_irq_set: hpd 2\n");
5787 hpd2
|= DC_HPDx_INT_EN
;
5789 if (rdev
->irq
.hpd
[2]) {
5790 DRM_DEBUG("si_irq_set: hpd 3\n");
5791 hpd3
|= DC_HPDx_INT_EN
;
5793 if (rdev
->irq
.hpd
[3]) {
5794 DRM_DEBUG("si_irq_set: hpd 4\n");
5795 hpd4
|= DC_HPDx_INT_EN
;
5797 if (rdev
->irq
.hpd
[4]) {
5798 DRM_DEBUG("si_irq_set: hpd 5\n");
5799 hpd5
|= DC_HPDx_INT_EN
;
5801 if (rdev
->irq
.hpd
[5]) {
5802 DRM_DEBUG("si_irq_set: hpd 6\n");
5803 hpd6
|= DC_HPDx_INT_EN
;
5806 WREG32(CP_INT_CNTL_RING0
, cp_int_cntl
);
5807 WREG32(CP_INT_CNTL_RING1
, cp_int_cntl1
);
5808 WREG32(CP_INT_CNTL_RING2
, cp_int_cntl2
);
5810 WREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
, dma_cntl
);
5811 WREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
, dma_cntl1
);
5813 WREG32(GRBM_INT_CNTL
, grbm_int_cntl
);
5815 if (rdev
->irq
.dpm_thermal
) {
5816 DRM_DEBUG("dpm thermal\n");
5817 thermal_int
|= THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
;
5820 if (rdev
->num_crtc
>= 2) {
5821 WREG32(INT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, crtc1
);
5822 WREG32(INT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, crtc2
);
5824 if (rdev
->num_crtc
>= 4) {
5825 WREG32(INT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, crtc3
);
5826 WREG32(INT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, crtc4
);
5828 if (rdev
->num_crtc
>= 6) {
5829 WREG32(INT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, crtc5
);
5830 WREG32(INT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, crtc6
);
5833 if (rdev
->num_crtc
>= 2) {
5834 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, grph1
);
5835 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, grph2
);
5837 if (rdev
->num_crtc
>= 4) {
5838 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, grph3
);
5839 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, grph4
);
5841 if (rdev
->num_crtc
>= 6) {
5842 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, grph5
);
5843 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, grph6
);
5846 if (!ASIC_IS_NODCE(rdev
)) {
5847 WREG32(DC_HPD1_INT_CONTROL
, hpd1
);
5848 WREG32(DC_HPD2_INT_CONTROL
, hpd2
);
5849 WREG32(DC_HPD3_INT_CONTROL
, hpd3
);
5850 WREG32(DC_HPD4_INT_CONTROL
, hpd4
);
5851 WREG32(DC_HPD5_INT_CONTROL
, hpd5
);
5852 WREG32(DC_HPD6_INT_CONTROL
, hpd6
);
5855 WREG32(CG_THERMAL_INT
, thermal_int
);
5860 static inline void si_irq_ack(struct radeon_device
*rdev
)
5864 if (ASIC_IS_NODCE(rdev
))
5867 rdev
->irq
.stat_regs
.evergreen
.disp_int
= RREG32(DISP_INTERRUPT_STATUS
);
5868 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE
);
5869 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE2
);
5870 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE3
);
5871 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE4
);
5872 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE5
);
5873 rdev
->irq
.stat_regs
.evergreen
.d1grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
);
5874 rdev
->irq
.stat_regs
.evergreen
.d2grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
);
5875 if (rdev
->num_crtc
>= 4) {
5876 rdev
->irq
.stat_regs
.evergreen
.d3grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
);
5877 rdev
->irq
.stat_regs
.evergreen
.d4grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
);
5879 if (rdev
->num_crtc
>= 6) {
5880 rdev
->irq
.stat_regs
.evergreen
.d5grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
);
5881 rdev
->irq
.stat_regs
.evergreen
.d6grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
);
5884 if (rdev
->irq
.stat_regs
.evergreen
.d1grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5885 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5886 if (rdev
->irq
.stat_regs
.evergreen
.d2grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5887 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5888 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VBLANK_INTERRUPT
)
5889 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VBLANK_ACK
);
5890 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VLINE_INTERRUPT
)
5891 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VLINE_ACK
);
5892 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
)
5893 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VBLANK_ACK
);
5894 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
)
5895 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VLINE_ACK
);
5897 if (rdev
->num_crtc
>= 4) {
5898 if (rdev
->irq
.stat_regs
.evergreen
.d3grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5899 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5900 if (rdev
->irq
.stat_regs
.evergreen
.d4grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5901 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5902 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
)
5903 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VBLANK_ACK
);
5904 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
)
5905 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VLINE_ACK
);
5906 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
)
5907 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VBLANK_ACK
);
5908 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
)
5909 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VLINE_ACK
);
5912 if (rdev
->num_crtc
>= 6) {
5913 if (rdev
->irq
.stat_regs
.evergreen
.d5grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5914 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5915 if (rdev
->irq
.stat_regs
.evergreen
.d6grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5916 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5917 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
)
5918 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VBLANK_ACK
);
5919 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
)
5920 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VLINE_ACK
);
5921 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
)
5922 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VBLANK_ACK
);
5923 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
)
5924 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VLINE_ACK
);
5927 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& DC_HPD1_INTERRUPT
) {
5928 tmp
= RREG32(DC_HPD1_INT_CONTROL
);
5929 tmp
|= DC_HPDx_INT_ACK
;
5930 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
5932 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
5933 tmp
= RREG32(DC_HPD2_INT_CONTROL
);
5934 tmp
|= DC_HPDx_INT_ACK
;
5935 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
5937 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
5938 tmp
= RREG32(DC_HPD3_INT_CONTROL
);
5939 tmp
|= DC_HPDx_INT_ACK
;
5940 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
5942 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
5943 tmp
= RREG32(DC_HPD4_INT_CONTROL
);
5944 tmp
|= DC_HPDx_INT_ACK
;
5945 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
5947 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
5948 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
5949 tmp
|= DC_HPDx_INT_ACK
;
5950 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
5952 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
5953 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
5954 tmp
|= DC_HPDx_INT_ACK
;
5955 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
5959 static void si_irq_disable(struct radeon_device
*rdev
)
5961 si_disable_interrupts(rdev
);
5962 /* Wait and acknowledge irq */
5965 si_disable_interrupt_state(rdev
);
5968 static void si_irq_suspend(struct radeon_device
*rdev
)
5970 si_irq_disable(rdev
);
5974 static void si_irq_fini(struct radeon_device
*rdev
)
5976 si_irq_suspend(rdev
);
5977 r600_ih_ring_fini(rdev
);
5980 static inline u32
si_get_ih_wptr(struct radeon_device
*rdev
)
5984 if (rdev
->wb
.enabled
)
5985 wptr
= le32_to_cpu(rdev
->wb
.wb
[R600_WB_IH_WPTR_OFFSET
/4]);
5987 wptr
= RREG32(IH_RB_WPTR
);
5989 if (wptr
& RB_OVERFLOW
) {
5990 /* When a ring buffer overflow happen start parsing interrupt
5991 * from the last not overwritten vector (wptr + 16). Hopefully
5992 * this should allow us to catchup.
5994 dev_warn(rdev
->dev
, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5995 wptr
, rdev
->ih
.rptr
, (wptr
+ 16) + rdev
->ih
.ptr_mask
);
5996 rdev
->ih
.rptr
= (wptr
+ 16) & rdev
->ih
.ptr_mask
;
5997 tmp
= RREG32(IH_RB_CNTL
);
5998 tmp
|= IH_WPTR_OVERFLOW_CLEAR
;
5999 WREG32(IH_RB_CNTL
, tmp
);
6001 return (wptr
& rdev
->ih
.ptr_mask
);
6005 * Each IV ring entry is 128 bits:
6006 * [7:0] - interrupt source id
6008 * [59:32] - interrupt source data
6009 * [63:60] - reserved
6012 * [127:80] - reserved
6014 int si_irq_process(struct radeon_device
*rdev
)
6018 u32 src_id
, src_data
, ring_id
;
6020 bool queue_hotplug
= false;
6021 bool queue_thermal
= false;
6024 if (!rdev
->ih
.enabled
|| rdev
->shutdown
)
6027 wptr
= si_get_ih_wptr(rdev
);
6030 /* is somebody else already processing irqs? */
6031 if (atomic_xchg(&rdev
->ih
.lock
, 1))
6034 rptr
= rdev
->ih
.rptr
;
6035 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr
, wptr
);
6037 /* Order reading of wptr vs. reading of IH ring data */
6040 /* display interrupts */
6043 while (rptr
!= wptr
) {
6044 /* wptr/rptr are in bytes! */
6045 ring_index
= rptr
/ 4;
6046 src_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
]) & 0xff;
6047 src_data
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 1]) & 0xfffffff;
6048 ring_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 2]) & 0xff;
6051 case 1: /* D1 vblank/vline */
6053 case 0: /* D1 vblank */
6054 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VBLANK_INTERRUPT
) {
6055 if (rdev
->irq
.crtc_vblank_int
[0]) {
6056 drm_handle_vblank(rdev
->ddev
, 0);
6057 rdev
->pm
.vblank_sync
= true;
6058 wake_up(&rdev
->irq
.vblank_queue
);
6060 if (atomic_read(&rdev
->irq
.pflip
[0]))
6061 radeon_crtc_handle_flip(rdev
, 0);
6062 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~LB_D1_VBLANK_INTERRUPT
;
6063 DRM_DEBUG("IH: D1 vblank\n");
6066 case 1: /* D1 vline */
6067 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VLINE_INTERRUPT
) {
6068 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~LB_D1_VLINE_INTERRUPT
;
6069 DRM_DEBUG("IH: D1 vline\n");
6073 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6077 case 2: /* D2 vblank/vline */
6079 case 0: /* D2 vblank */
6080 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
) {
6081 if (rdev
->irq
.crtc_vblank_int
[1]) {
6082 drm_handle_vblank(rdev
->ddev
, 1);
6083 rdev
->pm
.vblank_sync
= true;
6084 wake_up(&rdev
->irq
.vblank_queue
);
6086 if (atomic_read(&rdev
->irq
.pflip
[1]))
6087 radeon_crtc_handle_flip(rdev
, 1);
6088 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~LB_D2_VBLANK_INTERRUPT
;
6089 DRM_DEBUG("IH: D2 vblank\n");
6092 case 1: /* D2 vline */
6093 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
) {
6094 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~LB_D2_VLINE_INTERRUPT
;
6095 DRM_DEBUG("IH: D2 vline\n");
6099 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6103 case 3: /* D3 vblank/vline */
6105 case 0: /* D3 vblank */
6106 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
) {
6107 if (rdev
->irq
.crtc_vblank_int
[2]) {
6108 drm_handle_vblank(rdev
->ddev
, 2);
6109 rdev
->pm
.vblank_sync
= true;
6110 wake_up(&rdev
->irq
.vblank_queue
);
6112 if (atomic_read(&rdev
->irq
.pflip
[2]))
6113 radeon_crtc_handle_flip(rdev
, 2);
6114 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~LB_D3_VBLANK_INTERRUPT
;
6115 DRM_DEBUG("IH: D3 vblank\n");
6118 case 1: /* D3 vline */
6119 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
) {
6120 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~LB_D3_VLINE_INTERRUPT
;
6121 DRM_DEBUG("IH: D3 vline\n");
6125 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6129 case 4: /* D4 vblank/vline */
6131 case 0: /* D4 vblank */
6132 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
) {
6133 if (rdev
->irq
.crtc_vblank_int
[3]) {
6134 drm_handle_vblank(rdev
->ddev
, 3);
6135 rdev
->pm
.vblank_sync
= true;
6136 wake_up(&rdev
->irq
.vblank_queue
);
6138 if (atomic_read(&rdev
->irq
.pflip
[3]))
6139 radeon_crtc_handle_flip(rdev
, 3);
6140 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~LB_D4_VBLANK_INTERRUPT
;
6141 DRM_DEBUG("IH: D4 vblank\n");
6144 case 1: /* D4 vline */
6145 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
) {
6146 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~LB_D4_VLINE_INTERRUPT
;
6147 DRM_DEBUG("IH: D4 vline\n");
6151 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6155 case 5: /* D5 vblank/vline */
6157 case 0: /* D5 vblank */
6158 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
) {
6159 if (rdev
->irq
.crtc_vblank_int
[4]) {
6160 drm_handle_vblank(rdev
->ddev
, 4);
6161 rdev
->pm
.vblank_sync
= true;
6162 wake_up(&rdev
->irq
.vblank_queue
);
6164 if (atomic_read(&rdev
->irq
.pflip
[4]))
6165 radeon_crtc_handle_flip(rdev
, 4);
6166 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~LB_D5_VBLANK_INTERRUPT
;
6167 DRM_DEBUG("IH: D5 vblank\n");
6170 case 1: /* D5 vline */
6171 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
) {
6172 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~LB_D5_VLINE_INTERRUPT
;
6173 DRM_DEBUG("IH: D5 vline\n");
6177 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6181 case 6: /* D6 vblank/vline */
6183 case 0: /* D6 vblank */
6184 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
) {
6185 if (rdev
->irq
.crtc_vblank_int
[5]) {
6186 drm_handle_vblank(rdev
->ddev
, 5);
6187 rdev
->pm
.vblank_sync
= true;
6188 wake_up(&rdev
->irq
.vblank_queue
);
6190 if (atomic_read(&rdev
->irq
.pflip
[5]))
6191 radeon_crtc_handle_flip(rdev
, 5);
6192 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~LB_D6_VBLANK_INTERRUPT
;
6193 DRM_DEBUG("IH: D6 vblank\n");
6196 case 1: /* D6 vline */
6197 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
) {
6198 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~LB_D6_VLINE_INTERRUPT
;
6199 DRM_DEBUG("IH: D6 vline\n");
6203 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6207 case 42: /* HPD hotplug */
6210 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& DC_HPD1_INTERRUPT
) {
6211 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~DC_HPD1_INTERRUPT
;
6212 queue_hotplug
= true;
6213 DRM_DEBUG("IH: HPD1\n");
6217 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
6218 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~DC_HPD2_INTERRUPT
;
6219 queue_hotplug
= true;
6220 DRM_DEBUG("IH: HPD2\n");
6224 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
6225 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~DC_HPD3_INTERRUPT
;
6226 queue_hotplug
= true;
6227 DRM_DEBUG("IH: HPD3\n");
6231 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
6232 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~DC_HPD4_INTERRUPT
;
6233 queue_hotplug
= true;
6234 DRM_DEBUG("IH: HPD4\n");
6238 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
6239 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~DC_HPD5_INTERRUPT
;
6240 queue_hotplug
= true;
6241 DRM_DEBUG("IH: HPD5\n");
6245 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
6246 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~DC_HPD6_INTERRUPT
;
6247 queue_hotplug
= true;
6248 DRM_DEBUG("IH: HPD6\n");
6252 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6258 addr
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
);
6259 status
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
);
6260 dev_err(rdev
->dev
, "GPU fault detected: %d 0x%08x\n", src_id
, src_data
);
6261 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6263 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6265 si_vm_decode_fault(rdev
, status
, addr
);
6266 /* reset addr and status */
6267 WREG32_P(VM_CONTEXT1_CNTL2
, 1, ~1);
6269 case 176: /* RINGID0 CP_INT */
6270 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
6272 case 177: /* RINGID1 CP_INT */
6273 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
6275 case 178: /* RINGID2 CP_INT */
6276 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
6278 case 181: /* CP EOP event */
6279 DRM_DEBUG("IH: CP EOP\n");
6282 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
6285 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
6288 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
6292 case 224: /* DMA trap event */
6293 DRM_DEBUG("IH: DMA trap\n");
6294 radeon_fence_process(rdev
, R600_RING_TYPE_DMA_INDEX
);
6296 case 230: /* thermal low to high */
6297 DRM_DEBUG("IH: thermal low to high\n");
6298 rdev
->pm
.dpm
.thermal
.high_to_low
= false;
6299 queue_thermal
= true;
6301 case 231: /* thermal high to low */
6302 DRM_DEBUG("IH: thermal high to low\n");
6303 rdev
->pm
.dpm
.thermal
.high_to_low
= true;
6304 queue_thermal
= true;
6306 case 233: /* GUI IDLE */
6307 DRM_DEBUG("IH: GUI idle\n");
6309 case 244: /* DMA trap event */
6310 DRM_DEBUG("IH: DMA1 trap\n");
6311 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
6314 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6318 /* wptr/rptr are in bytes! */
6320 rptr
&= rdev
->ih
.ptr_mask
;
6323 schedule_work(&rdev
->hotplug_work
);
6324 if (queue_thermal
&& rdev
->pm
.dpm_enabled
)
6325 schedule_work(&rdev
->pm
.dpm
.thermal
.work
);
6326 rdev
->ih
.rptr
= rptr
;
6327 WREG32(IH_RB_RPTR
, rdev
->ih
.rptr
);
6328 atomic_set(&rdev
->ih
.lock
, 0);
6330 /* make sure wptr hasn't changed while processing */
6331 wptr
= si_get_ih_wptr(rdev
);
6339 * si_copy_dma - copy pages using the DMA engine
6341 * @rdev: radeon_device pointer
6342 * @src_offset: src GPU address
6343 * @dst_offset: dst GPU address
6344 * @num_gpu_pages: number of GPU pages to xfer
6345 * @fence: radeon fence object
6347 * Copy GPU paging using the DMA engine (SI).
6348 * Used by the radeon ttm implementation to move pages if
6349 * registered as the asic copy callback.
6351 int si_copy_dma(struct radeon_device
*rdev
,
6352 uint64_t src_offset
, uint64_t dst_offset
,
6353 unsigned num_gpu_pages
,
6354 struct radeon_fence
**fence
)
6356 struct radeon_semaphore
*sem
= NULL
;
6357 int ring_index
= rdev
->asic
->copy
.dma_ring_index
;
6358 struct radeon_ring
*ring
= &rdev
->ring
[ring_index
];
6359 u32 size_in_bytes
, cur_size_in_bytes
;
6363 r
= radeon_semaphore_create(rdev
, &sem
);
6365 DRM_ERROR("radeon: moving bo (%d).\n", r
);
6369 size_in_bytes
= (num_gpu_pages
<< RADEON_GPU_PAGE_SHIFT
);
6370 num_loops
= DIV_ROUND_UP(size_in_bytes
, 0xfffff);
6371 r
= radeon_ring_lock(rdev
, ring
, num_loops
* 5 + 11);
6373 DRM_ERROR("radeon: moving bo (%d).\n", r
);
6374 radeon_semaphore_free(rdev
, &sem
, NULL
);
6378 if (radeon_fence_need_sync(*fence
, ring
->idx
)) {
6379 radeon_semaphore_sync_rings(rdev
, sem
, (*fence
)->ring
,
6381 radeon_fence_note_sync(*fence
, ring
->idx
);
6383 radeon_semaphore_free(rdev
, &sem
, NULL
);
6386 for (i
= 0; i
< num_loops
; i
++) {
6387 cur_size_in_bytes
= size_in_bytes
;
6388 if (cur_size_in_bytes
> 0xFFFFF)
6389 cur_size_in_bytes
= 0xFFFFF;
6390 size_in_bytes
-= cur_size_in_bytes
;
6391 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_COPY
, 1, 0, 0, cur_size_in_bytes
));
6392 radeon_ring_write(ring
, dst_offset
& 0xffffffff);
6393 radeon_ring_write(ring
, src_offset
& 0xffffffff);
6394 radeon_ring_write(ring
, upper_32_bits(dst_offset
) & 0xff);
6395 radeon_ring_write(ring
, upper_32_bits(src_offset
) & 0xff);
6396 src_offset
+= cur_size_in_bytes
;
6397 dst_offset
+= cur_size_in_bytes
;
6400 r
= radeon_fence_emit(rdev
, fence
, ring
->idx
);
6402 radeon_ring_unlock_undo(rdev
, ring
);
6406 radeon_ring_unlock_commit(rdev
, ring
);
6407 radeon_semaphore_free(rdev
, &sem
, *fence
);
6413 * startup/shutdown callbacks
6415 static int si_startup(struct radeon_device
*rdev
)
6417 struct radeon_ring
*ring
;
6420 /* enable pcie gen2/3 link */
6421 si_pcie_gen3_enable(rdev
);
6423 si_program_aspm(rdev
);
6425 si_mc_program(rdev
);
6427 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
6428 !rdev
->rlc_fw
|| !rdev
->mc_fw
) {
6429 r
= si_init_microcode(rdev
);
6431 DRM_ERROR("Failed to load firmware!\n");
6436 r
= si_mc_load_microcode(rdev
);
6438 DRM_ERROR("Failed to load MC firmware!\n");
6442 r
= r600_vram_scratch_init(rdev
);
6446 r
= si_pcie_gart_enable(rdev
);
6451 /* allocate rlc buffers */
6452 r
= si_rlc_init(rdev
);
6454 DRM_ERROR("Failed to init rlc BOs!\n");
6458 /* allocate wb buffer */
6459 r
= radeon_wb_init(rdev
);
6463 r
= radeon_fence_driver_start_ring(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
6465 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
6469 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
6471 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
6475 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
6477 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
6481 r
= radeon_fence_driver_start_ring(rdev
, R600_RING_TYPE_DMA_INDEX
);
6483 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
6487 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
6489 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
6493 if (rdev
->has_uvd
) {
6494 r
= rv770_uvd_resume(rdev
);
6496 r
= radeon_fence_driver_start_ring(rdev
,
6497 R600_RING_TYPE_UVD_INDEX
);
6499 dev_err(rdev
->dev
, "UVD fences init error (%d).\n", r
);
6502 rdev
->ring
[R600_RING_TYPE_UVD_INDEX
].ring_size
= 0;
6506 if (!rdev
->irq
.installed
) {
6507 r
= radeon_irq_kms_init(rdev
);
6512 r
= si_irq_init(rdev
);
6514 DRM_ERROR("radeon: IH init failed (%d).\n", r
);
6515 radeon_irq_kms_fini(rdev
);
6520 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
6521 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP_RPTR_OFFSET
,
6522 CP_RB0_RPTR
, CP_RB0_WPTR
,
6523 0, 0xfffff, RADEON_CP_PACKET2
);
6527 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
6528 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP1_RPTR_OFFSET
,
6529 CP_RB1_RPTR
, CP_RB1_WPTR
,
6530 0, 0xfffff, RADEON_CP_PACKET2
);
6534 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
6535 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP2_RPTR_OFFSET
,
6536 CP_RB2_RPTR
, CP_RB2_WPTR
,
6537 0, 0xfffff, RADEON_CP_PACKET2
);
6541 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
6542 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, R600_WB_DMA_RPTR_OFFSET
,
6543 DMA_RB_RPTR
+ DMA0_REGISTER_OFFSET
,
6544 DMA_RB_WPTR
+ DMA0_REGISTER_OFFSET
,
6545 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0));
6549 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
6550 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, CAYMAN_WB_DMA1_RPTR_OFFSET
,
6551 DMA_RB_RPTR
+ DMA1_REGISTER_OFFSET
,
6552 DMA_RB_WPTR
+ DMA1_REGISTER_OFFSET
,
6553 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0));
6557 r
= si_cp_load_microcode(rdev
);
6560 r
= si_cp_resume(rdev
);
6564 r
= cayman_dma_resume(rdev
);
6568 if (rdev
->has_uvd
) {
6569 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
6570 if (ring
->ring_size
) {
6571 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
,
6572 R600_WB_UVD_RPTR_OFFSET
,
6573 UVD_RBC_RB_RPTR
, UVD_RBC_RB_WPTR
,
6574 0, 0xfffff, RADEON_CP_PACKET2
);
6576 r
= r600_uvd_init(rdev
);
6578 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r
);
6582 r
= radeon_ib_pool_init(rdev
);
6584 dev_err(rdev
->dev
, "IB initialization failed (%d).\n", r
);
6588 r
= radeon_vm_manager_init(rdev
);
6590 dev_err(rdev
->dev
, "vm manager initialization failed (%d).\n", r
);
6597 int si_resume(struct radeon_device
*rdev
)
6601 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6602 * posting will perform necessary task to bring back GPU into good
6606 atom_asic_init(rdev
->mode_info
.atom_context
);
6608 /* init golden registers */
6609 si_init_golden_registers(rdev
);
6611 rdev
->accel_working
= true;
6612 r
= si_startup(rdev
);
6614 DRM_ERROR("si startup failed on resume\n");
6615 rdev
->accel_working
= false;
6623 int si_suspend(struct radeon_device
*rdev
)
6625 radeon_vm_manager_fini(rdev
);
6626 si_cp_enable(rdev
, false);
6627 cayman_dma_stop(rdev
);
6628 if (rdev
->has_uvd
) {
6629 r600_uvd_stop(rdev
);
6630 radeon_uvd_suspend(rdev
);
6632 si_irq_suspend(rdev
);
6633 radeon_wb_disable(rdev
);
6634 si_pcie_gart_disable(rdev
);
6638 /* Plan is to move initialization in that function and use
6639 * helper function so that radeon_device_init pretty much
6640 * do nothing more than calling asic specific function. This
6641 * should also allow to remove a bunch of callback function
6644 int si_init(struct radeon_device
*rdev
)
6646 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
6650 if (!radeon_get_bios(rdev
)) {
6651 if (ASIC_IS_AVIVO(rdev
))
6654 /* Must be an ATOMBIOS */
6655 if (!rdev
->is_atom_bios
) {
6656 dev_err(rdev
->dev
, "Expecting atombios for cayman GPU\n");
6659 r
= radeon_atombios_init(rdev
);
6663 /* Post card if necessary */
6664 if (!radeon_card_posted(rdev
)) {
6666 dev_err(rdev
->dev
, "Card not posted and no BIOS - ignoring\n");
6669 DRM_INFO("GPU not posted. posting now...\n");
6670 atom_asic_init(rdev
->mode_info
.atom_context
);
6672 /* init golden registers */
6673 si_init_golden_registers(rdev
);
6674 /* Initialize scratch registers */
6675 si_scratch_init(rdev
);
6676 /* Initialize surface registers */
6677 radeon_surface_init(rdev
);
6678 /* Initialize clocks */
6679 radeon_get_clock_info(rdev
->ddev
);
6682 r
= radeon_fence_driver_init(rdev
);
6686 /* initialize memory controller */
6687 r
= si_mc_init(rdev
);
6690 /* Memory manager */
6691 r
= radeon_bo_init(rdev
);
6695 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
6696 ring
->ring_obj
= NULL
;
6697 r600_ring_init(rdev
, ring
, 1024 * 1024);
6699 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
6700 ring
->ring_obj
= NULL
;
6701 r600_ring_init(rdev
, ring
, 1024 * 1024);
6703 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
6704 ring
->ring_obj
= NULL
;
6705 r600_ring_init(rdev
, ring
, 1024 * 1024);
6707 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
6708 ring
->ring_obj
= NULL
;
6709 r600_ring_init(rdev
, ring
, 64 * 1024);
6711 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
6712 ring
->ring_obj
= NULL
;
6713 r600_ring_init(rdev
, ring
, 64 * 1024);
6715 if (rdev
->has_uvd
) {
6716 r
= radeon_uvd_init(rdev
);
6718 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
6719 ring
->ring_obj
= NULL
;
6720 r600_ring_init(rdev
, ring
, 4096);
6724 rdev
->ih
.ring_obj
= NULL
;
6725 r600_ih_ring_init(rdev
, 64 * 1024);
6727 r
= r600_pcie_gart_init(rdev
);
6731 rdev
->accel_working
= true;
6732 r
= si_startup(rdev
);
6734 dev_err(rdev
->dev
, "disabling GPU acceleration\n");
6736 cayman_dma_fini(rdev
);
6739 radeon_wb_fini(rdev
);
6740 radeon_ib_pool_fini(rdev
);
6741 radeon_vm_manager_fini(rdev
);
6742 radeon_irq_kms_fini(rdev
);
6743 si_pcie_gart_fini(rdev
);
6744 rdev
->accel_working
= false;
6747 /* Don't start up if the MC ucode is missing.
6748 * The default clocks and voltages before the MC ucode
6749 * is loaded are not suffient for advanced operations.
6752 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6759 void si_fini(struct radeon_device
*rdev
)
6762 cayman_dma_fini(rdev
);
6767 radeon_wb_fini(rdev
);
6768 radeon_vm_manager_fini(rdev
);
6769 radeon_ib_pool_fini(rdev
);
6770 radeon_irq_kms_fini(rdev
);
6771 if (rdev
->has_uvd
) {
6772 r600_uvd_stop(rdev
);
6773 radeon_uvd_fini(rdev
);
6775 si_pcie_gart_fini(rdev
);
6776 r600_vram_scratch_fini(rdev
);
6777 radeon_gem_fini(rdev
);
6778 radeon_fence_driver_fini(rdev
);
6779 radeon_bo_fini(rdev
);
6780 radeon_atombios_fini(rdev
);
6786 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6788 * @rdev: radeon_device pointer
6790 * Fetches a GPU clock counter snapshot (SI).
6791 * Returns the 64 bit clock counter snapshot.
6793 uint64_t si_get_gpu_clock_counter(struct radeon_device
*rdev
)
6797 mutex_lock(&rdev
->gpu_clock_mutex
);
6798 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT
, 1);
6799 clock
= (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB
) |
6800 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB
) << 32ULL);
6801 mutex_unlock(&rdev
->gpu_clock_mutex
);
6805 int si_set_uvd_clocks(struct radeon_device
*rdev
, u32 vclk
, u32 dclk
)
6807 unsigned fb_div
= 0, vclk_div
= 0, dclk_div
= 0;
6810 /* bypass vclk and dclk with bclk */
6811 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
6812 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6813 ~(VCLK_SRC_SEL_MASK
| DCLK_SRC_SEL_MASK
));
6815 /* put PLL in bypass mode */
6816 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_BYPASS_EN_MASK
, ~UPLL_BYPASS_EN_MASK
);
6818 if (!vclk
|| !dclk
) {
6819 /* keep the Bypass mode, put PLL to sleep */
6820 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_SLEEP_MASK
, ~UPLL_SLEEP_MASK
);
6824 r
= radeon_uvd_calc_upll_dividers(rdev
, vclk
, dclk
, 125000, 250000,
6825 16384, 0x03FFFFFF, 0, 128, 5,
6826 &fb_div
, &vclk_div
, &dclk_div
);
6830 /* set RESET_ANTI_MUX to 0 */
6831 WREG32_P(CG_UPLL_FUNC_CNTL_5
, 0, ~RESET_ANTI_MUX_MASK
);
6833 /* set VCO_MODE to 1 */
6834 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_VCO_MODE_MASK
, ~UPLL_VCO_MODE_MASK
);
6836 /* toggle UPLL_SLEEP to 1 then back to 0 */
6837 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_SLEEP_MASK
, ~UPLL_SLEEP_MASK
);
6838 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_SLEEP_MASK
);
6840 /* deassert UPLL_RESET */
6841 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_RESET_MASK
);
6845 r
= radeon_uvd_send_upll_ctlreq(rdev
, CG_UPLL_FUNC_CNTL
);
6849 /* assert UPLL_RESET again */
6850 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_RESET_MASK
, ~UPLL_RESET_MASK
);
6852 /* disable spread spectrum. */
6853 WREG32_P(CG_UPLL_SPREAD_SPECTRUM
, 0, ~SSEN_MASK
);
6855 /* set feedback divider */
6856 WREG32_P(CG_UPLL_FUNC_CNTL_3
, UPLL_FB_DIV(fb_div
), ~UPLL_FB_DIV_MASK
);
6858 /* set ref divider to 0 */
6859 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_REF_DIV_MASK
);
6861 if (fb_div
< 307200)
6862 WREG32_P(CG_UPLL_FUNC_CNTL_4
, 0, ~UPLL_SPARE_ISPARE9
);
6864 WREG32_P(CG_UPLL_FUNC_CNTL_4
, UPLL_SPARE_ISPARE9
, ~UPLL_SPARE_ISPARE9
);
6866 /* set PDIV_A and PDIV_B */
6867 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
6868 UPLL_PDIV_A(vclk_div
) | UPLL_PDIV_B(dclk_div
),
6869 ~(UPLL_PDIV_A_MASK
| UPLL_PDIV_B_MASK
));
6871 /* give the PLL some time to settle */
6874 /* deassert PLL_RESET */
6875 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_RESET_MASK
);
6879 /* switch from bypass mode to normal mode */
6880 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_BYPASS_EN_MASK
);
6882 r
= radeon_uvd_send_upll_ctlreq(rdev
, CG_UPLL_FUNC_CNTL
);
6886 /* switch VCLK and DCLK selection */
6887 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
6888 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6889 ~(VCLK_SRC_SEL_MASK
| DCLK_SRC_SEL_MASK
));
6896 static void si_pcie_gen3_enable(struct radeon_device
*rdev
)
6898 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
6899 int bridge_pos
, gpu_pos
;
6900 u32 speed_cntl
, mask
, current_data_rate
;
6904 if (radeon_pcie_gen2
== 0)
6907 if (rdev
->flags
& RADEON_IS_IGP
)
6910 if (!(rdev
->flags
& RADEON_IS_PCIE
))
6913 ret
= drm_pcie_get_speed_cap_mask(rdev
->ddev
, &mask
);
6917 if (!(mask
& (DRM_PCIE_SPEED_50
| DRM_PCIE_SPEED_80
)))
6920 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
6921 current_data_rate
= (speed_cntl
& LC_CURRENT_DATA_RATE_MASK
) >>
6922 LC_CURRENT_DATA_RATE_SHIFT
;
6923 if (mask
& DRM_PCIE_SPEED_80
) {
6924 if (current_data_rate
== 2) {
6925 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6928 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6929 } else if (mask
& DRM_PCIE_SPEED_50
) {
6930 if (current_data_rate
== 1) {
6931 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6934 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6937 bridge_pos
= pci_pcie_cap(root
);
6941 gpu_pos
= pci_pcie_cap(rdev
->pdev
);
6945 if (mask
& DRM_PCIE_SPEED_80
) {
6946 /* re-try equalization if gen3 is not already enabled */
6947 if (current_data_rate
!= 2) {
6948 u16 bridge_cfg
, gpu_cfg
;
6949 u16 bridge_cfg2
, gpu_cfg2
;
6950 u32 max_lw
, current_lw
, tmp
;
6952 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
6953 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
6955 tmp16
= bridge_cfg
| PCI_EXP_LNKCTL_HAWD
;
6956 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
6958 tmp16
= gpu_cfg
| PCI_EXP_LNKCTL_HAWD
;
6959 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
6961 tmp
= RREG32_PCIE(PCIE_LC_STATUS1
);
6962 max_lw
= (tmp
& LC_DETECTED_LINK_WIDTH_MASK
) >> LC_DETECTED_LINK_WIDTH_SHIFT
;
6963 current_lw
= (tmp
& LC_OPERATING_LINK_WIDTH_MASK
) >> LC_OPERATING_LINK_WIDTH_SHIFT
;
6965 if (current_lw
< max_lw
) {
6966 tmp
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
6967 if (tmp
& LC_RENEGOTIATION_SUPPORT
) {
6968 tmp
&= ~(LC_LINK_WIDTH_MASK
| LC_UPCONFIGURE_DIS
);
6969 tmp
|= (max_lw
<< LC_LINK_WIDTH_SHIFT
);
6970 tmp
|= LC_UPCONFIGURE_SUPPORT
| LC_RENEGOTIATE_EN
| LC_RECONFIG_NOW
;
6971 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, tmp
);
6975 for (i
= 0; i
< 10; i
++) {
6977 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_DEVSTA
, &tmp16
);
6978 if (tmp16
& PCI_EXP_DEVSTA_TRPND
)
6981 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
6982 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
6984 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &bridge_cfg2
);
6985 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &gpu_cfg2
);
6987 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
6988 tmp
|= LC_SET_QUIESCE
;
6989 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
6991 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
6993 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
6998 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
6999 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
7000 tmp16
|= (bridge_cfg
& PCI_EXP_LNKCTL_HAWD
);
7001 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
7003 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
7004 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
7005 tmp16
|= (gpu_cfg
& PCI_EXP_LNKCTL_HAWD
);
7006 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
7009 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
7010 tmp16
&= ~((1 << 4) | (7 << 9));
7011 tmp16
|= (bridge_cfg2
& ((1 << 4) | (7 << 9)));
7012 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
7014 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
7015 tmp16
&= ~((1 << 4) | (7 << 9));
7016 tmp16
|= (gpu_cfg2
& ((1 << 4) | (7 << 9)));
7017 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
7019 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
7020 tmp
&= ~LC_SET_QUIESCE
;
7021 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
7026 /* set the link speed */
7027 speed_cntl
|= LC_FORCE_EN_SW_SPEED_CHANGE
| LC_FORCE_DIS_HW_SPEED_CHANGE
;
7028 speed_cntl
&= ~LC_FORCE_DIS_SW_SPEED_CHANGE
;
7029 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
7031 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
7033 if (mask
& DRM_PCIE_SPEED_80
)
7034 tmp16
|= 3; /* gen3 */
7035 else if (mask
& DRM_PCIE_SPEED_50
)
7036 tmp16
|= 2; /* gen2 */
7038 tmp16
|= 1; /* gen1 */
7039 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
7041 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
7042 speed_cntl
|= LC_INITIATE_LINK_SPEED_CHANGE
;
7043 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
7045 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
7046 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
7047 if ((speed_cntl
& LC_INITIATE_LINK_SPEED_CHANGE
) == 0)
7053 static void si_program_aspm(struct radeon_device
*rdev
)
7056 bool disable_l0s
= false, disable_l1
= false, disable_plloff_in_l1
= false;
7057 bool disable_clkreq
= false;
7059 if (radeon_aspm
== 0)
7062 if (!(rdev
->flags
& RADEON_IS_PCIE
))
7065 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
7066 data
&= ~LC_XMIT_N_FTS_MASK
;
7067 data
|= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN
;
7069 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
, data
);
7071 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL3
);
7072 data
|= LC_GO_TO_RECOVERY
;
7074 WREG32_PCIE_PORT(PCIE_LC_CNTL3
, data
);
7076 orig
= data
= RREG32_PCIE(PCIE_P_CNTL
);
7077 data
|= P_IGNORE_EDB_ERR
;
7079 WREG32_PCIE(PCIE_P_CNTL
, data
);
7081 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
7082 data
&= ~(LC_L0S_INACTIVITY_MASK
| LC_L1_INACTIVITY_MASK
);
7083 data
|= LC_PMI_TO_L1_DIS
;
7085 data
|= LC_L0S_INACTIVITY(7);
7088 data
|= LC_L1_INACTIVITY(7);
7089 data
&= ~LC_PMI_TO_L1_DIS
;
7091 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
7093 if (!disable_plloff_in_l1
) {
7094 bool clk_req_support
;
7096 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0
);
7097 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
7098 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7100 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0
, data
);
7102 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1
);
7103 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
7104 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7106 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1
, data
);
7108 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0
);
7109 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
7110 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7112 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0
, data
);
7114 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1
);
7115 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
7116 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7118 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1
, data
);
7120 if ((rdev
->family
!= CHIP_OLAND
) && (rdev
->family
!= CHIP_HAINAN
)) {
7121 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0
);
7122 data
&= ~PLL_RAMP_UP_TIME_0_MASK
;
7124 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0
, data
);
7126 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1
);
7127 data
&= ~PLL_RAMP_UP_TIME_1_MASK
;
7129 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1
, data
);
7131 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2
);
7132 data
&= ~PLL_RAMP_UP_TIME_2_MASK
;
7134 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2
, data
);
7136 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3
);
7137 data
&= ~PLL_RAMP_UP_TIME_3_MASK
;
7139 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3
, data
);
7141 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0
);
7142 data
&= ~PLL_RAMP_UP_TIME_0_MASK
;
7144 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0
, data
);
7146 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1
);
7147 data
&= ~PLL_RAMP_UP_TIME_1_MASK
;
7149 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1
, data
);
7151 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2
);
7152 data
&= ~PLL_RAMP_UP_TIME_2_MASK
;
7154 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2
, data
);
7156 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3
);
7157 data
&= ~PLL_RAMP_UP_TIME_3_MASK
;
7159 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3
, data
);
7161 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
7162 data
&= ~LC_DYN_LANES_PWR_STATE_MASK
;
7163 data
|= LC_DYN_LANES_PWR_STATE(3);
7165 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, data
);
7167 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_CNTL
);
7168 data
&= ~LS2_EXIT_TIME_MASK
;
7169 if ((rdev
->family
== CHIP_OLAND
) || (rdev
->family
== CHIP_HAINAN
))
7170 data
|= LS2_EXIT_TIME(5);
7172 WREG32_PIF_PHY0(PB0_PIF_CNTL
, data
);
7174 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_CNTL
);
7175 data
&= ~LS2_EXIT_TIME_MASK
;
7176 if ((rdev
->family
== CHIP_OLAND
) || (rdev
->family
== CHIP_HAINAN
))
7177 data
|= LS2_EXIT_TIME(5);
7179 WREG32_PIF_PHY1(PB1_PIF_CNTL
, data
);
7181 if (!disable_clkreq
) {
7182 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
7185 clk_req_support
= false;
7186 pcie_capability_read_dword(root
, PCI_EXP_LNKCAP
, &lnkcap
);
7187 if (lnkcap
& PCI_EXP_LNKCAP_CLKPM
)
7188 clk_req_support
= true;
7190 clk_req_support
= false;
7193 if (clk_req_support
) {
7194 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL2
);
7195 data
|= LC_ALLOW_PDWN_IN_L1
| LC_ALLOW_PDWN_IN_L23
;
7197 WREG32_PCIE_PORT(PCIE_LC_CNTL2
, data
);
7199 orig
= data
= RREG32(THM_CLK_CNTL
);
7200 data
&= ~(CMON_CLK_SEL_MASK
| TMON_CLK_SEL_MASK
);
7201 data
|= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7203 WREG32(THM_CLK_CNTL
, data
);
7205 orig
= data
= RREG32(MISC_CLK_CNTL
);
7206 data
&= ~(DEEP_SLEEP_CLK_SEL_MASK
| ZCLK_SEL_MASK
);
7207 data
|= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7209 WREG32(MISC_CLK_CNTL
, data
);
7211 orig
= data
= RREG32(CG_CLKPIN_CNTL
);
7212 data
&= ~BCLK_AS_XCLK
;
7214 WREG32(CG_CLKPIN_CNTL
, data
);
7216 orig
= data
= RREG32(CG_CLKPIN_CNTL_2
);
7217 data
&= ~FORCE_BIF_REFCLK_EN
;
7219 WREG32(CG_CLKPIN_CNTL_2
, data
);
7221 orig
= data
= RREG32(MPLL_BYPASSCLK_SEL
);
7222 data
&= ~MPLL_CLKOUT_SEL_MASK
;
7223 data
|= MPLL_CLKOUT_SEL(4);
7225 WREG32(MPLL_BYPASSCLK_SEL
, data
);
7227 orig
= data
= RREG32(SPLL_CNTL_MODE
);
7228 data
&= ~SPLL_REFCLK_SEL_MASK
;
7230 WREG32(SPLL_CNTL_MODE
, data
);
7235 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
7238 orig
= data
= RREG32_PCIE(PCIE_CNTL2
);
7239 data
|= SLV_MEM_LS_EN
| MST_MEM_LS_EN
| REPLAY_MEM_LS_EN
;
7241 WREG32_PCIE(PCIE_CNTL2
, data
);
7244 data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
7245 if((data
& LC_N_FTS_MASK
) == LC_N_FTS_MASK
) {
7246 data
= RREG32_PCIE(PCIE_LC_STATUS1
);
7247 if ((data
& LC_REVERSE_XMIT
) && (data
& LC_REVERSE_RCVR
)) {
7248 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
7249 data
&= ~LC_L0S_INACTIVITY_MASK
;
7251 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);