2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
33 #include "si_blit_shaders.h"
34 #include "clearstate_si.h"
35 #include "radeon_ucode.h"
38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51 MODULE_FIRMWARE("radeon/VERDE_me.bin");
52 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
55 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57 MODULE_FIRMWARE("radeon/OLAND_me.bin");
58 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
61 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
69 static void si_pcie_gen3_enable(struct radeon_device
*rdev
);
70 static void si_program_aspm(struct radeon_device
*rdev
);
71 extern void sumo_rlc_fini(struct radeon_device
*rdev
);
72 extern int sumo_rlc_init(struct radeon_device
*rdev
);
73 extern int r600_ih_ring_alloc(struct radeon_device
*rdev
);
74 extern void r600_ih_ring_fini(struct radeon_device
*rdev
);
75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device
*rdev
);
76 extern void evergreen_mc_stop(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
77 extern void evergreen_mc_resume(struct radeon_device
*rdev
, struct evergreen_mc_save
*save
);
78 extern u32
evergreen_get_number_of_dram_channels(struct radeon_device
*rdev
);
79 extern void evergreen_print_gpu_status_regs(struct radeon_device
*rdev
);
80 extern bool evergreen_is_display_hung(struct radeon_device
*rdev
);
82 static const u32 verde_rlc_save_restore_register_list
[] =
84 (0x8000 << 16) | (0x98f4 >> 2),
86 (0x8040 << 16) | (0x98f4 >> 2),
88 (0x8000 << 16) | (0xe80 >> 2),
90 (0x8040 << 16) | (0xe80 >> 2),
92 (0x8000 << 16) | (0x89bc >> 2),
94 (0x8040 << 16) | (0x89bc >> 2),
96 (0x8000 << 16) | (0x8c1c >> 2),
98 (0x8040 << 16) | (0x8c1c >> 2),
100 (0x9c00 << 16) | (0x98f0 >> 2),
102 (0x9c00 << 16) | (0xe7c >> 2),
104 (0x8000 << 16) | (0x9148 >> 2),
106 (0x8040 << 16) | (0x9148 >> 2),
108 (0x9c00 << 16) | (0x9150 >> 2),
110 (0x9c00 << 16) | (0x897c >> 2),
112 (0x9c00 << 16) | (0x8d8c >> 2),
114 (0x9c00 << 16) | (0xac54 >> 2),
117 (0x9c00 << 16) | (0x98f8 >> 2),
119 (0x9c00 << 16) | (0x9910 >> 2),
121 (0x9c00 << 16) | (0x9914 >> 2),
123 (0x9c00 << 16) | (0x9918 >> 2),
125 (0x9c00 << 16) | (0x991c >> 2),
127 (0x9c00 << 16) | (0x9920 >> 2),
129 (0x9c00 << 16) | (0x9924 >> 2),
131 (0x9c00 << 16) | (0x9928 >> 2),
133 (0x9c00 << 16) | (0x992c >> 2),
135 (0x9c00 << 16) | (0x9930 >> 2),
137 (0x9c00 << 16) | (0x9934 >> 2),
139 (0x9c00 << 16) | (0x9938 >> 2),
141 (0x9c00 << 16) | (0x993c >> 2),
143 (0x9c00 << 16) | (0x9940 >> 2),
145 (0x9c00 << 16) | (0x9944 >> 2),
147 (0x9c00 << 16) | (0x9948 >> 2),
149 (0x9c00 << 16) | (0x994c >> 2),
151 (0x9c00 << 16) | (0x9950 >> 2),
153 (0x9c00 << 16) | (0x9954 >> 2),
155 (0x9c00 << 16) | (0x9958 >> 2),
157 (0x9c00 << 16) | (0x995c >> 2),
159 (0x9c00 << 16) | (0x9960 >> 2),
161 (0x9c00 << 16) | (0x9964 >> 2),
163 (0x9c00 << 16) | (0x9968 >> 2),
165 (0x9c00 << 16) | (0x996c >> 2),
167 (0x9c00 << 16) | (0x9970 >> 2),
169 (0x9c00 << 16) | (0x9974 >> 2),
171 (0x9c00 << 16) | (0x9978 >> 2),
173 (0x9c00 << 16) | (0x997c >> 2),
175 (0x9c00 << 16) | (0x9980 >> 2),
177 (0x9c00 << 16) | (0x9984 >> 2),
179 (0x9c00 << 16) | (0x9988 >> 2),
181 (0x9c00 << 16) | (0x998c >> 2),
183 (0x9c00 << 16) | (0x8c00 >> 2),
185 (0x9c00 << 16) | (0x8c14 >> 2),
187 (0x9c00 << 16) | (0x8c04 >> 2),
189 (0x9c00 << 16) | (0x8c08 >> 2),
191 (0x8000 << 16) | (0x9b7c >> 2),
193 (0x8040 << 16) | (0x9b7c >> 2),
195 (0x8000 << 16) | (0xe84 >> 2),
197 (0x8040 << 16) | (0xe84 >> 2),
199 (0x8000 << 16) | (0x89c0 >> 2),
201 (0x8040 << 16) | (0x89c0 >> 2),
203 (0x8000 << 16) | (0x914c >> 2),
205 (0x8040 << 16) | (0x914c >> 2),
207 (0x8000 << 16) | (0x8c20 >> 2),
209 (0x8040 << 16) | (0x8c20 >> 2),
211 (0x8000 << 16) | (0x9354 >> 2),
213 (0x8040 << 16) | (0x9354 >> 2),
215 (0x9c00 << 16) | (0x9060 >> 2),
217 (0x9c00 << 16) | (0x9364 >> 2),
219 (0x9c00 << 16) | (0x9100 >> 2),
221 (0x9c00 << 16) | (0x913c >> 2),
223 (0x8000 << 16) | (0x90e0 >> 2),
225 (0x8000 << 16) | (0x90e4 >> 2),
227 (0x8000 << 16) | (0x90e8 >> 2),
229 (0x8040 << 16) | (0x90e0 >> 2),
231 (0x8040 << 16) | (0x90e4 >> 2),
233 (0x8040 << 16) | (0x90e8 >> 2),
235 (0x9c00 << 16) | (0x8bcc >> 2),
237 (0x9c00 << 16) | (0x8b24 >> 2),
239 (0x9c00 << 16) | (0x88c4 >> 2),
241 (0x9c00 << 16) | (0x8e50 >> 2),
243 (0x9c00 << 16) | (0x8c0c >> 2),
245 (0x9c00 << 16) | (0x8e58 >> 2),
247 (0x9c00 << 16) | (0x8e5c >> 2),
249 (0x9c00 << 16) | (0x9508 >> 2),
251 (0x9c00 << 16) | (0x950c >> 2),
253 (0x9c00 << 16) | (0x9494 >> 2),
255 (0x9c00 << 16) | (0xac0c >> 2),
257 (0x9c00 << 16) | (0xac10 >> 2),
259 (0x9c00 << 16) | (0xac14 >> 2),
261 (0x9c00 << 16) | (0xae00 >> 2),
263 (0x9c00 << 16) | (0xac08 >> 2),
265 (0x9c00 << 16) | (0x88d4 >> 2),
267 (0x9c00 << 16) | (0x88c8 >> 2),
269 (0x9c00 << 16) | (0x88cc >> 2),
271 (0x9c00 << 16) | (0x89b0 >> 2),
273 (0x9c00 << 16) | (0x8b10 >> 2),
275 (0x9c00 << 16) | (0x8a14 >> 2),
277 (0x9c00 << 16) | (0x9830 >> 2),
279 (0x9c00 << 16) | (0x9834 >> 2),
281 (0x9c00 << 16) | (0x9838 >> 2),
283 (0x9c00 << 16) | (0x9a10 >> 2),
285 (0x8000 << 16) | (0x9870 >> 2),
287 (0x8000 << 16) | (0x9874 >> 2),
289 (0x8001 << 16) | (0x9870 >> 2),
291 (0x8001 << 16) | (0x9874 >> 2),
293 (0x8040 << 16) | (0x9870 >> 2),
295 (0x8040 << 16) | (0x9874 >> 2),
297 (0x8041 << 16) | (0x9870 >> 2),
299 (0x8041 << 16) | (0x9874 >> 2),
304 static const u32 tahiti_golden_rlc_registers
[] =
306 0xc424, 0xffffffff, 0x00601005,
307 0xc47c, 0xffffffff, 0x10104040,
308 0xc488, 0xffffffff, 0x0100000a,
309 0xc314, 0xffffffff, 0x00000800,
310 0xc30c, 0xffffffff, 0x800000f4,
311 0xf4a8, 0xffffffff, 0x00000000
314 static const u32 tahiti_golden_registers
[] =
316 0x9a10, 0x00010000, 0x00018208,
317 0x9830, 0xffffffff, 0x00000000,
318 0x9834, 0xf00fffff, 0x00000400,
319 0x9838, 0x0002021c, 0x00020200,
320 0xc78, 0x00000080, 0x00000000,
321 0xd030, 0x000300c0, 0x00800040,
322 0xd830, 0x000300c0, 0x00800040,
323 0x5bb0, 0x000000f0, 0x00000070,
324 0x5bc0, 0x00200000, 0x50100000,
325 0x7030, 0x31000311, 0x00000011,
326 0x277c, 0x00000003, 0x000007ff,
327 0x240c, 0x000007ff, 0x00000000,
328 0x8a14, 0xf000001f, 0x00000007,
329 0x8b24, 0xffffffff, 0x00ffffff,
330 0x8b10, 0x0000ff0f, 0x00000000,
331 0x28a4c, 0x07ffffff, 0x4e000000,
332 0x28350, 0x3f3f3fff, 0x2a00126a,
333 0x30, 0x000000ff, 0x0040,
334 0x34, 0x00000040, 0x00004040,
335 0x9100, 0x07ffffff, 0x03000000,
336 0x8e88, 0x01ff1f3f, 0x00000000,
337 0x8e84, 0x01ff1f3f, 0x00000000,
338 0x9060, 0x0000007f, 0x00000020,
339 0x9508, 0x00010000, 0x00010000,
340 0xac14, 0x00000200, 0x000002fb,
341 0xac10, 0xffffffff, 0x0000543b,
342 0xac0c, 0xffffffff, 0xa9210876,
343 0x88d0, 0xffffffff, 0x000fff40,
344 0x88d4, 0x0000001f, 0x00000010,
345 0x1410, 0x20000000, 0x20fffed8,
346 0x15c0, 0x000c0fc0, 0x000c0400
349 static const u32 tahiti_golden_registers2
[] =
351 0xc64, 0x00000001, 0x00000001
354 static const u32 pitcairn_golden_rlc_registers
[] =
356 0xc424, 0xffffffff, 0x00601004,
357 0xc47c, 0xffffffff, 0x10102020,
358 0xc488, 0xffffffff, 0x01000020,
359 0xc314, 0xffffffff, 0x00000800,
360 0xc30c, 0xffffffff, 0x800000a4
363 static const u32 pitcairn_golden_registers
[] =
365 0x9a10, 0x00010000, 0x00018208,
366 0x9830, 0xffffffff, 0x00000000,
367 0x9834, 0xf00fffff, 0x00000400,
368 0x9838, 0x0002021c, 0x00020200,
369 0xc78, 0x00000080, 0x00000000,
370 0xd030, 0x000300c0, 0x00800040,
371 0xd830, 0x000300c0, 0x00800040,
372 0x5bb0, 0x000000f0, 0x00000070,
373 0x5bc0, 0x00200000, 0x50100000,
374 0x7030, 0x31000311, 0x00000011,
375 0x2ae4, 0x00073ffe, 0x000022a2,
376 0x240c, 0x000007ff, 0x00000000,
377 0x8a14, 0xf000001f, 0x00000007,
378 0x8b24, 0xffffffff, 0x00ffffff,
379 0x8b10, 0x0000ff0f, 0x00000000,
380 0x28a4c, 0x07ffffff, 0x4e000000,
381 0x28350, 0x3f3f3fff, 0x2a00126a,
382 0x30, 0x000000ff, 0x0040,
383 0x34, 0x00000040, 0x00004040,
384 0x9100, 0x07ffffff, 0x03000000,
385 0x9060, 0x0000007f, 0x00000020,
386 0x9508, 0x00010000, 0x00010000,
387 0xac14, 0x000003ff, 0x000000f7,
388 0xac10, 0xffffffff, 0x00000000,
389 0xac0c, 0xffffffff, 0x32761054,
390 0x88d4, 0x0000001f, 0x00000010,
391 0x15c0, 0x000c0fc0, 0x000c0400
394 static const u32 verde_golden_rlc_registers
[] =
396 0xc424, 0xffffffff, 0x033f1005,
397 0xc47c, 0xffffffff, 0x10808020,
398 0xc488, 0xffffffff, 0x00800008,
399 0xc314, 0xffffffff, 0x00001000,
400 0xc30c, 0xffffffff, 0x80010014
403 static const u32 verde_golden_registers
[] =
405 0x9a10, 0x00010000, 0x00018208,
406 0x9830, 0xffffffff, 0x00000000,
407 0x9834, 0xf00fffff, 0x00000400,
408 0x9838, 0x0002021c, 0x00020200,
409 0xc78, 0x00000080, 0x00000000,
410 0xd030, 0x000300c0, 0x00800040,
411 0xd030, 0x000300c0, 0x00800040,
412 0xd830, 0x000300c0, 0x00800040,
413 0xd830, 0x000300c0, 0x00800040,
414 0x5bb0, 0x000000f0, 0x00000070,
415 0x5bc0, 0x00200000, 0x50100000,
416 0x7030, 0x31000311, 0x00000011,
417 0x2ae4, 0x00073ffe, 0x000022a2,
418 0x2ae4, 0x00073ffe, 0x000022a2,
419 0x2ae4, 0x00073ffe, 0x000022a2,
420 0x240c, 0x000007ff, 0x00000000,
421 0x240c, 0x000007ff, 0x00000000,
422 0x240c, 0x000007ff, 0x00000000,
423 0x8a14, 0xf000001f, 0x00000007,
424 0x8a14, 0xf000001f, 0x00000007,
425 0x8a14, 0xf000001f, 0x00000007,
426 0x8b24, 0xffffffff, 0x00ffffff,
427 0x8b10, 0x0000ff0f, 0x00000000,
428 0x28a4c, 0x07ffffff, 0x4e000000,
429 0x28350, 0x3f3f3fff, 0x0000124a,
430 0x28350, 0x3f3f3fff, 0x0000124a,
431 0x28350, 0x3f3f3fff, 0x0000124a,
432 0x30, 0x000000ff, 0x0040,
433 0x34, 0x00000040, 0x00004040,
434 0x9100, 0x07ffffff, 0x03000000,
435 0x9100, 0x07ffffff, 0x03000000,
436 0x8e88, 0x01ff1f3f, 0x00000000,
437 0x8e88, 0x01ff1f3f, 0x00000000,
438 0x8e88, 0x01ff1f3f, 0x00000000,
439 0x8e84, 0x01ff1f3f, 0x00000000,
440 0x8e84, 0x01ff1f3f, 0x00000000,
441 0x8e84, 0x01ff1f3f, 0x00000000,
442 0x9060, 0x0000007f, 0x00000020,
443 0x9508, 0x00010000, 0x00010000,
444 0xac14, 0x000003ff, 0x00000003,
445 0xac14, 0x000003ff, 0x00000003,
446 0xac14, 0x000003ff, 0x00000003,
447 0xac10, 0xffffffff, 0x00000000,
448 0xac10, 0xffffffff, 0x00000000,
449 0xac10, 0xffffffff, 0x00000000,
450 0xac0c, 0xffffffff, 0x00001032,
451 0xac0c, 0xffffffff, 0x00001032,
452 0xac0c, 0xffffffff, 0x00001032,
453 0x88d4, 0x0000001f, 0x00000010,
454 0x88d4, 0x0000001f, 0x00000010,
455 0x88d4, 0x0000001f, 0x00000010,
456 0x15c0, 0x000c0fc0, 0x000c0400
459 static const u32 oland_golden_rlc_registers
[] =
461 0xc424, 0xffffffff, 0x00601005,
462 0xc47c, 0xffffffff, 0x10104040,
463 0xc488, 0xffffffff, 0x0100000a,
464 0xc314, 0xffffffff, 0x00000800,
465 0xc30c, 0xffffffff, 0x800000f4
468 static const u32 oland_golden_registers
[] =
470 0x9a10, 0x00010000, 0x00018208,
471 0x9830, 0xffffffff, 0x00000000,
472 0x9834, 0xf00fffff, 0x00000400,
473 0x9838, 0x0002021c, 0x00020200,
474 0xc78, 0x00000080, 0x00000000,
475 0xd030, 0x000300c0, 0x00800040,
476 0xd830, 0x000300c0, 0x00800040,
477 0x5bb0, 0x000000f0, 0x00000070,
478 0x5bc0, 0x00200000, 0x50100000,
479 0x7030, 0x31000311, 0x00000011,
480 0x2ae4, 0x00073ffe, 0x000022a2,
481 0x240c, 0x000007ff, 0x00000000,
482 0x8a14, 0xf000001f, 0x00000007,
483 0x8b24, 0xffffffff, 0x00ffffff,
484 0x8b10, 0x0000ff0f, 0x00000000,
485 0x28a4c, 0x07ffffff, 0x4e000000,
486 0x28350, 0x3f3f3fff, 0x00000082,
487 0x30, 0x000000ff, 0x0040,
488 0x34, 0x00000040, 0x00004040,
489 0x9100, 0x07ffffff, 0x03000000,
490 0x9060, 0x0000007f, 0x00000020,
491 0x9508, 0x00010000, 0x00010000,
492 0xac14, 0x000003ff, 0x000000f3,
493 0xac10, 0xffffffff, 0x00000000,
494 0xac0c, 0xffffffff, 0x00003210,
495 0x88d4, 0x0000001f, 0x00000010,
496 0x15c0, 0x000c0fc0, 0x000c0400
499 static const u32 hainan_golden_registers
[] =
501 0x9a10, 0x00010000, 0x00018208,
502 0x9830, 0xffffffff, 0x00000000,
503 0x9834, 0xf00fffff, 0x00000400,
504 0x9838, 0x0002021c, 0x00020200,
505 0xd0c0, 0xff000fff, 0x00000100,
506 0xd030, 0x000300c0, 0x00800040,
507 0xd8c0, 0xff000fff, 0x00000100,
508 0xd830, 0x000300c0, 0x00800040,
509 0x2ae4, 0x00073ffe, 0x000022a2,
510 0x240c, 0x000007ff, 0x00000000,
511 0x8a14, 0xf000001f, 0x00000007,
512 0x8b24, 0xffffffff, 0x00ffffff,
513 0x8b10, 0x0000ff0f, 0x00000000,
514 0x28a4c, 0x07ffffff, 0x4e000000,
515 0x28350, 0x3f3f3fff, 0x00000000,
516 0x30, 0x000000ff, 0x0040,
517 0x34, 0x00000040, 0x00004040,
518 0x9100, 0x03e00000, 0x03600000,
519 0x9060, 0x0000007f, 0x00000020,
520 0x9508, 0x00010000, 0x00010000,
521 0xac14, 0x000003ff, 0x000000f1,
522 0xac10, 0xffffffff, 0x00000000,
523 0xac0c, 0xffffffff, 0x00003210,
524 0x88d4, 0x0000001f, 0x00000010,
525 0x15c0, 0x000c0fc0, 0x000c0400
528 static const u32 hainan_golden_registers2
[] =
530 0x98f8, 0xffffffff, 0x02010001
533 static const u32 tahiti_mgcg_cgcg_init
[] =
535 0xc400, 0xffffffff, 0xfffffffc,
536 0x802c, 0xffffffff, 0xe0000000,
537 0x9a60, 0xffffffff, 0x00000100,
538 0x92a4, 0xffffffff, 0x00000100,
539 0xc164, 0xffffffff, 0x00000100,
540 0x9774, 0xffffffff, 0x00000100,
541 0x8984, 0xffffffff, 0x06000100,
542 0x8a18, 0xffffffff, 0x00000100,
543 0x92a0, 0xffffffff, 0x00000100,
544 0xc380, 0xffffffff, 0x00000100,
545 0x8b28, 0xffffffff, 0x00000100,
546 0x9144, 0xffffffff, 0x00000100,
547 0x8d88, 0xffffffff, 0x00000100,
548 0x8d8c, 0xffffffff, 0x00000100,
549 0x9030, 0xffffffff, 0x00000100,
550 0x9034, 0xffffffff, 0x00000100,
551 0x9038, 0xffffffff, 0x00000100,
552 0x903c, 0xffffffff, 0x00000100,
553 0xad80, 0xffffffff, 0x00000100,
554 0xac54, 0xffffffff, 0x00000100,
555 0x897c, 0xffffffff, 0x06000100,
556 0x9868, 0xffffffff, 0x00000100,
557 0x9510, 0xffffffff, 0x00000100,
558 0xaf04, 0xffffffff, 0x00000100,
559 0xae04, 0xffffffff, 0x00000100,
560 0x949c, 0xffffffff, 0x00000100,
561 0x802c, 0xffffffff, 0xe0000000,
562 0x9160, 0xffffffff, 0x00010000,
563 0x9164, 0xffffffff, 0x00030002,
564 0x9168, 0xffffffff, 0x00040007,
565 0x916c, 0xffffffff, 0x00060005,
566 0x9170, 0xffffffff, 0x00090008,
567 0x9174, 0xffffffff, 0x00020001,
568 0x9178, 0xffffffff, 0x00040003,
569 0x917c, 0xffffffff, 0x00000007,
570 0x9180, 0xffffffff, 0x00060005,
571 0x9184, 0xffffffff, 0x00090008,
572 0x9188, 0xffffffff, 0x00030002,
573 0x918c, 0xffffffff, 0x00050004,
574 0x9190, 0xffffffff, 0x00000008,
575 0x9194, 0xffffffff, 0x00070006,
576 0x9198, 0xffffffff, 0x000a0009,
577 0x919c, 0xffffffff, 0x00040003,
578 0x91a0, 0xffffffff, 0x00060005,
579 0x91a4, 0xffffffff, 0x00000009,
580 0x91a8, 0xffffffff, 0x00080007,
581 0x91ac, 0xffffffff, 0x000b000a,
582 0x91b0, 0xffffffff, 0x00050004,
583 0x91b4, 0xffffffff, 0x00070006,
584 0x91b8, 0xffffffff, 0x0008000b,
585 0x91bc, 0xffffffff, 0x000a0009,
586 0x91c0, 0xffffffff, 0x000d000c,
587 0x91c4, 0xffffffff, 0x00060005,
588 0x91c8, 0xffffffff, 0x00080007,
589 0x91cc, 0xffffffff, 0x0000000b,
590 0x91d0, 0xffffffff, 0x000a0009,
591 0x91d4, 0xffffffff, 0x000d000c,
592 0x91d8, 0xffffffff, 0x00070006,
593 0x91dc, 0xffffffff, 0x00090008,
594 0x91e0, 0xffffffff, 0x0000000c,
595 0x91e4, 0xffffffff, 0x000b000a,
596 0x91e8, 0xffffffff, 0x000e000d,
597 0x91ec, 0xffffffff, 0x00080007,
598 0x91f0, 0xffffffff, 0x000a0009,
599 0x91f4, 0xffffffff, 0x0000000d,
600 0x91f8, 0xffffffff, 0x000c000b,
601 0x91fc, 0xffffffff, 0x000f000e,
602 0x9200, 0xffffffff, 0x00090008,
603 0x9204, 0xffffffff, 0x000b000a,
604 0x9208, 0xffffffff, 0x000c000f,
605 0x920c, 0xffffffff, 0x000e000d,
606 0x9210, 0xffffffff, 0x00110010,
607 0x9214, 0xffffffff, 0x000a0009,
608 0x9218, 0xffffffff, 0x000c000b,
609 0x921c, 0xffffffff, 0x0000000f,
610 0x9220, 0xffffffff, 0x000e000d,
611 0x9224, 0xffffffff, 0x00110010,
612 0x9228, 0xffffffff, 0x000b000a,
613 0x922c, 0xffffffff, 0x000d000c,
614 0x9230, 0xffffffff, 0x00000010,
615 0x9234, 0xffffffff, 0x000f000e,
616 0x9238, 0xffffffff, 0x00120011,
617 0x923c, 0xffffffff, 0x000c000b,
618 0x9240, 0xffffffff, 0x000e000d,
619 0x9244, 0xffffffff, 0x00000011,
620 0x9248, 0xffffffff, 0x0010000f,
621 0x924c, 0xffffffff, 0x00130012,
622 0x9250, 0xffffffff, 0x000d000c,
623 0x9254, 0xffffffff, 0x000f000e,
624 0x9258, 0xffffffff, 0x00100013,
625 0x925c, 0xffffffff, 0x00120011,
626 0x9260, 0xffffffff, 0x00150014,
627 0x9264, 0xffffffff, 0x000e000d,
628 0x9268, 0xffffffff, 0x0010000f,
629 0x926c, 0xffffffff, 0x00000013,
630 0x9270, 0xffffffff, 0x00120011,
631 0x9274, 0xffffffff, 0x00150014,
632 0x9278, 0xffffffff, 0x000f000e,
633 0x927c, 0xffffffff, 0x00110010,
634 0x9280, 0xffffffff, 0x00000014,
635 0x9284, 0xffffffff, 0x00130012,
636 0x9288, 0xffffffff, 0x00160015,
637 0x928c, 0xffffffff, 0x0010000f,
638 0x9290, 0xffffffff, 0x00120011,
639 0x9294, 0xffffffff, 0x00000015,
640 0x9298, 0xffffffff, 0x00140013,
641 0x929c, 0xffffffff, 0x00170016,
642 0x9150, 0xffffffff, 0x96940200,
643 0x8708, 0xffffffff, 0x00900100,
644 0xc478, 0xffffffff, 0x00000080,
645 0xc404, 0xffffffff, 0x0020003f,
646 0x30, 0xffffffff, 0x0000001c,
647 0x34, 0x000f0000, 0x000f0000,
648 0x160c, 0xffffffff, 0x00000100,
649 0x1024, 0xffffffff, 0x00000100,
650 0x102c, 0x00000101, 0x00000000,
651 0x20a8, 0xffffffff, 0x00000104,
652 0x264c, 0x000c0000, 0x000c0000,
653 0x2648, 0x000c0000, 0x000c0000,
654 0x55e4, 0xff000fff, 0x00000100,
655 0x55e8, 0x00000001, 0x00000001,
656 0x2f50, 0x00000001, 0x00000001,
657 0x30cc, 0xc0000fff, 0x00000104,
658 0xc1e4, 0x00000001, 0x00000001,
659 0xd0c0, 0xfffffff0, 0x00000100,
660 0xd8c0, 0xfffffff0, 0x00000100
663 static const u32 pitcairn_mgcg_cgcg_init
[] =
665 0xc400, 0xffffffff, 0xfffffffc,
666 0x802c, 0xffffffff, 0xe0000000,
667 0x9a60, 0xffffffff, 0x00000100,
668 0x92a4, 0xffffffff, 0x00000100,
669 0xc164, 0xffffffff, 0x00000100,
670 0x9774, 0xffffffff, 0x00000100,
671 0x8984, 0xffffffff, 0x06000100,
672 0x8a18, 0xffffffff, 0x00000100,
673 0x92a0, 0xffffffff, 0x00000100,
674 0xc380, 0xffffffff, 0x00000100,
675 0x8b28, 0xffffffff, 0x00000100,
676 0x9144, 0xffffffff, 0x00000100,
677 0x8d88, 0xffffffff, 0x00000100,
678 0x8d8c, 0xffffffff, 0x00000100,
679 0x9030, 0xffffffff, 0x00000100,
680 0x9034, 0xffffffff, 0x00000100,
681 0x9038, 0xffffffff, 0x00000100,
682 0x903c, 0xffffffff, 0x00000100,
683 0xad80, 0xffffffff, 0x00000100,
684 0xac54, 0xffffffff, 0x00000100,
685 0x897c, 0xffffffff, 0x06000100,
686 0x9868, 0xffffffff, 0x00000100,
687 0x9510, 0xffffffff, 0x00000100,
688 0xaf04, 0xffffffff, 0x00000100,
689 0xae04, 0xffffffff, 0x00000100,
690 0x949c, 0xffffffff, 0x00000100,
691 0x802c, 0xffffffff, 0xe0000000,
692 0x9160, 0xffffffff, 0x00010000,
693 0x9164, 0xffffffff, 0x00030002,
694 0x9168, 0xffffffff, 0x00040007,
695 0x916c, 0xffffffff, 0x00060005,
696 0x9170, 0xffffffff, 0x00090008,
697 0x9174, 0xffffffff, 0x00020001,
698 0x9178, 0xffffffff, 0x00040003,
699 0x917c, 0xffffffff, 0x00000007,
700 0x9180, 0xffffffff, 0x00060005,
701 0x9184, 0xffffffff, 0x00090008,
702 0x9188, 0xffffffff, 0x00030002,
703 0x918c, 0xffffffff, 0x00050004,
704 0x9190, 0xffffffff, 0x00000008,
705 0x9194, 0xffffffff, 0x00070006,
706 0x9198, 0xffffffff, 0x000a0009,
707 0x919c, 0xffffffff, 0x00040003,
708 0x91a0, 0xffffffff, 0x00060005,
709 0x91a4, 0xffffffff, 0x00000009,
710 0x91a8, 0xffffffff, 0x00080007,
711 0x91ac, 0xffffffff, 0x000b000a,
712 0x91b0, 0xffffffff, 0x00050004,
713 0x91b4, 0xffffffff, 0x00070006,
714 0x91b8, 0xffffffff, 0x0008000b,
715 0x91bc, 0xffffffff, 0x000a0009,
716 0x91c0, 0xffffffff, 0x000d000c,
717 0x9200, 0xffffffff, 0x00090008,
718 0x9204, 0xffffffff, 0x000b000a,
719 0x9208, 0xffffffff, 0x000c000f,
720 0x920c, 0xffffffff, 0x000e000d,
721 0x9210, 0xffffffff, 0x00110010,
722 0x9214, 0xffffffff, 0x000a0009,
723 0x9218, 0xffffffff, 0x000c000b,
724 0x921c, 0xffffffff, 0x0000000f,
725 0x9220, 0xffffffff, 0x000e000d,
726 0x9224, 0xffffffff, 0x00110010,
727 0x9228, 0xffffffff, 0x000b000a,
728 0x922c, 0xffffffff, 0x000d000c,
729 0x9230, 0xffffffff, 0x00000010,
730 0x9234, 0xffffffff, 0x000f000e,
731 0x9238, 0xffffffff, 0x00120011,
732 0x923c, 0xffffffff, 0x000c000b,
733 0x9240, 0xffffffff, 0x000e000d,
734 0x9244, 0xffffffff, 0x00000011,
735 0x9248, 0xffffffff, 0x0010000f,
736 0x924c, 0xffffffff, 0x00130012,
737 0x9250, 0xffffffff, 0x000d000c,
738 0x9254, 0xffffffff, 0x000f000e,
739 0x9258, 0xffffffff, 0x00100013,
740 0x925c, 0xffffffff, 0x00120011,
741 0x9260, 0xffffffff, 0x00150014,
742 0x9150, 0xffffffff, 0x96940200,
743 0x8708, 0xffffffff, 0x00900100,
744 0xc478, 0xffffffff, 0x00000080,
745 0xc404, 0xffffffff, 0x0020003f,
746 0x30, 0xffffffff, 0x0000001c,
747 0x34, 0x000f0000, 0x000f0000,
748 0x160c, 0xffffffff, 0x00000100,
749 0x1024, 0xffffffff, 0x00000100,
750 0x102c, 0x00000101, 0x00000000,
751 0x20a8, 0xffffffff, 0x00000104,
752 0x55e4, 0xff000fff, 0x00000100,
753 0x55e8, 0x00000001, 0x00000001,
754 0x2f50, 0x00000001, 0x00000001,
755 0x30cc, 0xc0000fff, 0x00000104,
756 0xc1e4, 0x00000001, 0x00000001,
757 0xd0c0, 0xfffffff0, 0x00000100,
758 0xd8c0, 0xfffffff0, 0x00000100
761 static const u32 verde_mgcg_cgcg_init
[] =
763 0xc400, 0xffffffff, 0xfffffffc,
764 0x802c, 0xffffffff, 0xe0000000,
765 0x9a60, 0xffffffff, 0x00000100,
766 0x92a4, 0xffffffff, 0x00000100,
767 0xc164, 0xffffffff, 0x00000100,
768 0x9774, 0xffffffff, 0x00000100,
769 0x8984, 0xffffffff, 0x06000100,
770 0x8a18, 0xffffffff, 0x00000100,
771 0x92a0, 0xffffffff, 0x00000100,
772 0xc380, 0xffffffff, 0x00000100,
773 0x8b28, 0xffffffff, 0x00000100,
774 0x9144, 0xffffffff, 0x00000100,
775 0x8d88, 0xffffffff, 0x00000100,
776 0x8d8c, 0xffffffff, 0x00000100,
777 0x9030, 0xffffffff, 0x00000100,
778 0x9034, 0xffffffff, 0x00000100,
779 0x9038, 0xffffffff, 0x00000100,
780 0x903c, 0xffffffff, 0x00000100,
781 0xad80, 0xffffffff, 0x00000100,
782 0xac54, 0xffffffff, 0x00000100,
783 0x897c, 0xffffffff, 0x06000100,
784 0x9868, 0xffffffff, 0x00000100,
785 0x9510, 0xffffffff, 0x00000100,
786 0xaf04, 0xffffffff, 0x00000100,
787 0xae04, 0xffffffff, 0x00000100,
788 0x949c, 0xffffffff, 0x00000100,
789 0x802c, 0xffffffff, 0xe0000000,
790 0x9160, 0xffffffff, 0x00010000,
791 0x9164, 0xffffffff, 0x00030002,
792 0x9168, 0xffffffff, 0x00040007,
793 0x916c, 0xffffffff, 0x00060005,
794 0x9170, 0xffffffff, 0x00090008,
795 0x9174, 0xffffffff, 0x00020001,
796 0x9178, 0xffffffff, 0x00040003,
797 0x917c, 0xffffffff, 0x00000007,
798 0x9180, 0xffffffff, 0x00060005,
799 0x9184, 0xffffffff, 0x00090008,
800 0x9188, 0xffffffff, 0x00030002,
801 0x918c, 0xffffffff, 0x00050004,
802 0x9190, 0xffffffff, 0x00000008,
803 0x9194, 0xffffffff, 0x00070006,
804 0x9198, 0xffffffff, 0x000a0009,
805 0x919c, 0xffffffff, 0x00040003,
806 0x91a0, 0xffffffff, 0x00060005,
807 0x91a4, 0xffffffff, 0x00000009,
808 0x91a8, 0xffffffff, 0x00080007,
809 0x91ac, 0xffffffff, 0x000b000a,
810 0x91b0, 0xffffffff, 0x00050004,
811 0x91b4, 0xffffffff, 0x00070006,
812 0x91b8, 0xffffffff, 0x0008000b,
813 0x91bc, 0xffffffff, 0x000a0009,
814 0x91c0, 0xffffffff, 0x000d000c,
815 0x9200, 0xffffffff, 0x00090008,
816 0x9204, 0xffffffff, 0x000b000a,
817 0x9208, 0xffffffff, 0x000c000f,
818 0x920c, 0xffffffff, 0x000e000d,
819 0x9210, 0xffffffff, 0x00110010,
820 0x9214, 0xffffffff, 0x000a0009,
821 0x9218, 0xffffffff, 0x000c000b,
822 0x921c, 0xffffffff, 0x0000000f,
823 0x9220, 0xffffffff, 0x000e000d,
824 0x9224, 0xffffffff, 0x00110010,
825 0x9228, 0xffffffff, 0x000b000a,
826 0x922c, 0xffffffff, 0x000d000c,
827 0x9230, 0xffffffff, 0x00000010,
828 0x9234, 0xffffffff, 0x000f000e,
829 0x9238, 0xffffffff, 0x00120011,
830 0x923c, 0xffffffff, 0x000c000b,
831 0x9240, 0xffffffff, 0x000e000d,
832 0x9244, 0xffffffff, 0x00000011,
833 0x9248, 0xffffffff, 0x0010000f,
834 0x924c, 0xffffffff, 0x00130012,
835 0x9250, 0xffffffff, 0x000d000c,
836 0x9254, 0xffffffff, 0x000f000e,
837 0x9258, 0xffffffff, 0x00100013,
838 0x925c, 0xffffffff, 0x00120011,
839 0x9260, 0xffffffff, 0x00150014,
840 0x9150, 0xffffffff, 0x96940200,
841 0x8708, 0xffffffff, 0x00900100,
842 0xc478, 0xffffffff, 0x00000080,
843 0xc404, 0xffffffff, 0x0020003f,
844 0x30, 0xffffffff, 0x0000001c,
845 0x34, 0x000f0000, 0x000f0000,
846 0x160c, 0xffffffff, 0x00000100,
847 0x1024, 0xffffffff, 0x00000100,
848 0x102c, 0x00000101, 0x00000000,
849 0x20a8, 0xffffffff, 0x00000104,
850 0x264c, 0x000c0000, 0x000c0000,
851 0x2648, 0x000c0000, 0x000c0000,
852 0x55e4, 0xff000fff, 0x00000100,
853 0x55e8, 0x00000001, 0x00000001,
854 0x2f50, 0x00000001, 0x00000001,
855 0x30cc, 0xc0000fff, 0x00000104,
856 0xc1e4, 0x00000001, 0x00000001,
857 0xd0c0, 0xfffffff0, 0x00000100,
858 0xd8c0, 0xfffffff0, 0x00000100
861 static const u32 oland_mgcg_cgcg_init
[] =
863 0xc400, 0xffffffff, 0xfffffffc,
864 0x802c, 0xffffffff, 0xe0000000,
865 0x9a60, 0xffffffff, 0x00000100,
866 0x92a4, 0xffffffff, 0x00000100,
867 0xc164, 0xffffffff, 0x00000100,
868 0x9774, 0xffffffff, 0x00000100,
869 0x8984, 0xffffffff, 0x06000100,
870 0x8a18, 0xffffffff, 0x00000100,
871 0x92a0, 0xffffffff, 0x00000100,
872 0xc380, 0xffffffff, 0x00000100,
873 0x8b28, 0xffffffff, 0x00000100,
874 0x9144, 0xffffffff, 0x00000100,
875 0x8d88, 0xffffffff, 0x00000100,
876 0x8d8c, 0xffffffff, 0x00000100,
877 0x9030, 0xffffffff, 0x00000100,
878 0x9034, 0xffffffff, 0x00000100,
879 0x9038, 0xffffffff, 0x00000100,
880 0x903c, 0xffffffff, 0x00000100,
881 0xad80, 0xffffffff, 0x00000100,
882 0xac54, 0xffffffff, 0x00000100,
883 0x897c, 0xffffffff, 0x06000100,
884 0x9868, 0xffffffff, 0x00000100,
885 0x9510, 0xffffffff, 0x00000100,
886 0xaf04, 0xffffffff, 0x00000100,
887 0xae04, 0xffffffff, 0x00000100,
888 0x949c, 0xffffffff, 0x00000100,
889 0x802c, 0xffffffff, 0xe0000000,
890 0x9160, 0xffffffff, 0x00010000,
891 0x9164, 0xffffffff, 0x00030002,
892 0x9168, 0xffffffff, 0x00040007,
893 0x916c, 0xffffffff, 0x00060005,
894 0x9170, 0xffffffff, 0x00090008,
895 0x9174, 0xffffffff, 0x00020001,
896 0x9178, 0xffffffff, 0x00040003,
897 0x917c, 0xffffffff, 0x00000007,
898 0x9180, 0xffffffff, 0x00060005,
899 0x9184, 0xffffffff, 0x00090008,
900 0x9188, 0xffffffff, 0x00030002,
901 0x918c, 0xffffffff, 0x00050004,
902 0x9190, 0xffffffff, 0x00000008,
903 0x9194, 0xffffffff, 0x00070006,
904 0x9198, 0xffffffff, 0x000a0009,
905 0x919c, 0xffffffff, 0x00040003,
906 0x91a0, 0xffffffff, 0x00060005,
907 0x91a4, 0xffffffff, 0x00000009,
908 0x91a8, 0xffffffff, 0x00080007,
909 0x91ac, 0xffffffff, 0x000b000a,
910 0x91b0, 0xffffffff, 0x00050004,
911 0x91b4, 0xffffffff, 0x00070006,
912 0x91b8, 0xffffffff, 0x0008000b,
913 0x91bc, 0xffffffff, 0x000a0009,
914 0x91c0, 0xffffffff, 0x000d000c,
915 0x91c4, 0xffffffff, 0x00060005,
916 0x91c8, 0xffffffff, 0x00080007,
917 0x91cc, 0xffffffff, 0x0000000b,
918 0x91d0, 0xffffffff, 0x000a0009,
919 0x91d4, 0xffffffff, 0x000d000c,
920 0x9150, 0xffffffff, 0x96940200,
921 0x8708, 0xffffffff, 0x00900100,
922 0xc478, 0xffffffff, 0x00000080,
923 0xc404, 0xffffffff, 0x0020003f,
924 0x30, 0xffffffff, 0x0000001c,
925 0x34, 0x000f0000, 0x000f0000,
926 0x160c, 0xffffffff, 0x00000100,
927 0x1024, 0xffffffff, 0x00000100,
928 0x102c, 0x00000101, 0x00000000,
929 0x20a8, 0xffffffff, 0x00000104,
930 0x264c, 0x000c0000, 0x000c0000,
931 0x2648, 0x000c0000, 0x000c0000,
932 0x55e4, 0xff000fff, 0x00000100,
933 0x55e8, 0x00000001, 0x00000001,
934 0x2f50, 0x00000001, 0x00000001,
935 0x30cc, 0xc0000fff, 0x00000104,
936 0xc1e4, 0x00000001, 0x00000001,
937 0xd0c0, 0xfffffff0, 0x00000100,
938 0xd8c0, 0xfffffff0, 0x00000100
941 static const u32 hainan_mgcg_cgcg_init
[] =
943 0xc400, 0xffffffff, 0xfffffffc,
944 0x802c, 0xffffffff, 0xe0000000,
945 0x9a60, 0xffffffff, 0x00000100,
946 0x92a4, 0xffffffff, 0x00000100,
947 0xc164, 0xffffffff, 0x00000100,
948 0x9774, 0xffffffff, 0x00000100,
949 0x8984, 0xffffffff, 0x06000100,
950 0x8a18, 0xffffffff, 0x00000100,
951 0x92a0, 0xffffffff, 0x00000100,
952 0xc380, 0xffffffff, 0x00000100,
953 0x8b28, 0xffffffff, 0x00000100,
954 0x9144, 0xffffffff, 0x00000100,
955 0x8d88, 0xffffffff, 0x00000100,
956 0x8d8c, 0xffffffff, 0x00000100,
957 0x9030, 0xffffffff, 0x00000100,
958 0x9034, 0xffffffff, 0x00000100,
959 0x9038, 0xffffffff, 0x00000100,
960 0x903c, 0xffffffff, 0x00000100,
961 0xad80, 0xffffffff, 0x00000100,
962 0xac54, 0xffffffff, 0x00000100,
963 0x897c, 0xffffffff, 0x06000100,
964 0x9868, 0xffffffff, 0x00000100,
965 0x9510, 0xffffffff, 0x00000100,
966 0xaf04, 0xffffffff, 0x00000100,
967 0xae04, 0xffffffff, 0x00000100,
968 0x949c, 0xffffffff, 0x00000100,
969 0x802c, 0xffffffff, 0xe0000000,
970 0x9160, 0xffffffff, 0x00010000,
971 0x9164, 0xffffffff, 0x00030002,
972 0x9168, 0xffffffff, 0x00040007,
973 0x916c, 0xffffffff, 0x00060005,
974 0x9170, 0xffffffff, 0x00090008,
975 0x9174, 0xffffffff, 0x00020001,
976 0x9178, 0xffffffff, 0x00040003,
977 0x917c, 0xffffffff, 0x00000007,
978 0x9180, 0xffffffff, 0x00060005,
979 0x9184, 0xffffffff, 0x00090008,
980 0x9188, 0xffffffff, 0x00030002,
981 0x918c, 0xffffffff, 0x00050004,
982 0x9190, 0xffffffff, 0x00000008,
983 0x9194, 0xffffffff, 0x00070006,
984 0x9198, 0xffffffff, 0x000a0009,
985 0x919c, 0xffffffff, 0x00040003,
986 0x91a0, 0xffffffff, 0x00060005,
987 0x91a4, 0xffffffff, 0x00000009,
988 0x91a8, 0xffffffff, 0x00080007,
989 0x91ac, 0xffffffff, 0x000b000a,
990 0x91b0, 0xffffffff, 0x00050004,
991 0x91b4, 0xffffffff, 0x00070006,
992 0x91b8, 0xffffffff, 0x0008000b,
993 0x91bc, 0xffffffff, 0x000a0009,
994 0x91c0, 0xffffffff, 0x000d000c,
995 0x91c4, 0xffffffff, 0x00060005,
996 0x91c8, 0xffffffff, 0x00080007,
997 0x91cc, 0xffffffff, 0x0000000b,
998 0x91d0, 0xffffffff, 0x000a0009,
999 0x91d4, 0xffffffff, 0x000d000c,
1000 0x9150, 0xffffffff, 0x96940200,
1001 0x8708, 0xffffffff, 0x00900100,
1002 0xc478, 0xffffffff, 0x00000080,
1003 0xc404, 0xffffffff, 0x0020003f,
1004 0x30, 0xffffffff, 0x0000001c,
1005 0x34, 0x000f0000, 0x000f0000,
1006 0x160c, 0xffffffff, 0x00000100,
1007 0x1024, 0xffffffff, 0x00000100,
1008 0x20a8, 0xffffffff, 0x00000104,
1009 0x264c, 0x000c0000, 0x000c0000,
1010 0x2648, 0x000c0000, 0x000c0000,
1011 0x2f50, 0x00000001, 0x00000001,
1012 0x30cc, 0xc0000fff, 0x00000104,
1013 0xc1e4, 0x00000001, 0x00000001,
1014 0xd0c0, 0xfffffff0, 0x00000100,
1015 0xd8c0, 0xfffffff0, 0x00000100
1018 static u32 verde_pg_init
[] =
1020 0x353c, 0xffffffff, 0x40000,
1021 0x3538, 0xffffffff, 0x200010ff,
1022 0x353c, 0xffffffff, 0x0,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x0,
1027 0x353c, 0xffffffff, 0x7007,
1028 0x3538, 0xffffffff, 0x300010ff,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x400000,
1035 0x3538, 0xffffffff, 0x100010ff,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x120200,
1042 0x3538, 0xffffffff, 0x500010ff,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x1e1e16,
1049 0x3538, 0xffffffff, 0x600010ff,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x171f1e,
1056 0x3538, 0xffffffff, 0x700010ff,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x0,
1063 0x3538, 0xffffffff, 0x9ff,
1064 0x3500, 0xffffffff, 0x0,
1065 0x3504, 0xffffffff, 0x10000800,
1066 0x3504, 0xffffffff, 0xf,
1067 0x3504, 0xffffffff, 0xf,
1068 0x3500, 0xffffffff, 0x4,
1069 0x3504, 0xffffffff, 0x1000051e,
1070 0x3504, 0xffffffff, 0xffff,
1071 0x3504, 0xffffffff, 0xffff,
1072 0x3500, 0xffffffff, 0x8,
1073 0x3504, 0xffffffff, 0x80500,
1074 0x3500, 0xffffffff, 0x12,
1075 0x3504, 0xffffffff, 0x9050c,
1076 0x3500, 0xffffffff, 0x1d,
1077 0x3504, 0xffffffff, 0xb052c,
1078 0x3500, 0xffffffff, 0x2a,
1079 0x3504, 0xffffffff, 0x1053e,
1080 0x3500, 0xffffffff, 0x2d,
1081 0x3504, 0xffffffff, 0x10546,
1082 0x3500, 0xffffffff, 0x30,
1083 0x3504, 0xffffffff, 0xa054e,
1084 0x3500, 0xffffffff, 0x3c,
1085 0x3504, 0xffffffff, 0x1055f,
1086 0x3500, 0xffffffff, 0x3f,
1087 0x3504, 0xffffffff, 0x10567,
1088 0x3500, 0xffffffff, 0x42,
1089 0x3504, 0xffffffff, 0x1056f,
1090 0x3500, 0xffffffff, 0x45,
1091 0x3504, 0xffffffff, 0x10572,
1092 0x3500, 0xffffffff, 0x48,
1093 0x3504, 0xffffffff, 0x20575,
1094 0x3500, 0xffffffff, 0x4c,
1095 0x3504, 0xffffffff, 0x190801,
1096 0x3500, 0xffffffff, 0x67,
1097 0x3504, 0xffffffff, 0x1082a,
1098 0x3500, 0xffffffff, 0x6a,
1099 0x3504, 0xffffffff, 0x1b082d,
1100 0x3500, 0xffffffff, 0x87,
1101 0x3504, 0xffffffff, 0x310851,
1102 0x3500, 0xffffffff, 0xba,
1103 0x3504, 0xffffffff, 0x891,
1104 0x3500, 0xffffffff, 0xbc,
1105 0x3504, 0xffffffff, 0x893,
1106 0x3500, 0xffffffff, 0xbe,
1107 0x3504, 0xffffffff, 0x20895,
1108 0x3500, 0xffffffff, 0xc2,
1109 0x3504, 0xffffffff, 0x20899,
1110 0x3500, 0xffffffff, 0xc6,
1111 0x3504, 0xffffffff, 0x2089d,
1112 0x3500, 0xffffffff, 0xca,
1113 0x3504, 0xffffffff, 0x8a1,
1114 0x3500, 0xffffffff, 0xcc,
1115 0x3504, 0xffffffff, 0x8a3,
1116 0x3500, 0xffffffff, 0xce,
1117 0x3504, 0xffffffff, 0x308a5,
1118 0x3500, 0xffffffff, 0xd3,
1119 0x3504, 0xffffffff, 0x6d08cd,
1120 0x3500, 0xffffffff, 0x142,
1121 0x3504, 0xffffffff, 0x2000095a,
1122 0x3504, 0xffffffff, 0x1,
1123 0x3500, 0xffffffff, 0x144,
1124 0x3504, 0xffffffff, 0x301f095b,
1125 0x3500, 0xffffffff, 0x165,
1126 0x3504, 0xffffffff, 0xc094d,
1127 0x3500, 0xffffffff, 0x173,
1128 0x3504, 0xffffffff, 0xf096d,
1129 0x3500, 0xffffffff, 0x184,
1130 0x3504, 0xffffffff, 0x15097f,
1131 0x3500, 0xffffffff, 0x19b,
1132 0x3504, 0xffffffff, 0xc0998,
1133 0x3500, 0xffffffff, 0x1a9,
1134 0x3504, 0xffffffff, 0x409a7,
1135 0x3500, 0xffffffff, 0x1af,
1136 0x3504, 0xffffffff, 0xcdc,
1137 0x3500, 0xffffffff, 0x1b1,
1138 0x3504, 0xffffffff, 0x800,
1139 0x3508, 0xffffffff, 0x6c9b2000,
1140 0x3510, 0xfc00, 0x2000,
1141 0x3544, 0xffffffff, 0xfc0,
1142 0x28d4, 0x00000100, 0x100
1145 static void si_init_golden_registers(struct radeon_device
*rdev
)
1147 switch (rdev
->family
) {
1149 radeon_program_register_sequence(rdev
,
1150 tahiti_golden_registers
,
1151 (const u32
)ARRAY_SIZE(tahiti_golden_registers
));
1152 radeon_program_register_sequence(rdev
,
1153 tahiti_golden_rlc_registers
,
1154 (const u32
)ARRAY_SIZE(tahiti_golden_rlc_registers
));
1155 radeon_program_register_sequence(rdev
,
1156 tahiti_mgcg_cgcg_init
,
1157 (const u32
)ARRAY_SIZE(tahiti_mgcg_cgcg_init
));
1158 radeon_program_register_sequence(rdev
,
1159 tahiti_golden_registers2
,
1160 (const u32
)ARRAY_SIZE(tahiti_golden_registers2
));
1163 radeon_program_register_sequence(rdev
,
1164 pitcairn_golden_registers
,
1165 (const u32
)ARRAY_SIZE(pitcairn_golden_registers
));
1166 radeon_program_register_sequence(rdev
,
1167 pitcairn_golden_rlc_registers
,
1168 (const u32
)ARRAY_SIZE(pitcairn_golden_rlc_registers
));
1169 radeon_program_register_sequence(rdev
,
1170 pitcairn_mgcg_cgcg_init
,
1171 (const u32
)ARRAY_SIZE(pitcairn_mgcg_cgcg_init
));
1174 radeon_program_register_sequence(rdev
,
1175 verde_golden_registers
,
1176 (const u32
)ARRAY_SIZE(verde_golden_registers
));
1177 radeon_program_register_sequence(rdev
,
1178 verde_golden_rlc_registers
,
1179 (const u32
)ARRAY_SIZE(verde_golden_rlc_registers
));
1180 radeon_program_register_sequence(rdev
,
1181 verde_mgcg_cgcg_init
,
1182 (const u32
)ARRAY_SIZE(verde_mgcg_cgcg_init
));
1183 radeon_program_register_sequence(rdev
,
1185 (const u32
)ARRAY_SIZE(verde_pg_init
));
1188 radeon_program_register_sequence(rdev
,
1189 oland_golden_registers
,
1190 (const u32
)ARRAY_SIZE(oland_golden_registers
));
1191 radeon_program_register_sequence(rdev
,
1192 oland_golden_rlc_registers
,
1193 (const u32
)ARRAY_SIZE(oland_golden_rlc_registers
));
1194 radeon_program_register_sequence(rdev
,
1195 oland_mgcg_cgcg_init
,
1196 (const u32
)ARRAY_SIZE(oland_mgcg_cgcg_init
));
1199 radeon_program_register_sequence(rdev
,
1200 hainan_golden_registers
,
1201 (const u32
)ARRAY_SIZE(hainan_golden_registers
));
1202 radeon_program_register_sequence(rdev
,
1203 hainan_golden_registers2
,
1204 (const u32
)ARRAY_SIZE(hainan_golden_registers2
));
1205 radeon_program_register_sequence(rdev
,
1206 hainan_mgcg_cgcg_init
,
1207 (const u32
)ARRAY_SIZE(hainan_mgcg_cgcg_init
));
1214 #define PCIE_BUS_CLK 10000
1215 #define TCLK (PCIE_BUS_CLK / 10)
1218 * si_get_xclk - get the xclk
1220 * @rdev: radeon_device pointer
1222 * Returns the reference clock used by the gfx engine
1225 u32
si_get_xclk(struct radeon_device
*rdev
)
1227 u32 reference_clock
= rdev
->clock
.spll
.reference_freq
;
1230 tmp
= RREG32(CG_CLKPIN_CNTL_2
);
1231 if (tmp
& MUX_TCLK_TO_XCLK
)
1234 tmp
= RREG32(CG_CLKPIN_CNTL
);
1235 if (tmp
& XTALIN_DIVIDE
)
1236 return reference_clock
/ 4;
1238 return reference_clock
;
1241 /* get temperature in millidegrees */
1242 int si_get_temp(struct radeon_device
*rdev
)
1245 int actual_temp
= 0;
1247 temp
= (RREG32(CG_MULT_THERMAL_STATUS
) & CTF_TEMP_MASK
) >>
1253 actual_temp
= temp
& 0x1ff;
1255 actual_temp
= (actual_temp
* 1000);
1260 #define TAHITI_IO_MC_REGS_SIZE 36
1262 static const u32 tahiti_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1263 {0x0000006f, 0x03044000},
1264 {0x00000070, 0x0480c018},
1265 {0x00000071, 0x00000040},
1266 {0x00000072, 0x01000000},
1267 {0x00000074, 0x000000ff},
1268 {0x00000075, 0x00143400},
1269 {0x00000076, 0x08ec0800},
1270 {0x00000077, 0x040000cc},
1271 {0x00000079, 0x00000000},
1272 {0x0000007a, 0x21000409},
1273 {0x0000007c, 0x00000000},
1274 {0x0000007d, 0xe8000000},
1275 {0x0000007e, 0x044408a8},
1276 {0x0000007f, 0x00000003},
1277 {0x00000080, 0x00000000},
1278 {0x00000081, 0x01000000},
1279 {0x00000082, 0x02000000},
1280 {0x00000083, 0x00000000},
1281 {0x00000084, 0xe3f3e4f4},
1282 {0x00000085, 0x00052024},
1283 {0x00000087, 0x00000000},
1284 {0x00000088, 0x66036603},
1285 {0x00000089, 0x01000000},
1286 {0x0000008b, 0x1c0a0000},
1287 {0x0000008c, 0xff010000},
1288 {0x0000008e, 0xffffefff},
1289 {0x0000008f, 0xfff3efff},
1290 {0x00000090, 0xfff3efbf},
1291 {0x00000094, 0x00101101},
1292 {0x00000095, 0x00000fff},
1293 {0x00000096, 0x00116fff},
1294 {0x00000097, 0x60010000},
1295 {0x00000098, 0x10010000},
1296 {0x00000099, 0x00006000},
1297 {0x0000009a, 0x00001000},
1298 {0x0000009f, 0x00a77400}
1301 static const u32 pitcairn_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1302 {0x0000006f, 0x03044000},
1303 {0x00000070, 0x0480c018},
1304 {0x00000071, 0x00000040},
1305 {0x00000072, 0x01000000},
1306 {0x00000074, 0x000000ff},
1307 {0x00000075, 0x00143400},
1308 {0x00000076, 0x08ec0800},
1309 {0x00000077, 0x040000cc},
1310 {0x00000079, 0x00000000},
1311 {0x0000007a, 0x21000409},
1312 {0x0000007c, 0x00000000},
1313 {0x0000007d, 0xe8000000},
1314 {0x0000007e, 0x044408a8},
1315 {0x0000007f, 0x00000003},
1316 {0x00000080, 0x00000000},
1317 {0x00000081, 0x01000000},
1318 {0x00000082, 0x02000000},
1319 {0x00000083, 0x00000000},
1320 {0x00000084, 0xe3f3e4f4},
1321 {0x00000085, 0x00052024},
1322 {0x00000087, 0x00000000},
1323 {0x00000088, 0x66036603},
1324 {0x00000089, 0x01000000},
1325 {0x0000008b, 0x1c0a0000},
1326 {0x0000008c, 0xff010000},
1327 {0x0000008e, 0xffffefff},
1328 {0x0000008f, 0xfff3efff},
1329 {0x00000090, 0xfff3efbf},
1330 {0x00000094, 0x00101101},
1331 {0x00000095, 0x00000fff},
1332 {0x00000096, 0x00116fff},
1333 {0x00000097, 0x60010000},
1334 {0x00000098, 0x10010000},
1335 {0x00000099, 0x00006000},
1336 {0x0000009a, 0x00001000},
1337 {0x0000009f, 0x00a47400}
1340 static const u32 verde_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1341 {0x0000006f, 0x03044000},
1342 {0x00000070, 0x0480c018},
1343 {0x00000071, 0x00000040},
1344 {0x00000072, 0x01000000},
1345 {0x00000074, 0x000000ff},
1346 {0x00000075, 0x00143400},
1347 {0x00000076, 0x08ec0800},
1348 {0x00000077, 0x040000cc},
1349 {0x00000079, 0x00000000},
1350 {0x0000007a, 0x21000409},
1351 {0x0000007c, 0x00000000},
1352 {0x0000007d, 0xe8000000},
1353 {0x0000007e, 0x044408a8},
1354 {0x0000007f, 0x00000003},
1355 {0x00000080, 0x00000000},
1356 {0x00000081, 0x01000000},
1357 {0x00000082, 0x02000000},
1358 {0x00000083, 0x00000000},
1359 {0x00000084, 0xe3f3e4f4},
1360 {0x00000085, 0x00052024},
1361 {0x00000087, 0x00000000},
1362 {0x00000088, 0x66036603},
1363 {0x00000089, 0x01000000},
1364 {0x0000008b, 0x1c0a0000},
1365 {0x0000008c, 0xff010000},
1366 {0x0000008e, 0xffffefff},
1367 {0x0000008f, 0xfff3efff},
1368 {0x00000090, 0xfff3efbf},
1369 {0x00000094, 0x00101101},
1370 {0x00000095, 0x00000fff},
1371 {0x00000096, 0x00116fff},
1372 {0x00000097, 0x60010000},
1373 {0x00000098, 0x10010000},
1374 {0x00000099, 0x00006000},
1375 {0x0000009a, 0x00001000},
1376 {0x0000009f, 0x00a37400}
1379 static const u32 oland_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1380 {0x0000006f, 0x03044000},
1381 {0x00000070, 0x0480c018},
1382 {0x00000071, 0x00000040},
1383 {0x00000072, 0x01000000},
1384 {0x00000074, 0x000000ff},
1385 {0x00000075, 0x00143400},
1386 {0x00000076, 0x08ec0800},
1387 {0x00000077, 0x040000cc},
1388 {0x00000079, 0x00000000},
1389 {0x0000007a, 0x21000409},
1390 {0x0000007c, 0x00000000},
1391 {0x0000007d, 0xe8000000},
1392 {0x0000007e, 0x044408a8},
1393 {0x0000007f, 0x00000003},
1394 {0x00000080, 0x00000000},
1395 {0x00000081, 0x01000000},
1396 {0x00000082, 0x02000000},
1397 {0x00000083, 0x00000000},
1398 {0x00000084, 0xe3f3e4f4},
1399 {0x00000085, 0x00052024},
1400 {0x00000087, 0x00000000},
1401 {0x00000088, 0x66036603},
1402 {0x00000089, 0x01000000},
1403 {0x0000008b, 0x1c0a0000},
1404 {0x0000008c, 0xff010000},
1405 {0x0000008e, 0xffffefff},
1406 {0x0000008f, 0xfff3efff},
1407 {0x00000090, 0xfff3efbf},
1408 {0x00000094, 0x00101101},
1409 {0x00000095, 0x00000fff},
1410 {0x00000096, 0x00116fff},
1411 {0x00000097, 0x60010000},
1412 {0x00000098, 0x10010000},
1413 {0x00000099, 0x00006000},
1414 {0x0000009a, 0x00001000},
1415 {0x0000009f, 0x00a17730}
1418 static const u32 hainan_io_mc_regs
[TAHITI_IO_MC_REGS_SIZE
][2] = {
1419 {0x0000006f, 0x03044000},
1420 {0x00000070, 0x0480c018},
1421 {0x00000071, 0x00000040},
1422 {0x00000072, 0x01000000},
1423 {0x00000074, 0x000000ff},
1424 {0x00000075, 0x00143400},
1425 {0x00000076, 0x08ec0800},
1426 {0x00000077, 0x040000cc},
1427 {0x00000079, 0x00000000},
1428 {0x0000007a, 0x21000409},
1429 {0x0000007c, 0x00000000},
1430 {0x0000007d, 0xe8000000},
1431 {0x0000007e, 0x044408a8},
1432 {0x0000007f, 0x00000003},
1433 {0x00000080, 0x00000000},
1434 {0x00000081, 0x01000000},
1435 {0x00000082, 0x02000000},
1436 {0x00000083, 0x00000000},
1437 {0x00000084, 0xe3f3e4f4},
1438 {0x00000085, 0x00052024},
1439 {0x00000087, 0x00000000},
1440 {0x00000088, 0x66036603},
1441 {0x00000089, 0x01000000},
1442 {0x0000008b, 0x1c0a0000},
1443 {0x0000008c, 0xff010000},
1444 {0x0000008e, 0xffffefff},
1445 {0x0000008f, 0xfff3efff},
1446 {0x00000090, 0xfff3efbf},
1447 {0x00000094, 0x00101101},
1448 {0x00000095, 0x00000fff},
1449 {0x00000096, 0x00116fff},
1450 {0x00000097, 0x60010000},
1451 {0x00000098, 0x10010000},
1452 {0x00000099, 0x00006000},
1453 {0x0000009a, 0x00001000},
1454 {0x0000009f, 0x00a07730}
1458 static int si_mc_load_microcode(struct radeon_device
*rdev
)
1460 const __be32
*fw_data
;
1461 u32 running
, blackout
= 0;
1463 int i
, ucode_size
, regs_size
;
1468 switch (rdev
->family
) {
1470 io_mc_regs
= (u32
*)&tahiti_io_mc_regs
;
1471 ucode_size
= SI_MC_UCODE_SIZE
;
1472 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1475 io_mc_regs
= (u32
*)&pitcairn_io_mc_regs
;
1476 ucode_size
= SI_MC_UCODE_SIZE
;
1477 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1481 io_mc_regs
= (u32
*)&verde_io_mc_regs
;
1482 ucode_size
= SI_MC_UCODE_SIZE
;
1483 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1486 io_mc_regs
= (u32
*)&oland_io_mc_regs
;
1487 ucode_size
= OLAND_MC_UCODE_SIZE
;
1488 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1491 io_mc_regs
= (u32
*)&hainan_io_mc_regs
;
1492 ucode_size
= OLAND_MC_UCODE_SIZE
;
1493 regs_size
= TAHITI_IO_MC_REGS_SIZE
;
1497 running
= RREG32(MC_SEQ_SUP_CNTL
) & RUN_MASK
;
1501 blackout
= RREG32(MC_SHARED_BLACKOUT_CNTL
);
1502 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
| 1);
1505 /* reset the engine and set to writable */
1506 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1507 WREG32(MC_SEQ_SUP_CNTL
, 0x00000010);
1509 /* load mc io regs */
1510 for (i
= 0; i
< regs_size
; i
++) {
1511 WREG32(MC_SEQ_IO_DEBUG_INDEX
, io_mc_regs
[(i
<< 1)]);
1512 WREG32(MC_SEQ_IO_DEBUG_DATA
, io_mc_regs
[(i
<< 1) + 1]);
1514 /* load the MC ucode */
1515 fw_data
= (const __be32
*)rdev
->mc_fw
->data
;
1516 for (i
= 0; i
< ucode_size
; i
++)
1517 WREG32(MC_SEQ_SUP_PGM
, be32_to_cpup(fw_data
++));
1519 /* put the engine back into the active state */
1520 WREG32(MC_SEQ_SUP_CNTL
, 0x00000008);
1521 WREG32(MC_SEQ_SUP_CNTL
, 0x00000004);
1522 WREG32(MC_SEQ_SUP_CNTL
, 0x00000001);
1524 /* wait for training to complete */
1525 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1526 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D0
)
1530 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
1531 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL
) & TRAIN_DONE_D1
)
1537 WREG32(MC_SHARED_BLACKOUT_CNTL
, blackout
);
1543 static int si_init_microcode(struct radeon_device
*rdev
)
1545 const char *chip_name
;
1546 const char *rlc_chip_name
;
1547 size_t pfp_req_size
, me_req_size
, ce_req_size
, rlc_req_size
, mc_req_size
;
1548 size_t smc_req_size
;
1554 switch (rdev
->family
) {
1556 chip_name
= "TAHITI";
1557 rlc_chip_name
= "TAHITI";
1558 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1559 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1560 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1561 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1562 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1563 smc_req_size
= ALIGN(TAHITI_SMC_UCODE_SIZE
, 4);
1566 chip_name
= "PITCAIRN";
1567 rlc_chip_name
= "PITCAIRN";
1568 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1569 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1570 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1571 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1572 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1573 smc_req_size
= ALIGN(PITCAIRN_SMC_UCODE_SIZE
, 4);
1576 chip_name
= "VERDE";
1577 rlc_chip_name
= "VERDE";
1578 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1579 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1580 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1581 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1582 mc_req_size
= SI_MC_UCODE_SIZE
* 4;
1583 smc_req_size
= ALIGN(VERDE_SMC_UCODE_SIZE
, 4);
1586 chip_name
= "OLAND";
1587 rlc_chip_name
= "OLAND";
1588 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1589 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1590 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1591 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1592 mc_req_size
= OLAND_MC_UCODE_SIZE
* 4;
1593 smc_req_size
= ALIGN(OLAND_SMC_UCODE_SIZE
, 4);
1596 chip_name
= "HAINAN";
1597 rlc_chip_name
= "HAINAN";
1598 pfp_req_size
= SI_PFP_UCODE_SIZE
* 4;
1599 me_req_size
= SI_PM4_UCODE_SIZE
* 4;
1600 ce_req_size
= SI_CE_UCODE_SIZE
* 4;
1601 rlc_req_size
= SI_RLC_UCODE_SIZE
* 4;
1602 mc_req_size
= OLAND_MC_UCODE_SIZE
* 4;
1603 smc_req_size
= ALIGN(HAINAN_SMC_UCODE_SIZE
, 4);
1608 DRM_INFO("Loading %s Microcode\n", chip_name
);
1610 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_pfp.bin", chip_name
);
1611 err
= request_firmware(&rdev
->pfp_fw
, fw_name
, rdev
->dev
);
1614 if (rdev
->pfp_fw
->size
!= pfp_req_size
) {
1616 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1617 rdev
->pfp_fw
->size
, fw_name
);
1622 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_me.bin", chip_name
);
1623 err
= request_firmware(&rdev
->me_fw
, fw_name
, rdev
->dev
);
1626 if (rdev
->me_fw
->size
!= me_req_size
) {
1628 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1629 rdev
->me_fw
->size
, fw_name
);
1633 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_ce.bin", chip_name
);
1634 err
= request_firmware(&rdev
->ce_fw
, fw_name
, rdev
->dev
);
1637 if (rdev
->ce_fw
->size
!= ce_req_size
) {
1639 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1640 rdev
->ce_fw
->size
, fw_name
);
1644 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_rlc.bin", rlc_chip_name
);
1645 err
= request_firmware(&rdev
->rlc_fw
, fw_name
, rdev
->dev
);
1648 if (rdev
->rlc_fw
->size
!= rlc_req_size
) {
1650 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1651 rdev
->rlc_fw
->size
, fw_name
);
1655 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_mc.bin", chip_name
);
1656 err
= request_firmware(&rdev
->mc_fw
, fw_name
, rdev
->dev
);
1659 if (rdev
->mc_fw
->size
!= mc_req_size
) {
1661 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1662 rdev
->mc_fw
->size
, fw_name
);
1666 snprintf(fw_name
, sizeof(fw_name
), "radeon/%s_smc.bin", chip_name
);
1667 err
= request_firmware(&rdev
->smc_fw
, fw_name
, rdev
->dev
);
1670 "smc: error loading firmware \"%s\"\n",
1672 release_firmware(rdev
->smc_fw
);
1673 rdev
->smc_fw
= NULL
;
1674 } else if (rdev
->smc_fw
->size
!= smc_req_size
) {
1676 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1677 rdev
->smc_fw
->size
, fw_name
);
1685 "si_cp: Failed to load firmware \"%s\"\n",
1687 release_firmware(rdev
->pfp_fw
);
1688 rdev
->pfp_fw
= NULL
;
1689 release_firmware(rdev
->me_fw
);
1691 release_firmware(rdev
->ce_fw
);
1693 release_firmware(rdev
->rlc_fw
);
1694 rdev
->rlc_fw
= NULL
;
1695 release_firmware(rdev
->mc_fw
);
1697 release_firmware(rdev
->smc_fw
);
1698 rdev
->smc_fw
= NULL
;
1703 /* watermark setup */
1704 static u32
dce6_line_buffer_adjust(struct radeon_device
*rdev
,
1705 struct radeon_crtc
*radeon_crtc
,
1706 struct drm_display_mode
*mode
,
1707 struct drm_display_mode
*other_mode
)
1712 * There are 3 line buffers, each one shared by 2 display controllers.
1713 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1714 * the display controllers. The paritioning is done via one of four
1715 * preset allocations specified in bits 21:20:
1717 * 2 - whole lb, other crtc must be disabled
1719 /* this can get tricky if we have two large displays on a paired group
1720 * of crtcs. Ideally for multiple large displays we'd assign them to
1721 * non-linked crtcs for maximum line buffer allocation.
1723 if (radeon_crtc
->base
.enabled
&& mode
) {
1727 tmp
= 2; /* whole */
1731 WREG32(DC_LB_MEMORY_SPLIT
+ radeon_crtc
->crtc_offset
,
1732 DC_LB_MEMORY_CONFIG(tmp
));
1734 if (radeon_crtc
->base
.enabled
&& mode
) {
1744 /* controller not enabled, so no lb used */
1748 static u32
si_get_number_of_dram_channels(struct radeon_device
*rdev
)
1750 u32 tmp
= RREG32(MC_SHARED_CHMAP
);
1752 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
1775 struct dce6_wm_params
{
1776 u32 dram_channels
; /* number of dram channels */
1777 u32 yclk
; /* bandwidth per dram data pin in kHz */
1778 u32 sclk
; /* engine clock in kHz */
1779 u32 disp_clk
; /* display clock in kHz */
1780 u32 src_width
; /* viewport width */
1781 u32 active_time
; /* active display time in ns */
1782 u32 blank_time
; /* blank time in ns */
1783 bool interlaced
; /* mode is interlaced */
1784 fixed20_12 vsc
; /* vertical scale ratio */
1785 u32 num_heads
; /* number of active crtcs */
1786 u32 bytes_per_pixel
; /* bytes per pixel display + overlay */
1787 u32 lb_size
; /* line buffer allocated to pipe */
1788 u32 vtaps
; /* vertical scaler taps */
1791 static u32
dce6_dram_bandwidth(struct dce6_wm_params
*wm
)
1793 /* Calculate raw DRAM Bandwidth */
1794 fixed20_12 dram_efficiency
; /* 0.7 */
1795 fixed20_12 yclk
, dram_channels
, bandwidth
;
1798 a
.full
= dfixed_const(1000);
1799 yclk
.full
= dfixed_const(wm
->yclk
);
1800 yclk
.full
= dfixed_div(yclk
, a
);
1801 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
1802 a
.full
= dfixed_const(10);
1803 dram_efficiency
.full
= dfixed_const(7);
1804 dram_efficiency
.full
= dfixed_div(dram_efficiency
, a
);
1805 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
1806 bandwidth
.full
= dfixed_mul(bandwidth
, dram_efficiency
);
1808 return dfixed_trunc(bandwidth
);
1811 static u32
dce6_dram_bandwidth_for_display(struct dce6_wm_params
*wm
)
1813 /* Calculate DRAM Bandwidth and the part allocated to display. */
1814 fixed20_12 disp_dram_allocation
; /* 0.3 to 0.7 */
1815 fixed20_12 yclk
, dram_channels
, bandwidth
;
1818 a
.full
= dfixed_const(1000);
1819 yclk
.full
= dfixed_const(wm
->yclk
);
1820 yclk
.full
= dfixed_div(yclk
, a
);
1821 dram_channels
.full
= dfixed_const(wm
->dram_channels
* 4);
1822 a
.full
= dfixed_const(10);
1823 disp_dram_allocation
.full
= dfixed_const(3); /* XXX worse case value 0.3 */
1824 disp_dram_allocation
.full
= dfixed_div(disp_dram_allocation
, a
);
1825 bandwidth
.full
= dfixed_mul(dram_channels
, yclk
);
1826 bandwidth
.full
= dfixed_mul(bandwidth
, disp_dram_allocation
);
1828 return dfixed_trunc(bandwidth
);
1831 static u32
dce6_data_return_bandwidth(struct dce6_wm_params
*wm
)
1833 /* Calculate the display Data return Bandwidth */
1834 fixed20_12 return_efficiency
; /* 0.8 */
1835 fixed20_12 sclk
, bandwidth
;
1838 a
.full
= dfixed_const(1000);
1839 sclk
.full
= dfixed_const(wm
->sclk
);
1840 sclk
.full
= dfixed_div(sclk
, a
);
1841 a
.full
= dfixed_const(10);
1842 return_efficiency
.full
= dfixed_const(8);
1843 return_efficiency
.full
= dfixed_div(return_efficiency
, a
);
1844 a
.full
= dfixed_const(32);
1845 bandwidth
.full
= dfixed_mul(a
, sclk
);
1846 bandwidth
.full
= dfixed_mul(bandwidth
, return_efficiency
);
1848 return dfixed_trunc(bandwidth
);
1851 static u32
dce6_get_dmif_bytes_per_request(struct dce6_wm_params
*wm
)
1856 static u32
dce6_dmif_request_bandwidth(struct dce6_wm_params
*wm
)
1858 /* Calculate the DMIF Request Bandwidth */
1859 fixed20_12 disp_clk_request_efficiency
; /* 0.8 */
1860 fixed20_12 disp_clk
, sclk
, bandwidth
;
1861 fixed20_12 a
, b1
, b2
;
1864 a
.full
= dfixed_const(1000);
1865 disp_clk
.full
= dfixed_const(wm
->disp_clk
);
1866 disp_clk
.full
= dfixed_div(disp_clk
, a
);
1867 a
.full
= dfixed_const(dce6_get_dmif_bytes_per_request(wm
) / 2);
1868 b1
.full
= dfixed_mul(a
, disp_clk
);
1870 a
.full
= dfixed_const(1000);
1871 sclk
.full
= dfixed_const(wm
->sclk
);
1872 sclk
.full
= dfixed_div(sclk
, a
);
1873 a
.full
= dfixed_const(dce6_get_dmif_bytes_per_request(wm
));
1874 b2
.full
= dfixed_mul(a
, sclk
);
1876 a
.full
= dfixed_const(10);
1877 disp_clk_request_efficiency
.full
= dfixed_const(8);
1878 disp_clk_request_efficiency
.full
= dfixed_div(disp_clk_request_efficiency
, a
);
1880 min_bandwidth
= min(dfixed_trunc(b1
), dfixed_trunc(b2
));
1882 a
.full
= dfixed_const(min_bandwidth
);
1883 bandwidth
.full
= dfixed_mul(a
, disp_clk_request_efficiency
);
1885 return dfixed_trunc(bandwidth
);
1888 static u32
dce6_available_bandwidth(struct dce6_wm_params
*wm
)
1890 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1891 u32 dram_bandwidth
= dce6_dram_bandwidth(wm
);
1892 u32 data_return_bandwidth
= dce6_data_return_bandwidth(wm
);
1893 u32 dmif_req_bandwidth
= dce6_dmif_request_bandwidth(wm
);
1895 return min(dram_bandwidth
, min(data_return_bandwidth
, dmif_req_bandwidth
));
1898 static u32
dce6_average_bandwidth(struct dce6_wm_params
*wm
)
1900 /* Calculate the display mode Average Bandwidth
1901 * DisplayMode should contain the source and destination dimensions,
1905 fixed20_12 line_time
;
1906 fixed20_12 src_width
;
1907 fixed20_12 bandwidth
;
1910 a
.full
= dfixed_const(1000);
1911 line_time
.full
= dfixed_const(wm
->active_time
+ wm
->blank_time
);
1912 line_time
.full
= dfixed_div(line_time
, a
);
1913 bpp
.full
= dfixed_const(wm
->bytes_per_pixel
);
1914 src_width
.full
= dfixed_const(wm
->src_width
);
1915 bandwidth
.full
= dfixed_mul(src_width
, bpp
);
1916 bandwidth
.full
= dfixed_mul(bandwidth
, wm
->vsc
);
1917 bandwidth
.full
= dfixed_div(bandwidth
, line_time
);
1919 return dfixed_trunc(bandwidth
);
1922 static u32
dce6_latency_watermark(struct dce6_wm_params
*wm
)
1924 /* First calcualte the latency in ns */
1925 u32 mc_latency
= 2000; /* 2000 ns. */
1926 u32 available_bandwidth
= dce6_available_bandwidth(wm
);
1927 u32 worst_chunk_return_time
= (512 * 8 * 1000) / available_bandwidth
;
1928 u32 cursor_line_pair_return_time
= (128 * 4 * 1000) / available_bandwidth
;
1929 u32 dc_latency
= 40000000 / wm
->disp_clk
; /* dc pipe latency */
1930 u32 other_heads_data_return_time
= ((wm
->num_heads
+ 1) * worst_chunk_return_time
) +
1931 (wm
->num_heads
* cursor_line_pair_return_time
);
1932 u32 latency
= mc_latency
+ other_heads_data_return_time
+ dc_latency
;
1933 u32 max_src_lines_per_dst_line
, lb_fill_bw
, line_fill_time
;
1934 u32 tmp
, dmif_size
= 12288;
1937 if (wm
->num_heads
== 0)
1940 a
.full
= dfixed_const(2);
1941 b
.full
= dfixed_const(1);
1942 if ((wm
->vsc
.full
> a
.full
) ||
1943 ((wm
->vsc
.full
> b
.full
) && (wm
->vtaps
>= 3)) ||
1945 ((wm
->vsc
.full
>= a
.full
) && wm
->interlaced
))
1946 max_src_lines_per_dst_line
= 4;
1948 max_src_lines_per_dst_line
= 2;
1950 a
.full
= dfixed_const(available_bandwidth
);
1951 b
.full
= dfixed_const(wm
->num_heads
);
1952 a
.full
= dfixed_div(a
, b
);
1954 b
.full
= dfixed_const(mc_latency
+ 512);
1955 c
.full
= dfixed_const(wm
->disp_clk
);
1956 b
.full
= dfixed_div(b
, c
);
1958 c
.full
= dfixed_const(dmif_size
);
1959 b
.full
= dfixed_div(c
, b
);
1961 tmp
= min(dfixed_trunc(a
), dfixed_trunc(b
));
1963 b
.full
= dfixed_const(1000);
1964 c
.full
= dfixed_const(wm
->disp_clk
);
1965 b
.full
= dfixed_div(c
, b
);
1966 c
.full
= dfixed_const(wm
->bytes_per_pixel
);
1967 b
.full
= dfixed_mul(b
, c
);
1969 lb_fill_bw
= min(tmp
, dfixed_trunc(b
));
1971 a
.full
= dfixed_const(max_src_lines_per_dst_line
* wm
->src_width
* wm
->bytes_per_pixel
);
1972 b
.full
= dfixed_const(1000);
1973 c
.full
= dfixed_const(lb_fill_bw
);
1974 b
.full
= dfixed_div(c
, b
);
1975 a
.full
= dfixed_div(a
, b
);
1976 line_fill_time
= dfixed_trunc(a
);
1978 if (line_fill_time
< wm
->active_time
)
1981 return latency
+ (line_fill_time
- wm
->active_time
);
1985 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params
*wm
)
1987 if (dce6_average_bandwidth(wm
) <=
1988 (dce6_dram_bandwidth_for_display(wm
) / wm
->num_heads
))
1994 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params
*wm
)
1996 if (dce6_average_bandwidth(wm
) <=
1997 (dce6_available_bandwidth(wm
) / wm
->num_heads
))
2003 static bool dce6_check_latency_hiding(struct dce6_wm_params
*wm
)
2005 u32 lb_partitions
= wm
->lb_size
/ wm
->src_width
;
2006 u32 line_time
= wm
->active_time
+ wm
->blank_time
;
2007 u32 latency_tolerant_lines
;
2011 a
.full
= dfixed_const(1);
2012 if (wm
->vsc
.full
> a
.full
)
2013 latency_tolerant_lines
= 1;
2015 if (lb_partitions
<= (wm
->vtaps
+ 1))
2016 latency_tolerant_lines
= 1;
2018 latency_tolerant_lines
= 2;
2021 latency_hiding
= (latency_tolerant_lines
* line_time
+ wm
->blank_time
);
2023 if (dce6_latency_watermark(wm
) <= latency_hiding
)
2029 static void dce6_program_watermarks(struct radeon_device
*rdev
,
2030 struct radeon_crtc
*radeon_crtc
,
2031 u32 lb_size
, u32 num_heads
)
2033 struct drm_display_mode
*mode
= &radeon_crtc
->base
.mode
;
2034 struct dce6_wm_params wm_low
, wm_high
;
2038 u32 latency_watermark_a
= 0, latency_watermark_b
= 0;
2039 u32 priority_a_mark
= 0, priority_b_mark
= 0;
2040 u32 priority_a_cnt
= PRIORITY_OFF
;
2041 u32 priority_b_cnt
= PRIORITY_OFF
;
2042 u32 tmp
, arb_control3
;
2045 if (radeon_crtc
->base
.enabled
&& num_heads
&& mode
) {
2046 pixel_period
= 1000000 / (u32
)mode
->clock
;
2047 line_time
= min((u32
)mode
->crtc_htotal
* pixel_period
, (u32
)65535);
2051 if (rdev
->family
== CHIP_ARUBA
)
2052 dram_channels
= evergreen_get_number_of_dram_channels(rdev
);
2054 dram_channels
= si_get_number_of_dram_channels(rdev
);
2056 /* watermark for high clocks */
2057 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) && rdev
->pm
.dpm_enabled
) {
2059 radeon_dpm_get_mclk(rdev
, false) * 10;
2061 radeon_dpm_get_sclk(rdev
, false) * 10;
2063 wm_high
.yclk
= rdev
->pm
.current_mclk
* 10;
2064 wm_high
.sclk
= rdev
->pm
.current_sclk
* 10;
2067 wm_high
.disp_clk
= mode
->clock
;
2068 wm_high
.src_width
= mode
->crtc_hdisplay
;
2069 wm_high
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
2070 wm_high
.blank_time
= line_time
- wm_high
.active_time
;
2071 wm_high
.interlaced
= false;
2072 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
2073 wm_high
.interlaced
= true;
2074 wm_high
.vsc
= radeon_crtc
->vsc
;
2076 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
2078 wm_high
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
2079 wm_high
.lb_size
= lb_size
;
2080 wm_high
.dram_channels
= dram_channels
;
2081 wm_high
.num_heads
= num_heads
;
2083 /* watermark for low clocks */
2084 if ((rdev
->pm
.pm_method
== PM_METHOD_DPM
) && rdev
->pm
.dpm_enabled
) {
2086 radeon_dpm_get_mclk(rdev
, true) * 10;
2088 radeon_dpm_get_sclk(rdev
, true) * 10;
2090 wm_low
.yclk
= rdev
->pm
.current_mclk
* 10;
2091 wm_low
.sclk
= rdev
->pm
.current_sclk
* 10;
2094 wm_low
.disp_clk
= mode
->clock
;
2095 wm_low
.src_width
= mode
->crtc_hdisplay
;
2096 wm_low
.active_time
= mode
->crtc_hdisplay
* pixel_period
;
2097 wm_low
.blank_time
= line_time
- wm_low
.active_time
;
2098 wm_low
.interlaced
= false;
2099 if (mode
->flags
& DRM_MODE_FLAG_INTERLACE
)
2100 wm_low
.interlaced
= true;
2101 wm_low
.vsc
= radeon_crtc
->vsc
;
2103 if (radeon_crtc
->rmx_type
!= RMX_OFF
)
2105 wm_low
.bytes_per_pixel
= 4; /* XXX: get this from fb config */
2106 wm_low
.lb_size
= lb_size
;
2107 wm_low
.dram_channels
= dram_channels
;
2108 wm_low
.num_heads
= num_heads
;
2110 /* set for high clocks */
2111 latency_watermark_a
= min(dce6_latency_watermark(&wm_high
), (u32
)65535);
2112 /* set for low clocks */
2113 latency_watermark_b
= min(dce6_latency_watermark(&wm_low
), (u32
)65535);
2115 /* possibly force display priority to high */
2116 /* should really do this at mode validation time... */
2117 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high
) ||
2118 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high
) ||
2119 !dce6_check_latency_hiding(&wm_high
) ||
2120 (rdev
->disp_priority
== 2)) {
2121 DRM_DEBUG_KMS("force priority to high\n");
2122 priority_a_cnt
|= PRIORITY_ALWAYS_ON
;
2123 priority_b_cnt
|= PRIORITY_ALWAYS_ON
;
2125 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low
) ||
2126 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low
) ||
2127 !dce6_check_latency_hiding(&wm_low
) ||
2128 (rdev
->disp_priority
== 2)) {
2129 DRM_DEBUG_KMS("force priority to high\n");
2130 priority_a_cnt
|= PRIORITY_ALWAYS_ON
;
2131 priority_b_cnt
|= PRIORITY_ALWAYS_ON
;
2134 a
.full
= dfixed_const(1000);
2135 b
.full
= dfixed_const(mode
->clock
);
2136 b
.full
= dfixed_div(b
, a
);
2137 c
.full
= dfixed_const(latency_watermark_a
);
2138 c
.full
= dfixed_mul(c
, b
);
2139 c
.full
= dfixed_mul(c
, radeon_crtc
->hsc
);
2140 c
.full
= dfixed_div(c
, a
);
2141 a
.full
= dfixed_const(16);
2142 c
.full
= dfixed_div(c
, a
);
2143 priority_a_mark
= dfixed_trunc(c
);
2144 priority_a_cnt
|= priority_a_mark
& PRIORITY_MARK_MASK
;
2146 a
.full
= dfixed_const(1000);
2147 b
.full
= dfixed_const(mode
->clock
);
2148 b
.full
= dfixed_div(b
, a
);
2149 c
.full
= dfixed_const(latency_watermark_b
);
2150 c
.full
= dfixed_mul(c
, b
);
2151 c
.full
= dfixed_mul(c
, radeon_crtc
->hsc
);
2152 c
.full
= dfixed_div(c
, a
);
2153 a
.full
= dfixed_const(16);
2154 c
.full
= dfixed_div(c
, a
);
2155 priority_b_mark
= dfixed_trunc(c
);
2156 priority_b_cnt
|= priority_b_mark
& PRIORITY_MARK_MASK
;
2160 arb_control3
= RREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
);
2162 tmp
&= ~LATENCY_WATERMARK_MASK(3);
2163 tmp
|= LATENCY_WATERMARK_MASK(1);
2164 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, tmp
);
2165 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
2166 (LATENCY_LOW_WATERMARK(latency_watermark_a
) |
2167 LATENCY_HIGH_WATERMARK(line_time
)));
2169 tmp
= RREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
);
2170 tmp
&= ~LATENCY_WATERMARK_MASK(3);
2171 tmp
|= LATENCY_WATERMARK_MASK(2);
2172 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, tmp
);
2173 WREG32(DPG_PIPE_LATENCY_CONTROL
+ radeon_crtc
->crtc_offset
,
2174 (LATENCY_LOW_WATERMARK(latency_watermark_b
) |
2175 LATENCY_HIGH_WATERMARK(line_time
)));
2176 /* restore original selection */
2177 WREG32(DPG_PIPE_ARBITRATION_CONTROL3
+ radeon_crtc
->crtc_offset
, arb_control3
);
2179 /* write the priority marks */
2180 WREG32(PRIORITY_A_CNT
+ radeon_crtc
->crtc_offset
, priority_a_cnt
);
2181 WREG32(PRIORITY_B_CNT
+ radeon_crtc
->crtc_offset
, priority_b_cnt
);
2183 /* save values for DPM */
2184 radeon_crtc
->line_time
= line_time
;
2185 radeon_crtc
->wm_high
= latency_watermark_a
;
2186 radeon_crtc
->wm_low
= latency_watermark_b
;
2189 void dce6_bandwidth_update(struct radeon_device
*rdev
)
2191 struct drm_display_mode
*mode0
= NULL
;
2192 struct drm_display_mode
*mode1
= NULL
;
2193 u32 num_heads
= 0, lb_size
;
2196 radeon_update_display_priority(rdev
);
2198 for (i
= 0; i
< rdev
->num_crtc
; i
++) {
2199 if (rdev
->mode_info
.crtcs
[i
]->base
.enabled
)
2202 for (i
= 0; i
< rdev
->num_crtc
; i
+= 2) {
2203 mode0
= &rdev
->mode_info
.crtcs
[i
]->base
.mode
;
2204 mode1
= &rdev
->mode_info
.crtcs
[i
+1]->base
.mode
;
2205 lb_size
= dce6_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
], mode0
, mode1
);
2206 dce6_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
], lb_size
, num_heads
);
2207 lb_size
= dce6_line_buffer_adjust(rdev
, rdev
->mode_info
.crtcs
[i
+1], mode1
, mode0
);
2208 dce6_program_watermarks(rdev
, rdev
->mode_info
.crtcs
[i
+1], lb_size
, num_heads
);
2215 static void si_tiling_mode_table_init(struct radeon_device
*rdev
)
2217 const u32 num_tile_mode_states
= 32;
2218 u32 reg_offset
, gb_tile_moden
, split_equal_to_row_size
;
2220 switch (rdev
->config
.si
.mem_row_size_in_kb
) {
2222 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_1KB
;
2226 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_2KB
;
2229 split_equal_to_row_size
= ADDR_SURF_TILE_SPLIT_4KB
;
2233 if ((rdev
->family
== CHIP_TAHITI
) ||
2234 (rdev
->family
== CHIP_PITCAIRN
)) {
2235 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2236 switch (reg_offset
) {
2237 case 0: /* non-AA compressed depth or any compressed stencil */
2238 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2239 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2240 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2241 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2242 NUM_BANKS(ADDR_SURF_16_BANK
) |
2243 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2244 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2245 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2247 case 1: /* 2xAA/4xAA compressed depth only */
2248 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2249 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2250 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2251 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2252 NUM_BANKS(ADDR_SURF_16_BANK
) |
2253 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2254 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2255 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2257 case 2: /* 8xAA compressed depth only */
2258 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2259 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2261 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2262 NUM_BANKS(ADDR_SURF_16_BANK
) |
2263 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2264 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2265 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2267 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2268 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2269 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2270 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2272 NUM_BANKS(ADDR_SURF_16_BANK
) |
2273 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2274 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2275 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2277 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2278 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2279 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2281 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2282 NUM_BANKS(ADDR_SURF_16_BANK
) |
2283 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2284 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2285 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2287 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2288 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2289 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2290 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2291 TILE_SPLIT(split_equal_to_row_size
) |
2292 NUM_BANKS(ADDR_SURF_16_BANK
) |
2293 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2294 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2295 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2297 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2298 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2299 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2300 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2301 TILE_SPLIT(split_equal_to_row_size
) |
2302 NUM_BANKS(ADDR_SURF_16_BANK
) |
2303 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2304 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2305 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2307 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2308 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2309 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2310 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2311 TILE_SPLIT(split_equal_to_row_size
) |
2312 NUM_BANKS(ADDR_SURF_16_BANK
) |
2313 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2317 case 8: /* 1D and 1D Array Surfaces */
2318 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2319 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2320 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2321 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2322 NUM_BANKS(ADDR_SURF_16_BANK
) |
2323 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2324 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2325 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2327 case 9: /* Displayable maps. */
2328 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2329 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2332 NUM_BANKS(ADDR_SURF_16_BANK
) |
2333 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2334 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2335 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2337 case 10: /* Display 8bpp. */
2338 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2339 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2340 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2341 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2342 NUM_BANKS(ADDR_SURF_16_BANK
) |
2343 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2347 case 11: /* Display 16bpp. */
2348 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2349 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2350 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2351 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2352 NUM_BANKS(ADDR_SURF_16_BANK
) |
2353 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2354 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2355 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2357 case 12: /* Display 32bpp. */
2358 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2359 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2360 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2361 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2362 NUM_BANKS(ADDR_SURF_16_BANK
) |
2363 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2364 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2365 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2367 case 13: /* Thin. */
2368 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2369 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2370 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2371 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2372 NUM_BANKS(ADDR_SURF_16_BANK
) |
2373 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2377 case 14: /* Thin 8 bpp. */
2378 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2379 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2380 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2381 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2382 NUM_BANKS(ADDR_SURF_16_BANK
) |
2383 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2384 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2385 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2387 case 15: /* Thin 16 bpp. */
2388 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2389 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2390 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2391 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2392 NUM_BANKS(ADDR_SURF_16_BANK
) |
2393 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2394 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2395 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2397 case 16: /* Thin 32 bpp. */
2398 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2399 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2400 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2401 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2402 NUM_BANKS(ADDR_SURF_16_BANK
) |
2403 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2407 case 17: /* Thin 64 bpp. */
2408 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2409 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2410 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2411 TILE_SPLIT(split_equal_to_row_size
) |
2412 NUM_BANKS(ADDR_SURF_16_BANK
) |
2413 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2414 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2415 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2417 case 21: /* 8 bpp PRT. */
2418 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2419 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2420 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2421 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2422 NUM_BANKS(ADDR_SURF_16_BANK
) |
2423 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2424 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2425 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2427 case 22: /* 16 bpp PRT */
2428 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2429 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2430 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2431 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2432 NUM_BANKS(ADDR_SURF_16_BANK
) |
2433 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2434 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2435 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2437 case 23: /* 32 bpp PRT */
2438 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2439 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2440 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2442 NUM_BANKS(ADDR_SURF_16_BANK
) |
2443 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2447 case 24: /* 64 bpp PRT */
2448 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2449 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2450 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2451 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2452 NUM_BANKS(ADDR_SURF_16_BANK
) |
2453 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2457 case 25: /* 128 bpp PRT */
2458 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2459 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2460 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2461 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB
) |
2462 NUM_BANKS(ADDR_SURF_8_BANK
) |
2463 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2471 rdev
->config
.si
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2472 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2474 } else if ((rdev
->family
== CHIP_VERDE
) ||
2475 (rdev
->family
== CHIP_OLAND
) ||
2476 (rdev
->family
== CHIP_HAINAN
)) {
2477 for (reg_offset
= 0; reg_offset
< num_tile_mode_states
; reg_offset
++) {
2478 switch (reg_offset
) {
2479 case 0: /* non-AA compressed depth or any compressed stencil */
2480 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2481 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2482 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2483 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2484 NUM_BANKS(ADDR_SURF_16_BANK
) |
2485 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2489 case 1: /* 2xAA/4xAA compressed depth only */
2490 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2491 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2492 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2493 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2494 NUM_BANKS(ADDR_SURF_16_BANK
) |
2495 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2496 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2497 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2499 case 2: /* 8xAA compressed depth only */
2500 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2501 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2502 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2503 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2504 NUM_BANKS(ADDR_SURF_16_BANK
) |
2505 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2509 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2510 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2511 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2512 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2513 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B
) |
2514 NUM_BANKS(ADDR_SURF_16_BANK
) |
2515 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2516 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2517 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2519 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2520 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2521 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2523 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2524 NUM_BANKS(ADDR_SURF_16_BANK
) |
2525 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2526 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2527 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2529 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2530 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2531 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2532 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2533 TILE_SPLIT(split_equal_to_row_size
) |
2534 NUM_BANKS(ADDR_SURF_16_BANK
) |
2535 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2536 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2537 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2539 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2540 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2541 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2542 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2543 TILE_SPLIT(split_equal_to_row_size
) |
2544 NUM_BANKS(ADDR_SURF_16_BANK
) |
2545 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2546 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2547 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2549 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2550 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2551 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING
) |
2552 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2553 TILE_SPLIT(split_equal_to_row_size
) |
2554 NUM_BANKS(ADDR_SURF_16_BANK
) |
2555 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2559 case 8: /* 1D and 1D Array Surfaces */
2560 gb_tile_moden
= (ARRAY_MODE(ARRAY_LINEAR_ALIGNED
) |
2561 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2562 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2564 NUM_BANKS(ADDR_SURF_16_BANK
) |
2565 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2566 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2567 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2569 case 9: /* Displayable maps. */
2570 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2571 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2573 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2574 NUM_BANKS(ADDR_SURF_16_BANK
) |
2575 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2576 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2577 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2579 case 10: /* Display 8bpp. */
2580 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2581 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2582 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2583 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2584 NUM_BANKS(ADDR_SURF_16_BANK
) |
2585 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2586 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2587 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2589 case 11: /* Display 16bpp. */
2590 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2591 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2592 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2593 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2594 NUM_BANKS(ADDR_SURF_16_BANK
) |
2595 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2596 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2597 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2599 case 12: /* Display 32bpp. */
2600 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2601 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING
) |
2602 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2603 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2604 NUM_BANKS(ADDR_SURF_16_BANK
) |
2605 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2606 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2607 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2609 case 13: /* Thin. */
2610 gb_tile_moden
= (ARRAY_MODE(ARRAY_1D_TILED_THIN1
) |
2611 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2612 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2613 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B
) |
2614 NUM_BANKS(ADDR_SURF_16_BANK
) |
2615 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2619 case 14: /* Thin 8 bpp. */
2620 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2621 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2622 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2623 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2624 NUM_BANKS(ADDR_SURF_16_BANK
) |
2625 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2626 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2627 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2629 case 15: /* Thin 16 bpp. */
2630 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2631 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2633 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2634 NUM_BANKS(ADDR_SURF_16_BANK
) |
2635 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2639 case 16: /* Thin 32 bpp. */
2640 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2641 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2643 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2644 NUM_BANKS(ADDR_SURF_16_BANK
) |
2645 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2646 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2647 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2649 case 17: /* Thin 64 bpp. */
2650 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2651 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2652 PIPE_CONFIG(ADDR_SURF_P4_8x16
) |
2653 TILE_SPLIT(split_equal_to_row_size
) |
2654 NUM_BANKS(ADDR_SURF_16_BANK
) |
2655 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2656 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2657 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2659 case 21: /* 8 bpp PRT. */
2660 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2661 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2662 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2664 NUM_BANKS(ADDR_SURF_16_BANK
) |
2665 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2
) |
2666 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2667 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2669 case 22: /* 16 bpp PRT */
2670 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2671 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2672 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2673 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2674 NUM_BANKS(ADDR_SURF_16_BANK
) |
2675 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2676 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4
) |
2677 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4
));
2679 case 23: /* 32 bpp PRT */
2680 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2681 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2682 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2683 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B
) |
2684 NUM_BANKS(ADDR_SURF_16_BANK
) |
2685 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2686 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2
) |
2687 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2689 case 24: /* 64 bpp PRT */
2690 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2691 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2692 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2693 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B
) |
2694 NUM_BANKS(ADDR_SURF_16_BANK
) |
2695 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2
));
2699 case 25: /* 128 bpp PRT */
2700 gb_tile_moden
= (ARRAY_MODE(ARRAY_2D_TILED_THIN1
) |
2701 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING
) |
2702 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16
) |
2703 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB
) |
2704 NUM_BANKS(ADDR_SURF_8_BANK
) |
2705 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1
) |
2706 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1
) |
2707 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1
));
2713 rdev
->config
.si
.tile_mode_array
[reg_offset
] = gb_tile_moden
;
2714 WREG32(GB_TILE_MODE0
+ (reg_offset
* 4), gb_tile_moden
);
2717 DRM_ERROR("unknown asic: 0x%x\n", rdev
->family
);
2720 static void si_select_se_sh(struct radeon_device
*rdev
,
2721 u32 se_num
, u32 sh_num
)
2723 u32 data
= INSTANCE_BROADCAST_WRITES
;
2725 if ((se_num
== 0xffffffff) && (sh_num
== 0xffffffff))
2726 data
|= SH_BROADCAST_WRITES
| SE_BROADCAST_WRITES
;
2727 else if (se_num
== 0xffffffff)
2728 data
|= SE_BROADCAST_WRITES
| SH_INDEX(sh_num
);
2729 else if (sh_num
== 0xffffffff)
2730 data
|= SH_BROADCAST_WRITES
| SE_INDEX(se_num
);
2732 data
|= SH_INDEX(sh_num
) | SE_INDEX(se_num
);
2733 WREG32(GRBM_GFX_INDEX
, data
);
2736 static u32
si_create_bitmask(u32 bit_width
)
2740 for (i
= 0; i
< bit_width
; i
++) {
2747 static u32
si_get_cu_enabled(struct radeon_device
*rdev
, u32 cu_per_sh
)
2751 data
= RREG32(CC_GC_SHADER_ARRAY_CONFIG
);
2753 data
&= INACTIVE_CUS_MASK
;
2756 data
|= RREG32(GC_USER_SHADER_ARRAY_CONFIG
);
2758 data
>>= INACTIVE_CUS_SHIFT
;
2760 mask
= si_create_bitmask(cu_per_sh
);
2762 return ~data
& mask
;
2765 static void si_setup_spi(struct radeon_device
*rdev
,
2766 u32 se_num
, u32 sh_per_se
,
2770 u32 data
, mask
, active_cu
;
2772 for (i
= 0; i
< se_num
; i
++) {
2773 for (j
= 0; j
< sh_per_se
; j
++) {
2774 si_select_se_sh(rdev
, i
, j
);
2775 data
= RREG32(SPI_STATIC_THREAD_MGMT_3
);
2776 active_cu
= si_get_cu_enabled(rdev
, cu_per_sh
);
2779 for (k
= 0; k
< 16; k
++) {
2781 if (active_cu
& mask
) {
2783 WREG32(SPI_STATIC_THREAD_MGMT_3
, data
);
2789 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2792 static u32
si_get_rb_disabled(struct radeon_device
*rdev
,
2793 u32 max_rb_num
, u32 se_num
,
2798 data
= RREG32(CC_RB_BACKEND_DISABLE
);
2800 data
&= BACKEND_DISABLE_MASK
;
2803 data
|= RREG32(GC_USER_RB_BACKEND_DISABLE
);
2805 data
>>= BACKEND_DISABLE_SHIFT
;
2807 mask
= si_create_bitmask(max_rb_num
/ se_num
/ sh_per_se
);
2812 static void si_setup_rb(struct radeon_device
*rdev
,
2813 u32 se_num
, u32 sh_per_se
,
2818 u32 disabled_rbs
= 0;
2819 u32 enabled_rbs
= 0;
2821 for (i
= 0; i
< se_num
; i
++) {
2822 for (j
= 0; j
< sh_per_se
; j
++) {
2823 si_select_se_sh(rdev
, i
, j
);
2824 data
= si_get_rb_disabled(rdev
, max_rb_num
, se_num
, sh_per_se
);
2825 disabled_rbs
|= data
<< ((i
* sh_per_se
+ j
) * TAHITI_RB_BITMAP_WIDTH_PER_SH
);
2828 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2831 for (i
= 0; i
< max_rb_num
; i
++) {
2832 if (!(disabled_rbs
& mask
))
2833 enabled_rbs
|= mask
;
2837 for (i
= 0; i
< se_num
; i
++) {
2838 si_select_se_sh(rdev
, i
, 0xffffffff);
2840 for (j
= 0; j
< sh_per_se
; j
++) {
2841 switch (enabled_rbs
& 3) {
2843 data
|= (RASTER_CONFIG_RB_MAP_0
<< (i
* sh_per_se
+ j
) * 2);
2846 data
|= (RASTER_CONFIG_RB_MAP_3
<< (i
* sh_per_se
+ j
) * 2);
2850 data
|= (RASTER_CONFIG_RB_MAP_2
<< (i
* sh_per_se
+ j
) * 2);
2855 WREG32(PA_SC_RASTER_CONFIG
, data
);
2857 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
2860 static void si_gpu_init(struct radeon_device
*rdev
)
2862 u32 gb_addr_config
= 0;
2863 u32 mc_shared_chmap
, mc_arb_ramcfg
;
2865 u32 hdp_host_path_cntl
;
2869 switch (rdev
->family
) {
2871 rdev
->config
.si
.max_shader_engines
= 2;
2872 rdev
->config
.si
.max_tile_pipes
= 12;
2873 rdev
->config
.si
.max_cu_per_sh
= 8;
2874 rdev
->config
.si
.max_sh_per_se
= 2;
2875 rdev
->config
.si
.max_backends_per_se
= 4;
2876 rdev
->config
.si
.max_texture_channel_caches
= 12;
2877 rdev
->config
.si
.max_gprs
= 256;
2878 rdev
->config
.si
.max_gs_threads
= 32;
2879 rdev
->config
.si
.max_hw_contexts
= 8;
2881 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2882 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x100;
2883 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2884 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2885 gb_addr_config
= TAHITI_GB_ADDR_CONFIG_GOLDEN
;
2888 rdev
->config
.si
.max_shader_engines
= 2;
2889 rdev
->config
.si
.max_tile_pipes
= 8;
2890 rdev
->config
.si
.max_cu_per_sh
= 5;
2891 rdev
->config
.si
.max_sh_per_se
= 2;
2892 rdev
->config
.si
.max_backends_per_se
= 4;
2893 rdev
->config
.si
.max_texture_channel_caches
= 8;
2894 rdev
->config
.si
.max_gprs
= 256;
2895 rdev
->config
.si
.max_gs_threads
= 32;
2896 rdev
->config
.si
.max_hw_contexts
= 8;
2898 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2899 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x100;
2900 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2901 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2902 gb_addr_config
= TAHITI_GB_ADDR_CONFIG_GOLDEN
;
2906 rdev
->config
.si
.max_shader_engines
= 1;
2907 rdev
->config
.si
.max_tile_pipes
= 4;
2908 rdev
->config
.si
.max_cu_per_sh
= 5;
2909 rdev
->config
.si
.max_sh_per_se
= 2;
2910 rdev
->config
.si
.max_backends_per_se
= 4;
2911 rdev
->config
.si
.max_texture_channel_caches
= 4;
2912 rdev
->config
.si
.max_gprs
= 256;
2913 rdev
->config
.si
.max_gs_threads
= 32;
2914 rdev
->config
.si
.max_hw_contexts
= 8;
2916 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2917 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2918 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2919 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2920 gb_addr_config
= VERDE_GB_ADDR_CONFIG_GOLDEN
;
2923 rdev
->config
.si
.max_shader_engines
= 1;
2924 rdev
->config
.si
.max_tile_pipes
= 4;
2925 rdev
->config
.si
.max_cu_per_sh
= 6;
2926 rdev
->config
.si
.max_sh_per_se
= 1;
2927 rdev
->config
.si
.max_backends_per_se
= 2;
2928 rdev
->config
.si
.max_texture_channel_caches
= 4;
2929 rdev
->config
.si
.max_gprs
= 256;
2930 rdev
->config
.si
.max_gs_threads
= 16;
2931 rdev
->config
.si
.max_hw_contexts
= 8;
2933 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2934 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2935 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2936 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2937 gb_addr_config
= VERDE_GB_ADDR_CONFIG_GOLDEN
;
2940 rdev
->config
.si
.max_shader_engines
= 1;
2941 rdev
->config
.si
.max_tile_pipes
= 4;
2942 rdev
->config
.si
.max_cu_per_sh
= 5;
2943 rdev
->config
.si
.max_sh_per_se
= 1;
2944 rdev
->config
.si
.max_backends_per_se
= 1;
2945 rdev
->config
.si
.max_texture_channel_caches
= 2;
2946 rdev
->config
.si
.max_gprs
= 256;
2947 rdev
->config
.si
.max_gs_threads
= 16;
2948 rdev
->config
.si
.max_hw_contexts
= 8;
2950 rdev
->config
.si
.sc_prim_fifo_size_frontend
= 0x20;
2951 rdev
->config
.si
.sc_prim_fifo_size_backend
= 0x40;
2952 rdev
->config
.si
.sc_hiz_tile_fifo_size
= 0x30;
2953 rdev
->config
.si
.sc_earlyz_tile_fifo_size
= 0x130;
2954 gb_addr_config
= HAINAN_GB_ADDR_CONFIG_GOLDEN
;
2958 /* Initialize HDP */
2959 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
2960 WREG32((0x2c14 + j
), 0x00000000);
2961 WREG32((0x2c18 + j
), 0x00000000);
2962 WREG32((0x2c1c + j
), 0x00000000);
2963 WREG32((0x2c20 + j
), 0x00000000);
2964 WREG32((0x2c24 + j
), 0x00000000);
2967 WREG32(GRBM_CNTL
, GRBM_READ_TIMEOUT(0xff));
2969 evergreen_fix_pci_max_read_req_size(rdev
);
2971 WREG32(BIF_FB_EN
, FB_READ_EN
| FB_WRITE_EN
);
2973 mc_shared_chmap
= RREG32(MC_SHARED_CHMAP
);
2974 mc_arb_ramcfg
= RREG32(MC_ARB_RAMCFG
);
2976 rdev
->config
.si
.num_tile_pipes
= rdev
->config
.si
.max_tile_pipes
;
2977 rdev
->config
.si
.mem_max_burst_length_bytes
= 256;
2978 tmp
= (mc_arb_ramcfg
& NOOFCOLS_MASK
) >> NOOFCOLS_SHIFT
;
2979 rdev
->config
.si
.mem_row_size_in_kb
= (4 * (1 << (8 + tmp
))) / 1024;
2980 if (rdev
->config
.si
.mem_row_size_in_kb
> 4)
2981 rdev
->config
.si
.mem_row_size_in_kb
= 4;
2982 /* XXX use MC settings? */
2983 rdev
->config
.si
.shader_engine_tile_size
= 32;
2984 rdev
->config
.si
.num_gpus
= 1;
2985 rdev
->config
.si
.multi_gpu_tile_size
= 64;
2987 /* fix up row size */
2988 gb_addr_config
&= ~ROW_SIZE_MASK
;
2989 switch (rdev
->config
.si
.mem_row_size_in_kb
) {
2992 gb_addr_config
|= ROW_SIZE(0);
2995 gb_addr_config
|= ROW_SIZE(1);
2998 gb_addr_config
|= ROW_SIZE(2);
3002 /* setup tiling info dword. gb_addr_config is not adequate since it does
3003 * not have bank info, so create a custom tiling dword.
3004 * bits 3:0 num_pipes
3005 * bits 7:4 num_banks
3006 * bits 11:8 group_size
3007 * bits 15:12 row_size
3009 rdev
->config
.si
.tile_config
= 0;
3010 switch (rdev
->config
.si
.num_tile_pipes
) {
3012 rdev
->config
.si
.tile_config
|= (0 << 0);
3015 rdev
->config
.si
.tile_config
|= (1 << 0);
3018 rdev
->config
.si
.tile_config
|= (2 << 0);
3022 /* XXX what about 12? */
3023 rdev
->config
.si
.tile_config
|= (3 << 0);
3026 switch ((mc_arb_ramcfg
& NOOFBANK_MASK
) >> NOOFBANK_SHIFT
) {
3027 case 0: /* four banks */
3028 rdev
->config
.si
.tile_config
|= 0 << 4;
3030 case 1: /* eight banks */
3031 rdev
->config
.si
.tile_config
|= 1 << 4;
3033 case 2: /* sixteen banks */
3035 rdev
->config
.si
.tile_config
|= 2 << 4;
3038 rdev
->config
.si
.tile_config
|=
3039 ((gb_addr_config
& PIPE_INTERLEAVE_SIZE_MASK
) >> PIPE_INTERLEAVE_SIZE_SHIFT
) << 8;
3040 rdev
->config
.si
.tile_config
|=
3041 ((gb_addr_config
& ROW_SIZE_MASK
) >> ROW_SIZE_SHIFT
) << 12;
3043 WREG32(GB_ADDR_CONFIG
, gb_addr_config
);
3044 WREG32(DMIF_ADDR_CONFIG
, gb_addr_config
);
3045 WREG32(DMIF_ADDR_CALC
, gb_addr_config
);
3046 WREG32(HDP_ADDR_CONFIG
, gb_addr_config
);
3047 WREG32(DMA_TILING_CONFIG
+ DMA0_REGISTER_OFFSET
, gb_addr_config
);
3048 WREG32(DMA_TILING_CONFIG
+ DMA1_REGISTER_OFFSET
, gb_addr_config
);
3049 if (rdev
->has_uvd
) {
3050 WREG32(UVD_UDEC_ADDR_CONFIG
, gb_addr_config
);
3051 WREG32(UVD_UDEC_DB_ADDR_CONFIG
, gb_addr_config
);
3052 WREG32(UVD_UDEC_DBW_ADDR_CONFIG
, gb_addr_config
);
3055 si_tiling_mode_table_init(rdev
);
3057 si_setup_rb(rdev
, rdev
->config
.si
.max_shader_engines
,
3058 rdev
->config
.si
.max_sh_per_se
,
3059 rdev
->config
.si
.max_backends_per_se
);
3061 si_setup_spi(rdev
, rdev
->config
.si
.max_shader_engines
,
3062 rdev
->config
.si
.max_sh_per_se
,
3063 rdev
->config
.si
.max_cu_per_sh
);
3066 /* set HW defaults for 3D engine */
3067 WREG32(CP_QUEUE_THRESHOLDS
, (ROQ_IB1_START(0x16) |
3068 ROQ_IB2_START(0x2b)));
3069 WREG32(CP_MEQ_THRESHOLDS
, MEQ1_START(0x30) | MEQ2_START(0x60));
3071 sx_debug_1
= RREG32(SX_DEBUG_1
);
3072 WREG32(SX_DEBUG_1
, sx_debug_1
);
3074 WREG32(SPI_CONFIG_CNTL_1
, VTX_DONE_DELAY(4));
3076 WREG32(PA_SC_FIFO_SIZE
, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev
->config
.si
.sc_prim_fifo_size_frontend
) |
3077 SC_BACKEND_PRIM_FIFO_SIZE(rdev
->config
.si
.sc_prim_fifo_size_backend
) |
3078 SC_HIZ_TILE_FIFO_SIZE(rdev
->config
.si
.sc_hiz_tile_fifo_size
) |
3079 SC_EARLYZ_TILE_FIFO_SIZE(rdev
->config
.si
.sc_earlyz_tile_fifo_size
)));
3081 WREG32(VGT_NUM_INSTANCES
, 1);
3083 WREG32(CP_PERFMON_CNTL
, 0);
3085 WREG32(SQ_CONFIG
, 0);
3087 WREG32(PA_SC_FORCE_EOV_MAX_CNTS
, (FORCE_EOV_MAX_CLK_CNT(4095) |
3088 FORCE_EOV_MAX_REZ_CNT(255)));
3090 WREG32(VGT_CACHE_INVALIDATION
, CACHE_INVALIDATION(VC_AND_TC
) |
3091 AUTO_INVLD_EN(ES_AND_GS_AUTO
));
3093 WREG32(VGT_GS_VERTEX_REUSE
, 16);
3094 WREG32(PA_SC_LINE_STIPPLE_STATE
, 0);
3096 WREG32(CB_PERFCOUNTER0_SELECT0
, 0);
3097 WREG32(CB_PERFCOUNTER0_SELECT1
, 0);
3098 WREG32(CB_PERFCOUNTER1_SELECT0
, 0);
3099 WREG32(CB_PERFCOUNTER1_SELECT1
, 0);
3100 WREG32(CB_PERFCOUNTER2_SELECT0
, 0);
3101 WREG32(CB_PERFCOUNTER2_SELECT1
, 0);
3102 WREG32(CB_PERFCOUNTER3_SELECT0
, 0);
3103 WREG32(CB_PERFCOUNTER3_SELECT1
, 0);
3105 tmp
= RREG32(HDP_MISC_CNTL
);
3106 tmp
|= HDP_FLUSH_INVALIDATE_CACHE
;
3107 WREG32(HDP_MISC_CNTL
, tmp
);
3109 hdp_host_path_cntl
= RREG32(HDP_HOST_PATH_CNTL
);
3110 WREG32(HDP_HOST_PATH_CNTL
, hdp_host_path_cntl
);
3112 WREG32(PA_CL_ENHANCE
, CLIP_VTX_REORDER_ENA
| NUM_CLIP_SEQ(3));
3118 * GPU scratch registers helpers function.
3120 static void si_scratch_init(struct radeon_device
*rdev
)
3124 rdev
->scratch
.num_reg
= 7;
3125 rdev
->scratch
.reg_base
= SCRATCH_REG0
;
3126 for (i
= 0; i
< rdev
->scratch
.num_reg
; i
++) {
3127 rdev
->scratch
.free
[i
] = true;
3128 rdev
->scratch
.reg
[i
] = rdev
->scratch
.reg_base
+ (i
* 4);
3132 void si_fence_ring_emit(struct radeon_device
*rdev
,
3133 struct radeon_fence
*fence
)
3135 struct radeon_ring
*ring
= &rdev
->ring
[fence
->ring
];
3136 u64 addr
= rdev
->fence_drv
[fence
->ring
].gpu_addr
;
3138 /* flush read cache over gart */
3139 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
3140 radeon_ring_write(ring
, (CP_COHER_CNTL2
- PACKET3_SET_CONFIG_REG_START
) >> 2);
3141 radeon_ring_write(ring
, 0);
3142 radeon_ring_write(ring
, PACKET3(PACKET3_SURFACE_SYNC
, 3));
3143 radeon_ring_write(ring
, PACKET3_TCL1_ACTION_ENA
|
3144 PACKET3_TC_ACTION_ENA
|
3145 PACKET3_SH_KCACHE_ACTION_ENA
|
3146 PACKET3_SH_ICACHE_ACTION_ENA
);
3147 radeon_ring_write(ring
, 0xFFFFFFFF);
3148 radeon_ring_write(ring
, 0);
3149 radeon_ring_write(ring
, 10); /* poll interval */
3150 /* EVENT_WRITE_EOP - flush caches, send int */
3151 radeon_ring_write(ring
, PACKET3(PACKET3_EVENT_WRITE_EOP
, 4));
3152 radeon_ring_write(ring
, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT
) | EVENT_INDEX(5));
3153 radeon_ring_write(ring
, addr
& 0xffffffff);
3154 radeon_ring_write(ring
, (upper_32_bits(addr
) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3155 radeon_ring_write(ring
, fence
->seq
);
3156 radeon_ring_write(ring
, 0);
3162 void si_ring_ib_execute(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
3164 struct radeon_ring
*ring
= &rdev
->ring
[ib
->ring
];
3167 if (ib
->is_const_ib
) {
3168 /* set switch buffer packet before const IB */
3169 radeon_ring_write(ring
, PACKET3(PACKET3_SWITCH_BUFFER
, 0));
3170 radeon_ring_write(ring
, 0);
3172 header
= PACKET3(PACKET3_INDIRECT_BUFFER_CONST
, 2);
3175 if (ring
->rptr_save_reg
) {
3176 next_rptr
= ring
->wptr
+ 3 + 4 + 8;
3177 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
3178 radeon_ring_write(ring
, ((ring
->rptr_save_reg
-
3179 PACKET3_SET_CONFIG_REG_START
) >> 2));
3180 radeon_ring_write(ring
, next_rptr
);
3181 } else if (rdev
->wb
.enabled
) {
3182 next_rptr
= ring
->wptr
+ 5 + 4 + 8;
3183 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
3184 radeon_ring_write(ring
, (1 << 8));
3185 radeon_ring_write(ring
, ring
->next_rptr_gpu_addr
& 0xfffffffc);
3186 radeon_ring_write(ring
, upper_32_bits(ring
->next_rptr_gpu_addr
) & 0xffffffff);
3187 radeon_ring_write(ring
, next_rptr
);
3190 header
= PACKET3(PACKET3_INDIRECT_BUFFER
, 2);
3193 radeon_ring_write(ring
, header
);
3194 radeon_ring_write(ring
,
3198 (ib
->gpu_addr
& 0xFFFFFFFC));
3199 radeon_ring_write(ring
, upper_32_bits(ib
->gpu_addr
) & 0xFFFF);
3200 radeon_ring_write(ring
, ib
->length_dw
|
3201 (ib
->vm
? (ib
->vm
->id
<< 24) : 0));
3203 if (!ib
->is_const_ib
) {
3204 /* flush read cache over gart for this vmid */
3205 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONFIG_REG
, 1));
3206 radeon_ring_write(ring
, (CP_COHER_CNTL2
- PACKET3_SET_CONFIG_REG_START
) >> 2);
3207 radeon_ring_write(ring
, ib
->vm
? ib
->vm
->id
: 0);
3208 radeon_ring_write(ring
, PACKET3(PACKET3_SURFACE_SYNC
, 3));
3209 radeon_ring_write(ring
, PACKET3_TCL1_ACTION_ENA
|
3210 PACKET3_TC_ACTION_ENA
|
3211 PACKET3_SH_KCACHE_ACTION_ENA
|
3212 PACKET3_SH_ICACHE_ACTION_ENA
);
3213 radeon_ring_write(ring
, 0xFFFFFFFF);
3214 radeon_ring_write(ring
, 0);
3215 radeon_ring_write(ring
, 10); /* poll interval */
3222 static void si_cp_enable(struct radeon_device
*rdev
, bool enable
)
3225 WREG32(CP_ME_CNTL
, 0);
3227 radeon_ttm_set_active_vram_size(rdev
, rdev
->mc
.visible_vram_size
);
3228 WREG32(CP_ME_CNTL
, (CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
));
3229 WREG32(SCRATCH_UMSK
, 0);
3230 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3231 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
3232 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
3237 static int si_cp_load_microcode(struct radeon_device
*rdev
)
3239 const __be32
*fw_data
;
3242 if (!rdev
->me_fw
|| !rdev
->pfp_fw
)
3245 si_cp_enable(rdev
, false);
3248 fw_data
= (const __be32
*)rdev
->pfp_fw
->data
;
3249 WREG32(CP_PFP_UCODE_ADDR
, 0);
3250 for (i
= 0; i
< SI_PFP_UCODE_SIZE
; i
++)
3251 WREG32(CP_PFP_UCODE_DATA
, be32_to_cpup(fw_data
++));
3252 WREG32(CP_PFP_UCODE_ADDR
, 0);
3255 fw_data
= (const __be32
*)rdev
->ce_fw
->data
;
3256 WREG32(CP_CE_UCODE_ADDR
, 0);
3257 for (i
= 0; i
< SI_CE_UCODE_SIZE
; i
++)
3258 WREG32(CP_CE_UCODE_DATA
, be32_to_cpup(fw_data
++));
3259 WREG32(CP_CE_UCODE_ADDR
, 0);
3262 fw_data
= (const __be32
*)rdev
->me_fw
->data
;
3263 WREG32(CP_ME_RAM_WADDR
, 0);
3264 for (i
= 0; i
< SI_PM4_UCODE_SIZE
; i
++)
3265 WREG32(CP_ME_RAM_DATA
, be32_to_cpup(fw_data
++));
3266 WREG32(CP_ME_RAM_WADDR
, 0);
3268 WREG32(CP_PFP_UCODE_ADDR
, 0);
3269 WREG32(CP_CE_UCODE_ADDR
, 0);
3270 WREG32(CP_ME_RAM_WADDR
, 0);
3271 WREG32(CP_ME_RAM_RADDR
, 0);
3275 static int si_cp_start(struct radeon_device
*rdev
)
3277 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3280 r
= radeon_ring_lock(rdev
, ring
, 7 + 4);
3282 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
3286 radeon_ring_write(ring
, PACKET3(PACKET3_ME_INITIALIZE
, 5));
3287 radeon_ring_write(ring
, 0x1);
3288 radeon_ring_write(ring
, 0x0);
3289 radeon_ring_write(ring
, rdev
->config
.si
.max_hw_contexts
- 1);
3290 radeon_ring_write(ring
, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3291 radeon_ring_write(ring
, 0);
3292 radeon_ring_write(ring
, 0);
3294 /* init the CE partitions */
3295 radeon_ring_write(ring
, PACKET3(PACKET3_SET_BASE
, 2));
3296 radeon_ring_write(ring
, PACKET3_BASE_INDEX(CE_PARTITION_BASE
));
3297 radeon_ring_write(ring
, 0xc000);
3298 radeon_ring_write(ring
, 0xe000);
3299 radeon_ring_unlock_commit(rdev
, ring
);
3301 si_cp_enable(rdev
, true);
3303 r
= radeon_ring_lock(rdev
, ring
, si_default_size
+ 10);
3305 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r
);
3309 /* setup clear context state */
3310 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3311 radeon_ring_write(ring
, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE
);
3313 for (i
= 0; i
< si_default_size
; i
++)
3314 radeon_ring_write(ring
, si_default_state
[i
]);
3316 radeon_ring_write(ring
, PACKET3(PACKET3_PREAMBLE_CNTL
, 0));
3317 radeon_ring_write(ring
, PACKET3_PREAMBLE_END_CLEAR_STATE
);
3319 /* set clear context state */
3320 radeon_ring_write(ring
, PACKET3(PACKET3_CLEAR_STATE
, 0));
3321 radeon_ring_write(ring
, 0);
3323 radeon_ring_write(ring
, PACKET3(PACKET3_SET_CONTEXT_REG
, 2));
3324 radeon_ring_write(ring
, 0x00000316);
3325 radeon_ring_write(ring
, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3326 radeon_ring_write(ring
, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3328 radeon_ring_unlock_commit(rdev
, ring
);
3330 for (i
= RADEON_RING_TYPE_GFX_INDEX
; i
<= CAYMAN_RING_TYPE_CP2_INDEX
; ++i
) {
3331 ring
= &rdev
->ring
[i
];
3332 r
= radeon_ring_lock(rdev
, ring
, 2);
3334 /* clear the compute context state */
3335 radeon_ring_write(ring
, PACKET3_COMPUTE(PACKET3_CLEAR_STATE
, 0));
3336 radeon_ring_write(ring
, 0);
3338 radeon_ring_unlock_commit(rdev
, ring
);
3344 static void si_cp_fini(struct radeon_device
*rdev
)
3346 struct radeon_ring
*ring
;
3347 si_cp_enable(rdev
, false);
3349 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3350 radeon_ring_fini(rdev
, ring
);
3351 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3353 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
3354 radeon_ring_fini(rdev
, ring
);
3355 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3357 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
3358 radeon_ring_fini(rdev
, ring
);
3359 radeon_scratch_free(rdev
, ring
->rptr_save_reg
);
3362 static int si_cp_resume(struct radeon_device
*rdev
)
3364 struct radeon_ring
*ring
;
3369 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3370 WREG32(GRBM_SOFT_RESET
, (SOFT_RESET_CP
|
3375 RREG32(GRBM_SOFT_RESET
);
3377 WREG32(GRBM_SOFT_RESET
, 0);
3378 RREG32(GRBM_SOFT_RESET
);
3380 WREG32(CP_SEM_WAIT_TIMER
, 0x0);
3381 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL
, 0x0);
3383 /* Set the write pointer delay */
3384 WREG32(CP_RB_WPTR_DELAY
, 0);
3386 WREG32(CP_DEBUG
, 0);
3387 WREG32(SCRATCH_ADDR
, ((rdev
->wb
.gpu_addr
+ RADEON_WB_SCRATCH_OFFSET
) >> 8) & 0xFFFFFFFF);
3389 /* ring 0 - compute and gfx */
3390 /* Set ring buffer size */
3391 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
3392 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3393 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3395 tmp
|= BUF_SWAP_32BIT
;
3397 WREG32(CP_RB0_CNTL
, tmp
);
3399 /* Initialize the ring buffer's read and write pointers */
3400 WREG32(CP_RB0_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3402 WREG32(CP_RB0_WPTR
, ring
->wptr
);
3404 /* set the wb address whether it's enabled or not */
3405 WREG32(CP_RB0_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFFFFFFFC);
3406 WREG32(CP_RB0_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP_RPTR_OFFSET
) & 0xFF);
3408 if (rdev
->wb
.enabled
)
3409 WREG32(SCRATCH_UMSK
, 0xff);
3411 tmp
|= RB_NO_UPDATE
;
3412 WREG32(SCRATCH_UMSK
, 0);
3416 WREG32(CP_RB0_CNTL
, tmp
);
3418 WREG32(CP_RB0_BASE
, ring
->gpu_addr
>> 8);
3420 ring
->rptr
= RREG32(CP_RB0_RPTR
);
3422 /* ring1 - compute only */
3423 /* Set ring buffer size */
3424 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
3425 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3426 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3428 tmp
|= BUF_SWAP_32BIT
;
3430 WREG32(CP_RB1_CNTL
, tmp
);
3432 /* Initialize the ring buffer's read and write pointers */
3433 WREG32(CP_RB1_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3435 WREG32(CP_RB1_WPTR
, ring
->wptr
);
3437 /* set the wb address whether it's enabled or not */
3438 WREG32(CP_RB1_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
) & 0xFFFFFFFC);
3439 WREG32(CP_RB1_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP1_RPTR_OFFSET
) & 0xFF);
3442 WREG32(CP_RB1_CNTL
, tmp
);
3444 WREG32(CP_RB1_BASE
, ring
->gpu_addr
>> 8);
3446 ring
->rptr
= RREG32(CP_RB1_RPTR
);
3448 /* ring2 - compute only */
3449 /* Set ring buffer size */
3450 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
3451 rb_bufsz
= drm_order(ring
->ring_size
/ 8);
3452 tmp
= (drm_order(RADEON_GPU_PAGE_SIZE
/8) << 8) | rb_bufsz
;
3454 tmp
|= BUF_SWAP_32BIT
;
3456 WREG32(CP_RB2_CNTL
, tmp
);
3458 /* Initialize the ring buffer's read and write pointers */
3459 WREG32(CP_RB2_CNTL
, tmp
| RB_RPTR_WR_ENA
);
3461 WREG32(CP_RB2_WPTR
, ring
->wptr
);
3463 /* set the wb address whether it's enabled or not */
3464 WREG32(CP_RB2_RPTR_ADDR
, (rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
) & 0xFFFFFFFC);
3465 WREG32(CP_RB2_RPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ RADEON_WB_CP2_RPTR_OFFSET
) & 0xFF);
3468 WREG32(CP_RB2_CNTL
, tmp
);
3470 WREG32(CP_RB2_BASE
, ring
->gpu_addr
>> 8);
3472 ring
->rptr
= RREG32(CP_RB2_RPTR
);
3474 /* start the rings */
3476 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= true;
3477 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= true;
3478 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= true;
3479 r
= radeon_ring_test(rdev
, RADEON_RING_TYPE_GFX_INDEX
, &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
]);
3481 rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
].ready
= false;
3482 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
3483 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
3486 r
= radeon_ring_test(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
, &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
]);
3488 rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
].ready
= false;
3490 r
= radeon_ring_test(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
, &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
]);
3492 rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
].ready
= false;
3498 static u32
si_gpu_check_soft_reset(struct radeon_device
*rdev
)
3504 tmp
= RREG32(GRBM_STATUS
);
3505 if (tmp
& (PA_BUSY
| SC_BUSY
|
3506 BCI_BUSY
| SX_BUSY
|
3507 TA_BUSY
| VGT_BUSY
|
3509 GDS_BUSY
| SPI_BUSY
|
3510 IA_BUSY
| IA_BUSY_NO_DMA
))
3511 reset_mask
|= RADEON_RESET_GFX
;
3513 if (tmp
& (CF_RQ_PENDING
| PF_RQ_PENDING
|
3514 CP_BUSY
| CP_COHERENCY_BUSY
))
3515 reset_mask
|= RADEON_RESET_CP
;
3517 if (tmp
& GRBM_EE_BUSY
)
3518 reset_mask
|= RADEON_RESET_GRBM
| RADEON_RESET_GFX
| RADEON_RESET_CP
;
3521 tmp
= RREG32(GRBM_STATUS2
);
3522 if (tmp
& (RLC_RQ_PENDING
| RLC_BUSY
))
3523 reset_mask
|= RADEON_RESET_RLC
;
3525 /* DMA_STATUS_REG 0 */
3526 tmp
= RREG32(DMA_STATUS_REG
+ DMA0_REGISTER_OFFSET
);
3527 if (!(tmp
& DMA_IDLE
))
3528 reset_mask
|= RADEON_RESET_DMA
;
3530 /* DMA_STATUS_REG 1 */
3531 tmp
= RREG32(DMA_STATUS_REG
+ DMA1_REGISTER_OFFSET
);
3532 if (!(tmp
& DMA_IDLE
))
3533 reset_mask
|= RADEON_RESET_DMA1
;
3536 tmp
= RREG32(SRBM_STATUS2
);
3538 reset_mask
|= RADEON_RESET_DMA
;
3540 if (tmp
& DMA1_BUSY
)
3541 reset_mask
|= RADEON_RESET_DMA1
;
3544 tmp
= RREG32(SRBM_STATUS
);
3547 reset_mask
|= RADEON_RESET_IH
;
3550 reset_mask
|= RADEON_RESET_SEM
;
3552 if (tmp
& GRBM_RQ_PENDING
)
3553 reset_mask
|= RADEON_RESET_GRBM
;
3556 reset_mask
|= RADEON_RESET_VMC
;
3558 if (tmp
& (MCB_BUSY
| MCB_NON_DISPLAY_BUSY
|
3559 MCC_BUSY
| MCD_BUSY
))
3560 reset_mask
|= RADEON_RESET_MC
;
3562 if (evergreen_is_display_hung(rdev
))
3563 reset_mask
|= RADEON_RESET_DISPLAY
;
3566 tmp
= RREG32(VM_L2_STATUS
);
3568 reset_mask
|= RADEON_RESET_VMC
;
3570 /* Skip MC reset as it's mostly likely not hung, just busy */
3571 if (reset_mask
& RADEON_RESET_MC
) {
3572 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask
);
3573 reset_mask
&= ~RADEON_RESET_MC
;
3579 static void si_gpu_soft_reset(struct radeon_device
*rdev
, u32 reset_mask
)
3581 struct evergreen_mc_save save
;
3582 u32 grbm_soft_reset
= 0, srbm_soft_reset
= 0;
3585 if (reset_mask
== 0)
3588 dev_info(rdev
->dev
, "GPU softreset: 0x%08X\n", reset_mask
);
3590 evergreen_print_gpu_status_regs(rdev
);
3591 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3592 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
));
3593 dev_info(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3594 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
));
3596 /* Disable CP parsing/prefetching */
3597 WREG32(CP_ME_CNTL
, CP_ME_HALT
| CP_PFP_HALT
| CP_CE_HALT
);
3599 if (reset_mask
& RADEON_RESET_DMA
) {
3601 tmp
= RREG32(DMA_RB_CNTL
+ DMA0_REGISTER_OFFSET
);
3602 tmp
&= ~DMA_RB_ENABLE
;
3603 WREG32(DMA_RB_CNTL
+ DMA0_REGISTER_OFFSET
, tmp
);
3605 if (reset_mask
& RADEON_RESET_DMA1
) {
3607 tmp
= RREG32(DMA_RB_CNTL
+ DMA1_REGISTER_OFFSET
);
3608 tmp
&= ~DMA_RB_ENABLE
;
3609 WREG32(DMA_RB_CNTL
+ DMA1_REGISTER_OFFSET
, tmp
);
3614 evergreen_mc_stop(rdev
, &save
);
3615 if (evergreen_mc_wait_for_idle(rdev
)) {
3616 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3619 if (reset_mask
& (RADEON_RESET_GFX
| RADEON_RESET_COMPUTE
| RADEON_RESET_CP
)) {
3620 grbm_soft_reset
= SOFT_RESET_CB
|
3634 if (reset_mask
& RADEON_RESET_CP
) {
3635 grbm_soft_reset
|= SOFT_RESET_CP
| SOFT_RESET_VGT
;
3637 srbm_soft_reset
|= SOFT_RESET_GRBM
;
3640 if (reset_mask
& RADEON_RESET_DMA
)
3641 srbm_soft_reset
|= SOFT_RESET_DMA
;
3643 if (reset_mask
& RADEON_RESET_DMA1
)
3644 srbm_soft_reset
|= SOFT_RESET_DMA1
;
3646 if (reset_mask
& RADEON_RESET_DISPLAY
)
3647 srbm_soft_reset
|= SOFT_RESET_DC
;
3649 if (reset_mask
& RADEON_RESET_RLC
)
3650 grbm_soft_reset
|= SOFT_RESET_RLC
;
3652 if (reset_mask
& RADEON_RESET_SEM
)
3653 srbm_soft_reset
|= SOFT_RESET_SEM
;
3655 if (reset_mask
& RADEON_RESET_IH
)
3656 srbm_soft_reset
|= SOFT_RESET_IH
;
3658 if (reset_mask
& RADEON_RESET_GRBM
)
3659 srbm_soft_reset
|= SOFT_RESET_GRBM
;
3661 if (reset_mask
& RADEON_RESET_VMC
)
3662 srbm_soft_reset
|= SOFT_RESET_VMC
;
3664 if (reset_mask
& RADEON_RESET_MC
)
3665 srbm_soft_reset
|= SOFT_RESET_MC
;
3667 if (grbm_soft_reset
) {
3668 tmp
= RREG32(GRBM_SOFT_RESET
);
3669 tmp
|= grbm_soft_reset
;
3670 dev_info(rdev
->dev
, "GRBM_SOFT_RESET=0x%08X\n", tmp
);
3671 WREG32(GRBM_SOFT_RESET
, tmp
);
3672 tmp
= RREG32(GRBM_SOFT_RESET
);
3676 tmp
&= ~grbm_soft_reset
;
3677 WREG32(GRBM_SOFT_RESET
, tmp
);
3678 tmp
= RREG32(GRBM_SOFT_RESET
);
3681 if (srbm_soft_reset
) {
3682 tmp
= RREG32(SRBM_SOFT_RESET
);
3683 tmp
|= srbm_soft_reset
;
3684 dev_info(rdev
->dev
, "SRBM_SOFT_RESET=0x%08X\n", tmp
);
3685 WREG32(SRBM_SOFT_RESET
, tmp
);
3686 tmp
= RREG32(SRBM_SOFT_RESET
);
3690 tmp
&= ~srbm_soft_reset
;
3691 WREG32(SRBM_SOFT_RESET
, tmp
);
3692 tmp
= RREG32(SRBM_SOFT_RESET
);
3695 /* Wait a little for things to settle down */
3698 evergreen_mc_resume(rdev
, &save
);
3701 evergreen_print_gpu_status_regs(rdev
);
3704 int si_asic_reset(struct radeon_device
*rdev
)
3708 reset_mask
= si_gpu_check_soft_reset(rdev
);
3711 r600_set_bios_scratch_engine_hung(rdev
, true);
3713 si_gpu_soft_reset(rdev
, reset_mask
);
3715 reset_mask
= si_gpu_check_soft_reset(rdev
);
3718 r600_set_bios_scratch_engine_hung(rdev
, false);
3724 * si_gfx_is_lockup - Check if the GFX engine is locked up
3726 * @rdev: radeon_device pointer
3727 * @ring: radeon_ring structure holding ring information
3729 * Check if the GFX engine is locked up.
3730 * Returns true if the engine appears to be locked up, false if not.
3732 bool si_gfx_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3734 u32 reset_mask
= si_gpu_check_soft_reset(rdev
);
3736 if (!(reset_mask
& (RADEON_RESET_GFX
|
3737 RADEON_RESET_COMPUTE
|
3738 RADEON_RESET_CP
))) {
3739 radeon_ring_lockup_update(ring
);
3742 /* force CP activities */
3743 radeon_ring_force_activity(rdev
, ring
);
3744 return radeon_ring_test_lockup(rdev
, ring
);
3748 * si_dma_is_lockup - Check if the DMA engine is locked up
3750 * @rdev: radeon_device pointer
3751 * @ring: radeon_ring structure holding ring information
3753 * Check if the async DMA engine is locked up.
3754 * Returns true if the engine appears to be locked up, false if not.
3756 bool si_dma_is_lockup(struct radeon_device
*rdev
, struct radeon_ring
*ring
)
3758 u32 reset_mask
= si_gpu_check_soft_reset(rdev
);
3761 if (ring
->idx
== R600_RING_TYPE_DMA_INDEX
)
3762 mask
= RADEON_RESET_DMA
;
3764 mask
= RADEON_RESET_DMA1
;
3766 if (!(reset_mask
& mask
)) {
3767 radeon_ring_lockup_update(ring
);
3770 /* force ring activities */
3771 radeon_ring_force_activity(rdev
, ring
);
3772 return radeon_ring_test_lockup(rdev
, ring
);
3776 static void si_mc_program(struct radeon_device
*rdev
)
3778 struct evergreen_mc_save save
;
3782 /* Initialize HDP */
3783 for (i
= 0, j
= 0; i
< 32; i
++, j
+= 0x18) {
3784 WREG32((0x2c14 + j
), 0x00000000);
3785 WREG32((0x2c18 + j
), 0x00000000);
3786 WREG32((0x2c1c + j
), 0x00000000);
3787 WREG32((0x2c20 + j
), 0x00000000);
3788 WREG32((0x2c24 + j
), 0x00000000);
3790 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL
, 0);
3792 evergreen_mc_stop(rdev
, &save
);
3793 if (radeon_mc_wait_for_idle(rdev
)) {
3794 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3796 if (!ASIC_IS_NODCE(rdev
))
3797 /* Lockout access through VGA aperture*/
3798 WREG32(VGA_HDP_CONTROL
, VGA_MEMORY_DISABLE
);
3799 /* Update configuration */
3800 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR
,
3801 rdev
->mc
.vram_start
>> 12);
3802 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR
,
3803 rdev
->mc
.vram_end
>> 12);
3804 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR
,
3805 rdev
->vram_scratch
.gpu_addr
>> 12);
3806 tmp
= ((rdev
->mc
.vram_end
>> 24) & 0xFFFF) << 16;
3807 tmp
|= ((rdev
->mc
.vram_start
>> 24) & 0xFFFF);
3808 WREG32(MC_VM_FB_LOCATION
, tmp
);
3809 /* XXX double check these! */
3810 WREG32(HDP_NONSURFACE_BASE
, (rdev
->mc
.vram_start
>> 8));
3811 WREG32(HDP_NONSURFACE_INFO
, (2 << 7) | (1 << 30));
3812 WREG32(HDP_NONSURFACE_SIZE
, 0x3FFFFFFF);
3813 WREG32(MC_VM_AGP_BASE
, 0);
3814 WREG32(MC_VM_AGP_TOP
, 0x0FFFFFFF);
3815 WREG32(MC_VM_AGP_BOT
, 0x0FFFFFFF);
3816 if (radeon_mc_wait_for_idle(rdev
)) {
3817 dev_warn(rdev
->dev
, "Wait for MC idle timedout !\n");
3819 evergreen_mc_resume(rdev
, &save
);
3820 if (!ASIC_IS_NODCE(rdev
)) {
3821 /* we need to own VRAM, so turn off the VGA renderer here
3822 * to stop it overwriting our objects */
3823 rv515_vga_render_disable(rdev
);
3827 void si_vram_gtt_location(struct radeon_device
*rdev
,
3828 struct radeon_mc
*mc
)
3830 if (mc
->mc_vram_size
> 0xFFC0000000ULL
) {
3831 /* leave room for at least 1024M GTT */
3832 dev_warn(rdev
->dev
, "limiting VRAM\n");
3833 mc
->real_vram_size
= 0xFFC0000000ULL
;
3834 mc
->mc_vram_size
= 0xFFC0000000ULL
;
3836 radeon_vram_location(rdev
, &rdev
->mc
, 0);
3837 rdev
->mc
.gtt_base_align
= 0;
3838 radeon_gtt_location(rdev
, mc
);
3841 static int si_mc_init(struct radeon_device
*rdev
)
3844 int chansize
, numchan
;
3846 /* Get VRAM informations */
3847 rdev
->mc
.vram_is_ddr
= true;
3848 tmp
= RREG32(MC_ARB_RAMCFG
);
3849 if (tmp
& CHANSIZE_OVERRIDE
) {
3851 } else if (tmp
& CHANSIZE_MASK
) {
3856 tmp
= RREG32(MC_SHARED_CHMAP
);
3857 switch ((tmp
& NOOFCHAN_MASK
) >> NOOFCHAN_SHIFT
) {
3887 rdev
->mc
.vram_width
= numchan
* chansize
;
3888 /* Could aper size report 0 ? */
3889 rdev
->mc
.aper_base
= pci_resource_start(rdev
->pdev
, 0);
3890 rdev
->mc
.aper_size
= pci_resource_len(rdev
->pdev
, 0);
3891 /* size in MB on si */
3892 rdev
->mc
.mc_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
3893 rdev
->mc
.real_vram_size
= RREG32(CONFIG_MEMSIZE
) * 1024ULL * 1024ULL;
3894 rdev
->mc
.visible_vram_size
= rdev
->mc
.aper_size
;
3895 si_vram_gtt_location(rdev
, &rdev
->mc
);
3896 radeon_update_bandwidth_info(rdev
);
3904 void si_pcie_gart_tlb_flush(struct radeon_device
*rdev
)
3906 /* flush hdp cache */
3907 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL
, 0x1);
3909 /* bits 0-15 are the VM contexts0-15 */
3910 WREG32(VM_INVALIDATE_REQUEST
, 1);
3913 static int si_pcie_gart_enable(struct radeon_device
*rdev
)
3917 if (rdev
->gart
.robj
== NULL
) {
3918 dev_err(rdev
->dev
, "No VRAM object for PCIE GART.\n");
3921 r
= radeon_gart_table_vram_pin(rdev
);
3924 radeon_gart_restore(rdev
);
3925 /* Setup TLB control */
3926 WREG32(MC_VM_MX_L1_TLB_CNTL
,
3929 SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
3930 ENABLE_ADVANCED_DRIVER_MODEL
|
3931 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
3932 /* Setup L2 cache */
3933 WREG32(VM_L2_CNTL
, ENABLE_L2_CACHE
|
3934 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
3935 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
3936 EFFECTIVE_L2_QUEUE_SIZE(7) |
3937 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3938 WREG32(VM_L2_CNTL2
, INVALIDATE_ALL_L1_TLBS
| INVALIDATE_L2_CACHE
);
3939 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
3940 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3941 /* setup context0 */
3942 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR
, rdev
->mc
.gtt_start
>> 12);
3943 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR
, rdev
->mc
.gtt_end
>> 12);
3944 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
, rdev
->gart
.table_addr
>> 12);
3945 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR
,
3946 (u32
)(rdev
->dummy_page
.addr
>> 12));
3947 WREG32(VM_CONTEXT0_CNTL2
, 0);
3948 WREG32(VM_CONTEXT0_CNTL
, (ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(0) |
3949 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
));
3955 /* empty context1-15 */
3956 /* set vm size, must be a multiple of 4 */
3957 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR
, 0);
3958 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR
, rdev
->vm_manager
.max_pfn
);
3959 /* Assign the pt base to something valid for now; the pts used for
3960 * the VMs are determined by the application and setup and assigned
3961 * on the fly in the vm part of radeon_gart.c
3963 for (i
= 1; i
< 16; i
++) {
3965 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (i
<< 2),
3966 rdev
->gart
.table_addr
>> 12);
3968 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((i
- 8) << 2),
3969 rdev
->gart
.table_addr
>> 12);
3972 /* enable context1-15 */
3973 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR
,
3974 (u32
)(rdev
->dummy_page
.addr
>> 12));
3975 WREG32(VM_CONTEXT1_CNTL2
, 4);
3976 WREG32(VM_CONTEXT1_CNTL
, ENABLE_CONTEXT
| PAGE_TABLE_DEPTH(1) |
3977 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3978 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
3979 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3980 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT
|
3981 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3982 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT
|
3983 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3984 VALID_PROTECTION_FAULT_ENABLE_DEFAULT
|
3985 READ_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3986 READ_PROTECTION_FAULT_ENABLE_DEFAULT
|
3987 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT
|
3988 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT
);
3990 si_pcie_gart_tlb_flush(rdev
);
3991 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3992 (unsigned)(rdev
->mc
.gtt_size
>> 20),
3993 (unsigned long long)rdev
->gart
.table_addr
);
3994 rdev
->gart
.ready
= true;
3998 static void si_pcie_gart_disable(struct radeon_device
*rdev
)
4000 /* Disable all tables */
4001 WREG32(VM_CONTEXT0_CNTL
, 0);
4002 WREG32(VM_CONTEXT1_CNTL
, 0);
4003 /* Setup TLB control */
4004 WREG32(MC_VM_MX_L1_TLB_CNTL
, SYSTEM_ACCESS_MODE_NOT_IN_SYS
|
4005 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU
);
4006 /* Setup L2 cache */
4007 WREG32(VM_L2_CNTL
, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE
|
4008 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE
|
4009 EFFECTIVE_L2_QUEUE_SIZE(7) |
4010 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4011 WREG32(VM_L2_CNTL2
, 0);
4012 WREG32(VM_L2_CNTL3
, L2_CACHE_BIGK_ASSOCIATIVITY
|
4013 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4014 radeon_gart_table_vram_unpin(rdev
);
4017 static void si_pcie_gart_fini(struct radeon_device
*rdev
)
4019 si_pcie_gart_disable(rdev
);
4020 radeon_gart_table_vram_free(rdev
);
4021 radeon_gart_fini(rdev
);
4025 static bool si_vm_reg_valid(u32 reg
)
4027 /* context regs are fine */
4031 /* check config regs */
4033 case GRBM_GFX_INDEX
:
4034 case CP_STRMOUT_CNTL
:
4035 case VGT_VTX_VECT_EJECT_REG
:
4036 case VGT_CACHE_INVALIDATION
:
4037 case VGT_ESGS_RING_SIZE
:
4038 case VGT_GSVS_RING_SIZE
:
4039 case VGT_GS_VERTEX_REUSE
:
4040 case VGT_PRIMITIVE_TYPE
:
4041 case VGT_INDEX_TYPE
:
4042 case VGT_NUM_INDICES
:
4043 case VGT_NUM_INSTANCES
:
4044 case VGT_TF_RING_SIZE
:
4045 case VGT_HS_OFFCHIP_PARAM
:
4046 case VGT_TF_MEMORY_BASE
:
4048 case PA_SU_LINE_STIPPLE_VALUE
:
4049 case PA_SC_LINE_STIPPLE_STATE
:
4052 case SPI_STATIC_THREAD_MGMT_1
:
4053 case SPI_STATIC_THREAD_MGMT_2
:
4054 case SPI_STATIC_THREAD_MGMT_3
:
4055 case SPI_PS_MAX_WAVE_ID
:
4056 case SPI_CONFIG_CNTL
:
4057 case SPI_CONFIG_CNTL_1
:
4061 DRM_ERROR("Invalid register 0x%x in CS\n", reg
);
4066 static int si_vm_packet3_ce_check(struct radeon_device
*rdev
,
4067 u32
*ib
, struct radeon_cs_packet
*pkt
)
4069 switch (pkt
->opcode
) {
4071 case PACKET3_SET_BASE
:
4072 case PACKET3_SET_CE_DE_COUNTERS
:
4073 case PACKET3_LOAD_CONST_RAM
:
4074 case PACKET3_WRITE_CONST_RAM
:
4075 case PACKET3_WRITE_CONST_RAM_OFFSET
:
4076 case PACKET3_DUMP_CONST_RAM
:
4077 case PACKET3_INCREMENT_CE_COUNTER
:
4078 case PACKET3_WAIT_ON_DE_COUNTER
:
4079 case PACKET3_CE_WRITE
:
4082 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt
->opcode
);
4088 static int si_vm_packet3_gfx_check(struct radeon_device
*rdev
,
4089 u32
*ib
, struct radeon_cs_packet
*pkt
)
4091 u32 idx
= pkt
->idx
+ 1;
4092 u32 idx_value
= ib
[idx
];
4093 u32 start_reg
, end_reg
, reg
, i
;
4096 switch (pkt
->opcode
) {
4098 case PACKET3_SET_BASE
:
4099 case PACKET3_CLEAR_STATE
:
4100 case PACKET3_INDEX_BUFFER_SIZE
:
4101 case PACKET3_DISPATCH_DIRECT
:
4102 case PACKET3_DISPATCH_INDIRECT
:
4103 case PACKET3_ALLOC_GDS
:
4104 case PACKET3_WRITE_GDS_RAM
:
4105 case PACKET3_ATOMIC_GDS
:
4106 case PACKET3_ATOMIC
:
4107 case PACKET3_OCCLUSION_QUERY
:
4108 case PACKET3_SET_PREDICATION
:
4109 case PACKET3_COND_EXEC
:
4110 case PACKET3_PRED_EXEC
:
4111 case PACKET3_DRAW_INDIRECT
:
4112 case PACKET3_DRAW_INDEX_INDIRECT
:
4113 case PACKET3_INDEX_BASE
:
4114 case PACKET3_DRAW_INDEX_2
:
4115 case PACKET3_CONTEXT_CONTROL
:
4116 case PACKET3_INDEX_TYPE
:
4117 case PACKET3_DRAW_INDIRECT_MULTI
:
4118 case PACKET3_DRAW_INDEX_AUTO
:
4119 case PACKET3_DRAW_INDEX_IMMD
:
4120 case PACKET3_NUM_INSTANCES
:
4121 case PACKET3_DRAW_INDEX_MULTI_AUTO
:
4122 case PACKET3_STRMOUT_BUFFER_UPDATE
:
4123 case PACKET3_DRAW_INDEX_OFFSET_2
:
4124 case PACKET3_DRAW_INDEX_MULTI_ELEMENT
:
4125 case PACKET3_DRAW_INDEX_INDIRECT_MULTI
:
4126 case PACKET3_MPEG_INDEX
:
4127 case PACKET3_WAIT_REG_MEM
:
4128 case PACKET3_MEM_WRITE
:
4129 case PACKET3_PFP_SYNC_ME
:
4130 case PACKET3_SURFACE_SYNC
:
4131 case PACKET3_EVENT_WRITE
:
4132 case PACKET3_EVENT_WRITE_EOP
:
4133 case PACKET3_EVENT_WRITE_EOS
:
4134 case PACKET3_SET_CONTEXT_REG
:
4135 case PACKET3_SET_CONTEXT_REG_INDIRECT
:
4136 case PACKET3_SET_SH_REG
:
4137 case PACKET3_SET_SH_REG_OFFSET
:
4138 case PACKET3_INCREMENT_DE_COUNTER
:
4139 case PACKET3_WAIT_ON_CE_COUNTER
:
4140 case PACKET3_WAIT_ON_AVAIL_BUFFER
:
4141 case PACKET3_ME_WRITE
:
4143 case PACKET3_COPY_DATA
:
4144 if ((idx_value
& 0xf00) == 0) {
4145 reg
= ib
[idx
+ 3] * 4;
4146 if (!si_vm_reg_valid(reg
))
4150 case PACKET3_WRITE_DATA
:
4151 if ((idx_value
& 0xf00) == 0) {
4152 start_reg
= ib
[idx
+ 1] * 4;
4153 if (idx_value
& 0x10000) {
4154 if (!si_vm_reg_valid(start_reg
))
4157 for (i
= 0; i
< (pkt
->count
- 2); i
++) {
4158 reg
= start_reg
+ (4 * i
);
4159 if (!si_vm_reg_valid(reg
))
4165 case PACKET3_COND_WRITE
:
4166 if (idx_value
& 0x100) {
4167 reg
= ib
[idx
+ 5] * 4;
4168 if (!si_vm_reg_valid(reg
))
4172 case PACKET3_COPY_DW
:
4173 if (idx_value
& 0x2) {
4174 reg
= ib
[idx
+ 3] * 4;
4175 if (!si_vm_reg_valid(reg
))
4179 case PACKET3_SET_CONFIG_REG
:
4180 start_reg
= (idx_value
<< 2) + PACKET3_SET_CONFIG_REG_START
;
4181 end_reg
= 4 * pkt
->count
+ start_reg
- 4;
4182 if ((start_reg
< PACKET3_SET_CONFIG_REG_START
) ||
4183 (start_reg
>= PACKET3_SET_CONFIG_REG_END
) ||
4184 (end_reg
>= PACKET3_SET_CONFIG_REG_END
)) {
4185 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4188 for (i
= 0; i
< pkt
->count
; i
++) {
4189 reg
= start_reg
+ (4 * i
);
4190 if (!si_vm_reg_valid(reg
))
4194 case PACKET3_CP_DMA
:
4195 command
= ib
[idx
+ 4];
4197 if (command
& PACKET3_CP_DMA_CMD_SAS
) {
4198 /* src address space is register */
4199 if (((info
& 0x60000000) >> 29) == 0) {
4200 start_reg
= idx_value
<< 2;
4201 if (command
& PACKET3_CP_DMA_CMD_SAIC
) {
4203 if (!si_vm_reg_valid(reg
)) {
4204 DRM_ERROR("CP DMA Bad SRC register\n");
4208 for (i
= 0; i
< (command
& 0x1fffff); i
++) {
4209 reg
= start_reg
+ (4 * i
);
4210 if (!si_vm_reg_valid(reg
)) {
4211 DRM_ERROR("CP DMA Bad SRC register\n");
4218 if (command
& PACKET3_CP_DMA_CMD_DAS
) {
4219 /* dst address space is register */
4220 if (((info
& 0x00300000) >> 20) == 0) {
4221 start_reg
= ib
[idx
+ 2];
4222 if (command
& PACKET3_CP_DMA_CMD_DAIC
) {
4224 if (!si_vm_reg_valid(reg
)) {
4225 DRM_ERROR("CP DMA Bad DST register\n");
4229 for (i
= 0; i
< (command
& 0x1fffff); i
++) {
4230 reg
= start_reg
+ (4 * i
);
4231 if (!si_vm_reg_valid(reg
)) {
4232 DRM_ERROR("CP DMA Bad DST register\n");
4241 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt
->opcode
);
4247 static int si_vm_packet3_compute_check(struct radeon_device
*rdev
,
4248 u32
*ib
, struct radeon_cs_packet
*pkt
)
4250 u32 idx
= pkt
->idx
+ 1;
4251 u32 idx_value
= ib
[idx
];
4252 u32 start_reg
, reg
, i
;
4254 switch (pkt
->opcode
) {
4256 case PACKET3_SET_BASE
:
4257 case PACKET3_CLEAR_STATE
:
4258 case PACKET3_DISPATCH_DIRECT
:
4259 case PACKET3_DISPATCH_INDIRECT
:
4260 case PACKET3_ALLOC_GDS
:
4261 case PACKET3_WRITE_GDS_RAM
:
4262 case PACKET3_ATOMIC_GDS
:
4263 case PACKET3_ATOMIC
:
4264 case PACKET3_OCCLUSION_QUERY
:
4265 case PACKET3_SET_PREDICATION
:
4266 case PACKET3_COND_EXEC
:
4267 case PACKET3_PRED_EXEC
:
4268 case PACKET3_CONTEXT_CONTROL
:
4269 case PACKET3_STRMOUT_BUFFER_UPDATE
:
4270 case PACKET3_WAIT_REG_MEM
:
4271 case PACKET3_MEM_WRITE
:
4272 case PACKET3_PFP_SYNC_ME
:
4273 case PACKET3_SURFACE_SYNC
:
4274 case PACKET3_EVENT_WRITE
:
4275 case PACKET3_EVENT_WRITE_EOP
:
4276 case PACKET3_EVENT_WRITE_EOS
:
4277 case PACKET3_SET_CONTEXT_REG
:
4278 case PACKET3_SET_CONTEXT_REG_INDIRECT
:
4279 case PACKET3_SET_SH_REG
:
4280 case PACKET3_SET_SH_REG_OFFSET
:
4281 case PACKET3_INCREMENT_DE_COUNTER
:
4282 case PACKET3_WAIT_ON_CE_COUNTER
:
4283 case PACKET3_WAIT_ON_AVAIL_BUFFER
:
4284 case PACKET3_ME_WRITE
:
4286 case PACKET3_COPY_DATA
:
4287 if ((idx_value
& 0xf00) == 0) {
4288 reg
= ib
[idx
+ 3] * 4;
4289 if (!si_vm_reg_valid(reg
))
4293 case PACKET3_WRITE_DATA
:
4294 if ((idx_value
& 0xf00) == 0) {
4295 start_reg
= ib
[idx
+ 1] * 4;
4296 if (idx_value
& 0x10000) {
4297 if (!si_vm_reg_valid(start_reg
))
4300 for (i
= 0; i
< (pkt
->count
- 2); i
++) {
4301 reg
= start_reg
+ (4 * i
);
4302 if (!si_vm_reg_valid(reg
))
4308 case PACKET3_COND_WRITE
:
4309 if (idx_value
& 0x100) {
4310 reg
= ib
[idx
+ 5] * 4;
4311 if (!si_vm_reg_valid(reg
))
4315 case PACKET3_COPY_DW
:
4316 if (idx_value
& 0x2) {
4317 reg
= ib
[idx
+ 3] * 4;
4318 if (!si_vm_reg_valid(reg
))
4323 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt
->opcode
);
4329 int si_ib_parse(struct radeon_device
*rdev
, struct radeon_ib
*ib
)
4333 struct radeon_cs_packet pkt
;
4337 pkt
.type
= RADEON_CP_PACKET_GET_TYPE(ib
->ptr
[idx
]);
4338 pkt
.count
= RADEON_CP_PACKET_GET_COUNT(ib
->ptr
[idx
]);
4341 case RADEON_PACKET_TYPE0
:
4342 dev_err(rdev
->dev
, "Packet0 not allowed!\n");
4345 case RADEON_PACKET_TYPE2
:
4348 case RADEON_PACKET_TYPE3
:
4349 pkt
.opcode
= RADEON_CP_PACKET3_GET_OPCODE(ib
->ptr
[idx
]);
4350 if (ib
->is_const_ib
)
4351 ret
= si_vm_packet3_ce_check(rdev
, ib
->ptr
, &pkt
);
4354 case RADEON_RING_TYPE_GFX_INDEX
:
4355 ret
= si_vm_packet3_gfx_check(rdev
, ib
->ptr
, &pkt
);
4357 case CAYMAN_RING_TYPE_CP1_INDEX
:
4358 case CAYMAN_RING_TYPE_CP2_INDEX
:
4359 ret
= si_vm_packet3_compute_check(rdev
, ib
->ptr
, &pkt
);
4362 dev_err(rdev
->dev
, "Non-PM4 ring %d !\n", ib
->ring
);
4367 idx
+= pkt
.count
+ 2;
4370 dev_err(rdev
->dev
, "Unknown packet type %d !\n", pkt
.type
);
4376 } while (idx
< ib
->length_dw
);
4384 int si_vm_init(struct radeon_device
*rdev
)
4387 rdev
->vm_manager
.nvm
= 16;
4388 /* base offset of vram pages */
4389 rdev
->vm_manager
.vram_base_offset
= 0;
4394 void si_vm_fini(struct radeon_device
*rdev
)
4399 * si_vm_decode_fault - print human readable fault info
4401 * @rdev: radeon_device pointer
4402 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4403 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4405 * Print human readable fault information (SI).
4407 static void si_vm_decode_fault(struct radeon_device
*rdev
,
4408 u32 status
, u32 addr
)
4410 u32 mc_id
= (status
& MEMORY_CLIENT_ID_MASK
) >> MEMORY_CLIENT_ID_SHIFT
;
4411 u32 vmid
= (status
& FAULT_VMID_MASK
) >> FAULT_VMID_SHIFT
;
4412 u32 protections
= (status
& PROTECTIONS_MASK
) >> PROTECTIONS_SHIFT
;
4415 if (rdev
->family
== CHIP_TAHITI
) {
4656 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4657 protections
, vmid
, addr
,
4658 (status
& MEMORY_CLIENT_RW_MASK
) ? "write" : "read",
4663 * si_vm_set_page - update the page tables using the CP
4665 * @rdev: radeon_device pointer
4666 * @ib: indirect buffer to fill with commands
4667 * @pe: addr of the page entry
4668 * @addr: dst addr to write into pe
4669 * @count: number of page entries to update
4670 * @incr: increase next addr by incr bytes
4671 * @flags: access flags
4673 * Update the page tables using the CP (SI).
4675 void si_vm_set_page(struct radeon_device
*rdev
,
4676 struct radeon_ib
*ib
,
4678 uint64_t addr
, unsigned count
,
4679 uint32_t incr
, uint32_t flags
)
4681 uint32_t r600_flags
= cayman_vm_page_flags(rdev
, flags
);
4685 if (rdev
->asic
->vm
.pt_ring_index
== RADEON_RING_TYPE_GFX_INDEX
) {
4687 ndw
= 2 + count
* 2;
4691 ib
->ptr
[ib
->length_dw
++] = PACKET3(PACKET3_WRITE_DATA
, ndw
);
4692 ib
->ptr
[ib
->length_dw
++] = (WRITE_DATA_ENGINE_SEL(0) |
4693 WRITE_DATA_DST_SEL(1));
4694 ib
->ptr
[ib
->length_dw
++] = pe
;
4695 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
);
4696 for (; ndw
> 2; ndw
-= 2, --count
, pe
+= 8) {
4697 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4698 value
= radeon_vm_map_gart(rdev
, addr
);
4699 value
&= 0xFFFFFFFFFFFFF000ULL
;
4700 } else if (flags
& RADEON_VM_PAGE_VALID
) {
4706 value
|= r600_flags
;
4707 ib
->ptr
[ib
->length_dw
++] = value
;
4708 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4713 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4719 /* for non-physically contiguous pages (system) */
4720 ib
->ptr
[ib
->length_dw
++] = DMA_PACKET(DMA_PACKET_WRITE
, 0, 0, 0, ndw
);
4721 ib
->ptr
[ib
->length_dw
++] = pe
;
4722 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
4723 for (; ndw
> 0; ndw
-= 2, --count
, pe
+= 8) {
4724 if (flags
& RADEON_VM_PAGE_SYSTEM
) {
4725 value
= radeon_vm_map_gart(rdev
, addr
);
4726 value
&= 0xFFFFFFFFFFFFF000ULL
;
4727 } else if (flags
& RADEON_VM_PAGE_VALID
) {
4733 value
|= r600_flags
;
4734 ib
->ptr
[ib
->length_dw
++] = value
;
4735 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4744 if (flags
& RADEON_VM_PAGE_VALID
)
4748 /* for physically contiguous pages (vram) */
4749 ib
->ptr
[ib
->length_dw
++] = DMA_PTE_PDE_PACKET(ndw
);
4750 ib
->ptr
[ib
->length_dw
++] = pe
; /* dst addr */
4751 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(pe
) & 0xff;
4752 ib
->ptr
[ib
->length_dw
++] = r600_flags
; /* mask */
4753 ib
->ptr
[ib
->length_dw
++] = 0;
4754 ib
->ptr
[ib
->length_dw
++] = value
; /* value */
4755 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(value
);
4756 ib
->ptr
[ib
->length_dw
++] = incr
; /* increment size */
4757 ib
->ptr
[ib
->length_dw
++] = 0;
4759 addr
+= (ndw
/ 2) * incr
;
4763 while (ib
->length_dw
& 0x7)
4764 ib
->ptr
[ib
->length_dw
++] = DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0);
4768 void si_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
4770 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
4775 /* write new base address */
4776 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4777 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4778 WRITE_DATA_DST_SEL(0)));
4781 radeon_ring_write(ring
,
4782 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2);
4784 radeon_ring_write(ring
,
4785 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2);
4787 radeon_ring_write(ring
, 0);
4788 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
4790 /* flush hdp cache */
4791 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4792 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4793 WRITE_DATA_DST_SEL(0)));
4794 radeon_ring_write(ring
, HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2);
4795 radeon_ring_write(ring
, 0);
4796 radeon_ring_write(ring
, 0x1);
4798 /* bits 0-15 are the VM contexts0-15 */
4799 radeon_ring_write(ring
, PACKET3(PACKET3_WRITE_DATA
, 3));
4800 radeon_ring_write(ring
, (WRITE_DATA_ENGINE_SEL(0) |
4801 WRITE_DATA_DST_SEL(0)));
4802 radeon_ring_write(ring
, VM_INVALIDATE_REQUEST
>> 2);
4803 radeon_ring_write(ring
, 0);
4804 radeon_ring_write(ring
, 1 << vm
->id
);
4806 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4807 radeon_ring_write(ring
, PACKET3(PACKET3_PFP_SYNC_ME
, 0));
4808 radeon_ring_write(ring
, 0x0);
4811 void si_dma_vm_flush(struct radeon_device
*rdev
, int ridx
, struct radeon_vm
*vm
)
4813 struct radeon_ring
*ring
= &rdev
->ring
[ridx
];
4818 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4820 radeon_ring_write(ring
, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+ (vm
->id
<< 2)) >> 2));
4822 radeon_ring_write(ring
, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
+ ((vm
->id
- 8) << 2)) >> 2));
4824 radeon_ring_write(ring
, vm
->pd_gpu_addr
>> 12);
4826 /* flush hdp cache */
4827 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4828 radeon_ring_write(ring
, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL
>> 2));
4829 radeon_ring_write(ring
, 1);
4831 /* bits 0-7 are the VM contexts0-7 */
4832 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_SRBM_WRITE
, 0, 0, 0, 0));
4833 radeon_ring_write(ring
, (0xf << 16) | (VM_INVALIDATE_REQUEST
>> 2));
4834 radeon_ring_write(ring
, 1 << vm
->id
);
4838 * Power and clock gating
4840 static void si_wait_for_rlc_serdes(struct radeon_device
*rdev
)
4844 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4845 if (RREG32(RLC_SERDES_MASTER_BUSY_0
) == 0)
4850 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4851 if (RREG32(RLC_SERDES_MASTER_BUSY_1
) == 0)
4857 static void si_enable_gui_idle_interrupt(struct radeon_device
*rdev
,
4860 u32 tmp
= RREG32(CP_INT_CNTL_RING0
);
4865 tmp
|= (CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
4867 tmp
&= ~(CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
4868 WREG32(CP_INT_CNTL_RING0
, tmp
);
4871 /* read a gfx register */
4872 tmp
= RREG32(DB_DEPTH_INFO
);
4874 mask
= RLC_BUSY_STATUS
| GFX_POWER_STATUS
| GFX_CLOCK_STATUS
| GFX_LS_STATUS
;
4875 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
4876 if ((RREG32(RLC_STAT
) & mask
) == (GFX_CLOCK_STATUS
| GFX_POWER_STATUS
))
4883 static void si_set_uvd_dcm(struct radeon_device
*rdev
,
4888 tmp
= RREG32(UVD_CGC_CTRL
);
4889 tmp
&= ~(CLK_OD_MASK
| CG_DT_MASK
);
4890 tmp
|= DCM
| CG_DT(1) | CLK_OD(4);
4894 tmp2
= DYN_OR_EN
| DYN_RR_EN
| G_DIV_ID(7);
4900 WREG32(UVD_CGC_CTRL
, tmp
);
4901 WREG32_UVD_CTX(UVD_CGC_CTRL2
, tmp2
);
4904 void si_init_uvd_internal_cg(struct radeon_device
*rdev
)
4906 bool hw_mode
= true;
4909 si_set_uvd_dcm(rdev
, false);
4911 u32 tmp
= RREG32(UVD_CGC_CTRL
);
4913 WREG32(UVD_CGC_CTRL
, tmp
);
4917 static u32
si_halt_rlc(struct radeon_device
*rdev
)
4921 orig
= data
= RREG32(RLC_CNTL
);
4923 if (data
& RLC_ENABLE
) {
4924 data
&= ~RLC_ENABLE
;
4925 WREG32(RLC_CNTL
, data
);
4927 si_wait_for_rlc_serdes(rdev
);
4933 static void si_update_rlc(struct radeon_device
*rdev
, u32 rlc
)
4937 tmp
= RREG32(RLC_CNTL
);
4939 WREG32(RLC_CNTL
, rlc
);
4942 static void si_enable_dma_pg(struct radeon_device
*rdev
, bool enable
)
4946 orig
= data
= RREG32(DMA_PG
);
4948 data
|= PG_CNTL_ENABLE
;
4950 data
&= ~PG_CNTL_ENABLE
;
4952 WREG32(DMA_PG
, data
);
4955 static void si_init_dma_pg(struct radeon_device
*rdev
)
4959 WREG32(DMA_PGFSM_WRITE
, 0x00002000);
4960 WREG32(DMA_PGFSM_CONFIG
, 0x100010ff);
4962 for (tmp
= 0; tmp
< 5; tmp
++)
4963 WREG32(DMA_PGFSM_WRITE
, 0);
4966 static void si_enable_gfx_cgpg(struct radeon_device
*rdev
,
4972 tmp
= RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4973 WREG32(RLC_TTOP_D
, tmp
);
4975 tmp
= RREG32(RLC_PG_CNTL
);
4976 tmp
|= GFX_PG_ENABLE
;
4977 WREG32(RLC_PG_CNTL
, tmp
);
4979 tmp
= RREG32(RLC_AUTO_PG_CTRL
);
4981 WREG32(RLC_AUTO_PG_CTRL
, tmp
);
4983 tmp
= RREG32(RLC_AUTO_PG_CTRL
);
4985 WREG32(RLC_AUTO_PG_CTRL
, tmp
);
4987 tmp
= RREG32(DB_RENDER_CONTROL
);
4991 static void si_init_gfx_cgpg(struct radeon_device
*rdev
)
4995 WREG32(RLC_SAVE_AND_RESTORE_BASE
, rdev
->rlc
.save_restore_gpu_addr
>> 8);
4997 tmp
= RREG32(RLC_PG_CNTL
);
4999 WREG32(RLC_PG_CNTL
, tmp
);
5001 WREG32(RLC_CLEAR_STATE_RESTORE_BASE
, rdev
->rlc
.clear_state_gpu_addr
>> 8);
5003 tmp
= RREG32(RLC_AUTO_PG_CTRL
);
5005 tmp
&= ~GRBM_REG_SGIT_MASK
;
5006 tmp
|= GRBM_REG_SGIT(0x700);
5007 tmp
&= ~PG_AFTER_GRBM_REG_ST_MASK
;
5008 WREG32(RLC_AUTO_PG_CTRL
, tmp
);
5011 static u32
si_get_cu_active_bitmap(struct radeon_device
*rdev
, u32 se
, u32 sh
)
5013 u32 mask
= 0, tmp
, tmp1
;
5016 si_select_se_sh(rdev
, se
, sh
);
5017 tmp
= RREG32(CC_GC_SHADER_ARRAY_CONFIG
);
5018 tmp1
= RREG32(GC_USER_SHADER_ARRAY_CONFIG
);
5019 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5026 for (i
= 0; i
< rdev
->config
.si
.max_cu_per_sh
; i
++) {
5031 return (~tmp
) & mask
;
5034 static void si_init_ao_cu_mask(struct radeon_device
*rdev
)
5036 u32 i
, j
, k
, active_cu_number
= 0;
5037 u32 mask
, counter
, cu_bitmap
;
5040 for (i
= 0; i
< rdev
->config
.si
.max_shader_engines
; i
++) {
5041 for (j
= 0; j
< rdev
->config
.si
.max_sh_per_se
; j
++) {
5045 for (k
= 0; k
< rdev
->config
.si
.max_cu_per_sh
; k
++) {
5046 if (si_get_cu_active_bitmap(rdev
, i
, j
) & mask
) {
5054 active_cu_number
+= counter
;
5055 tmp
|= (cu_bitmap
<< (i
* 16 + j
* 8));
5059 WREG32(RLC_PG_AO_CU_MASK
, tmp
);
5061 tmp
= RREG32(RLC_MAX_PG_CU
);
5062 tmp
&= ~MAX_PU_CU_MASK
;
5063 tmp
|= MAX_PU_CU(active_cu_number
);
5064 WREG32(RLC_MAX_PG_CU
, tmp
);
5067 static void si_enable_cgcg(struct radeon_device
*rdev
,
5070 u32 data
, orig
, tmp
;
5072 orig
= data
= RREG32(RLC_CGCG_CGLS_CTRL
);
5074 si_enable_gui_idle_interrupt(rdev
, enable
);
5077 WREG32(RLC_GCPM_GENERAL_3
, 0x00000080);
5079 tmp
= si_halt_rlc(rdev
);
5081 WREG32(RLC_SERDES_WR_MASTER_MASK_0
, 0xffffffff);
5082 WREG32(RLC_SERDES_WR_MASTER_MASK_1
, 0xffffffff);
5083 WREG32(RLC_SERDES_WR_CTRL
, 0x00b000ff);
5085 si_wait_for_rlc_serdes(rdev
);
5087 si_update_rlc(rdev
, tmp
);
5089 WREG32(RLC_SERDES_WR_CTRL
, 0x007000ff);
5091 data
|= CGCG_EN
| CGLS_EN
;
5093 RREG32(CB_CGTT_SCLK_CTRL
);
5094 RREG32(CB_CGTT_SCLK_CTRL
);
5095 RREG32(CB_CGTT_SCLK_CTRL
);
5096 RREG32(CB_CGTT_SCLK_CTRL
);
5098 data
&= ~(CGCG_EN
| CGLS_EN
);
5102 WREG32(RLC_CGCG_CGLS_CTRL
, data
);
5105 static void si_enable_mgcg(struct radeon_device
*rdev
,
5108 u32 data
, orig
, tmp
= 0;
5111 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5114 WREG32(CGTS_SM_CTRL_REG
, data
);
5116 orig
= data
= RREG32(CP_MEM_SLP_CNTL
);
5117 data
|= CP_MEM_LS_EN
;
5119 WREG32(CP_MEM_SLP_CNTL
, data
);
5121 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5124 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5126 tmp
= si_halt_rlc(rdev
);
5128 WREG32(RLC_SERDES_WR_MASTER_MASK_0
, 0xffffffff);
5129 WREG32(RLC_SERDES_WR_MASTER_MASK_1
, 0xffffffff);
5130 WREG32(RLC_SERDES_WR_CTRL
, 0x00d000ff);
5132 si_update_rlc(rdev
, tmp
);
5134 orig
= data
= RREG32(RLC_CGTT_MGCG_OVERRIDE
);
5137 WREG32(RLC_CGTT_MGCG_OVERRIDE
, data
);
5139 data
= RREG32(CP_MEM_SLP_CNTL
);
5140 if (data
& CP_MEM_LS_EN
) {
5141 data
&= ~CP_MEM_LS_EN
;
5142 WREG32(CP_MEM_SLP_CNTL
, data
);
5144 orig
= data
= RREG32(CGTS_SM_CTRL_REG
);
5145 data
|= LS_OVERRIDE
| OVERRIDE
;
5147 WREG32(CGTS_SM_CTRL_REG
, data
);
5149 tmp
= si_halt_rlc(rdev
);
5151 WREG32(RLC_SERDES_WR_MASTER_MASK_0
, 0xffffffff);
5152 WREG32(RLC_SERDES_WR_MASTER_MASK_1
, 0xffffffff);
5153 WREG32(RLC_SERDES_WR_CTRL
, 0x00e000ff);
5155 si_update_rlc(rdev
, tmp
);
5159 static void si_enable_uvd_mgcg(struct radeon_device
*rdev
,
5162 u32 orig
, data
, tmp
;
5165 tmp
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
5167 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, tmp
);
5169 orig
= data
= RREG32(UVD_CGC_CTRL
);
5172 WREG32(UVD_CGC_CTRL
, data
);
5174 WREG32_SMC(SMC_CG_IND_START
+ CG_CGTT_LOCAL_0
, 0);
5175 WREG32_SMC(SMC_CG_IND_START
+ CG_CGTT_LOCAL_1
, 0);
5177 tmp
= RREG32_UVD_CTX(UVD_CGC_MEM_CTRL
);
5179 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL
, tmp
);
5181 orig
= data
= RREG32(UVD_CGC_CTRL
);
5184 WREG32(UVD_CGC_CTRL
, data
);
5186 WREG32_SMC(SMC_CG_IND_START
+ CG_CGTT_LOCAL_0
, 0xffffffff);
5187 WREG32_SMC(SMC_CG_IND_START
+ CG_CGTT_LOCAL_1
, 0xffffffff);
5191 static const u32 mc_cg_registers
[] =
5204 static void si_enable_mc_ls(struct radeon_device
*rdev
,
5210 for (i
= 0; i
< ARRAY_SIZE(mc_cg_registers
); i
++) {
5211 orig
= data
= RREG32(mc_cg_registers
[i
]);
5213 data
|= MC_LS_ENABLE
;
5215 data
&= ~MC_LS_ENABLE
;
5217 WREG32(mc_cg_registers
[i
], data
);
5222 static void si_init_cg(struct radeon_device
*rdev
)
5224 si_enable_mgcg(rdev
, true);
5225 si_enable_cgcg(rdev
, false);
5226 /* disable MC LS on Tahiti */
5227 if (rdev
->family
== CHIP_TAHITI
)
5228 si_enable_mc_ls(rdev
, false);
5229 if (rdev
->has_uvd
) {
5230 si_enable_uvd_mgcg(rdev
, true);
5231 si_init_uvd_internal_cg(rdev
);
5235 static void si_fini_cg(struct radeon_device
*rdev
)
5238 si_enable_uvd_mgcg(rdev
, false);
5239 si_enable_cgcg(rdev
, false);
5240 si_enable_mgcg(rdev
, false);
5243 static void si_init_pg(struct radeon_device
*rdev
)
5245 bool has_pg
= false;
5247 /* only cape verde supports PG */
5248 if (rdev
->family
== CHIP_VERDE
)
5252 si_init_ao_cu_mask(rdev
);
5253 si_init_dma_pg(rdev
);
5254 si_enable_dma_pg(rdev
, true);
5255 si_init_gfx_cgpg(rdev
);
5256 si_enable_gfx_cgpg(rdev
, true);
5258 WREG32(RLC_SAVE_AND_RESTORE_BASE
, rdev
->rlc
.save_restore_gpu_addr
>> 8);
5259 WREG32(RLC_CLEAR_STATE_RESTORE_BASE
, rdev
->rlc
.clear_state_gpu_addr
>> 8);
5263 static void si_fini_pg(struct radeon_device
*rdev
)
5265 bool has_pg
= false;
5267 /* only cape verde supports PG */
5268 if (rdev
->family
== CHIP_VERDE
)
5272 si_enable_dma_pg(rdev
, false);
5273 si_enable_gfx_cgpg(rdev
, false);
5280 void si_rlc_reset(struct radeon_device
*rdev
)
5282 u32 tmp
= RREG32(GRBM_SOFT_RESET
);
5284 tmp
|= SOFT_RESET_RLC
;
5285 WREG32(GRBM_SOFT_RESET
, tmp
);
5287 tmp
&= ~SOFT_RESET_RLC
;
5288 WREG32(GRBM_SOFT_RESET
, tmp
);
5292 static void si_rlc_stop(struct radeon_device
*rdev
)
5294 WREG32(RLC_CNTL
, 0);
5296 si_enable_gui_idle_interrupt(rdev
, false);
5298 si_wait_for_rlc_serdes(rdev
);
5301 static void si_rlc_start(struct radeon_device
*rdev
)
5303 WREG32(RLC_CNTL
, RLC_ENABLE
);
5305 si_enable_gui_idle_interrupt(rdev
, true);
5310 static bool si_lbpw_supported(struct radeon_device
*rdev
)
5314 /* Enable LBPW only for DDR3 */
5315 tmp
= RREG32(MC_SEQ_MISC0
);
5316 if ((tmp
& 0xF0000000) == 0xB0000000)
5321 static void si_enable_lbpw(struct radeon_device
*rdev
, bool enable
)
5325 tmp
= RREG32(RLC_LB_CNTL
);
5327 tmp
|= LOAD_BALANCE_ENABLE
;
5329 tmp
&= ~LOAD_BALANCE_ENABLE
;
5330 WREG32(RLC_LB_CNTL
, tmp
);
5333 si_select_se_sh(rdev
, 0xffffffff, 0xffffffff);
5334 WREG32(SPI_LB_CU_MASK
, 0x00ff);
5338 static int si_rlc_resume(struct radeon_device
*rdev
)
5341 const __be32
*fw_data
;
5354 WREG32(RLC_RL_BASE
, 0);
5355 WREG32(RLC_RL_SIZE
, 0);
5356 WREG32(RLC_LB_CNTL
, 0);
5357 WREG32(RLC_LB_CNTR_MAX
, 0xffffffff);
5358 WREG32(RLC_LB_CNTR_INIT
, 0);
5359 WREG32(RLC_LB_INIT_CU_MASK
, 0xffffffff);
5361 WREG32(RLC_MC_CNTL
, 0);
5362 WREG32(RLC_UCODE_CNTL
, 0);
5364 fw_data
= (const __be32
*)rdev
->rlc_fw
->data
;
5365 for (i
= 0; i
< SI_RLC_UCODE_SIZE
; i
++) {
5366 WREG32(RLC_UCODE_ADDR
, i
);
5367 WREG32(RLC_UCODE_DATA
, be32_to_cpup(fw_data
++));
5369 WREG32(RLC_UCODE_ADDR
, 0);
5371 si_enable_lbpw(rdev
, si_lbpw_supported(rdev
));
5378 static void si_enable_interrupts(struct radeon_device
*rdev
)
5380 u32 ih_cntl
= RREG32(IH_CNTL
);
5381 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
5383 ih_cntl
|= ENABLE_INTR
;
5384 ih_rb_cntl
|= IH_RB_ENABLE
;
5385 WREG32(IH_CNTL
, ih_cntl
);
5386 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
5387 rdev
->ih
.enabled
= true;
5390 static void si_disable_interrupts(struct radeon_device
*rdev
)
5392 u32 ih_rb_cntl
= RREG32(IH_RB_CNTL
);
5393 u32 ih_cntl
= RREG32(IH_CNTL
);
5395 ih_rb_cntl
&= ~IH_RB_ENABLE
;
5396 ih_cntl
&= ~ENABLE_INTR
;
5397 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
5398 WREG32(IH_CNTL
, ih_cntl
);
5399 /* set rptr, wptr to 0 */
5400 WREG32(IH_RB_RPTR
, 0);
5401 WREG32(IH_RB_WPTR
, 0);
5402 rdev
->ih
.enabled
= false;
5406 static void si_disable_interrupt_state(struct radeon_device
*rdev
)
5410 WREG32(CP_INT_CNTL_RING0
, CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
);
5411 WREG32(CP_INT_CNTL_RING1
, 0);
5412 WREG32(CP_INT_CNTL_RING2
, 0);
5413 tmp
= RREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
5414 WREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
, tmp
);
5415 tmp
= RREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
5416 WREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
, tmp
);
5417 WREG32(GRBM_INT_CNTL
, 0);
5418 if (rdev
->num_crtc
>= 2) {
5419 WREG32(INT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
5420 WREG32(INT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
5422 if (rdev
->num_crtc
>= 4) {
5423 WREG32(INT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
5424 WREG32(INT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
5426 if (rdev
->num_crtc
>= 6) {
5427 WREG32(INT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
5428 WREG32(INT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
5431 if (rdev
->num_crtc
>= 2) {
5432 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, 0);
5433 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, 0);
5435 if (rdev
->num_crtc
>= 4) {
5436 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, 0);
5437 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, 0);
5439 if (rdev
->num_crtc
>= 6) {
5440 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, 0);
5441 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, 0);
5444 if (!ASIC_IS_NODCE(rdev
)) {
5445 WREG32(DACA_AUTODETECT_INT_CONTROL
, 0);
5447 tmp
= RREG32(DC_HPD1_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5448 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
5449 tmp
= RREG32(DC_HPD2_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5450 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
5451 tmp
= RREG32(DC_HPD3_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5452 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
5453 tmp
= RREG32(DC_HPD4_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5454 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
5455 tmp
= RREG32(DC_HPD5_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5456 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
5457 tmp
= RREG32(DC_HPD6_INT_CONTROL
) & DC_HPDx_INT_POLARITY
;
5458 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
5462 static int si_irq_init(struct radeon_device
*rdev
)
5466 u32 interrupt_cntl
, ih_cntl
, ih_rb_cntl
;
5469 ret
= r600_ih_ring_alloc(rdev
);
5474 si_disable_interrupts(rdev
);
5477 ret
= si_rlc_resume(rdev
);
5479 r600_ih_ring_fini(rdev
);
5483 /* setup interrupt control */
5484 /* set dummy read address to ring address */
5485 WREG32(INTERRUPT_CNTL2
, rdev
->ih
.gpu_addr
>> 8);
5486 interrupt_cntl
= RREG32(INTERRUPT_CNTL
);
5487 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5488 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5490 interrupt_cntl
&= ~IH_DUMMY_RD_OVERRIDE
;
5491 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5492 interrupt_cntl
&= ~IH_REQ_NONSNOOP_EN
;
5493 WREG32(INTERRUPT_CNTL
, interrupt_cntl
);
5495 WREG32(IH_RB_BASE
, rdev
->ih
.gpu_addr
>> 8);
5496 rb_bufsz
= drm_order(rdev
->ih
.ring_size
/ 4);
5498 ih_rb_cntl
= (IH_WPTR_OVERFLOW_ENABLE
|
5499 IH_WPTR_OVERFLOW_CLEAR
|
5502 if (rdev
->wb
.enabled
)
5503 ih_rb_cntl
|= IH_WPTR_WRITEBACK_ENABLE
;
5505 /* set the writeback address whether it's enabled or not */
5506 WREG32(IH_RB_WPTR_ADDR_LO
, (rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFFFFFFFC);
5507 WREG32(IH_RB_WPTR_ADDR_HI
, upper_32_bits(rdev
->wb
.gpu_addr
+ R600_WB_IH_WPTR_OFFSET
) & 0xFF);
5509 WREG32(IH_RB_CNTL
, ih_rb_cntl
);
5511 /* set rptr, wptr to 0 */
5512 WREG32(IH_RB_RPTR
, 0);
5513 WREG32(IH_RB_WPTR
, 0);
5515 /* Default settings for IH_CNTL (disabled at first) */
5516 ih_cntl
= MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5517 /* RPTR_REARM only works if msi's are enabled */
5518 if (rdev
->msi_enabled
)
5519 ih_cntl
|= RPTR_REARM
;
5520 WREG32(IH_CNTL
, ih_cntl
);
5522 /* force the active interrupt state to all disabled */
5523 si_disable_interrupt_state(rdev
);
5525 pci_set_master(rdev
->pdev
);
5528 si_enable_interrupts(rdev
);
5533 int si_irq_set(struct radeon_device
*rdev
)
5535 u32 cp_int_cntl
= CNTX_BUSY_INT_ENABLE
| CNTX_EMPTY_INT_ENABLE
;
5536 u32 cp_int_cntl1
= 0, cp_int_cntl2
= 0;
5537 u32 crtc1
= 0, crtc2
= 0, crtc3
= 0, crtc4
= 0, crtc5
= 0, crtc6
= 0;
5538 u32 hpd1
= 0, hpd2
= 0, hpd3
= 0, hpd4
= 0, hpd5
= 0, hpd6
= 0;
5539 u32 grbm_int_cntl
= 0;
5540 u32 grph1
= 0, grph2
= 0, grph3
= 0, grph4
= 0, grph5
= 0, grph6
= 0;
5541 u32 dma_cntl
, dma_cntl1
;
5542 u32 thermal_int
= 0;
5544 if (!rdev
->irq
.installed
) {
5545 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5548 /* don't enable anything if the ih is disabled */
5549 if (!rdev
->ih
.enabled
) {
5550 si_disable_interrupts(rdev
);
5551 /* force the active interrupt state to all disabled */
5552 si_disable_interrupt_state(rdev
);
5556 if (!ASIC_IS_NODCE(rdev
)) {
5557 hpd1
= RREG32(DC_HPD1_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5558 hpd2
= RREG32(DC_HPD2_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5559 hpd3
= RREG32(DC_HPD3_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5560 hpd4
= RREG32(DC_HPD4_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5561 hpd5
= RREG32(DC_HPD5_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5562 hpd6
= RREG32(DC_HPD6_INT_CONTROL
) & ~DC_HPDx_INT_EN
;
5565 dma_cntl
= RREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
5566 dma_cntl1
= RREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
) & ~TRAP_ENABLE
;
5568 thermal_int
= RREG32(CG_THERMAL_INT
) &
5569 ~(THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
);
5571 /* enable CP interrupts on all rings */
5572 if (atomic_read(&rdev
->irq
.ring_int
[RADEON_RING_TYPE_GFX_INDEX
])) {
5573 DRM_DEBUG("si_irq_set: sw int gfx\n");
5574 cp_int_cntl
|= TIME_STAMP_INT_ENABLE
;
5576 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP1_INDEX
])) {
5577 DRM_DEBUG("si_irq_set: sw int cp1\n");
5578 cp_int_cntl1
|= TIME_STAMP_INT_ENABLE
;
5580 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_CP2_INDEX
])) {
5581 DRM_DEBUG("si_irq_set: sw int cp2\n");
5582 cp_int_cntl2
|= TIME_STAMP_INT_ENABLE
;
5584 if (atomic_read(&rdev
->irq
.ring_int
[R600_RING_TYPE_DMA_INDEX
])) {
5585 DRM_DEBUG("si_irq_set: sw int dma\n");
5586 dma_cntl
|= TRAP_ENABLE
;
5589 if (atomic_read(&rdev
->irq
.ring_int
[CAYMAN_RING_TYPE_DMA1_INDEX
])) {
5590 DRM_DEBUG("si_irq_set: sw int dma1\n");
5591 dma_cntl1
|= TRAP_ENABLE
;
5593 if (rdev
->irq
.crtc_vblank_int
[0] ||
5594 atomic_read(&rdev
->irq
.pflip
[0])) {
5595 DRM_DEBUG("si_irq_set: vblank 0\n");
5596 crtc1
|= VBLANK_INT_MASK
;
5598 if (rdev
->irq
.crtc_vblank_int
[1] ||
5599 atomic_read(&rdev
->irq
.pflip
[1])) {
5600 DRM_DEBUG("si_irq_set: vblank 1\n");
5601 crtc2
|= VBLANK_INT_MASK
;
5603 if (rdev
->irq
.crtc_vblank_int
[2] ||
5604 atomic_read(&rdev
->irq
.pflip
[2])) {
5605 DRM_DEBUG("si_irq_set: vblank 2\n");
5606 crtc3
|= VBLANK_INT_MASK
;
5608 if (rdev
->irq
.crtc_vblank_int
[3] ||
5609 atomic_read(&rdev
->irq
.pflip
[3])) {
5610 DRM_DEBUG("si_irq_set: vblank 3\n");
5611 crtc4
|= VBLANK_INT_MASK
;
5613 if (rdev
->irq
.crtc_vblank_int
[4] ||
5614 atomic_read(&rdev
->irq
.pflip
[4])) {
5615 DRM_DEBUG("si_irq_set: vblank 4\n");
5616 crtc5
|= VBLANK_INT_MASK
;
5618 if (rdev
->irq
.crtc_vblank_int
[5] ||
5619 atomic_read(&rdev
->irq
.pflip
[5])) {
5620 DRM_DEBUG("si_irq_set: vblank 5\n");
5621 crtc6
|= VBLANK_INT_MASK
;
5623 if (rdev
->irq
.hpd
[0]) {
5624 DRM_DEBUG("si_irq_set: hpd 1\n");
5625 hpd1
|= DC_HPDx_INT_EN
;
5627 if (rdev
->irq
.hpd
[1]) {
5628 DRM_DEBUG("si_irq_set: hpd 2\n");
5629 hpd2
|= DC_HPDx_INT_EN
;
5631 if (rdev
->irq
.hpd
[2]) {
5632 DRM_DEBUG("si_irq_set: hpd 3\n");
5633 hpd3
|= DC_HPDx_INT_EN
;
5635 if (rdev
->irq
.hpd
[3]) {
5636 DRM_DEBUG("si_irq_set: hpd 4\n");
5637 hpd4
|= DC_HPDx_INT_EN
;
5639 if (rdev
->irq
.hpd
[4]) {
5640 DRM_DEBUG("si_irq_set: hpd 5\n");
5641 hpd5
|= DC_HPDx_INT_EN
;
5643 if (rdev
->irq
.hpd
[5]) {
5644 DRM_DEBUG("si_irq_set: hpd 6\n");
5645 hpd6
|= DC_HPDx_INT_EN
;
5648 WREG32(CP_INT_CNTL_RING0
, cp_int_cntl
);
5649 WREG32(CP_INT_CNTL_RING1
, cp_int_cntl1
);
5650 WREG32(CP_INT_CNTL_RING2
, cp_int_cntl2
);
5652 WREG32(DMA_CNTL
+ DMA0_REGISTER_OFFSET
, dma_cntl
);
5653 WREG32(DMA_CNTL
+ DMA1_REGISTER_OFFSET
, dma_cntl1
);
5655 WREG32(GRBM_INT_CNTL
, grbm_int_cntl
);
5657 if (rdev
->irq
.dpm_thermal
) {
5658 DRM_DEBUG("dpm thermal\n");
5659 thermal_int
|= THERM_INT_MASK_HIGH
| THERM_INT_MASK_LOW
;
5662 if (rdev
->num_crtc
>= 2) {
5663 WREG32(INT_MASK
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, crtc1
);
5664 WREG32(INT_MASK
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, crtc2
);
5666 if (rdev
->num_crtc
>= 4) {
5667 WREG32(INT_MASK
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, crtc3
);
5668 WREG32(INT_MASK
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, crtc4
);
5670 if (rdev
->num_crtc
>= 6) {
5671 WREG32(INT_MASK
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, crtc5
);
5672 WREG32(INT_MASK
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, crtc6
);
5675 if (rdev
->num_crtc
>= 2) {
5676 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, grph1
);
5677 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, grph2
);
5679 if (rdev
->num_crtc
>= 4) {
5680 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, grph3
);
5681 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, grph4
);
5683 if (rdev
->num_crtc
>= 6) {
5684 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, grph5
);
5685 WREG32(GRPH_INT_CONTROL
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, grph6
);
5688 if (!ASIC_IS_NODCE(rdev
)) {
5689 WREG32(DC_HPD1_INT_CONTROL
, hpd1
);
5690 WREG32(DC_HPD2_INT_CONTROL
, hpd2
);
5691 WREG32(DC_HPD3_INT_CONTROL
, hpd3
);
5692 WREG32(DC_HPD4_INT_CONTROL
, hpd4
);
5693 WREG32(DC_HPD5_INT_CONTROL
, hpd5
);
5694 WREG32(DC_HPD6_INT_CONTROL
, hpd6
);
5697 WREG32(CG_THERMAL_INT
, thermal_int
);
5702 static inline void si_irq_ack(struct radeon_device
*rdev
)
5706 if (ASIC_IS_NODCE(rdev
))
5709 rdev
->irq
.stat_regs
.evergreen
.disp_int
= RREG32(DISP_INTERRUPT_STATUS
);
5710 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE
);
5711 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE2
);
5712 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE3
);
5713 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE4
);
5714 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
= RREG32(DISP_INTERRUPT_STATUS_CONTINUE5
);
5715 rdev
->irq
.stat_regs
.evergreen
.d1grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
);
5716 rdev
->irq
.stat_regs
.evergreen
.d2grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
);
5717 if (rdev
->num_crtc
>= 4) {
5718 rdev
->irq
.stat_regs
.evergreen
.d3grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
);
5719 rdev
->irq
.stat_regs
.evergreen
.d4grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
);
5721 if (rdev
->num_crtc
>= 6) {
5722 rdev
->irq
.stat_regs
.evergreen
.d5grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
);
5723 rdev
->irq
.stat_regs
.evergreen
.d6grph_int
= RREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
);
5726 if (rdev
->irq
.stat_regs
.evergreen
.d1grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5727 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5728 if (rdev
->irq
.stat_regs
.evergreen
.d2grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5729 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5730 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VBLANK_INTERRUPT
)
5731 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VBLANK_ACK
);
5732 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VLINE_INTERRUPT
)
5733 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC0_REGISTER_OFFSET
, VLINE_ACK
);
5734 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
)
5735 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VBLANK_ACK
);
5736 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
)
5737 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC1_REGISTER_OFFSET
, VLINE_ACK
);
5739 if (rdev
->num_crtc
>= 4) {
5740 if (rdev
->irq
.stat_regs
.evergreen
.d3grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5741 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5742 if (rdev
->irq
.stat_regs
.evergreen
.d4grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5743 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5744 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
)
5745 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VBLANK_ACK
);
5746 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
)
5747 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC2_REGISTER_OFFSET
, VLINE_ACK
);
5748 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
)
5749 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VBLANK_ACK
);
5750 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
)
5751 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC3_REGISTER_OFFSET
, VLINE_ACK
);
5754 if (rdev
->num_crtc
>= 6) {
5755 if (rdev
->irq
.stat_regs
.evergreen
.d5grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5756 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5757 if (rdev
->irq
.stat_regs
.evergreen
.d6grph_int
& GRPH_PFLIP_INT_OCCURRED
)
5758 WREG32(GRPH_INT_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, GRPH_PFLIP_INT_CLEAR
);
5759 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
)
5760 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VBLANK_ACK
);
5761 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
)
5762 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC4_REGISTER_OFFSET
, VLINE_ACK
);
5763 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
)
5764 WREG32(VBLANK_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VBLANK_ACK
);
5765 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
)
5766 WREG32(VLINE_STATUS
+ EVERGREEN_CRTC5_REGISTER_OFFSET
, VLINE_ACK
);
5769 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& DC_HPD1_INTERRUPT
) {
5770 tmp
= RREG32(DC_HPD1_INT_CONTROL
);
5771 tmp
|= DC_HPDx_INT_ACK
;
5772 WREG32(DC_HPD1_INT_CONTROL
, tmp
);
5774 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
5775 tmp
= RREG32(DC_HPD2_INT_CONTROL
);
5776 tmp
|= DC_HPDx_INT_ACK
;
5777 WREG32(DC_HPD2_INT_CONTROL
, tmp
);
5779 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
5780 tmp
= RREG32(DC_HPD3_INT_CONTROL
);
5781 tmp
|= DC_HPDx_INT_ACK
;
5782 WREG32(DC_HPD3_INT_CONTROL
, tmp
);
5784 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
5785 tmp
= RREG32(DC_HPD4_INT_CONTROL
);
5786 tmp
|= DC_HPDx_INT_ACK
;
5787 WREG32(DC_HPD4_INT_CONTROL
, tmp
);
5789 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
5790 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
5791 tmp
|= DC_HPDx_INT_ACK
;
5792 WREG32(DC_HPD5_INT_CONTROL
, tmp
);
5794 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
5795 tmp
= RREG32(DC_HPD5_INT_CONTROL
);
5796 tmp
|= DC_HPDx_INT_ACK
;
5797 WREG32(DC_HPD6_INT_CONTROL
, tmp
);
5801 static void si_irq_disable(struct radeon_device
*rdev
)
5803 si_disable_interrupts(rdev
);
5804 /* Wait and acknowledge irq */
5807 si_disable_interrupt_state(rdev
);
5810 static void si_irq_suspend(struct radeon_device
*rdev
)
5812 si_irq_disable(rdev
);
5816 static void si_irq_fini(struct radeon_device
*rdev
)
5818 si_irq_suspend(rdev
);
5819 r600_ih_ring_fini(rdev
);
5822 static inline u32
si_get_ih_wptr(struct radeon_device
*rdev
)
5826 if (rdev
->wb
.enabled
)
5827 wptr
= le32_to_cpu(rdev
->wb
.wb
[R600_WB_IH_WPTR_OFFSET
/4]);
5829 wptr
= RREG32(IH_RB_WPTR
);
5831 if (wptr
& RB_OVERFLOW
) {
5832 /* When a ring buffer overflow happen start parsing interrupt
5833 * from the last not overwritten vector (wptr + 16). Hopefully
5834 * this should allow us to catchup.
5836 dev_warn(rdev
->dev
, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5837 wptr
, rdev
->ih
.rptr
, (wptr
+ 16) + rdev
->ih
.ptr_mask
);
5838 rdev
->ih
.rptr
= (wptr
+ 16) & rdev
->ih
.ptr_mask
;
5839 tmp
= RREG32(IH_RB_CNTL
);
5840 tmp
|= IH_WPTR_OVERFLOW_CLEAR
;
5841 WREG32(IH_RB_CNTL
, tmp
);
5843 return (wptr
& rdev
->ih
.ptr_mask
);
5847 * Each IV ring entry is 128 bits:
5848 * [7:0] - interrupt source id
5850 * [59:32] - interrupt source data
5851 * [63:60] - reserved
5854 * [127:80] - reserved
5856 int si_irq_process(struct radeon_device
*rdev
)
5860 u32 src_id
, src_data
, ring_id
;
5862 bool queue_hotplug
= false;
5863 bool queue_thermal
= false;
5866 if (!rdev
->ih
.enabled
|| rdev
->shutdown
)
5869 wptr
= si_get_ih_wptr(rdev
);
5872 /* is somebody else already processing irqs? */
5873 if (atomic_xchg(&rdev
->ih
.lock
, 1))
5876 rptr
= rdev
->ih
.rptr
;
5877 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr
, wptr
);
5879 /* Order reading of wptr vs. reading of IH ring data */
5882 /* display interrupts */
5885 while (rptr
!= wptr
) {
5886 /* wptr/rptr are in bytes! */
5887 ring_index
= rptr
/ 4;
5888 src_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
]) & 0xff;
5889 src_data
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 1]) & 0xfffffff;
5890 ring_id
= le32_to_cpu(rdev
->ih
.ring
[ring_index
+ 2]) & 0xff;
5893 case 1: /* D1 vblank/vline */
5895 case 0: /* D1 vblank */
5896 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VBLANK_INTERRUPT
) {
5897 if (rdev
->irq
.crtc_vblank_int
[0]) {
5898 drm_handle_vblank(rdev
->ddev
, 0);
5899 rdev
->pm
.vblank_sync
= true;
5900 wake_up(&rdev
->irq
.vblank_queue
);
5902 if (atomic_read(&rdev
->irq
.pflip
[0]))
5903 radeon_crtc_handle_flip(rdev
, 0);
5904 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~LB_D1_VBLANK_INTERRUPT
;
5905 DRM_DEBUG("IH: D1 vblank\n");
5908 case 1: /* D1 vline */
5909 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& LB_D1_VLINE_INTERRUPT
) {
5910 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~LB_D1_VLINE_INTERRUPT
;
5911 DRM_DEBUG("IH: D1 vline\n");
5915 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5919 case 2: /* D2 vblank/vline */
5921 case 0: /* D2 vblank */
5922 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VBLANK_INTERRUPT
) {
5923 if (rdev
->irq
.crtc_vblank_int
[1]) {
5924 drm_handle_vblank(rdev
->ddev
, 1);
5925 rdev
->pm
.vblank_sync
= true;
5926 wake_up(&rdev
->irq
.vblank_queue
);
5928 if (atomic_read(&rdev
->irq
.pflip
[1]))
5929 radeon_crtc_handle_flip(rdev
, 1);
5930 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~LB_D2_VBLANK_INTERRUPT
;
5931 DRM_DEBUG("IH: D2 vblank\n");
5934 case 1: /* D2 vline */
5935 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& LB_D2_VLINE_INTERRUPT
) {
5936 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~LB_D2_VLINE_INTERRUPT
;
5937 DRM_DEBUG("IH: D2 vline\n");
5941 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5945 case 3: /* D3 vblank/vline */
5947 case 0: /* D3 vblank */
5948 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VBLANK_INTERRUPT
) {
5949 if (rdev
->irq
.crtc_vblank_int
[2]) {
5950 drm_handle_vblank(rdev
->ddev
, 2);
5951 rdev
->pm
.vblank_sync
= true;
5952 wake_up(&rdev
->irq
.vblank_queue
);
5954 if (atomic_read(&rdev
->irq
.pflip
[2]))
5955 radeon_crtc_handle_flip(rdev
, 2);
5956 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~LB_D3_VBLANK_INTERRUPT
;
5957 DRM_DEBUG("IH: D3 vblank\n");
5960 case 1: /* D3 vline */
5961 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& LB_D3_VLINE_INTERRUPT
) {
5962 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~LB_D3_VLINE_INTERRUPT
;
5963 DRM_DEBUG("IH: D3 vline\n");
5967 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5971 case 4: /* D4 vblank/vline */
5973 case 0: /* D4 vblank */
5974 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VBLANK_INTERRUPT
) {
5975 if (rdev
->irq
.crtc_vblank_int
[3]) {
5976 drm_handle_vblank(rdev
->ddev
, 3);
5977 rdev
->pm
.vblank_sync
= true;
5978 wake_up(&rdev
->irq
.vblank_queue
);
5980 if (atomic_read(&rdev
->irq
.pflip
[3]))
5981 radeon_crtc_handle_flip(rdev
, 3);
5982 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~LB_D4_VBLANK_INTERRUPT
;
5983 DRM_DEBUG("IH: D4 vblank\n");
5986 case 1: /* D4 vline */
5987 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& LB_D4_VLINE_INTERRUPT
) {
5988 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~LB_D4_VLINE_INTERRUPT
;
5989 DRM_DEBUG("IH: D4 vline\n");
5993 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
5997 case 5: /* D5 vblank/vline */
5999 case 0: /* D5 vblank */
6000 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VBLANK_INTERRUPT
) {
6001 if (rdev
->irq
.crtc_vblank_int
[4]) {
6002 drm_handle_vblank(rdev
->ddev
, 4);
6003 rdev
->pm
.vblank_sync
= true;
6004 wake_up(&rdev
->irq
.vblank_queue
);
6006 if (atomic_read(&rdev
->irq
.pflip
[4]))
6007 radeon_crtc_handle_flip(rdev
, 4);
6008 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~LB_D5_VBLANK_INTERRUPT
;
6009 DRM_DEBUG("IH: D5 vblank\n");
6012 case 1: /* D5 vline */
6013 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& LB_D5_VLINE_INTERRUPT
) {
6014 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~LB_D5_VLINE_INTERRUPT
;
6015 DRM_DEBUG("IH: D5 vline\n");
6019 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6023 case 6: /* D6 vblank/vline */
6025 case 0: /* D6 vblank */
6026 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VBLANK_INTERRUPT
) {
6027 if (rdev
->irq
.crtc_vblank_int
[5]) {
6028 drm_handle_vblank(rdev
->ddev
, 5);
6029 rdev
->pm
.vblank_sync
= true;
6030 wake_up(&rdev
->irq
.vblank_queue
);
6032 if (atomic_read(&rdev
->irq
.pflip
[5]))
6033 radeon_crtc_handle_flip(rdev
, 5);
6034 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~LB_D6_VBLANK_INTERRUPT
;
6035 DRM_DEBUG("IH: D6 vblank\n");
6038 case 1: /* D6 vline */
6039 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& LB_D6_VLINE_INTERRUPT
) {
6040 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~LB_D6_VLINE_INTERRUPT
;
6041 DRM_DEBUG("IH: D6 vline\n");
6045 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6049 case 42: /* HPD hotplug */
6052 if (rdev
->irq
.stat_regs
.evergreen
.disp_int
& DC_HPD1_INTERRUPT
) {
6053 rdev
->irq
.stat_regs
.evergreen
.disp_int
&= ~DC_HPD1_INTERRUPT
;
6054 queue_hotplug
= true;
6055 DRM_DEBUG("IH: HPD1\n");
6059 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
& DC_HPD2_INTERRUPT
) {
6060 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont
&= ~DC_HPD2_INTERRUPT
;
6061 queue_hotplug
= true;
6062 DRM_DEBUG("IH: HPD2\n");
6066 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
& DC_HPD3_INTERRUPT
) {
6067 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont2
&= ~DC_HPD3_INTERRUPT
;
6068 queue_hotplug
= true;
6069 DRM_DEBUG("IH: HPD3\n");
6073 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
& DC_HPD4_INTERRUPT
) {
6074 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont3
&= ~DC_HPD4_INTERRUPT
;
6075 queue_hotplug
= true;
6076 DRM_DEBUG("IH: HPD4\n");
6080 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
& DC_HPD5_INTERRUPT
) {
6081 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont4
&= ~DC_HPD5_INTERRUPT
;
6082 queue_hotplug
= true;
6083 DRM_DEBUG("IH: HPD5\n");
6087 if (rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
& DC_HPD6_INTERRUPT
) {
6088 rdev
->irq
.stat_regs
.evergreen
.disp_int_cont5
&= ~DC_HPD6_INTERRUPT
;
6089 queue_hotplug
= true;
6090 DRM_DEBUG("IH: HPD6\n");
6094 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6100 addr
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR
);
6101 status
= RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS
);
6102 dev_err(rdev
->dev
, "GPU fault detected: %d 0x%08x\n", src_id
, src_data
);
6103 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6105 dev_err(rdev
->dev
, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6107 si_vm_decode_fault(rdev
, status
, addr
);
6108 /* reset addr and status */
6109 WREG32_P(VM_CONTEXT1_CNTL2
, 1, ~1);
6111 case 176: /* RINGID0 CP_INT */
6112 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
6114 case 177: /* RINGID1 CP_INT */
6115 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
6117 case 178: /* RINGID2 CP_INT */
6118 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
6120 case 181: /* CP EOP event */
6121 DRM_DEBUG("IH: CP EOP\n");
6124 radeon_fence_process(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
6127 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
6130 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
6134 case 224: /* DMA trap event */
6135 DRM_DEBUG("IH: DMA trap\n");
6136 radeon_fence_process(rdev
, R600_RING_TYPE_DMA_INDEX
);
6138 case 230: /* thermal low to high */
6139 DRM_DEBUG("IH: thermal low to high\n");
6140 rdev
->pm
.dpm
.thermal
.high_to_low
= false;
6141 queue_thermal
= true;
6143 case 231: /* thermal high to low */
6144 DRM_DEBUG("IH: thermal high to low\n");
6145 rdev
->pm
.dpm
.thermal
.high_to_low
= true;
6146 queue_thermal
= true;
6148 case 233: /* GUI IDLE */
6149 DRM_DEBUG("IH: GUI idle\n");
6151 case 244: /* DMA trap event */
6152 DRM_DEBUG("IH: DMA1 trap\n");
6153 radeon_fence_process(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
6156 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id
, src_data
);
6160 /* wptr/rptr are in bytes! */
6162 rptr
&= rdev
->ih
.ptr_mask
;
6165 schedule_work(&rdev
->hotplug_work
);
6166 if (queue_thermal
&& rdev
->pm
.dpm_enabled
)
6167 schedule_work(&rdev
->pm
.dpm
.thermal
.work
);
6168 rdev
->ih
.rptr
= rptr
;
6169 WREG32(IH_RB_RPTR
, rdev
->ih
.rptr
);
6170 atomic_set(&rdev
->ih
.lock
, 0);
6172 /* make sure wptr hasn't changed while processing */
6173 wptr
= si_get_ih_wptr(rdev
);
6181 * si_copy_dma - copy pages using the DMA engine
6183 * @rdev: radeon_device pointer
6184 * @src_offset: src GPU address
6185 * @dst_offset: dst GPU address
6186 * @num_gpu_pages: number of GPU pages to xfer
6187 * @fence: radeon fence object
6189 * Copy GPU paging using the DMA engine (SI).
6190 * Used by the radeon ttm implementation to move pages if
6191 * registered as the asic copy callback.
6193 int si_copy_dma(struct radeon_device
*rdev
,
6194 uint64_t src_offset
, uint64_t dst_offset
,
6195 unsigned num_gpu_pages
,
6196 struct radeon_fence
**fence
)
6198 struct radeon_semaphore
*sem
= NULL
;
6199 int ring_index
= rdev
->asic
->copy
.dma_ring_index
;
6200 struct radeon_ring
*ring
= &rdev
->ring
[ring_index
];
6201 u32 size_in_bytes
, cur_size_in_bytes
;
6205 r
= radeon_semaphore_create(rdev
, &sem
);
6207 DRM_ERROR("radeon: moving bo (%d).\n", r
);
6211 size_in_bytes
= (num_gpu_pages
<< RADEON_GPU_PAGE_SHIFT
);
6212 num_loops
= DIV_ROUND_UP(size_in_bytes
, 0xfffff);
6213 r
= radeon_ring_lock(rdev
, ring
, num_loops
* 5 + 11);
6215 DRM_ERROR("radeon: moving bo (%d).\n", r
);
6216 radeon_semaphore_free(rdev
, &sem
, NULL
);
6220 if (radeon_fence_need_sync(*fence
, ring
->idx
)) {
6221 radeon_semaphore_sync_rings(rdev
, sem
, (*fence
)->ring
,
6223 radeon_fence_note_sync(*fence
, ring
->idx
);
6225 radeon_semaphore_free(rdev
, &sem
, NULL
);
6228 for (i
= 0; i
< num_loops
; i
++) {
6229 cur_size_in_bytes
= size_in_bytes
;
6230 if (cur_size_in_bytes
> 0xFFFFF)
6231 cur_size_in_bytes
= 0xFFFFF;
6232 size_in_bytes
-= cur_size_in_bytes
;
6233 radeon_ring_write(ring
, DMA_PACKET(DMA_PACKET_COPY
, 1, 0, 0, cur_size_in_bytes
));
6234 radeon_ring_write(ring
, dst_offset
& 0xffffffff);
6235 radeon_ring_write(ring
, src_offset
& 0xffffffff);
6236 radeon_ring_write(ring
, upper_32_bits(dst_offset
) & 0xff);
6237 radeon_ring_write(ring
, upper_32_bits(src_offset
) & 0xff);
6238 src_offset
+= cur_size_in_bytes
;
6239 dst_offset
+= cur_size_in_bytes
;
6242 r
= radeon_fence_emit(rdev
, fence
, ring
->idx
);
6244 radeon_ring_unlock_undo(rdev
, ring
);
6248 radeon_ring_unlock_commit(rdev
, ring
);
6249 radeon_semaphore_free(rdev
, &sem
, *fence
);
6255 * startup/shutdown callbacks
6257 static int si_startup(struct radeon_device
*rdev
)
6259 struct radeon_ring
*ring
;
6262 /* enable pcie gen2/3 link */
6263 si_pcie_gen3_enable(rdev
);
6265 si_program_aspm(rdev
);
6267 si_mc_program(rdev
);
6269 if (!rdev
->me_fw
|| !rdev
->pfp_fw
|| !rdev
->ce_fw
||
6270 !rdev
->rlc_fw
|| !rdev
->mc_fw
) {
6271 r
= si_init_microcode(rdev
);
6273 DRM_ERROR("Failed to load firmware!\n");
6278 r
= si_mc_load_microcode(rdev
);
6280 DRM_ERROR("Failed to load MC firmware!\n");
6284 r
= r600_vram_scratch_init(rdev
);
6288 r
= si_pcie_gart_enable(rdev
);
6293 /* allocate rlc buffers */
6294 if (rdev
->family
== CHIP_VERDE
) {
6295 rdev
->rlc
.reg_list
= verde_rlc_save_restore_register_list
;
6296 rdev
->rlc
.reg_list_size
=
6297 (u32
)ARRAY_SIZE(verde_rlc_save_restore_register_list
);
6299 rdev
->rlc
.cs_data
= si_cs_data
;
6300 r
= sumo_rlc_init(rdev
);
6302 DRM_ERROR("Failed to init rlc BOs!\n");
6306 /* allocate wb buffer */
6307 r
= radeon_wb_init(rdev
);
6311 r
= radeon_fence_driver_start_ring(rdev
, RADEON_RING_TYPE_GFX_INDEX
);
6313 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
6317 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP1_INDEX
);
6319 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
6323 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_CP2_INDEX
);
6325 dev_err(rdev
->dev
, "failed initializing CP fences (%d).\n", r
);
6329 r
= radeon_fence_driver_start_ring(rdev
, R600_RING_TYPE_DMA_INDEX
);
6331 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
6335 r
= radeon_fence_driver_start_ring(rdev
, CAYMAN_RING_TYPE_DMA1_INDEX
);
6337 dev_err(rdev
->dev
, "failed initializing DMA fences (%d).\n", r
);
6341 if (rdev
->has_uvd
) {
6342 r
= uvd_v2_2_resume(rdev
);
6344 r
= radeon_fence_driver_start_ring(rdev
,
6345 R600_RING_TYPE_UVD_INDEX
);
6347 dev_err(rdev
->dev
, "UVD fences init error (%d).\n", r
);
6350 rdev
->ring
[R600_RING_TYPE_UVD_INDEX
].ring_size
= 0;
6354 if (!rdev
->irq
.installed
) {
6355 r
= radeon_irq_kms_init(rdev
);
6360 r
= si_irq_init(rdev
);
6362 DRM_ERROR("radeon: IH init failed (%d).\n", r
);
6363 radeon_irq_kms_fini(rdev
);
6368 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
6369 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP_RPTR_OFFSET
,
6370 CP_RB0_RPTR
, CP_RB0_WPTR
,
6375 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
6376 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP1_RPTR_OFFSET
,
6377 CP_RB1_RPTR
, CP_RB1_WPTR
,
6382 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
6383 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, RADEON_WB_CP2_RPTR_OFFSET
,
6384 CP_RB2_RPTR
, CP_RB2_WPTR
,
6389 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
6390 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, R600_WB_DMA_RPTR_OFFSET
,
6391 DMA_RB_RPTR
+ DMA0_REGISTER_OFFSET
,
6392 DMA_RB_WPTR
+ DMA0_REGISTER_OFFSET
,
6393 DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0));
6397 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
6398 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, CAYMAN_WB_DMA1_RPTR_OFFSET
,
6399 DMA_RB_RPTR
+ DMA1_REGISTER_OFFSET
,
6400 DMA_RB_WPTR
+ DMA1_REGISTER_OFFSET
,
6401 DMA_PACKET(DMA_PACKET_NOP
, 0, 0, 0, 0));
6405 r
= si_cp_load_microcode(rdev
);
6408 r
= si_cp_resume(rdev
);
6412 r
= cayman_dma_resume(rdev
);
6416 if (rdev
->has_uvd
) {
6417 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
6418 if (ring
->ring_size
) {
6419 r
= radeon_ring_init(rdev
, ring
, ring
->ring_size
, 0,
6420 UVD_RBC_RB_RPTR
, UVD_RBC_RB_WPTR
,
6423 r
= uvd_v1_0_init(rdev
);
6425 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r
);
6429 r
= radeon_ib_pool_init(rdev
);
6431 dev_err(rdev
->dev
, "IB initialization failed (%d).\n", r
);
6435 r
= radeon_vm_manager_init(rdev
);
6437 dev_err(rdev
->dev
, "vm manager initialization failed (%d).\n", r
);
6444 int si_resume(struct radeon_device
*rdev
)
6448 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6449 * posting will perform necessary task to bring back GPU into good
6453 atom_asic_init(rdev
->mode_info
.atom_context
);
6455 /* init golden registers */
6456 si_init_golden_registers(rdev
);
6458 rdev
->accel_working
= true;
6459 r
= si_startup(rdev
);
6461 DRM_ERROR("si startup failed on resume\n");
6462 rdev
->accel_working
= false;
6470 int si_suspend(struct radeon_device
*rdev
)
6472 radeon_vm_manager_fini(rdev
);
6473 si_cp_enable(rdev
, false);
6474 cayman_dma_stop(rdev
);
6475 if (rdev
->has_uvd
) {
6476 uvd_v1_0_fini(rdev
);
6477 radeon_uvd_suspend(rdev
);
6479 si_irq_suspend(rdev
);
6480 radeon_wb_disable(rdev
);
6481 si_pcie_gart_disable(rdev
);
6485 /* Plan is to move initialization in that function and use
6486 * helper function so that radeon_device_init pretty much
6487 * do nothing more than calling asic specific function. This
6488 * should also allow to remove a bunch of callback function
6491 int si_init(struct radeon_device
*rdev
)
6493 struct radeon_ring
*ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
6497 if (!radeon_get_bios(rdev
)) {
6498 if (ASIC_IS_AVIVO(rdev
))
6501 /* Must be an ATOMBIOS */
6502 if (!rdev
->is_atom_bios
) {
6503 dev_err(rdev
->dev
, "Expecting atombios for cayman GPU\n");
6506 r
= radeon_atombios_init(rdev
);
6510 /* Post card if necessary */
6511 if (!radeon_card_posted(rdev
)) {
6513 dev_err(rdev
->dev
, "Card not posted and no BIOS - ignoring\n");
6516 DRM_INFO("GPU not posted. posting now...\n");
6517 atom_asic_init(rdev
->mode_info
.atom_context
);
6519 /* init golden registers */
6520 si_init_golden_registers(rdev
);
6521 /* Initialize scratch registers */
6522 si_scratch_init(rdev
);
6523 /* Initialize surface registers */
6524 radeon_surface_init(rdev
);
6525 /* Initialize clocks */
6526 radeon_get_clock_info(rdev
->ddev
);
6529 r
= radeon_fence_driver_init(rdev
);
6533 /* initialize memory controller */
6534 r
= si_mc_init(rdev
);
6537 /* Memory manager */
6538 r
= radeon_bo_init(rdev
);
6542 ring
= &rdev
->ring
[RADEON_RING_TYPE_GFX_INDEX
];
6543 ring
->ring_obj
= NULL
;
6544 r600_ring_init(rdev
, ring
, 1024 * 1024);
6546 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP1_INDEX
];
6547 ring
->ring_obj
= NULL
;
6548 r600_ring_init(rdev
, ring
, 1024 * 1024);
6550 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_CP2_INDEX
];
6551 ring
->ring_obj
= NULL
;
6552 r600_ring_init(rdev
, ring
, 1024 * 1024);
6554 ring
= &rdev
->ring
[R600_RING_TYPE_DMA_INDEX
];
6555 ring
->ring_obj
= NULL
;
6556 r600_ring_init(rdev
, ring
, 64 * 1024);
6558 ring
= &rdev
->ring
[CAYMAN_RING_TYPE_DMA1_INDEX
];
6559 ring
->ring_obj
= NULL
;
6560 r600_ring_init(rdev
, ring
, 64 * 1024);
6562 if (rdev
->has_uvd
) {
6563 r
= radeon_uvd_init(rdev
);
6565 ring
= &rdev
->ring
[R600_RING_TYPE_UVD_INDEX
];
6566 ring
->ring_obj
= NULL
;
6567 r600_ring_init(rdev
, ring
, 4096);
6571 rdev
->ih
.ring_obj
= NULL
;
6572 r600_ih_ring_init(rdev
, 64 * 1024);
6574 r
= r600_pcie_gart_init(rdev
);
6578 rdev
->accel_working
= true;
6579 r
= si_startup(rdev
);
6581 dev_err(rdev
->dev
, "disabling GPU acceleration\n");
6583 cayman_dma_fini(rdev
);
6585 sumo_rlc_fini(rdev
);
6586 radeon_wb_fini(rdev
);
6587 radeon_ib_pool_fini(rdev
);
6588 radeon_vm_manager_fini(rdev
);
6589 radeon_irq_kms_fini(rdev
);
6590 si_pcie_gart_fini(rdev
);
6591 rdev
->accel_working
= false;
6594 /* Don't start up if the MC ucode is missing.
6595 * The default clocks and voltages before the MC ucode
6596 * is loaded are not suffient for advanced operations.
6599 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6606 void si_fini(struct radeon_device
*rdev
)
6609 cayman_dma_fini(rdev
);
6611 sumo_rlc_fini(rdev
);
6614 radeon_wb_fini(rdev
);
6615 radeon_vm_manager_fini(rdev
);
6616 radeon_ib_pool_fini(rdev
);
6617 radeon_irq_kms_fini(rdev
);
6618 if (rdev
->has_uvd
) {
6619 uvd_v1_0_fini(rdev
);
6620 radeon_uvd_fini(rdev
);
6622 si_pcie_gart_fini(rdev
);
6623 r600_vram_scratch_fini(rdev
);
6624 radeon_gem_fini(rdev
);
6625 radeon_fence_driver_fini(rdev
);
6626 radeon_bo_fini(rdev
);
6627 radeon_atombios_fini(rdev
);
6633 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6635 * @rdev: radeon_device pointer
6637 * Fetches a GPU clock counter snapshot (SI).
6638 * Returns the 64 bit clock counter snapshot.
6640 uint64_t si_get_gpu_clock_counter(struct radeon_device
*rdev
)
6644 mutex_lock(&rdev
->gpu_clock_mutex
);
6645 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT
, 1);
6646 clock
= (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB
) |
6647 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB
) << 32ULL);
6648 mutex_unlock(&rdev
->gpu_clock_mutex
);
6652 int si_set_uvd_clocks(struct radeon_device
*rdev
, u32 vclk
, u32 dclk
)
6654 unsigned fb_div
= 0, vclk_div
= 0, dclk_div
= 0;
6657 /* bypass vclk and dclk with bclk */
6658 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
6659 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6660 ~(VCLK_SRC_SEL_MASK
| DCLK_SRC_SEL_MASK
));
6662 /* put PLL in bypass mode */
6663 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_BYPASS_EN_MASK
, ~UPLL_BYPASS_EN_MASK
);
6665 if (!vclk
|| !dclk
) {
6666 /* keep the Bypass mode, put PLL to sleep */
6667 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_SLEEP_MASK
, ~UPLL_SLEEP_MASK
);
6671 r
= radeon_uvd_calc_upll_dividers(rdev
, vclk
, dclk
, 125000, 250000,
6672 16384, 0x03FFFFFF, 0, 128, 5,
6673 &fb_div
, &vclk_div
, &dclk_div
);
6677 /* set RESET_ANTI_MUX to 0 */
6678 WREG32_P(CG_UPLL_FUNC_CNTL_5
, 0, ~RESET_ANTI_MUX_MASK
);
6680 /* set VCO_MODE to 1 */
6681 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_VCO_MODE_MASK
, ~UPLL_VCO_MODE_MASK
);
6683 /* toggle UPLL_SLEEP to 1 then back to 0 */
6684 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_SLEEP_MASK
, ~UPLL_SLEEP_MASK
);
6685 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_SLEEP_MASK
);
6687 /* deassert UPLL_RESET */
6688 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_RESET_MASK
);
6692 r
= radeon_uvd_send_upll_ctlreq(rdev
, CG_UPLL_FUNC_CNTL
);
6696 /* assert UPLL_RESET again */
6697 WREG32_P(CG_UPLL_FUNC_CNTL
, UPLL_RESET_MASK
, ~UPLL_RESET_MASK
);
6699 /* disable spread spectrum. */
6700 WREG32_P(CG_UPLL_SPREAD_SPECTRUM
, 0, ~SSEN_MASK
);
6702 /* set feedback divider */
6703 WREG32_P(CG_UPLL_FUNC_CNTL_3
, UPLL_FB_DIV(fb_div
), ~UPLL_FB_DIV_MASK
);
6705 /* set ref divider to 0 */
6706 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_REF_DIV_MASK
);
6708 if (fb_div
< 307200)
6709 WREG32_P(CG_UPLL_FUNC_CNTL_4
, 0, ~UPLL_SPARE_ISPARE9
);
6711 WREG32_P(CG_UPLL_FUNC_CNTL_4
, UPLL_SPARE_ISPARE9
, ~UPLL_SPARE_ISPARE9
);
6713 /* set PDIV_A and PDIV_B */
6714 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
6715 UPLL_PDIV_A(vclk_div
) | UPLL_PDIV_B(dclk_div
),
6716 ~(UPLL_PDIV_A_MASK
| UPLL_PDIV_B_MASK
));
6718 /* give the PLL some time to settle */
6721 /* deassert PLL_RESET */
6722 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_RESET_MASK
);
6726 /* switch from bypass mode to normal mode */
6727 WREG32_P(CG_UPLL_FUNC_CNTL
, 0, ~UPLL_BYPASS_EN_MASK
);
6729 r
= radeon_uvd_send_upll_ctlreq(rdev
, CG_UPLL_FUNC_CNTL
);
6733 /* switch VCLK and DCLK selection */
6734 WREG32_P(CG_UPLL_FUNC_CNTL_2
,
6735 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6736 ~(VCLK_SRC_SEL_MASK
| DCLK_SRC_SEL_MASK
));
6743 static void si_pcie_gen3_enable(struct radeon_device
*rdev
)
6745 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
6746 int bridge_pos
, gpu_pos
;
6747 u32 speed_cntl
, mask
, current_data_rate
;
6751 if (radeon_pcie_gen2
== 0)
6754 if (rdev
->flags
& RADEON_IS_IGP
)
6757 if (!(rdev
->flags
& RADEON_IS_PCIE
))
6760 ret
= drm_pcie_get_speed_cap_mask(rdev
->ddev
, &mask
);
6764 if (!(mask
& (DRM_PCIE_SPEED_50
| DRM_PCIE_SPEED_80
)))
6767 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
6768 current_data_rate
= (speed_cntl
& LC_CURRENT_DATA_RATE_MASK
) >>
6769 LC_CURRENT_DATA_RATE_SHIFT
;
6770 if (mask
& DRM_PCIE_SPEED_80
) {
6771 if (current_data_rate
== 2) {
6772 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6775 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6776 } else if (mask
& DRM_PCIE_SPEED_50
) {
6777 if (current_data_rate
== 1) {
6778 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6781 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6784 bridge_pos
= pci_pcie_cap(root
);
6788 gpu_pos
= pci_pcie_cap(rdev
->pdev
);
6792 if (mask
& DRM_PCIE_SPEED_80
) {
6793 /* re-try equalization if gen3 is not already enabled */
6794 if (current_data_rate
!= 2) {
6795 u16 bridge_cfg
, gpu_cfg
;
6796 u16 bridge_cfg2
, gpu_cfg2
;
6797 u32 max_lw
, current_lw
, tmp
;
6799 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
6800 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
6802 tmp16
= bridge_cfg
| PCI_EXP_LNKCTL_HAWD
;
6803 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
6805 tmp16
= gpu_cfg
| PCI_EXP_LNKCTL_HAWD
;
6806 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
6808 tmp
= RREG32_PCIE(PCIE_LC_STATUS1
);
6809 max_lw
= (tmp
& LC_DETECTED_LINK_WIDTH_MASK
) >> LC_DETECTED_LINK_WIDTH_SHIFT
;
6810 current_lw
= (tmp
& LC_OPERATING_LINK_WIDTH_MASK
) >> LC_OPERATING_LINK_WIDTH_SHIFT
;
6812 if (current_lw
< max_lw
) {
6813 tmp
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
6814 if (tmp
& LC_RENEGOTIATION_SUPPORT
) {
6815 tmp
&= ~(LC_LINK_WIDTH_MASK
| LC_UPCONFIGURE_DIS
);
6816 tmp
|= (max_lw
<< LC_LINK_WIDTH_SHIFT
);
6817 tmp
|= LC_UPCONFIGURE_SUPPORT
| LC_RENEGOTIATE_EN
| LC_RECONFIG_NOW
;
6818 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, tmp
);
6822 for (i
= 0; i
< 10; i
++) {
6824 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_DEVSTA
, &tmp16
);
6825 if (tmp16
& PCI_EXP_DEVSTA_TRPND
)
6828 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &bridge_cfg
);
6829 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &gpu_cfg
);
6831 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &bridge_cfg2
);
6832 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &gpu_cfg2
);
6834 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
6835 tmp
|= LC_SET_QUIESCE
;
6836 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
6838 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
6840 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
6845 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
6846 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
6847 tmp16
|= (bridge_cfg
& PCI_EXP_LNKCTL_HAWD
);
6848 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL
, tmp16
);
6850 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, &tmp16
);
6851 tmp16
&= ~PCI_EXP_LNKCTL_HAWD
;
6852 tmp16
|= (gpu_cfg
& PCI_EXP_LNKCTL_HAWD
);
6853 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL
, tmp16
);
6856 pci_read_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
6857 tmp16
&= ~((1 << 4) | (7 << 9));
6858 tmp16
|= (bridge_cfg2
& ((1 << 4) | (7 << 9)));
6859 pci_write_config_word(root
, bridge_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
6861 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
6862 tmp16
&= ~((1 << 4) | (7 << 9));
6863 tmp16
|= (gpu_cfg2
& ((1 << 4) | (7 << 9)));
6864 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
6866 tmp
= RREG32_PCIE_PORT(PCIE_LC_CNTL4
);
6867 tmp
&= ~LC_SET_QUIESCE
;
6868 WREG32_PCIE_PORT(PCIE_LC_CNTL4
, tmp
);
6873 /* set the link speed */
6874 speed_cntl
|= LC_FORCE_EN_SW_SPEED_CHANGE
| LC_FORCE_DIS_HW_SPEED_CHANGE
;
6875 speed_cntl
&= ~LC_FORCE_DIS_SW_SPEED_CHANGE
;
6876 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
6878 pci_read_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, &tmp16
);
6880 if (mask
& DRM_PCIE_SPEED_80
)
6881 tmp16
|= 3; /* gen3 */
6882 else if (mask
& DRM_PCIE_SPEED_50
)
6883 tmp16
|= 2; /* gen2 */
6885 tmp16
|= 1; /* gen1 */
6886 pci_write_config_word(rdev
->pdev
, gpu_pos
+ PCI_EXP_LNKCTL2
, tmp16
);
6888 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
6889 speed_cntl
|= LC_INITIATE_LINK_SPEED_CHANGE
;
6890 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
, speed_cntl
);
6892 for (i
= 0; i
< rdev
->usec_timeout
; i
++) {
6893 speed_cntl
= RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL
);
6894 if ((speed_cntl
& LC_INITIATE_LINK_SPEED_CHANGE
) == 0)
6900 static void si_program_aspm(struct radeon_device
*rdev
)
6903 bool disable_l0s
= false, disable_l1
= false, disable_plloff_in_l1
= false;
6904 bool disable_clkreq
= false;
6906 if (radeon_aspm
== 0)
6909 if (!(rdev
->flags
& RADEON_IS_PCIE
))
6912 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
6913 data
&= ~LC_XMIT_N_FTS_MASK
;
6914 data
|= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN
;
6916 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
, data
);
6918 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL3
);
6919 data
|= LC_GO_TO_RECOVERY
;
6921 WREG32_PCIE_PORT(PCIE_LC_CNTL3
, data
);
6923 orig
= data
= RREG32_PCIE(PCIE_P_CNTL
);
6924 data
|= P_IGNORE_EDB_ERR
;
6926 WREG32_PCIE(PCIE_P_CNTL
, data
);
6928 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
6929 data
&= ~(LC_L0S_INACTIVITY_MASK
| LC_L1_INACTIVITY_MASK
);
6930 data
|= LC_PMI_TO_L1_DIS
;
6932 data
|= LC_L0S_INACTIVITY(7);
6935 data
|= LC_L1_INACTIVITY(7);
6936 data
&= ~LC_PMI_TO_L1_DIS
;
6938 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
6940 if (!disable_plloff_in_l1
) {
6941 bool clk_req_support
;
6943 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0
);
6944 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
6945 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6947 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0
, data
);
6949 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1
);
6950 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
6951 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6953 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1
, data
);
6955 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0
);
6956 data
&= ~(PLL_POWER_STATE_IN_OFF_0_MASK
| PLL_POWER_STATE_IN_TXS2_0_MASK
);
6957 data
|= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6959 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0
, data
);
6961 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1
);
6962 data
&= ~(PLL_POWER_STATE_IN_OFF_1_MASK
| PLL_POWER_STATE_IN_TXS2_1_MASK
);
6963 data
|= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6965 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1
, data
);
6967 if ((rdev
->family
!= CHIP_OLAND
) && (rdev
->family
!= CHIP_HAINAN
)) {
6968 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0
);
6969 data
&= ~PLL_RAMP_UP_TIME_0_MASK
;
6971 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0
, data
);
6973 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1
);
6974 data
&= ~PLL_RAMP_UP_TIME_1_MASK
;
6976 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1
, data
);
6978 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2
);
6979 data
&= ~PLL_RAMP_UP_TIME_2_MASK
;
6981 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2
, data
);
6983 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3
);
6984 data
&= ~PLL_RAMP_UP_TIME_3_MASK
;
6986 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3
, data
);
6988 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0
);
6989 data
&= ~PLL_RAMP_UP_TIME_0_MASK
;
6991 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0
, data
);
6993 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1
);
6994 data
&= ~PLL_RAMP_UP_TIME_1_MASK
;
6996 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1
, data
);
6998 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2
);
6999 data
&= ~PLL_RAMP_UP_TIME_2_MASK
;
7001 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2
, data
);
7003 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3
);
7004 data
&= ~PLL_RAMP_UP_TIME_3_MASK
;
7006 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3
, data
);
7008 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
);
7009 data
&= ~LC_DYN_LANES_PWR_STATE_MASK
;
7010 data
|= LC_DYN_LANES_PWR_STATE(3);
7012 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL
, data
);
7014 orig
= data
= RREG32_PIF_PHY0(PB0_PIF_CNTL
);
7015 data
&= ~LS2_EXIT_TIME_MASK
;
7016 if ((rdev
->family
== CHIP_OLAND
) || (rdev
->family
== CHIP_HAINAN
))
7017 data
|= LS2_EXIT_TIME(5);
7019 WREG32_PIF_PHY0(PB0_PIF_CNTL
, data
);
7021 orig
= data
= RREG32_PIF_PHY1(PB1_PIF_CNTL
);
7022 data
&= ~LS2_EXIT_TIME_MASK
;
7023 if ((rdev
->family
== CHIP_OLAND
) || (rdev
->family
== CHIP_HAINAN
))
7024 data
|= LS2_EXIT_TIME(5);
7026 WREG32_PIF_PHY1(PB1_PIF_CNTL
, data
);
7028 if (!disable_clkreq
) {
7029 struct pci_dev
*root
= rdev
->pdev
->bus
->self
;
7032 clk_req_support
= false;
7033 pcie_capability_read_dword(root
, PCI_EXP_LNKCAP
, &lnkcap
);
7034 if (lnkcap
& PCI_EXP_LNKCAP_CLKPM
)
7035 clk_req_support
= true;
7037 clk_req_support
= false;
7040 if (clk_req_support
) {
7041 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL2
);
7042 data
|= LC_ALLOW_PDWN_IN_L1
| LC_ALLOW_PDWN_IN_L23
;
7044 WREG32_PCIE_PORT(PCIE_LC_CNTL2
, data
);
7046 orig
= data
= RREG32(THM_CLK_CNTL
);
7047 data
&= ~(CMON_CLK_SEL_MASK
| TMON_CLK_SEL_MASK
);
7048 data
|= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7050 WREG32(THM_CLK_CNTL
, data
);
7052 orig
= data
= RREG32(MISC_CLK_CNTL
);
7053 data
&= ~(DEEP_SLEEP_CLK_SEL_MASK
| ZCLK_SEL_MASK
);
7054 data
|= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7056 WREG32(MISC_CLK_CNTL
, data
);
7058 orig
= data
= RREG32(CG_CLKPIN_CNTL
);
7059 data
&= ~BCLK_AS_XCLK
;
7061 WREG32(CG_CLKPIN_CNTL
, data
);
7063 orig
= data
= RREG32(CG_CLKPIN_CNTL_2
);
7064 data
&= ~FORCE_BIF_REFCLK_EN
;
7066 WREG32(CG_CLKPIN_CNTL_2
, data
);
7068 orig
= data
= RREG32(MPLL_BYPASSCLK_SEL
);
7069 data
&= ~MPLL_CLKOUT_SEL_MASK
;
7070 data
|= MPLL_CLKOUT_SEL(4);
7072 WREG32(MPLL_BYPASSCLK_SEL
, data
);
7074 orig
= data
= RREG32(SPLL_CNTL_MODE
);
7075 data
&= ~SPLL_REFCLK_SEL_MASK
;
7077 WREG32(SPLL_CNTL_MODE
, data
);
7082 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);
7085 orig
= data
= RREG32_PCIE(PCIE_CNTL2
);
7086 data
|= SLV_MEM_LS_EN
| MST_MEM_LS_EN
| REPLAY_MEM_LS_EN
;
7088 WREG32_PCIE(PCIE_CNTL2
, data
);
7091 data
= RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL
);
7092 if((data
& LC_N_FTS_MASK
) == LC_N_FTS_MASK
) {
7093 data
= RREG32_PCIE(PCIE_LC_STATUS1
);
7094 if ((data
& LC_REVERSE_XMIT
) && (data
& LC_REVERSE_RCVR
)) {
7095 orig
= data
= RREG32_PCIE_PORT(PCIE_LC_CNTL
);
7096 data
&= ~LC_L0S_INACTIVITY_MASK
;
7098 WREG32_PCIE_PORT(PCIE_LC_CNTL
, data
);