[deliverable/linux.git] / drivers / gpu / drm / radeon / si_dma.c

/*
 * Copyright 2013 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Alex Deucher
 */
#include <drm/drmP.h>
#include "radeon.h"
#include "radeon_asic.h"
#include "radeon_trace.h"
#include "sid.h"

u32 si_gpu_check_soft_reset(struct radeon_device *rdev);

/**
 * si_dma_is_lockup - Check if the DMA engine is locked up
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Check if the async DMA engine is locked up.
 * Returns true if the engine appears to be locked up, false if not.
 */
bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
{
	u32 reset_mask = si_gpu_check_soft_reset(rdev);
	u32 mask;

	if (ring->idx == R600_RING_TYPE_DMA_INDEX)
		mask = RADEON_RESET_DMA;
	else
		mask = RADEON_RESET_DMA1;

	if (!(reset_mask & mask)) {
		radeon_ring_lockup_update(rdev, ring);
		return false;
	}
	return radeon_ring_test_lockup(rdev, ring);
}

/**
 * si_dma_vm_copy_pages - update PTEs by copying them from the GART
 *
 * @rdev: radeon_device pointer
 * @ib: indirect buffer to fill with commands
 * @pe: addr of the page entry
 * @src: src addr where to copy from
 * @count: number of page entries to update
 *
 * Update PTEs by copying them from the GART using the DMA (SI).
 */
void si_dma_vm_copy_pages(struct radeon_device *rdev,
			  struct radeon_ib *ib,
			  uint64_t pe, uint64_t src,
			  unsigned count)
{
	while (count) {
		unsigned bytes = count * 8;
		if (bytes > 0xFFFF8)
			bytes = 0xFFFF8;

		ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
						      1, 0, 0, bytes);
		ib->ptr[ib->length_dw++] = lower_32_bits(pe);
		ib->ptr[ib->length_dw++] = lower_32_bits(src);
		ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
		ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff;

		pe += bytes;
		src += bytes;
		count -= bytes / 8;
	}
}

/**
 * si_dma_vm_write_pages - update PTEs by writing them manually
 *
 * @rdev: radeon_device pointer
 * @ib: indirect buffer to fill with commands
 * @pe: addr of the page entry
 * @addr: dst addr to write into pe
 * @count: number of page entries to update
 * @incr: increase next addr by incr bytes
 * @flags: access flags
 *
 * Update PTEs by writing them manually using the DMA (SI).
 */
void si_dma_vm_write_pages(struct radeon_device *rdev,
			   struct radeon_ib *ib,
			   uint64_t pe,
			   uint64_t addr, unsigned count,
			   uint32_t incr, uint32_t flags)
{
	uint64_t value;
	unsigned ndw;

	while (count) {
		ndw = count * 2;
		if (ndw > 0xFFFFE)
			ndw = 0xFFFFE;

		/* for non-physically contiguous pages (system) */
		ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
		ib->ptr[ib->length_dw++] = pe;
		ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
		for (; ndw > 0; ndw -= 2, --count, pe += 8) {
			if (flags & R600_PTE_SYSTEM) {
				value = radeon_vm_map_gart(rdev, addr);
			} else if (flags & R600_PTE_VALID) {
				value = addr;
			} else {
				value = 0;
			}
			addr += incr;
			value |= flags;
			ib->ptr[ib->length_dw++] = value;
			ib->ptr[ib->length_dw++] = upper_32_bits(value);
		}
	}
}

/**
 * si_dma_vm_set_pages - update the page tables using the DMA
 *
 * @rdev: radeon_device pointer
 * @ib: indirect buffer to fill with commands
 * @pe: addr of the page entry
 * @addr: dst addr to write into pe
 * @count: number of page entries to update
 * @incr: increase next addr by incr bytes
 * @flags: access flags
 *
 * Update the page tables using the DMA (SI).
 */
void si_dma_vm_set_pages(struct radeon_device *rdev,
			 struct radeon_ib *ib,
			 uint64_t pe,
			 uint64_t addr, unsigned count,
			 uint32_t incr, uint32_t flags)
{
	uint64_t value;
	unsigned ndw;

	while (count) {
		ndw = count * 2;
		if (ndw > 0xFFFFE)
			ndw = 0xFFFFE;

		if (flags & R600_PTE_VALID)
			value = addr;
		else
			value = 0;

		/* for physically contiguous pages (vram) */
		ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
		ib->ptr[ib->length_dw++] = pe; /* dst addr */
		ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
		ib->ptr[ib->length_dw++] = flags; /* mask */
		ib->ptr[ib->length_dw++] = 0;
		ib->ptr[ib->length_dw++] = value; /* value */
		ib->ptr[ib->length_dw++] = upper_32_bits(value);
		ib->ptr[ib->length_dw++] = incr; /* increment size */
		ib->ptr[ib->length_dw++] = 0;
		pe += ndw * 4;
		addr += (ndw / 2) * incr;
		count -= ndw / 2;
	}
}

void si_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
		     unsigned vm_id, uint64_t pd_addr)

{
	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
	if (vm_id < 8) {
		radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2));
	} else {
		radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2));
	}
	radeon_ring_write(ring, pd_addr >> 12);

	/* flush hdp cache */
	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
	radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
	radeon_ring_write(ring, 1);

	/* bits 0-7 are the VM contexts0-7 */
	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
	radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
	radeon_ring_write(ring, 1 << vm_id);

	/* wait for invalidate to complete */
	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
	radeon_ring_write(ring, VM_INVALIDATE_REQUEST);
	radeon_ring_write(ring, 0xff << 16); /* retry */
	radeon_ring_write(ring, 1 << vm_id); /* mask */
	radeon_ring_write(ring, 0); /* value */
	radeon_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */
}

/**
 * si_copy_dma - copy pages using the DMA engine
 *
 * @rdev: radeon_device pointer
 * @src_offset: src GPU address
 * @dst_offset: dst GPU address
 * @num_gpu_pages: number of GPU pages to xfer
 * @resv: reservation object to sync to
 *
 * Copy GPU paging using the DMA engine (SI).
 * Used by the radeon ttm implementation to move pages if
 * registered as the asic copy callback.
 */
struct radeon_fence *si_copy_dma(struct radeon_device *rdev,
				 uint64_t src_offset, uint64_t dst_offset,
				 unsigned num_gpu_pages,
				 struct reservation_object *resv)
{
	struct radeon_fence *fence;
	struct radeon_sync sync;
	int ring_index = rdev->asic->copy.dma_ring_index;
	struct radeon_ring *ring = &rdev->ring[ring_index];
	u32 size_in_bytes, cur_size_in_bytes;
	int i, num_loops;
	int r = 0;

	radeon_sync_create(&sync);

	size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
	num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
	r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
	if (r) {
		DRM_ERROR("radeon: moving bo (%d).\n", r);
		radeon_sync_free(rdev, &sync, NULL);
		return ERR_PTR(r);
	}

	radeon_sync_resv(rdev, &sync, resv, false);
	radeon_sync_rings(rdev, &sync, ring->idx);

	for (i = 0; i < num_loops; i++) {
		cur_size_in_bytes = size_in_bytes;
		if (cur_size_in_bytes > 0xFFFFF)
			cur_size_in_bytes = 0xFFFFF;
		size_in_bytes -= cur_size_in_bytes;
		radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
		radeon_ring_write(ring, lower_32_bits(dst_offset));
		radeon_ring_write(ring, lower_32_bits(src_offset));
		radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
		radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
		src_offset += cur_size_in_bytes;
		dst_offset += cur_size_in_bytes;
	}

	r = radeon_fence_emit(rdev, &fence, ring->idx);
	if (r) {
		radeon_ring_unlock_undo(rdev, ring);
		radeon_sync_free(rdev, &sync, NULL);
		return ERR_PTR(r);
	}

	radeon_ring_unlock_commit(rdev, ring, false);
	radeon_sync_free(rdev, &sync, fence);

	return fence;
}
Commit	Line	Data
2483b4ea CK	1	/*
	2	* Copyright 2013 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	* Authors: Alex Deucher
	23	*/
	24	#include <drm/drmP.h>
	25	#include "radeon.h"
	26	#include "radeon_asic.h"
74d360f6	27	#include "radeon_trace.h"
2483b4ea CK	28	#include "sid.h"
	29
	30	u32 si_gpu_check_soft_reset(struct radeon_device *rdev);
	31
	32	/**
	33	* si_dma_is_lockup - Check if the DMA engine is locked up
	34	*
	35	* @rdev: radeon_device pointer
	36	* @ring: radeon_ring structure holding ring information
	37	*
	38	* Check if the async DMA engine is locked up.
	39	* Returns true if the engine appears to be locked up, false if not.
	40	*/
	41	bool si_dma_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
	42	{
	43	u32 reset_mask = si_gpu_check_soft_reset(rdev);
	44	u32 mask;
	45
	46	if (ring->idx == R600_RING_TYPE_DMA_INDEX)
	47	mask = RADEON_RESET_DMA;
	48	else
	49	mask = RADEON_RESET_DMA1;
	50
	51	if (!(reset_mask & mask)) {
ff212f25	52	radeon_ring_lockup_update(rdev, ring);
2483b4ea CK	53	return false;
2483b4ea CK	54	}
2483b4ea CK	55	return radeon_ring_test_lockup(rdev, ring);
	56	}
	57
	58	/**
03f62abd CK	59	* si_dma_vm_copy_pages - update PTEs by copying them from the GART
	60	*
	61	* @rdev: radeon_device pointer
	62	* @ib: indirect buffer to fill with commands
	63	* @pe: addr of the page entry
	64	* @src: src addr where to copy from
	65	* @count: number of page entries to update
	66	*
	67	* Update PTEs by copying them from the GART using the DMA (SI).
	68	*/
	69	void si_dma_vm_copy_pages(struct radeon_device *rdev,
	70	struct radeon_ib *ib,
	71	uint64_t pe, uint64_t src,
	72	unsigned count)
	73	{
	74	while (count) {
	75	unsigned bytes = count * 8;
	76	if (bytes > 0xFFFF8)
	77	bytes = 0xFFFF8;
	78
	79	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
	80	1, 0, 0, bytes);
	81	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
	82	ib->ptr[ib->length_dw++] = lower_32_bits(src);
	83	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
	84	ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff;
	85
	86	pe += bytes;
	87	src += bytes;
	88	count -= bytes / 8;
	89	}
	90	}
	91
	92	/**
	93	* si_dma_vm_write_pages - update PTEs by writing them manually
2483b4ea CK	94	*
	95	* @rdev: radeon_device pointer
	96	* @ib: indirect buffer to fill with commands
	97	* @pe: addr of the page entry
	98	* @addr: dst addr to write into pe
	99	* @count: number of page entries to update
	100	* @incr: increase next addr by incr bytes
	101	* @flags: access flags
	102	*
03f62abd	103	* Update PTEs by writing them manually using the DMA (SI).
2483b4ea	104	*/
03f62abd CK	105	void si_dma_vm_write_pages(struct radeon_device *rdev,
	106	struct radeon_ib *ib,
	107	uint64_t pe,
	108	uint64_t addr, unsigned count,
	109	uint32_t incr, uint32_t flags)
2483b4ea	110	{
2483b4ea CK	111	uint64_t value;
	112	unsigned ndw;
	113
03f62abd CK	114	while (count) {
	115	ndw = count * 2;
	116	if (ndw > 0xFFFFE)
	117	ndw = 0xFFFFE;
	118
	119	/* for non-physically contiguous pages (system) */
	120	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
	121	ib->ptr[ib->length_dw++] = pe;
	122	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
	123	for (; ndw > 0; ndw -= 2, --count, pe += 8) {
	124	if (flags & R600_PTE_SYSTEM) {
24c16439	125	value = radeon_vm_map_gart(rdev, addr);
03f62abd	126	} else if (flags & R600_PTE_VALID) {
2483b4ea	127	value = addr;
03f62abd	128	} else {
2483b4ea	129	value = 0;
03f62abd CK	130	}
	131	addr += incr;
	132	value \|= flags;
	133	ib->ptr[ib->length_dw++] = value;
2483b4ea	134	ib->ptr[ib->length_dw++] = upper_32_bits(value);
2483b4ea CK	135	}
2483b4ea CK	136	}
03f62abd CK	137	}
	138
	139	/**
	140	* si_dma_vm_set_pages - update the page tables using the DMA
	141	*
	142	* @rdev: radeon_device pointer
	143	* @ib: indirect buffer to fill with commands
	144	* @pe: addr of the page entry
	145	* @addr: dst addr to write into pe
	146	* @count: number of page entries to update
	147	* @incr: increase next addr by incr bytes
	148	* @flags: access flags
	149	*
	150	* Update the page tables using the DMA (SI).
	151	*/
	152	void si_dma_vm_set_pages(struct radeon_device *rdev,
	153	struct radeon_ib *ib,
	154	uint64_t pe,
	155	uint64_t addr, unsigned count,
	156	uint32_t incr, uint32_t flags)
	157	{
	158	uint64_t value;
	159	unsigned ndw;
	160
	161	while (count) {
	162	ndw = count * 2;
	163	if (ndw > 0xFFFFE)
	164	ndw = 0xFFFFE;
	165
	166	if (flags & R600_PTE_VALID)
	167	value = addr;
	168	else
	169	value = 0;
	170
	171	/* for physically contiguous pages (vram) */
	172	ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
	173	ib->ptr[ib->length_dw++] = pe; /* dst addr */
	174	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
	175	ib->ptr[ib->length_dw++] = flags; /* mask */
	176	ib->ptr[ib->length_dw++] = 0;
	177	ib->ptr[ib->length_dw++] = value; /* value */
	178	ib->ptr[ib->length_dw++] = upper_32_bits(value);
	179	ib->ptr[ib->length_dw++] = incr; /* increment size */
	180	ib->ptr[ib->length_dw++] = 0;
	181	pe += ndw * 4;
	182	addr += (ndw / 2) * incr;
	183	count -= ndw / 2;
	184	}
2483b4ea CK	185	}
2483b4ea CK	186
faffaf62 CK	187	void si_dma_vm_flush(struct radeon_device rdev, struct radeon_ring ring,
faffaf62 CK	188	unsigned vm_id, uint64_t pd_addr)
2483b4ea	189
faffaf62	190	{
2483b4ea	191	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
faffaf62 CK	192	if (vm_id < 8) {
faffaf62 CK	193	radeon_ring_write(ring, (0xf << 16) \| ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2));
2483b4ea	194	} else {
faffaf62	195	radeon_ring_write(ring, (0xf << 16) \| ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2));
2483b4ea	196	}
faffaf62	197	radeon_ring_write(ring, pd_addr >> 12);
2483b4ea CK	198
	199	/* flush hdp cache */
	200	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
	201	radeon_ring_write(ring, (0xf << 16) \| (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
	202	radeon_ring_write(ring, 1);
	203
	204	/* bits 0-7 are the VM contexts0-7 */
	205	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
	206	radeon_ring_write(ring, (0xf << 16) \| (VM_INVALIDATE_REQUEST >> 2));
faffaf62	207	radeon_ring_write(ring, 1 << vm_id);
d474ea7e AD	208
	209	/* wait for invalidate to complete */
	210	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
	211	radeon_ring_write(ring, VM_INVALIDATE_REQUEST);
	212	radeon_ring_write(ring, 0xff << 16); /* retry */
	213	radeon_ring_write(ring, 1 << vm_id); /* mask */
	214	radeon_ring_write(ring, 0); /* value */
	215	radeon_ring_write(ring, (0 << 28) \| 0x20); /* func(always) \| poll interval */
2483b4ea CK	216	}
	217
	218	/**
	219	* si_copy_dma - copy pages using the DMA engine
	220	*
	221	* @rdev: radeon_device pointer
	222	* @src_offset: src GPU address
	223	* @dst_offset: dst GPU address
	224	* @num_gpu_pages: number of GPU pages to xfer
57d20a43	225	* @resv: reservation object to sync to
2483b4ea CK	226	*
	227	* Copy GPU paging using the DMA engine (SI).
	228	* Used by the radeon ttm implementation to move pages if
	229	* registered as the asic copy callback.
	230	*/
57d20a43 CK	231	struct radeon_fence si_copy_dma(struct radeon_device rdev,
	232	uint64_t src_offset, uint64_t dst_offset,
	233	unsigned num_gpu_pages,
	234	struct reservation_object *resv)
2483b4ea	235	{
57d20a43	236	struct radeon_fence *fence;
975700d2	237	struct radeon_sync sync;
2483b4ea CK	238	int ring_index = rdev->asic->copy.dma_ring_index;
	239	struct radeon_ring *ring = &rdev->ring[ring_index];
	240	u32 size_in_bytes, cur_size_in_bytes;
	241	int i, num_loops;
	242	int r = 0;
	243
975700d2	244	radeon_sync_create(&sync);
2483b4ea CK	245
	246	size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
	247	num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
	248	r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
	249	if (r) {
	250	DRM_ERROR("radeon: moving bo (%d).\n", r);
975700d2	251	radeon_sync_free(rdev, &sync, NULL);
57d20a43	252	return ERR_PTR(r);
2483b4ea CK	253	}
2483b4ea CK	254
975700d2 CK	255	radeon_sync_resv(rdev, &sync, resv, false);
975700d2 CK	256	radeon_sync_rings(rdev, &sync, ring->idx);
2483b4ea CK	257
	258	for (i = 0; i < num_loops; i++) {
	259	cur_size_in_bytes = size_in_bytes;
	260	if (cur_size_in_bytes > 0xFFFFF)
	261	cur_size_in_bytes = 0xFFFFF;
	262	size_in_bytes -= cur_size_in_bytes;
	263	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5e167cdb CK	264	radeon_ring_write(ring, lower_32_bits(dst_offset));
5e167cdb CK	265	radeon_ring_write(ring, lower_32_bits(src_offset));
2483b4ea CK	266	radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
	267	radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
	268	src_offset += cur_size_in_bytes;
	269	dst_offset += cur_size_in_bytes;
	270	}
	271
57d20a43	272	r = radeon_fence_emit(rdev, &fence, ring->idx);
2483b4ea CK	273	if (r) {
2483b4ea CK	274	radeon_ring_unlock_undo(rdev, ring);
975700d2	275	radeon_sync_free(rdev, &sync, NULL);
57d20a43	276	return ERR_PTR(r);
2483b4ea CK	277	}
2483b4ea CK	278
1538a9e0	279	radeon_ring_unlock_commit(rdev, ring, false);
975700d2	280	radeon_sync_free(rdev, &sync, fence);
2483b4ea	281
57d20a43	282	return fence;
2483b4ea CK	283	}
2483b4ea CK	284