[deliverable/linux.git] / drivers / gpu / host1x / cdma.c

/*
 * Tegra host1x Command DMA
 *
 * Copyright (c) 2010-2013, NVIDIA Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */


#include <asm/cacheflush.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/host1x.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/kfifo.h>
#include <linux/slab.h>
#include <trace/events/host1x.h>

#include "cdma.h"
#include "channel.h"
#include "dev.h"
#include "debug.h"
#include "job.h"

/*
 * push_buffer
 *
 * The push buffer is a circular array of words to be fetched by command DMA.
 * Note that it works slightly differently to the sync queue; fence == pos
 * means that the push buffer is full, not empty.
 */

#define HOST1X_PUSHBUFFER_SLOTS	512

/*
 * Clean up push buffer resources
 */
static void host1x_pushbuffer_destroy(struct push_buffer *pb)
{
	struct host1x_cdma *cdma = pb_to_cdma(pb);
	struct host1x *host1x = cdma_to_host1x(cdma);

	if (pb->phys != 0)
		dma_free_wc(host1x->dev, pb->size_bytes + 4, pb->mapped,
			    pb->phys);

	pb->mapped = NULL;
	pb->phys = 0;
}

/*
 * Init push buffer resources
 */
static int host1x_pushbuffer_init(struct push_buffer *pb)
{
	struct host1x_cdma *cdma = pb_to_cdma(pb);
	struct host1x *host1x = cdma_to_host1x(cdma);

	pb->mapped = NULL;
	pb->phys = 0;
	pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8;

	/* initialize buffer pointers */
	pb->fence = pb->size_bytes - 8;
	pb->pos = 0;

	/* allocate and map pushbuffer memory */
	pb->mapped = dma_alloc_wc(host1x->dev, pb->size_bytes + 4, &pb->phys,
				  GFP_KERNEL);
	if (!pb->mapped)
		goto fail;

	host1x_hw_pushbuffer_init(host1x, pb);

	return 0;

fail:
	host1x_pushbuffer_destroy(pb);
	return -ENOMEM;
}

/*
 * Push two words to the push buffer
 * Caller must ensure push buffer is not full
 */
static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
{
	u32 *p = (u32 *)((void *)pb->mapped + pb->pos);

	WARN_ON(pb->pos == pb->fence);
	*(p++) = op1;
	*(p++) = op2;
	pb->pos = (pb->pos + 8) & (pb->size_bytes - 1);
}

/*
 * Pop a number of two word slots from the push buffer
 * Caller must ensure push buffer is not empty
 */
static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
{
	/* Advance the next write position */
	pb->fence = (pb->fence + slots * 8) & (pb->size_bytes - 1);
}

/*
 * Return the number of two word slots free in the push buffer
 */
static u32 host1x_pushbuffer_space(struct push_buffer *pb)
{
	return ((pb->fence - pb->pos) & (pb->size_bytes - 1)) / 8;
}

/*
 * Sleep (if necessary) until the requested event happens
 *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
 *     - Returns 1
 *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
 *     - Return the amount of space (> 0)
 * Must be called with the cdma lock held.
 */
unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
				     enum cdma_event event)
{
	for (;;) {
		struct push_buffer *pb = &cdma->push_buffer;
		unsigned int space;

		switch (event) {
		case CDMA_EVENT_SYNC_QUEUE_EMPTY:
			space = list_empty(&cdma->sync_queue) ? 1 : 0;
			break;

		case CDMA_EVENT_PUSH_BUFFER_SPACE:
			space = host1x_pushbuffer_space(pb);
			break;

		default:
			WARN_ON(1);
			return -EINVAL;
		}

		if (space)
			return space;

		trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
				       event);

		/* If somebody has managed to already start waiting, yield */
		if (cdma->event != CDMA_EVENT_NONE) {
			mutex_unlock(&cdma->lock);
			schedule();
			mutex_lock(&cdma->lock);
			continue;
		}

		cdma->event = event;

		mutex_unlock(&cdma->lock);
		down(&cdma->sem);
		mutex_lock(&cdma->lock);
	}

	return 0;
}

/*
 * Start timer that tracks the time spent by the job.
 * Must be called with the cdma lock held.
 */
static void cdma_start_timer_locked(struct host1x_cdma *cdma,
				    struct host1x_job *job)
{
	struct host1x *host = cdma_to_host1x(cdma);

	if (cdma->timeout.client) {
		/* timer already started */
		return;
	}

	cdma->timeout.client = job->client;
	cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
	cdma->timeout.syncpt_val = job->syncpt_end;
	cdma->timeout.start_ktime = ktime_get();

	schedule_delayed_work(&cdma->timeout.wq,
			      msecs_to_jiffies(job->timeout));
}

/*
 * Stop timer when a buffer submission completes.
 * Must be called with the cdma lock held.
 */
static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
{
	cancel_delayed_work(&cdma->timeout.wq);
	cdma->timeout.client = 0;
}

/*
 * For all sync queue entries that have already finished according to the
 * current sync point registers:
 *  - unpin & unref their mems
 *  - pop their push buffer slots
 *  - remove them from the sync queue
 * This is normally called from the host code's worker thread, but can be
 * called manually if necessary.
 * Must be called with the cdma lock held.
 */
static void update_cdma_locked(struct host1x_cdma *cdma)
{
	bool signal = false;
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct host1x_job *job, *n;

	/* If CDMA is stopped, queue is cleared and we can return */
	if (!cdma->running)
		return;

	/*
	 * Walk the sync queue, reading the sync point registers as necessary,
	 * to consume as many sync queue entries as possible without blocking
	 */
	list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
		struct host1x_syncpt *sp =
			host1x_syncpt_get(host1x, job->syncpt_id);

		/* Check whether this syncpt has completed, and bail if not */
		if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
			/* Start timer on next pending syncpt */
			if (job->timeout)
				cdma_start_timer_locked(cdma, job);

			break;
		}

		/* Cancel timeout, when a buffer completes */
		if (cdma->timeout.client)
			stop_cdma_timer_locked(cdma);

		/* Unpin the memory */
		host1x_job_unpin(job);

		/* Pop push buffer slots */
		if (job->num_slots) {
			struct push_buffer *pb = &cdma->push_buffer;

			host1x_pushbuffer_pop(pb, job->num_slots);

			if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
				signal = true;
		}

		list_del(&job->list);
		host1x_job_put(job);
	}

	if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
	    list_empty(&cdma->sync_queue))
		signal = true;

	if (signal) {
		cdma->event = CDMA_EVENT_NONE;
		up(&cdma->sem);
	}
}

void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
				   struct device *dev)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	u32 restart_addr, syncpt_incrs, syncpt_val;
	struct host1x_job *job = NULL;

	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);

	dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
		__func__, syncpt_val);

	/*
	 * Move the sync_queue read pointer to the first entry that hasn't
	 * completed based on the current HW syncpt value. It's likely there
	 * won't be any (i.e. we're still at the head), but covers the case
	 * where a syncpt incr happens just prior/during the teardown.
	 */

	dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
		__func__);

	list_for_each_entry(job, &cdma->sync_queue, list) {
		if (syncpt_val < job->syncpt_end)
			break;

		host1x_job_dump(dev, job);
	}

	/*
	 * Walk the sync_queue, first incrementing with the CPU syncpts that
	 * are partially executed (the first buffer) or fully skipped while
	 * still in the current context (slots are also NOP-ed).
	 *
	 * At the point contexts are interleaved, syncpt increments must be
	 * done inline with the pushbuffer from a GATHER buffer to maintain
	 * the order (slots are modified to be a GATHER of syncpt incrs).
	 *
	 * Note: save in restart_addr the location where the timed out buffer
	 * started in the PB, so we can start the refetch from there (with the
	 * modified NOP-ed PB slots). This lets things appear to have completed
	 * properly for this buffer and resources are freed.
	 */

	dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n",
		__func__);

	if (!list_empty(&cdma->sync_queue))
		restart_addr = job->first_get;
	else
		restart_addr = cdma->last_pos;

	/* do CPU increments as long as this context continues */
	list_for_each_entry_from(job, &cdma->sync_queue, list) {
		/* different context, gets us out of this loop */
		if (job->client != cdma->timeout.client)
			break;

		/* won't need a timeout when replayed */
		job->timeout = 0;

		syncpt_incrs = job->syncpt_end - syncpt_val;
		dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);

		host1x_job_dump(dev, job);

		/* safe to use CPU to incr syncpts */
		host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
						syncpt_incrs, job->syncpt_end,
						job->num_slots);

		syncpt_val += syncpt_incrs;
	}

	/*
	 * The following sumbits from the same client may be dependent on the
	 * failed submit and therefore they may fail. Force a small timeout
	 * to make the queue cleanup faster.
	 */

	list_for_each_entry_from(job, &cdma->sync_queue, list)
		if (job->client == cdma->timeout.client)
			job->timeout = min_t(unsigned int, job->timeout, 500);

	dev_dbg(dev, "%s: finished sync_queue modification\n", __func__);

	/* roll back DMAGET and start up channel again */
	host1x_hw_cdma_resume(host1x, cdma, restart_addr);
}

/*
 * Create a cdma
 */
int host1x_cdma_init(struct host1x_cdma *cdma)
{
	int err;

	mutex_init(&cdma->lock);
	sema_init(&cdma->sem, 0);

	INIT_LIST_HEAD(&cdma->sync_queue);

	cdma->event = CDMA_EVENT_NONE;
	cdma->running = false;
	cdma->torndown = false;

	err = host1x_pushbuffer_init(&cdma->push_buffer);
	if (err)
		return err;

	return 0;
}

/*
 * Destroy a cdma
 */
int host1x_cdma_deinit(struct host1x_cdma *cdma)
{
	struct push_buffer *pb = &cdma->push_buffer;
	struct host1x *host1x = cdma_to_host1x(cdma);

	if (cdma->running) {
		pr_warn("%s: CDMA still running\n", __func__);
		return -EBUSY;
	}

	host1x_pushbuffer_destroy(pb);
	host1x_hw_cdma_timeout_destroy(host1x, cdma);

	return 0;
}

/*
 * Begin a cdma submit
 */
int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
{
	struct host1x *host1x = cdma_to_host1x(cdma);

	mutex_lock(&cdma->lock);

	if (job->timeout) {
		/* init state on first submit with timeout value */
		if (!cdma->timeout.initialized) {
			int err;

			err = host1x_hw_cdma_timeout_init(host1x, cdma,
							  job->syncpt_id);
			if (err) {
				mutex_unlock(&cdma->lock);
				return err;
			}
		}
	}

	if (!cdma->running)
		host1x_hw_cdma_start(host1x, cdma);

	cdma->slots_free = 0;
	cdma->slots_used = 0;
	cdma->first_get = cdma->push_buffer.pos;

	trace_host1x_cdma_begin(dev_name(job->channel->dev));
	return 0;
}

/*
 * Push two words into a push buffer slot
 * Blocks as necessary if the push buffer is full.
 */
void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct push_buffer *pb = &cdma->push_buffer;
	u32 slots_free = cdma->slots_free;

	if (host1x_debug_trace_cmdbuf)
		trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
				       op1, op2);

	if (slots_free == 0) {
		host1x_hw_cdma_flush(host1x, cdma);
		slots_free = host1x_cdma_wait_locked(cdma,
						CDMA_EVENT_PUSH_BUFFER_SPACE);
	}

	cdma->slots_free = slots_free - 1;
	cdma->slots_used++;
	host1x_pushbuffer_push(pb, op1, op2);
}

/*
 * End a cdma submit
 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
 * from the pushbuffer. The handles for a submit must all be pinned at the same
 * time, but they can be unpinned in smaller chunks.
 */
void host1x_cdma_end(struct host1x_cdma *cdma,
		     struct host1x_job *job)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	bool idle = list_empty(&cdma->sync_queue);

	host1x_hw_cdma_flush(host1x, cdma);

	job->first_get = cdma->first_get;
	job->num_slots = cdma->slots_used;
	host1x_job_get(job);
	list_add_tail(&job->list, &cdma->sync_queue);

	/* start timer on idle -> active transitions */
	if (job->timeout && idle)
		cdma_start_timer_locked(cdma, job);

	trace_host1x_cdma_end(dev_name(job->channel->dev));
	mutex_unlock(&cdma->lock);
}

/*
 * Update cdma state according to current sync point values
 */
void host1x_cdma_update(struct host1x_cdma *cdma)
{
	mutex_lock(&cdma->lock);
	update_cdma_locked(cdma);
	mutex_unlock(&cdma->lock);
}
Commit	Line	Data
6579324a TB	1	/*
	2	* Tegra host1x Command DMA
	3	*
	4	* Copyright (c) 2010-2013, NVIDIA Corporation.
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms and conditions of the GNU General Public License,
	8	* version 2, as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope it will be useful, but WITHOUT
	11	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	12	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	13	* more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18
	19
	20	#include <asm/cacheflush.h>
	21	#include <linux/device.h>
	22	#include <linux/dma-mapping.h>
35d747a8	23	#include <linux/host1x.h>
6579324a TB	24	#include <linux/interrupt.h>
	25	#include <linux/kernel.h>
	26	#include <linux/kfifo.h>
	27	#include <linux/slab.h>
	28	#include <trace/events/host1x.h>
	29
	30	#include "cdma.h"
	31	#include "channel.h"
	32	#include "dev.h"
	33	#include "debug.h"
6579324a TB	34	#include "job.h"
	35
	36	/*
	37	* push_buffer
	38	*
	39	* The push buffer is a circular array of words to be fetched by command DMA.
	40	* Note that it works slightly differently to the sync queue; fence == pos
	41	* means that the push buffer is full, not empty.
	42	*/
	43
	44	#define HOST1X_PUSHBUFFER_SLOTS 512
	45
	46	/*
	47	* Clean up push buffer resources
	48	*/
	49	static void host1x_pushbuffer_destroy(struct push_buffer *pb)
	50	{
	51	struct host1x_cdma *cdma = pb_to_cdma(pb);
	52	struct host1x *host1x = cdma_to_host1x(cdma);
	53
	54	if (pb->phys != 0)
f6e45661 LR	55	dma_free_wc(host1x->dev, pb->size_bytes + 4, pb->mapped,
f6e45661 LR	56	pb->phys);
6579324a TB	57
	58	pb->mapped = NULL;
	59	pb->phys = 0;
	60	}
	61
	62	/*
	63	* Init push buffer resources
	64	*/
	65	static int host1x_pushbuffer_init(struct push_buffer *pb)
	66	{
	67	struct host1x_cdma *cdma = pb_to_cdma(pb);
	68	struct host1x *host1x = cdma_to_host1x(cdma);
	69
	70	pb->mapped = NULL;
	71	pb->phys = 0;
	72	pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8;
	73
	74	/* initialize buffer pointers */
	75	pb->fence = pb->size_bytes - 8;
	76	pb->pos = 0;
	77
	78	/* allocate and map pushbuffer memory */
f6e45661 LR	79	pb->mapped = dma_alloc_wc(host1x->dev, pb->size_bytes + 4, &pb->phys,
f6e45661 LR	80	GFP_KERNEL);
6579324a TB	81	if (!pb->mapped)
	82	goto fail;
	83
	84	host1x_hw_pushbuffer_init(host1x, pb);
	85
	86	return 0;
	87
	88	fail:
	89	host1x_pushbuffer_destroy(pb);
	90	return -ENOMEM;
	91	}
	92
	93	/*
	94	* Push two words to the push buffer
	95	* Caller must ensure push buffer is not full
	96	*/
	97	static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
	98	{
ebb2475c TR	99	u32 p = (u32 )((void *)pb->mapped + pb->pos);
	100
	101	WARN_ON(pb->pos == pb->fence);
6579324a TB	102	*(p++) = op1;
6579324a TB	103	*(p++) = op2;
ebb2475c	104	pb->pos = (pb->pos + 8) & (pb->size_bytes - 1);
6579324a TB	105	}
	106
	107	/*
	108	* Pop a number of two word slots from the push buffer
	109	* Caller must ensure push buffer is not empty
	110	*/
	111	static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
	112	{
	113	/* Advance the next write position */
	114	pb->fence = (pb->fence + slots * 8) & (pb->size_bytes - 1);
	115	}
	116
	117	/*
	118	* Return the number of two word slots free in the push buffer
	119	*/
	120	static u32 host1x_pushbuffer_space(struct push_buffer *pb)
	121	{
	122	return ((pb->fence - pb->pos) & (pb->size_bytes - 1)) / 8;
	123	}
	124
	125	/*
	126	* Sleep (if necessary) until the requested event happens
	127	* - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
	128	* - Returns 1
	129	* - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
	130	* - Return the amount of space (> 0)
	131	* Must be called with the cdma lock held.
	132	*/
	133	unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
	134	enum cdma_event event)
	135	{
	136	for (;;) {
0b8070d1	137	struct push_buffer *pb = &cdma->push_buffer;
6579324a TB	138	unsigned int space;
6579324a TB	139
0b8070d1 TR	140	switch (event) {
0b8070d1 TR	141	case CDMA_EVENT_SYNC_QUEUE_EMPTY:
6579324a	142	space = list_empty(&cdma->sync_queue) ? 1 : 0;
0b8070d1 TR	143	break;
	144
	145	case CDMA_EVENT_PUSH_BUFFER_SPACE:
6579324a	146	space = host1x_pushbuffer_space(pb);
0b8070d1 TR	147	break;
	148
	149	default:
6579324a TB	150	WARN_ON(1);
	151	return -EINVAL;
	152	}
	153
	154	if (space)
	155	return space;
	156
	157	trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
	158	event);
	159
	160	/* If somebody has managed to already start waiting, yield */
	161	if (cdma->event != CDMA_EVENT_NONE) {
	162	mutex_unlock(&cdma->lock);
	163	schedule();
	164	mutex_lock(&cdma->lock);
	165	continue;
	166	}
0b8070d1	167
6579324a TB	168	cdma->event = event;
	169
	170	mutex_unlock(&cdma->lock);
	171	down(&cdma->sem);
	172	mutex_lock(&cdma->lock);
	173	}
0b8070d1	174
6579324a TB	175	return 0;
	176	}
	177
	178	/*
	179	* Start timer that tracks the time spent by the job.
	180	* Must be called with the cdma lock held.
	181	*/
	182	static void cdma_start_timer_locked(struct host1x_cdma *cdma,
	183	struct host1x_job *job)
	184	{
	185	struct host1x *host = cdma_to_host1x(cdma);
	186
	187	if (cdma->timeout.client) {
	188	/* timer already started */
	189	return;
	190	}
	191
	192	cdma->timeout.client = job->client;
	193	cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
	194	cdma->timeout.syncpt_val = job->syncpt_end;
	195	cdma->timeout.start_ktime = ktime_get();
	196
	197	schedule_delayed_work(&cdma->timeout.wq,
	198	msecs_to_jiffies(job->timeout));
	199	}
	200
	201	/*
	202	* Stop timer when a buffer submission completes.
	203	* Must be called with the cdma lock held.
	204	*/
	205	static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
	206	{
	207	cancel_delayed_work(&cdma->timeout.wq);
	208	cdma->timeout.client = 0;
	209	}
	210
	211	/*
	212	* For all sync queue entries that have already finished according to the
	213	* current sync point registers:
	214	* - unpin & unref their mems
	215	* - pop their push buffer slots
	216	* - remove them from the sync queue
	217	* This is normally called from the host code's worker thread, but can be
	218	* called manually if necessary.
	219	* Must be called with the cdma lock held.
	220	*/
	221	static void update_cdma_locked(struct host1x_cdma *cdma)
	222	{
	223	bool signal = false;
	224	struct host1x *host1x = cdma_to_host1x(cdma);
	225	struct host1x_job job, n;
	226
	227	/* If CDMA is stopped, queue is cleared and we can return */
	228	if (!cdma->running)
	229	return;
	230
	231	/*
	232	* Walk the sync queue, reading the sync point registers as necessary,
	233	* to consume as many sync queue entries as possible without blocking
	234	*/
	235	list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
	236	struct host1x_syncpt *sp =
	237	host1x_syncpt_get(host1x, job->syncpt_id);
	238
239	/* Check whether this syncpt has completed, and bail if not */
240	if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
241	/* Start timer on next pending syncpt */
242	if (job->timeout)
243	cdma_start_timer_locked(cdma, job);
0b8070d1	244
6579324a TB	245	break;
	246	}
	247
	248	/* Cancel timeout, when a buffer completes */
	249	if (cdma->timeout.client)
	250	stop_cdma_timer_locked(cdma);
	251
	252	/* Unpin the memory */
	253	host1x_job_unpin(job);
	254
	255	/* Pop push buffer slots */
	256	if (job->num_slots) {
	257	struct push_buffer *pb = &cdma->push_buffer;
6df633d0	258
6579324a	259	host1x_pushbuffer_pop(pb, job->num_slots);
0b8070d1	260
6579324a TB	261	if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
	262	signal = true;
	263	}
	264
	265	list_del(&job->list);
	266	host1x_job_put(job);
	267	}
	268
	269	if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
	270	list_empty(&cdma->sync_queue))
	271	signal = true;
	272
	273	if (signal) {
	274	cdma->event = CDMA_EVENT_NONE;
	275	up(&cdma->sem);
	276	}
	277	}
	278
	279	void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
	280	struct device *dev)
	281	{
6579324a	282	struct host1x *host1x = cdma_to_host1x(cdma);
0b8070d1 TR	283	u32 restart_addr, syncpt_incrs, syncpt_val;
0b8070d1 TR	284	struct host1x_job *job = NULL;
6579324a TB	285
	286	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
	287
	288	dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
	289	__func__, syncpt_val);
	290
	291	/*
	292	* Move the sync_queue read pointer to the first entry that hasn't
	293	* completed based on the current HW syncpt value. It's likely there
	294	* won't be any (i.e. we're still at the head), but covers the case
	295	* where a syncpt incr happens just prior/during the teardown.
	296	*/
	297
	298	dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
	299	__func__);
	300
	301	list_for_each_entry(job, &cdma->sync_queue, list) {
	302	if (syncpt_val < job->syncpt_end)
	303	break;
	304
	305	host1x_job_dump(dev, job);
	306	}
	307
	308	/*
	309	* Walk the sync_queue, first incrementing with the CPU syncpts that
	310	* are partially executed (the first buffer) or fully skipped while
	311	* still in the current context (slots are also NOP-ed).
	312	*
	313	* At the point contexts are interleaved, syncpt increments must be
	314	* done inline with the pushbuffer from a GATHER buffer to maintain
	315	* the order (slots are modified to be a GATHER of syncpt incrs).
	316	*
	317	* Note: save in restart_addr the location where the timed out buffer
	318	* started in the PB, so we can start the refetch from there (with the
	319	* modified NOP-ed PB slots). This lets things appear to have completed
	320	* properly for this buffer and resources are freed.
	321	*/
	322
	323	dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n",
	324	__func__);
	325
	326	if (!list_empty(&cdma->sync_queue))
	327	restart_addr = job->first_get;
	328	else
	329	restart_addr = cdma->last_pos;
	330
	331	/* do CPU increments as long as this context continues */
	332	list_for_each_entry_from(job, &cdma->sync_queue, list) {
	333	/* different context, gets us out of this loop */
	334	if (job->client != cdma->timeout.client)
	335	break;
	336
	337	/* won't need a timeout when replayed */
	338	job->timeout = 0;
	339
	340	syncpt_incrs = job->syncpt_end - syncpt_val;
	341	dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
	342
	343	host1x_job_dump(dev, job);
	344
	345	/* safe to use CPU to incr syncpts */
	346	host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
	347	syncpt_incrs, job->syncpt_end,
	348	job->num_slots);
349
350	syncpt_val += syncpt_incrs;
351	}
352
6df633d0 TR	353	/*
6df633d0 TR	354	* The following sumbits from the same client may be dependent on the
6579324a	355	* failed submit and therefore they may fail. Force a small timeout
6df633d0 TR	356	* to make the queue cleanup faster.
6df633d0 TR	357	*/
6579324a TB	358
	359	list_for_each_entry_from(job, &cdma->sync_queue, list)
	360	if (job->client == cdma->timeout.client)
	361	job->timeout = min_t(unsigned int, job->timeout, 500);
	362
	363	dev_dbg(dev, "%s: finished sync_queue modification\n", __func__);
	364
	365	/* roll back DMAGET and start up channel again */
	366	host1x_hw_cdma_resume(host1x, cdma, restart_addr);
	367	}
	368
	369	/*
	370	* Create a cdma
	371	*/
	372	int host1x_cdma_init(struct host1x_cdma *cdma)
	373	{
	374	int err;
	375
	376	mutex_init(&cdma->lock);
	377	sema_init(&cdma->sem, 0);
	378
	379	INIT_LIST_HEAD(&cdma->sync_queue);
	380
	381	cdma->event = CDMA_EVENT_NONE;
	382	cdma->running = false;
	383	cdma->torndown = false;
	384
	385	err = host1x_pushbuffer_init(&cdma->push_buffer);
	386	if (err)
	387	return err;
0b8070d1	388
6579324a TB	389	return 0;
	390	}
	391
	392	/*
	393	* Destroy a cdma
	394	*/
	395	int host1x_cdma_deinit(struct host1x_cdma *cdma)
	396	{
	397	struct push_buffer *pb = &cdma->push_buffer;
	398	struct host1x *host1x = cdma_to_host1x(cdma);
	399
	400	if (cdma->running) {
	401	pr_warn("%s: CDMA still running\n", __func__);
	402	return -EBUSY;
	403	}
	404
	405	host1x_pushbuffer_destroy(pb);
	406	host1x_hw_cdma_timeout_destroy(host1x, cdma);
	407
	408	return 0;
	409	}
	410
	411	/*
	412	* Begin a cdma submit
	413	*/
	414	int host1x_cdma_begin(struct host1x_cdma cdma, struct host1x_job job)
	415	{
	416	struct host1x *host1x = cdma_to_host1x(cdma);
	417
	418	mutex_lock(&cdma->lock);
	419
	420	if (job->timeout) {
	421	/* init state on first submit with timeout value */
	422	if (!cdma->timeout.initialized) {
	423	int err;
6df633d0	424
6579324a TB	425	err = host1x_hw_cdma_timeout_init(host1x, cdma,
	426	job->syncpt_id);
	427	if (err) {
	428	mutex_unlock(&cdma->lock);
	429	return err;
	430	}
	431	}
	432	}
0b8070d1	433
6579324a TB	434	if (!cdma->running)
	435	host1x_hw_cdma_start(host1x, cdma);
	436
	437	cdma->slots_free = 0;
	438	cdma->slots_used = 0;
	439	cdma->first_get = cdma->push_buffer.pos;
	440
	441	trace_host1x_cdma_begin(dev_name(job->channel->dev));
	442	return 0;
	443	}
	444
	445	/*
	446	* Push two words into a push buffer slot
	447	* Blocks as necessary if the push buffer is full.
	448	*/
	449	void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
	450	{
	451	struct host1x *host1x = cdma_to_host1x(cdma);
	452	struct push_buffer *pb = &cdma->push_buffer;
	453	u32 slots_free = cdma->slots_free;
	454
6236451d TB	455	if (host1x_debug_trace_cmdbuf)
	456	trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
	457	op1, op2);
	458
6579324a TB	459	if (slots_free == 0) {
	460	host1x_hw_cdma_flush(host1x, cdma);
	461	slots_free = host1x_cdma_wait_locked(cdma,
	462	CDMA_EVENT_PUSH_BUFFER_SPACE);
	463	}
0b8070d1	464
6579324a TB	465	cdma->slots_free = slots_free - 1;
	466	cdma->slots_used++;
	467	host1x_pushbuffer_push(pb, op1, op2);
	468	}
	469
	470	/*
	471	* End a cdma submit
	472	* Kick off DMA, add job to the sync queue, and a number of slots to be freed
	473	* from the pushbuffer. The handles for a submit must all be pinned at the same
	474	* time, but they can be unpinned in smaller chunks.
	475	*/
	476	void host1x_cdma_end(struct host1x_cdma *cdma,
	477	struct host1x_job *job)
	478	{
	479	struct host1x *host1x = cdma_to_host1x(cdma);
	480	bool idle = list_empty(&cdma->sync_queue);
	481
	482	host1x_hw_cdma_flush(host1x, cdma);
	483
	484	job->first_get = cdma->first_get;
	485	job->num_slots = cdma->slots_used;
	486	host1x_job_get(job);
	487	list_add_tail(&job->list, &cdma->sync_queue);
	488
	489	/* start timer on idle -> active transitions */
	490	if (job->timeout && idle)
	491	cdma_start_timer_locked(cdma, job);
	492
	493	trace_host1x_cdma_end(dev_name(job->channel->dev));
	494	mutex_unlock(&cdma->lock);
	495	}
	496
	497	/*
	498	* Update cdma state according to current sync point values
	499	*/
	500	void host1x_cdma_update(struct host1x_cdma *cdma)
	501	{
	502	mutex_lock(&cdma->lock);
	503	update_cdma_locked(cdma);
	504	mutex_unlock(&cdma->lock);
	505	}