[deliverable/linux.git] / drivers / gpu / drm / nouveau / nv50_fifo.c

/*
 * Copyright (C) 2007 Ben Skeggs.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include "drmP.h"
#include "drm.h"
#include "nouveau_drv.h"
#include "nouveau_ramht.h"
#include "nouveau_vm.h"

static void
nv50_fifo_playlist_update(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
	struct nouveau_gpuobj *cur;
	int i, nr;

	NV_DEBUG(dev, "\n");

	cur = pfifo->playlist[pfifo->cur_playlist];
	pfifo->cur_playlist = !pfifo->cur_playlist;

	/* We never schedule channel 0 or 127 */
	for (i = 1, nr = 0; i < 127; i++) {
		if (dev_priv->channels.ptr[i] &&
		    dev_priv->channels.ptr[i]->ramfc) {
			nv_wo32(cur, (nr * 4), i);
			nr++;
		}
	}
	dev_priv->engine.instmem.flush(dev);

	nv_wr32(dev, 0x32f4, cur->vinst >> 12);
	nv_wr32(dev, 0x32ec, nr);
	nv_wr32(dev, 0x2500, 0x101);
}

static void
nv50_fifo_channel_enable(struct drm_device *dev, int channel)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_channel *chan = dev_priv->channels.ptr[channel];
	uint32_t inst;

	NV_DEBUG(dev, "ch%d\n", channel);

	if (dev_priv->chipset == 0x50)
		inst = chan->ramfc->vinst >> 12;
	else
		inst = chan->ramfc->vinst >> 8;

	nv_wr32(dev, NV50_PFIFO_CTX_TABLE(channel), inst |
		     NV50_PFIFO_CTX_TABLE_CHANNEL_ENABLED);
}

static void
nv50_fifo_channel_disable(struct drm_device *dev, int channel)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	uint32_t inst;

	NV_DEBUG(dev, "ch%d\n", channel);

	if (dev_priv->chipset == 0x50)
		inst = NV50_PFIFO_CTX_TABLE_INSTANCE_MASK_G80;
	else
		inst = NV50_PFIFO_CTX_TABLE_INSTANCE_MASK_G84;
	nv_wr32(dev, NV50_PFIFO_CTX_TABLE(channel), inst);
}

static void
nv50_fifo_init_reset(struct drm_device *dev)
{
	uint32_t pmc_e = NV_PMC_ENABLE_PFIFO;

	NV_DEBUG(dev, "\n");

	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) & ~pmc_e);
	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) |  pmc_e);
}

static void
nv50_fifo_init_intr(struct drm_device *dev)
{
	NV_DEBUG(dev, "\n");

	nouveau_irq_register(dev, 8, nv04_fifo_isr);
	nv_wr32(dev, NV03_PFIFO_INTR_0, 0xFFFFFFFF);
	nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0xFFFFFFFF);
}

static void
nv50_fifo_init_context_table(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	int i;

	NV_DEBUG(dev, "\n");

	for (i = 0; i < NV50_PFIFO_CTX_TABLE__SIZE; i++) {
		if (dev_priv->channels.ptr[i])
			nv50_fifo_channel_enable(dev, i);
		else
			nv50_fifo_channel_disable(dev, i);
	}

	nv50_fifo_playlist_update(dev);
}

static void
nv50_fifo_init_regs__nv(struct drm_device *dev)
{
	NV_DEBUG(dev, "\n");

	nv_wr32(dev, 0x250c, 0x6f3cfc34);
}

static void
nv50_fifo_init_regs(struct drm_device *dev)
{
	NV_DEBUG(dev, "\n");

	nv_wr32(dev, 0x2500, 0);
	nv_wr32(dev, 0x3250, 0);
	nv_wr32(dev, 0x3220, 0);
	nv_wr32(dev, 0x3204, 0);
	nv_wr32(dev, 0x3210, 0);
	nv_wr32(dev, 0x3270, 0);
	nv_wr32(dev, 0x2044, 0x01003fff);

	/* Enable dummy channels setup by nv50_instmem.c */
	nv50_fifo_channel_enable(dev, 0);
	nv50_fifo_channel_enable(dev, 127);
}

int
nv50_fifo_init(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
	int ret;

	NV_DEBUG(dev, "\n");

	if (pfifo->playlist[0]) {
		pfifo->cur_playlist = !pfifo->cur_playlist;
		goto just_reset;
	}

	ret = nouveau_gpuobj_new(dev, NULL, 128*4, 0x1000,
				 NVOBJ_FLAG_ZERO_ALLOC,
				 &pfifo->playlist[0]);
	if (ret) {
		NV_ERROR(dev, "error creating playlist 0: %d\n", ret);
		return ret;
	}

	ret = nouveau_gpuobj_new(dev, NULL, 128*4, 0x1000,
				 NVOBJ_FLAG_ZERO_ALLOC,
				 &pfifo->playlist[1]);
	if (ret) {
		nouveau_gpuobj_ref(NULL, &pfifo->playlist[0]);
		NV_ERROR(dev, "error creating playlist 1: %d\n", ret);
		return ret;
	}

just_reset:
	nv50_fifo_init_reset(dev);
	nv50_fifo_init_intr(dev);
	nv50_fifo_init_context_table(dev);
	nv50_fifo_init_regs__nv(dev);
	nv50_fifo_init_regs(dev);
	dev_priv->engine.fifo.enable(dev);
	dev_priv->engine.fifo.reassign(dev, true);

	return 0;
}

void
nv50_fifo_takedown(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;

	NV_DEBUG(dev, "\n");

	if (!pfifo->playlist[0])
		return;

	nv_wr32(dev, 0x2140, 0x00000000);
	nouveau_irq_unregister(dev, 8);

	nouveau_gpuobj_ref(NULL, &pfifo->playlist[0]);
	nouveau_gpuobj_ref(NULL, &pfifo->playlist[1]);
}

int
nv50_fifo_channel_id(struct drm_device *dev)
{
	return nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH1) &
			NV50_PFIFO_CACHE1_PUSH1_CHID_MASK;
}

int
nv50_fifo_create_context(struct nouveau_channel *chan)
{
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_gpuobj *ramfc = NULL;
	unsigned long flags;
	int ret;

	NV_DEBUG(dev, "ch%d\n", chan->id);

	if (dev_priv->chipset == 0x50) {
		ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst,
					      chan->ramin->vinst, 0x100,
					      NVOBJ_FLAG_ZERO_ALLOC |
					      NVOBJ_FLAG_ZERO_FREE,
					      &chan->ramfc);
		if (ret)
			return ret;

		ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst + 0x0400,
					      chan->ramin->vinst + 0x0400,
					      4096, 0, &chan->cache);
		if (ret)
			return ret;
	} else {
		ret = nouveau_gpuobj_new(dev, chan, 0x100, 256,
					 NVOBJ_FLAG_ZERO_ALLOC |
					 NVOBJ_FLAG_ZERO_FREE, &chan->ramfc);
		if (ret)
			return ret;

		ret = nouveau_gpuobj_new(dev, chan, 4096, 1024,
					 0, &chan->cache);
		if (ret)
			return ret;
	}
	ramfc = chan->ramfc;

	chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
			     NV50_USER(chan->id), PAGE_SIZE);
	if (!chan->user)
		return -ENOMEM;

	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);

	nv_wo32(ramfc, 0x48, chan->pushbuf->cinst >> 4);
	nv_wo32(ramfc, 0x80, ((chan->ramht->bits - 9) << 27) |
			     (4 << 24) /* SEARCH_FULL */ |
			     (chan->ramht->gpuobj->cinst >> 4));
	nv_wo32(ramfc, 0x44, 0x01003fff);
	nv_wo32(ramfc, 0x60, 0x7fffffff);
	nv_wo32(ramfc, 0x40, 0x00000000);
	nv_wo32(ramfc, 0x7c, 0x30000001);
	nv_wo32(ramfc, 0x78, 0x00000000);
	nv_wo32(ramfc, 0x3c, 0x403f6078);
	nv_wo32(ramfc, 0x50, chan->pushbuf_base + chan->dma.ib_base * 4);
	nv_wo32(ramfc, 0x54, drm_order(chan->dma.ib_max + 1) << 16);

	if (dev_priv->chipset != 0x50) {
		nv_wo32(chan->ramin, 0, chan->id);
		nv_wo32(chan->ramin, 4, chan->ramfc->vinst >> 8);

		nv_wo32(ramfc, 0x88, chan->cache->vinst >> 10);
		nv_wo32(ramfc, 0x98, chan->ramin->vinst >> 12);
	}

	dev_priv->engine.instmem.flush(dev);

	nv50_fifo_channel_enable(dev, chan->id);
	nv50_fifo_playlist_update(dev);
	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
	return 0;
}

void
nv50_fifo_destroy_context(struct nouveau_channel *chan)
{
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
	struct nouveau_gpuobj *ramfc = NULL;
	unsigned long flags;

	NV_DEBUG(dev, "ch%d\n", chan->id);

	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
	pfifo->reassign(dev, false);

	/* Unload the context if it's the currently active one */
	if (pfifo->channel_id(dev) == chan->id) {
		pfifo->disable(dev);
		pfifo->unload_context(dev);
		pfifo->enable(dev);
	}

	/* This will ensure the channel is seen as disabled. */
	nouveau_gpuobj_ref(chan->ramfc, &ramfc);
	nouveau_gpuobj_ref(NULL, &chan->ramfc);
	nv50_fifo_channel_disable(dev, chan->id);

	/* Dummy channel, also used on ch 127 */
	if (chan->id == 0)
		nv50_fifo_channel_disable(dev, 127);
	nv50_fifo_playlist_update(dev);

	pfifo->reassign(dev, true);
	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);

	/* Free the channel resources */
	if (chan->user) {
		iounmap(chan->user);
		chan->user = NULL;
	}
	nouveau_gpuobj_ref(NULL, &ramfc);
	nouveau_gpuobj_ref(NULL, &chan->cache);
}

int
nv50_fifo_load_context(struct nouveau_channel *chan)
{
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_gpuobj *ramfc = chan->ramfc;
	struct nouveau_gpuobj *cache = chan->cache;
	int ptr, cnt;

	NV_DEBUG(dev, "ch%d\n", chan->id);

	nv_wr32(dev, 0x3330, nv_ro32(ramfc, 0x00));
	nv_wr32(dev, 0x3334, nv_ro32(ramfc, 0x04));
	nv_wr32(dev, 0x3240, nv_ro32(ramfc, 0x08));
	nv_wr32(dev, 0x3320, nv_ro32(ramfc, 0x0c));
	nv_wr32(dev, 0x3244, nv_ro32(ramfc, 0x10));
	nv_wr32(dev, 0x3328, nv_ro32(ramfc, 0x14));
	nv_wr32(dev, 0x3368, nv_ro32(ramfc, 0x18));
	nv_wr32(dev, 0x336c, nv_ro32(ramfc, 0x1c));
	nv_wr32(dev, 0x3370, nv_ro32(ramfc, 0x20));
	nv_wr32(dev, 0x3374, nv_ro32(ramfc, 0x24));
	nv_wr32(dev, 0x3378, nv_ro32(ramfc, 0x28));
	nv_wr32(dev, 0x337c, nv_ro32(ramfc, 0x2c));
	nv_wr32(dev, 0x3228, nv_ro32(ramfc, 0x30));
	nv_wr32(dev, 0x3364, nv_ro32(ramfc, 0x34));
	nv_wr32(dev, 0x32a0, nv_ro32(ramfc, 0x38));
	nv_wr32(dev, 0x3224, nv_ro32(ramfc, 0x3c));
	nv_wr32(dev, 0x324c, nv_ro32(ramfc, 0x40));
	nv_wr32(dev, 0x2044, nv_ro32(ramfc, 0x44));
	nv_wr32(dev, 0x322c, nv_ro32(ramfc, 0x48));
	nv_wr32(dev, 0x3234, nv_ro32(ramfc, 0x4c));
	nv_wr32(dev, 0x3340, nv_ro32(ramfc, 0x50));
	nv_wr32(dev, 0x3344, nv_ro32(ramfc, 0x54));
	nv_wr32(dev, 0x3280, nv_ro32(ramfc, 0x58));
	nv_wr32(dev, 0x3254, nv_ro32(ramfc, 0x5c));
	nv_wr32(dev, 0x3260, nv_ro32(ramfc, 0x60));
	nv_wr32(dev, 0x3264, nv_ro32(ramfc, 0x64));
	nv_wr32(dev, 0x3268, nv_ro32(ramfc, 0x68));
	nv_wr32(dev, 0x326c, nv_ro32(ramfc, 0x6c));
	nv_wr32(dev, 0x32e4, nv_ro32(ramfc, 0x70));
	nv_wr32(dev, 0x3248, nv_ro32(ramfc, 0x74));
	nv_wr32(dev, 0x2088, nv_ro32(ramfc, 0x78));
	nv_wr32(dev, 0x2058, nv_ro32(ramfc, 0x7c));
	nv_wr32(dev, 0x2210, nv_ro32(ramfc, 0x80));

	cnt = nv_ro32(ramfc, 0x84);
	for (ptr = 0; ptr < cnt; ptr++) {
		nv_wr32(dev, NV40_PFIFO_CACHE1_METHOD(ptr),
			nv_ro32(cache, (ptr * 8) + 0));
		nv_wr32(dev, NV40_PFIFO_CACHE1_DATA(ptr),
			nv_ro32(cache, (ptr * 8) + 4));
	}
	nv_wr32(dev, NV03_PFIFO_CACHE1_PUT, cnt << 2);
	nv_wr32(dev, NV03_PFIFO_CACHE1_GET, 0);

	/* guessing that all the 0x34xx regs aren't on NV50 */
	if (dev_priv->chipset != 0x50) {
		nv_wr32(dev, 0x340c, nv_ro32(ramfc, 0x88));
		nv_wr32(dev, 0x3400, nv_ro32(ramfc, 0x8c));
		nv_wr32(dev, 0x3404, nv_ro32(ramfc, 0x90));
		nv_wr32(dev, 0x3408, nv_ro32(ramfc, 0x94));
		nv_wr32(dev, 0x3410, nv_ro32(ramfc, 0x98));
	}

	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, chan->id | (1<<16));
	return 0;
}

int
nv50_fifo_unload_context(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
	struct nouveau_gpuobj *ramfc, *cache;
	struct nouveau_channel *chan = NULL;
	int chid, get, put, ptr;

	NV_DEBUG(dev, "\n");

	chid = pfifo->channel_id(dev);
	if (chid < 1 || chid >= dev_priv->engine.fifo.channels - 1)
		return 0;

	chan = dev_priv->channels.ptr[chid];
	if (!chan) {
		NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid);
		return -EINVAL;
	}
	NV_DEBUG(dev, "ch%d\n", chan->id);
	ramfc = chan->ramfc;
	cache = chan->cache;

	nv_wo32(ramfc, 0x00, nv_rd32(dev, 0x3330));
	nv_wo32(ramfc, 0x04, nv_rd32(dev, 0x3334));
	nv_wo32(ramfc, 0x08, nv_rd32(dev, 0x3240));
	nv_wo32(ramfc, 0x0c, nv_rd32(dev, 0x3320));
	nv_wo32(ramfc, 0x10, nv_rd32(dev, 0x3244));
	nv_wo32(ramfc, 0x14, nv_rd32(dev, 0x3328));
	nv_wo32(ramfc, 0x18, nv_rd32(dev, 0x3368));
	nv_wo32(ramfc, 0x1c, nv_rd32(dev, 0x336c));
	nv_wo32(ramfc, 0x20, nv_rd32(dev, 0x3370));
	nv_wo32(ramfc, 0x24, nv_rd32(dev, 0x3374));
	nv_wo32(ramfc, 0x28, nv_rd32(dev, 0x3378));
	nv_wo32(ramfc, 0x2c, nv_rd32(dev, 0x337c));
	nv_wo32(ramfc, 0x30, nv_rd32(dev, 0x3228));
	nv_wo32(ramfc, 0x34, nv_rd32(dev, 0x3364));
	nv_wo32(ramfc, 0x38, nv_rd32(dev, 0x32a0));
	nv_wo32(ramfc, 0x3c, nv_rd32(dev, 0x3224));
	nv_wo32(ramfc, 0x40, nv_rd32(dev, 0x324c));
	nv_wo32(ramfc, 0x44, nv_rd32(dev, 0x2044));
	nv_wo32(ramfc, 0x48, nv_rd32(dev, 0x322c));
	nv_wo32(ramfc, 0x4c, nv_rd32(dev, 0x3234));
	nv_wo32(ramfc, 0x50, nv_rd32(dev, 0x3340));
	nv_wo32(ramfc, 0x54, nv_rd32(dev, 0x3344));
	nv_wo32(ramfc, 0x58, nv_rd32(dev, 0x3280));
	nv_wo32(ramfc, 0x5c, nv_rd32(dev, 0x3254));
	nv_wo32(ramfc, 0x60, nv_rd32(dev, 0x3260));
	nv_wo32(ramfc, 0x64, nv_rd32(dev, 0x3264));
	nv_wo32(ramfc, 0x68, nv_rd32(dev, 0x3268));
	nv_wo32(ramfc, 0x6c, nv_rd32(dev, 0x326c));
	nv_wo32(ramfc, 0x70, nv_rd32(dev, 0x32e4));
	nv_wo32(ramfc, 0x74, nv_rd32(dev, 0x3248));
	nv_wo32(ramfc, 0x78, nv_rd32(dev, 0x2088));
	nv_wo32(ramfc, 0x7c, nv_rd32(dev, 0x2058));
	nv_wo32(ramfc, 0x80, nv_rd32(dev, 0x2210));

	put = (nv_rd32(dev, NV03_PFIFO_CACHE1_PUT) & 0x7ff) >> 2;
	get = (nv_rd32(dev, NV03_PFIFO_CACHE1_GET) & 0x7ff) >> 2;
	ptr = 0;
	while (put != get) {
		nv_wo32(cache, ptr + 0,
			nv_rd32(dev, NV40_PFIFO_CACHE1_METHOD(get)));
		nv_wo32(cache, ptr + 4,
			nv_rd32(dev, NV40_PFIFO_CACHE1_DATA(get)));
		get = (get + 1) & 0x1ff;
		ptr += 8;
	}

	/* guessing that all the 0x34xx regs aren't on NV50 */
	if (dev_priv->chipset != 0x50) {
		nv_wo32(ramfc, 0x84, ptr >> 3);
		nv_wo32(ramfc, 0x88, nv_rd32(dev, 0x340c));
		nv_wo32(ramfc, 0x8c, nv_rd32(dev, 0x3400));
		nv_wo32(ramfc, 0x90, nv_rd32(dev, 0x3404));
		nv_wo32(ramfc, 0x94, nv_rd32(dev, 0x3408));
		nv_wo32(ramfc, 0x98, nv_rd32(dev, 0x3410));
	}

	dev_priv->engine.instmem.flush(dev);

	/*XXX: probably reload ch127 (NULL) state back too */
	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, 127);
	return 0;
}

void
nv50_fifo_tlb_flush(struct drm_device *dev)
{
	nv50_vm_flush_engine(dev, 5);
}
Commit	Line	Data
6ee73861 BS	1	/*
	2	* Copyright (C) 2007 Ben Skeggs.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining
	6	* a copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sublicense, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* The above copyright notice and this permission notice (including the
	14	* next paragraph) shall be included in all copies or substantial
	15	* portions of the Software.
	16	*
	17	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	18	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	19	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	20	* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
	21	* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
	22	* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
	23	* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	24	*
	25	*/
	26
	27	#include "drmP.h"
	28	#include "drm.h"
	29	#include "nouveau_drv.h"
a8eaebc6	30	#include "nouveau_ramht.h"
a11c3198	31	#include "nouveau_vm.h"
6ee73861	32
6ee73861	33	static void
ac94a343	34	nv50_fifo_playlist_update(struct drm_device *dev)
6ee73861 BS	35	{
6ee73861 BS	36	struct drm_nouveau_private *dev_priv = dev->dev_private;
ac94a343	37	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
a8eaebc6	38	struct nouveau_gpuobj *cur;
6ee73861 BS	39	int i, nr;
	40
	41	NV_DEBUG(dev, "\n");
	42
ac94a343 BS	43	cur = pfifo->playlist[pfifo->cur_playlist];
ac94a343 BS	44	pfifo->cur_playlist = !pfifo->cur_playlist;
6ee73861 BS	45
6ee73861 BS	46	/* We never schedule channel 0 or 127 */
6ee73861	47	for (i = 1, nr = 0; i < 127; i++) {
cff5c133 BS	48	if (dev_priv->channels.ptr[i] &&
cff5c133 BS	49	dev_priv->channels.ptr[i]->ramfc) {
a8eaebc6	50	nv_wo32(cur, (nr * 4), i);
b3beb167 BS	51	nr++;
b3beb167 BS	52	}
6ee73861	53	}
f56cb86f	54	dev_priv->engine.instmem.flush(dev);
6ee73861	55
a8eaebc6	56	nv_wr32(dev, 0x32f4, cur->vinst >> 12);
6ee73861 BS	57	nv_wr32(dev, 0x32ec, nr);
	58	nv_wr32(dev, 0x2500, 0x101);
	59	}
	60
ac94a343 BS	61	static void
ac94a343 BS	62	nv50_fifo_channel_enable(struct drm_device *dev, int channel)
6ee73861 BS	63	{
6ee73861 BS	64	struct drm_nouveau_private *dev_priv = dev->dev_private;
cff5c133	65	struct nouveau_channel *chan = dev_priv->channels.ptr[channel];
6ee73861 BS	66	uint32_t inst;
	67
	68	NV_DEBUG(dev, "ch%d\n", channel);
	69
ac94a343	70	if (dev_priv->chipset == 0x50)
a8eaebc6	71	inst = chan->ramfc->vinst >> 12;
6ee73861	72	else
a8eaebc6	73	inst = chan->ramfc->vinst >> 8;
6ee73861	74
ac94a343 BS	75	nv_wr32(dev, NV50_PFIFO_CTX_TABLE(channel), inst \|
ac94a343 BS	76	NV50_PFIFO_CTX_TABLE_CHANNEL_ENABLED);
6ee73861 BS	77	}
	78
	79	static void
ac94a343	80	nv50_fifo_channel_disable(struct drm_device *dev, int channel)
6ee73861 BS	81	{
	82	struct drm_nouveau_private *dev_priv = dev->dev_private;
	83	uint32_t inst;
	84
ac94a343	85	NV_DEBUG(dev, "ch%d\n", channel);
6ee73861	86
ac94a343	87	if (dev_priv->chipset == 0x50)
6ee73861 BS	88	inst = NV50_PFIFO_CTX_TABLE_INSTANCE_MASK_G80;
	89	else
	90	inst = NV50_PFIFO_CTX_TABLE_INSTANCE_MASK_G84;
	91	nv_wr32(dev, NV50_PFIFO_CTX_TABLE(channel), inst);
6ee73861 BS	92	}
	93
	94	static void
	95	nv50_fifo_init_reset(struct drm_device *dev)
	96	{
	97	uint32_t pmc_e = NV_PMC_ENABLE_PFIFO;
	98
	99	NV_DEBUG(dev, "\n");
	100
	101	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) & ~pmc_e);
	102	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) \| pmc_e);
	103	}
	104
	105	static void
	106	nv50_fifo_init_intr(struct drm_device *dev)
	107	{
	108	NV_DEBUG(dev, "\n");
	109
5178d40d	110	nouveau_irq_register(dev, 8, nv04_fifo_isr);
6ee73861 BS	111	nv_wr32(dev, NV03_PFIFO_INTR_0, 0xFFFFFFFF);
	112	nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0xFFFFFFFF);
	113	}
	114
	115	static void
	116	nv50_fifo_init_context_table(struct drm_device *dev)
	117	{
	118	struct drm_nouveau_private *dev_priv = dev->dev_private;
	119	int i;
	120
	121	NV_DEBUG(dev, "\n");
	122
	123	for (i = 0; i < NV50_PFIFO_CTX_TABLE__SIZE; i++) {
cff5c133	124	if (dev_priv->channels.ptr[i])
ac94a343	125	nv50_fifo_channel_enable(dev, i);
6ee73861	126	else
ac94a343	127	nv50_fifo_channel_disable(dev, i);
6ee73861 BS	128	}
6ee73861 BS	129
ac94a343	130	nv50_fifo_playlist_update(dev);
6ee73861 BS	131	}
	132
	133	static void
	134	nv50_fifo_init_regs__nv(struct drm_device *dev)
	135	{
	136	NV_DEBUG(dev, "\n");
	137
	138	nv_wr32(dev, 0x250c, 0x6f3cfc34);
	139	}
	140
	141	static void
	142	nv50_fifo_init_regs(struct drm_device *dev)
	143	{
	144	NV_DEBUG(dev, "\n");
	145
	146	nv_wr32(dev, 0x2500, 0);
	147	nv_wr32(dev, 0x3250, 0);
	148	nv_wr32(dev, 0x3220, 0);
	149	nv_wr32(dev, 0x3204, 0);
	150	nv_wr32(dev, 0x3210, 0);
	151	nv_wr32(dev, 0x3270, 0);
ec23802d	152	nv_wr32(dev, 0x2044, 0x01003fff);
6ee73861 BS	153
6ee73861 BS	154	/* Enable dummy channels setup by nv50_instmem.c */
ac94a343 BS	155	nv50_fifo_channel_enable(dev, 0);
ac94a343 BS	156	nv50_fifo_channel_enable(dev, 127);
6ee73861 BS	157	}
	158
	159	int
	160	nv50_fifo_init(struct drm_device *dev)
	161	{
	162	struct drm_nouveau_private *dev_priv = dev->dev_private;
ac94a343	163	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
6ee73861 BS	164	int ret;
	165
	166	NV_DEBUG(dev, "\n");
	167
ac94a343 BS	168	if (pfifo->playlist[0]) {
ac94a343 BS	169	pfifo->cur_playlist = !pfifo->cur_playlist;
6ee73861 BS	170	goto just_reset;
	171	}
	172
a8eaebc6 BS	173	ret = nouveau_gpuobj_new(dev, NULL, 128*4, 0x1000,
	174	NVOBJ_FLAG_ZERO_ALLOC,
	175	&pfifo->playlist[0]);
6ee73861	176	if (ret) {
ac94a343	177	NV_ERROR(dev, "error creating playlist 0: %d\n", ret);
6ee73861 BS	178	return ret;
	179	}
	180
a8eaebc6 BS	181	ret = nouveau_gpuobj_new(dev, NULL, 128*4, 0x1000,
	182	NVOBJ_FLAG_ZERO_ALLOC,
	183	&pfifo->playlist[1]);
6ee73861	184	if (ret) {
a8eaebc6	185	nouveau_gpuobj_ref(NULL, &pfifo->playlist[0]);
ac94a343	186	NV_ERROR(dev, "error creating playlist 1: %d\n", ret);
6ee73861 BS	187	return ret;
	188	}
	189
	190	just_reset:
	191	nv50_fifo_init_reset(dev);
	192	nv50_fifo_init_intr(dev);
	193	nv50_fifo_init_context_table(dev);
	194	nv50_fifo_init_regs__nv(dev);
	195	nv50_fifo_init_regs(dev);
	196	dev_priv->engine.fifo.enable(dev);
	197	dev_priv->engine.fifo.reassign(dev, true);
	198
	199	return 0;
	200	}
	201
	202	void
	203	nv50_fifo_takedown(struct drm_device *dev)
	204	{
	205	struct drm_nouveau_private *dev_priv = dev->dev_private;
ac94a343	206	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
6ee73861 BS	207
	208	NV_DEBUG(dev, "\n");
	209
ac94a343	210	if (!pfifo->playlist[0])
6ee73861 BS	211	return;
6ee73861 BS	212
5178d40d BS	213	nv_wr32(dev, 0x2140, 0x00000000);
	214	nouveau_irq_unregister(dev, 8);
	215
a8eaebc6 BS	216	nouveau_gpuobj_ref(NULL, &pfifo->playlist[0]);
a8eaebc6 BS	217	nouveau_gpuobj_ref(NULL, &pfifo->playlist[1]);
6ee73861 BS	218	}
	219
	220	int
	221	nv50_fifo_channel_id(struct drm_device *dev)
	222	{
	223	return nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH1) &
	224	NV50_PFIFO_CACHE1_PUSH1_CHID_MASK;
	225	}
	226
	227	int
	228	nv50_fifo_create_context(struct nouveau_channel *chan)
	229	{
	230	struct drm_device *dev = chan->dev;
	231	struct drm_nouveau_private *dev_priv = dev->dev_private;
	232	struct nouveau_gpuobj *ramfc = NULL;
ff9e5279	233	unsigned long flags;
6ee73861 BS	234	int ret;
	235
	236	NV_DEBUG(dev, "ch%d\n", chan->id);
	237
ac94a343	238	if (dev_priv->chipset == 0x50) {
a8eaebc6 BS	239	ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst,
a8eaebc6 BS	240	chan->ramin->vinst, 0x100,
de3a6c0a	241	NVOBJ_FLAG_ZERO_ALLOC \|
a8eaebc6	242	NVOBJ_FLAG_ZERO_FREE,
6ee73861 BS	243	&chan->ramfc);
	244	if (ret)
	245	return ret;
	246
a8eaebc6 BS	247	ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst + 0x0400,
	248	chan->ramin->vinst + 0x0400,
	249	4096, 0, &chan->cache);
6ee73861 BS	250	if (ret)
	251	return ret;
	252	} else {
a8eaebc6 BS	253	ret = nouveau_gpuobj_new(dev, chan, 0x100, 256,
	254	NVOBJ_FLAG_ZERO_ALLOC \|
	255	NVOBJ_FLAG_ZERO_FREE, &chan->ramfc);
6ee73861 BS	256	if (ret)
6ee73861 BS	257	return ret;
6ee73861	258
a8eaebc6 BS	259	ret = nouveau_gpuobj_new(dev, chan, 4096, 1024,
a8eaebc6 BS	260	0, &chan->cache);
6ee73861 BS	261	if (ret)
	262	return ret;
	263	}
a8eaebc6	264	ramfc = chan->ramfc;
6ee73861	265
d908175c BS	266	chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
	267	NV50_USER(chan->id), PAGE_SIZE);
	268	if (!chan->user)
	269	return -ENOMEM;
	270
ff9e5279 MM	271	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
ff9e5279 MM	272
a8eaebc6	273	nv_wo32(ramfc, 0x48, chan->pushbuf->cinst >> 4);
e05c5a31	274	nv_wo32(ramfc, 0x80, ((chan->ramht->bits - 9) << 27) \|
b3beb167	275	(4 << 24) /* SEARCH_FULL */ \|
a8eaebc6	276	(chan->ramht->gpuobj->cinst >> 4));
ec23802d	277	nv_wo32(ramfc, 0x44, 0x01003fff);
b3beb167 BS	278	nv_wo32(ramfc, 0x60, 0x7fffffff);
	279	nv_wo32(ramfc, 0x40, 0x00000000);
	280	nv_wo32(ramfc, 0x7c, 0x30000001);
	281	nv_wo32(ramfc, 0x78, 0x00000000);
	282	nv_wo32(ramfc, 0x3c, 0x403f6078);
	283	nv_wo32(ramfc, 0x50, chan->pushbuf_base + chan->dma.ib_base * 4);
	284	nv_wo32(ramfc, 0x54, drm_order(chan->dma.ib_max + 1) << 16);
6ee73861	285
ac94a343	286	if (dev_priv->chipset != 0x50) {
a8eaebc6 BS	287	nv_wo32(chan->ramin, 0, chan->id);
a8eaebc6 BS	288	nv_wo32(chan->ramin, 4, chan->ramfc->vinst >> 8);
6ee73861	289
a8eaebc6 BS	290	nv_wo32(ramfc, 0x88, chan->cache->vinst >> 10);
a8eaebc6 BS	291	nv_wo32(ramfc, 0x98, chan->ramin->vinst >> 12);
6ee73861 BS	292	}
6ee73861 BS	293
f56cb86f	294	dev_priv->engine.instmem.flush(dev);
6ee73861	295
ac94a343 BS	296	nv50_fifo_channel_enable(dev, chan->id);
ac94a343 BS	297	nv50_fifo_playlist_update(dev);
ff9e5279	298	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
6ee73861 BS	299	return 0;
	300	}
	301
	302	void
	303	nv50_fifo_destroy_context(struct nouveau_channel *chan)
	304	{
	305	struct drm_device *dev = chan->dev;
3945e475 FJ	306	struct drm_nouveau_private *dev_priv = dev->dev_private;
3945e475 FJ	307	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
a8eaebc6	308	struct nouveau_gpuobj *ramfc = NULL;
3945e475	309	unsigned long flags;
6ee73861 BS	310
	311	NV_DEBUG(dev, "ch%d\n", chan->id);
	312
3945e475 FJ	313	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
	314	pfifo->reassign(dev, false);
	315
	316	/* Unload the context if it's the currently active one */
	317	if (pfifo->channel_id(dev) == chan->id) {
	318	pfifo->disable(dev);
	319	pfifo->unload_context(dev);
	320	pfifo->enable(dev);
	321	}
	322
a87ff62a	323	/* This will ensure the channel is seen as disabled. */
a8eaebc6 BS	324	nouveau_gpuobj_ref(chan->ramfc, &ramfc);
a8eaebc6 BS	325	nouveau_gpuobj_ref(NULL, &chan->ramfc);
ac94a343	326	nv50_fifo_channel_disable(dev, chan->id);
6ee73861 BS	327
	328	/* Dummy channel, also used on ch 127 */
	329	if (chan->id == 0)
ac94a343 BS	330	nv50_fifo_channel_disable(dev, 127);
ac94a343 BS	331	nv50_fifo_playlist_update(dev);
a87ff62a	332
3945e475 FJ	333	pfifo->reassign(dev, true);
	334	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
	335
	336	/* Free the channel resources */
d908175c BS	337	if (chan->user) {
	338	iounmap(chan->user);
	339	chan->user = NULL;
	340	}
a8eaebc6 BS	341	nouveau_gpuobj_ref(NULL, &ramfc);
a8eaebc6 BS	342	nouveau_gpuobj_ref(NULL, &chan->cache);
6ee73861 BS	343	}
	344
	345	int
	346	nv50_fifo_load_context(struct nouveau_channel *chan)
	347	{
	348	struct drm_device *dev = chan->dev;
	349	struct drm_nouveau_private *dev_priv = dev->dev_private;
a8eaebc6 BS	350	struct nouveau_gpuobj *ramfc = chan->ramfc;
a8eaebc6 BS	351	struct nouveau_gpuobj *cache = chan->cache;
6ee73861 BS	352	int ptr, cnt;
	353
	354	NV_DEBUG(dev, "ch%d\n", chan->id);
	355
b3beb167 BS	356	nv_wr32(dev, 0x3330, nv_ro32(ramfc, 0x00));
	357	nv_wr32(dev, 0x3334, nv_ro32(ramfc, 0x04));
	358	nv_wr32(dev, 0x3240, nv_ro32(ramfc, 0x08));
	359	nv_wr32(dev, 0x3320, nv_ro32(ramfc, 0x0c));
	360	nv_wr32(dev, 0x3244, nv_ro32(ramfc, 0x10));
	361	nv_wr32(dev, 0x3328, nv_ro32(ramfc, 0x14));
	362	nv_wr32(dev, 0x3368, nv_ro32(ramfc, 0x18));
	363	nv_wr32(dev, 0x336c, nv_ro32(ramfc, 0x1c));
	364	nv_wr32(dev, 0x3370, nv_ro32(ramfc, 0x20));
	365	nv_wr32(dev, 0x3374, nv_ro32(ramfc, 0x24));
	366	nv_wr32(dev, 0x3378, nv_ro32(ramfc, 0x28));
	367	nv_wr32(dev, 0x337c, nv_ro32(ramfc, 0x2c));
	368	nv_wr32(dev, 0x3228, nv_ro32(ramfc, 0x30));
	369	nv_wr32(dev, 0x3364, nv_ro32(ramfc, 0x34));
	370	nv_wr32(dev, 0x32a0, nv_ro32(ramfc, 0x38));
	371	nv_wr32(dev, 0x3224, nv_ro32(ramfc, 0x3c));
	372	nv_wr32(dev, 0x324c, nv_ro32(ramfc, 0x40));
	373	nv_wr32(dev, 0x2044, nv_ro32(ramfc, 0x44));
	374	nv_wr32(dev, 0x322c, nv_ro32(ramfc, 0x48));
	375	nv_wr32(dev, 0x3234, nv_ro32(ramfc, 0x4c));
	376	nv_wr32(dev, 0x3340, nv_ro32(ramfc, 0x50));
	377	nv_wr32(dev, 0x3344, nv_ro32(ramfc, 0x54));
	378	nv_wr32(dev, 0x3280, nv_ro32(ramfc, 0x58));
	379	nv_wr32(dev, 0x3254, nv_ro32(ramfc, 0x5c));
	380	nv_wr32(dev, 0x3260, nv_ro32(ramfc, 0x60));
	381	nv_wr32(dev, 0x3264, nv_ro32(ramfc, 0x64));
	382	nv_wr32(dev, 0x3268, nv_ro32(ramfc, 0x68));
	383	nv_wr32(dev, 0x326c, nv_ro32(ramfc, 0x6c));
	384	nv_wr32(dev, 0x32e4, nv_ro32(ramfc, 0x70));
	385	nv_wr32(dev, 0x3248, nv_ro32(ramfc, 0x74));
	386	nv_wr32(dev, 0x2088, nv_ro32(ramfc, 0x78));
	387	nv_wr32(dev, 0x2058, nv_ro32(ramfc, 0x7c));
	388	nv_wr32(dev, 0x2210, nv_ro32(ramfc, 0x80));
	389
	390	cnt = nv_ro32(ramfc, 0x84);
6ee73861 BS	391	for (ptr = 0; ptr < cnt; ptr++) {
6ee73861 BS	392	nv_wr32(dev, NV40_PFIFO_CACHE1_METHOD(ptr),
b3beb167	393	nv_ro32(cache, (ptr * 8) + 0));
6ee73861	394	nv_wr32(dev, NV40_PFIFO_CACHE1_DATA(ptr),
b3beb167	395	nv_ro32(cache, (ptr * 8) + 4));
6ee73861	396	}
7fb8ec8e BS	397	nv_wr32(dev, NV03_PFIFO_CACHE1_PUT, cnt << 2);
7fb8ec8e BS	398	nv_wr32(dev, NV03_PFIFO_CACHE1_GET, 0);
6ee73861 BS	399
6ee73861 BS	400	/* guessing that all the 0x34xx regs aren't on NV50 */
ac94a343	401	if (dev_priv->chipset != 0x50) {
b3beb167 BS	402	nv_wr32(dev, 0x340c, nv_ro32(ramfc, 0x88));
	403	nv_wr32(dev, 0x3400, nv_ro32(ramfc, 0x8c));
	404	nv_wr32(dev, 0x3404, nv_ro32(ramfc, 0x90));
	405	nv_wr32(dev, 0x3408, nv_ro32(ramfc, 0x94));
	406	nv_wr32(dev, 0x3410, nv_ro32(ramfc, 0x98));
6ee73861 BS	407	}
6ee73861 BS	408
6ee73861 BS	409	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, chan->id \| (1<<16));
	410	return 0;
	411	}
	412
	413	int
	414	nv50_fifo_unload_context(struct drm_device *dev)
	415	{
	416	struct drm_nouveau_private *dev_priv = dev->dev_private;
	417	struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
	418	struct nouveau_gpuobj ramfc, cache;
	419	struct nouveau_channel *chan = NULL;
	420	int chid, get, put, ptr;
	421
	422	NV_DEBUG(dev, "\n");
	423
	424	chid = pfifo->channel_id(dev);
3c8868d3	425	if (chid < 1 \|\| chid >= dev_priv->engine.fifo.channels - 1)
6ee73861 BS	426	return 0;
6ee73861 BS	427
cff5c133	428	chan = dev_priv->channels.ptr[chid];
6ee73861 BS	429	if (!chan) {
	430	NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid);
	431	return -EINVAL;
	432	}
	433	NV_DEBUG(dev, "ch%d\n", chan->id);
a8eaebc6 BS	434	ramfc = chan->ramfc;
a8eaebc6 BS	435	cache = chan->cache;
6ee73861	436
b3beb167 BS	437	nv_wo32(ramfc, 0x00, nv_rd32(dev, 0x3330));
	438	nv_wo32(ramfc, 0x04, nv_rd32(dev, 0x3334));
	439	nv_wo32(ramfc, 0x08, nv_rd32(dev, 0x3240));
	440	nv_wo32(ramfc, 0x0c, nv_rd32(dev, 0x3320));
	441	nv_wo32(ramfc, 0x10, nv_rd32(dev, 0x3244));
	442	nv_wo32(ramfc, 0x14, nv_rd32(dev, 0x3328));
	443	nv_wo32(ramfc, 0x18, nv_rd32(dev, 0x3368));
	444	nv_wo32(ramfc, 0x1c, nv_rd32(dev, 0x336c));
	445	nv_wo32(ramfc, 0x20, nv_rd32(dev, 0x3370));
	446	nv_wo32(ramfc, 0x24, nv_rd32(dev, 0x3374));
	447	nv_wo32(ramfc, 0x28, nv_rd32(dev, 0x3378));
	448	nv_wo32(ramfc, 0x2c, nv_rd32(dev, 0x337c));
	449	nv_wo32(ramfc, 0x30, nv_rd32(dev, 0x3228));
	450	nv_wo32(ramfc, 0x34, nv_rd32(dev, 0x3364));
	451	nv_wo32(ramfc, 0x38, nv_rd32(dev, 0x32a0));
	452	nv_wo32(ramfc, 0x3c, nv_rd32(dev, 0x3224));
	453	nv_wo32(ramfc, 0x40, nv_rd32(dev, 0x324c));
	454	nv_wo32(ramfc, 0x44, nv_rd32(dev, 0x2044));
	455	nv_wo32(ramfc, 0x48, nv_rd32(dev, 0x322c));
	456	nv_wo32(ramfc, 0x4c, nv_rd32(dev, 0x3234));
	457	nv_wo32(ramfc, 0x50, nv_rd32(dev, 0x3340));
	458	nv_wo32(ramfc, 0x54, nv_rd32(dev, 0x3344));
	459	nv_wo32(ramfc, 0x58, nv_rd32(dev, 0x3280));
	460	nv_wo32(ramfc, 0x5c, nv_rd32(dev, 0x3254));
	461	nv_wo32(ramfc, 0x60, nv_rd32(dev, 0x3260));
	462	nv_wo32(ramfc, 0x64, nv_rd32(dev, 0x3264));
	463	nv_wo32(ramfc, 0x68, nv_rd32(dev, 0x3268));
	464	nv_wo32(ramfc, 0x6c, nv_rd32(dev, 0x326c));
	465	nv_wo32(ramfc, 0x70, nv_rd32(dev, 0x32e4));
	466	nv_wo32(ramfc, 0x74, nv_rd32(dev, 0x3248));
	467	nv_wo32(ramfc, 0x78, nv_rd32(dev, 0x2088));
	468	nv_wo32(ramfc, 0x7c, nv_rd32(dev, 0x2058));
	469	nv_wo32(ramfc, 0x80, nv_rd32(dev, 0x2210));
6ee73861 BS	470
	471	put = (nv_rd32(dev, NV03_PFIFO_CACHE1_PUT) & 0x7ff) >> 2;
	472	get = (nv_rd32(dev, NV03_PFIFO_CACHE1_GET) & 0x7ff) >> 2;
	473	ptr = 0;
	474	while (put != get) {
b3beb167 BS	475	nv_wo32(cache, ptr + 0,
	476	nv_rd32(dev, NV40_PFIFO_CACHE1_METHOD(get)));
	477	nv_wo32(cache, ptr + 4,
	478	nv_rd32(dev, NV40_PFIFO_CACHE1_DATA(get)));
6ee73861	479	get = (get + 1) & 0x1ff;
b3beb167	480	ptr += 8;
6ee73861 BS	481	}
	482
	483	/* guessing that all the 0x34xx regs aren't on NV50 */
ac94a343	484	if (dev_priv->chipset != 0x50) {
b3beb167 BS	485	nv_wo32(ramfc, 0x84, ptr >> 3);
	486	nv_wo32(ramfc, 0x88, nv_rd32(dev, 0x340c));
	487	nv_wo32(ramfc, 0x8c, nv_rd32(dev, 0x3400));
	488	nv_wo32(ramfc, 0x90, nv_rd32(dev, 0x3404));
	489	nv_wo32(ramfc, 0x94, nv_rd32(dev, 0x3408));
	490	nv_wo32(ramfc, 0x98, nv_rd32(dev, 0x3410));
6ee73861 BS	491	}
6ee73861 BS	492
f56cb86f	493	dev_priv->engine.instmem.flush(dev);
6ee73861 BS	494
	495	/XXX: probably reload ch127 (NULL) state back too /
	496	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, 127);
	497	return 0;
	498	}
	499
56ac7475 BS	500	void
	501	nv50_fifo_tlb_flush(struct drm_device *dev)
	502	{
a11c3198	503	nv50_vm_flush_engine(dev, 5);
56ac7475	504	}