[deliverable/linux.git] / drivers / s390 / block / scm_blk.c

/*
 * Block driver for s390 storage class memory.
 *
 * Copyright IBM Corp. 2012
 * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
 */

#define KMSG_COMPONENT "scm_block"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/blkdev.h>
#include <linux/genhd.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <asm/eadm.h>
#include "scm_blk.h"

debug_info_t *scm_debug;
static int scm_major;
static mempool_t *aidaw_pool;
static DEFINE_SPINLOCK(list_lock);
static LIST_HEAD(inactive_requests);
static unsigned int nr_requests = 64;
static atomic_t nr_devices = ATOMIC_INIT(0);
module_param(nr_requests, uint, S_IRUGO);
MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");

MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
MODULE_LICENSE("GPL");
MODULE_ALIAS("scm:scmdev*");

static void __scm_free_rq(struct scm_request *scmrq)
{
	struct aob_rq_header *aobrq = to_aobrq(scmrq);

	free_page((unsigned long) scmrq->aob);
	__scm_free_rq_cluster(scmrq);
	kfree(aobrq);
}

static void scm_free_rqs(void)
{
	struct list_head *iter, *safe;
	struct scm_request *scmrq;

	spin_lock_irq(&list_lock);
	list_for_each_safe(iter, safe, &inactive_requests) {
		scmrq = list_entry(iter, struct scm_request, list);
		list_del(&scmrq->list);
		__scm_free_rq(scmrq);
	}
	spin_unlock_irq(&list_lock);

	mempool_destroy(aidaw_pool);
}

static int __scm_alloc_rq(void)
{
	struct aob_rq_header *aobrq;
	struct scm_request *scmrq;

	aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
	if (!aobrq)
		return -ENOMEM;

	scmrq = (void *) aobrq->data;
	scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
	if (!scmrq->aob) {
		__scm_free_rq(scmrq);
		return -ENOMEM;
	}

	if (__scm_alloc_rq_cluster(scmrq)) {
		__scm_free_rq(scmrq);
		return -ENOMEM;
	}

	INIT_LIST_HEAD(&scmrq->list);
	spin_lock_irq(&list_lock);
	list_add(&scmrq->list, &inactive_requests);
	spin_unlock_irq(&list_lock);

	return 0;
}

static int scm_alloc_rqs(unsigned int nrqs)
{
	int ret = 0;

	aidaw_pool = mempool_create_page_pool(max(nrqs/8, 1U), 0);
	if (!aidaw_pool)
		return -ENOMEM;

	while (nrqs-- && !ret)
		ret = __scm_alloc_rq();

	return ret;
}

static struct scm_request *scm_request_fetch(void)
{
	struct scm_request *scmrq = NULL;

	spin_lock(&list_lock);
	if (list_empty(&inactive_requests))
		goto out;
	scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
	list_del(&scmrq->list);
out:
	spin_unlock(&list_lock);
	return scmrq;
}

static void scm_request_done(struct scm_request *scmrq)
{
	unsigned long flags;
	struct msb *msb;
	u64 aidaw;
	int i;

	for (i = 0; i < SCM_RQ_PER_IO && scmrq->request[i]; i++) {
		msb = &scmrq->aob->msb[i];
		aidaw = msb->data_addr;

		if ((msb->flags & MSB_FLAG_IDA) && aidaw &&
		    IS_ALIGNED(aidaw, PAGE_SIZE))
			mempool_free(virt_to_page(aidaw), aidaw_pool);
	}

	spin_lock_irqsave(&list_lock, flags);
	list_add(&scmrq->list, &inactive_requests);
	spin_unlock_irqrestore(&list_lock, flags);
}

static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
{
	return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
}

static inline struct aidaw *scm_aidaw_alloc(void)
{
	struct page *page = mempool_alloc(aidaw_pool, GFP_ATOMIC);

	return page ? page_address(page) : NULL;
}

static inline unsigned long scm_aidaw_bytes(struct aidaw *aidaw)
{
	unsigned long _aidaw = (unsigned long) aidaw;
	unsigned long bytes = ALIGN(_aidaw, PAGE_SIZE) - _aidaw;

	return (bytes / sizeof(*aidaw)) * PAGE_SIZE;
}

struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes)
{
	struct aidaw *aidaw;

	if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)
		return scmrq->next_aidaw;

	aidaw = scm_aidaw_alloc();
	if (aidaw)
		memset(aidaw, 0, PAGE_SIZE);
	return aidaw;
}

static int scm_request_prepare(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	struct scm_device *scmdev = bdev->gendisk->private_data;
	int pos = scmrq->aob->request.msb_count;
	struct msb *msb = &scmrq->aob->msb[pos];
	struct request *req = scmrq->request[pos];
	struct req_iterator iter;
	struct aidaw *aidaw;
	struct bio_vec bv;

	aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));
	if (!aidaw)
		return -ENOMEM;

	msb->bs = MSB_BS_4K;
	scmrq->aob->request.msb_count++;
	msb->scm_addr = scmdev->address + ((u64) blk_rq_pos(req) << 9);
	msb->oc = (rq_data_dir(req) == READ) ? MSB_OC_READ : MSB_OC_WRITE;
	msb->flags |= MSB_FLAG_IDA;
	msb->data_addr = (u64) aidaw;

	rq_for_each_segment(bv, req, iter) {
		WARN_ON(bv.bv_offset);
		msb->blk_count += bv.bv_len >> 12;
		aidaw->data_addr = (u64) page_address(bv.bv_page);
		aidaw++;
	}

	scmrq->next_aidaw = aidaw;
	return 0;
}

static inline void scm_request_set(struct scm_request *scmrq,
				   struct request *req)
{
	scmrq->request[scmrq->aob->request.msb_count] = req;
}

static inline void scm_request_init(struct scm_blk_dev *bdev,
				    struct scm_request *scmrq)
{
	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	struct aob *aob = scmrq->aob;

	memset(scmrq->request, 0, sizeof(scmrq->request));
	memset(aob, 0, sizeof(*aob));
	aobrq->scmdev = bdev->scmdev;
	aob->request.cmd_code = ARQB_CMD_MOVE;
	aob->request.data = (u64) aobrq;
	scmrq->bdev = bdev;
	scmrq->retries = 4;
	scmrq->error = 0;
	/* We don't use all msbs - place aidaws at the end of the aob page. */
	scmrq->next_aidaw = (void *) &aob->msb[SCM_RQ_PER_IO];
	scm_request_cluster_init(scmrq);
}

static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
{
	if (atomic_read(&bdev->queued_reqs)) {
		/* Queue restart is triggered by the next interrupt. */
		return;
	}
	blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
}

void scm_request_requeue(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	int i;

	scm_release_cluster(scmrq);
	for (i = 0; i < SCM_RQ_PER_IO && scmrq->request[i]; i++)
		blk_requeue_request(bdev->rq, scmrq->request[i]);

	atomic_dec(&bdev->queued_reqs);
	scm_request_done(scmrq);
	scm_ensure_queue_restart(bdev);
}

void scm_request_finish(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	int i;

	scm_release_cluster(scmrq);
	for (i = 0; i < SCM_RQ_PER_IO && scmrq->request[i]; i++)
		blk_end_request_all(scmrq->request[i], scmrq->error);

	atomic_dec(&bdev->queued_reqs);
	scm_request_done(scmrq);
}

static int scm_request_start(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	int ret;

	atomic_inc(&bdev->queued_reqs);
	if (!scmrq->aob->request.msb_count) {
		scm_request_requeue(scmrq);
		return -EINVAL;
	}

	ret = eadm_start_aob(scmrq->aob);
	if (ret) {
		SCM_LOG(5, "no subchannel");
		scm_request_requeue(scmrq);
	}
	return ret;
}

static void scm_blk_request(struct request_queue *rq)
{
	struct scm_device *scmdev = rq->queuedata;
	struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
	struct scm_request *scmrq = NULL;
	struct request *req;

	while ((req = blk_peek_request(rq))) {
		if (req->cmd_type != REQ_TYPE_FS) {
			blk_start_request(req);
			blk_dump_rq_flags(req, KMSG_COMPONENT " bad request");
			blk_end_request_all(req, -EIO);
			continue;
		}

		if (!scm_permit_request(bdev, req))
			goto out;

		if (!scmrq) {
			scmrq = scm_request_fetch();
			if (!scmrq) {
				SCM_LOG(5, "no request");
				goto out;
			}
			scm_request_init(bdev, scmrq);
		}
		scm_request_set(scmrq, req);

		if (!scm_reserve_cluster(scmrq)) {
			SCM_LOG(5, "cluster busy");
			scm_request_set(scmrq, NULL);
			if (scmrq->aob->request.msb_count)
				goto out;

			scm_request_done(scmrq);
			return;
		}

		if (scm_need_cluster_request(scmrq)) {
			if (scmrq->aob->request.msb_count) {
				/* Start cluster requests separately. */
				scm_request_set(scmrq, NULL);
				if (scm_request_start(scmrq))
					return;
			} else {
				atomic_inc(&bdev->queued_reqs);
				blk_start_request(req);
				scm_initiate_cluster_request(scmrq);
			}
			scmrq = NULL;
			continue;
		}

		if (scm_request_prepare(scmrq)) {
			SCM_LOG(5, "aidaw alloc failed");
			scm_request_set(scmrq, NULL);
			goto out;
		}
		blk_start_request(req);

		if (scmrq->aob->request.msb_count < SCM_RQ_PER_IO)
			continue;

		if (scm_request_start(scmrq))
			return;

		scmrq = NULL;
	}
out:
	if (scmrq)
		scm_request_start(scmrq);
	else
		scm_ensure_queue_restart(bdev);
}

static void __scmrq_log_error(struct scm_request *scmrq)
{
	struct aob *aob = scmrq->aob;

	if (scmrq->error == -ETIMEDOUT)
		SCM_LOG(1, "Request timeout");
	else {
		SCM_LOG(1, "Request error");
		SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
	}
	if (scmrq->retries)
		SCM_LOG(1, "Retry request");
	else
		pr_err("An I/O operation to SCM failed with rc=%d\n",
		       scmrq->error);
}

void scm_blk_irq(struct scm_device *scmdev, void *data, int error)
{
	struct scm_request *scmrq = data;
	struct scm_blk_dev *bdev = scmrq->bdev;

	scmrq->error = error;
	if (error)
		__scmrq_log_error(scmrq);

	spin_lock(&bdev->lock);
	list_add_tail(&scmrq->list, &bdev->finished_requests);
	spin_unlock(&bdev->lock);
	tasklet_hi_schedule(&bdev->tasklet);
}

static void scm_blk_handle_error(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	unsigned long flags;

	if (scmrq->error != -EIO)
		goto restart;

	/* For -EIO the response block is valid. */
	switch (scmrq->aob->response.eqc) {
	case EQC_WR_PROHIBIT:
		spin_lock_irqsave(&bdev->lock, flags);
		if (bdev->state != SCM_WR_PROHIBIT)
			pr_info("%lx: Write access to the SCM increment is suspended\n",
				(unsigned long) bdev->scmdev->address);
		bdev->state = SCM_WR_PROHIBIT;
		spin_unlock_irqrestore(&bdev->lock, flags);
		goto requeue;
	default:
		break;
	}

restart:
	if (!eadm_start_aob(scmrq->aob))
		return;

requeue:
	spin_lock_irqsave(&bdev->rq_lock, flags);
	scm_request_requeue(scmrq);
	spin_unlock_irqrestore(&bdev->rq_lock, flags);
}

static void scm_blk_tasklet(struct scm_blk_dev *bdev)
{
	struct scm_request *scmrq;
	unsigned long flags;

	spin_lock_irqsave(&bdev->lock, flags);
	while (!list_empty(&bdev->finished_requests)) {
		scmrq = list_first_entry(&bdev->finished_requests,
					 struct scm_request, list);
		list_del(&scmrq->list);
		spin_unlock_irqrestore(&bdev->lock, flags);

		if (scmrq->error && scmrq->retries-- > 0) {
			scm_blk_handle_error(scmrq);

			/* Request restarted or requeued, handle next. */
			spin_lock_irqsave(&bdev->lock, flags);
			continue;
		}

		if (scm_test_cluster_request(scmrq)) {
			scm_cluster_request_irq(scmrq);
			spin_lock_irqsave(&bdev->lock, flags);
			continue;
		}

		scm_request_finish(scmrq);
		spin_lock_irqsave(&bdev->lock, flags);
	}
	spin_unlock_irqrestore(&bdev->lock, flags);
	/* Look out for more requests. */
	blk_run_queue(bdev->rq);
}

static const struct block_device_operations scm_blk_devops = {
	.owner = THIS_MODULE,
};

int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
{
	struct request_queue *rq;
	int len, ret = -ENOMEM;
	unsigned int devindex, nr_max_blk;

	devindex = atomic_inc_return(&nr_devices) - 1;
	/* scma..scmz + scmaa..scmzz */
	if (devindex > 701) {
		ret = -ENODEV;
		goto out;
	}

	bdev->scmdev = scmdev;
	bdev->state = SCM_OPER;
	spin_lock_init(&bdev->rq_lock);
	spin_lock_init(&bdev->lock);
	INIT_LIST_HEAD(&bdev->finished_requests);
	atomic_set(&bdev->queued_reqs, 0);
	tasklet_init(&bdev->tasklet,
		     (void (*)(unsigned long)) scm_blk_tasklet,
		     (unsigned long) bdev);

	rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
	if (!rq)
		goto out;

	bdev->rq = rq;
	nr_max_blk = min(scmdev->nr_max_block,
			 (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));

	blk_queue_logical_block_size(rq, 1 << 12);
	blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
	blk_queue_max_segments(rq, nr_max_blk);
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
	queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, rq);
	scm_blk_dev_cluster_setup(bdev);

	bdev->gendisk = alloc_disk(SCM_NR_PARTS);
	if (!bdev->gendisk)
		goto out_queue;

	rq->queuedata = scmdev;
	bdev->gendisk->driverfs_dev = &scmdev->dev;
	bdev->gendisk->private_data = scmdev;
	bdev->gendisk->fops = &scm_blk_devops;
	bdev->gendisk->queue = rq;
	bdev->gendisk->major = scm_major;
	bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;

	len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
	if (devindex > 25) {
		len += snprintf(bdev->gendisk->disk_name + len,
				DISK_NAME_LEN - len, "%c",
				'a' + (devindex / 26) - 1);
		devindex = devindex % 26;
	}
	snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
		 'a' + devindex);

	/* 512 byte sectors */
	set_capacity(bdev->gendisk, scmdev->size >> 9);
	add_disk(bdev->gendisk);
	return 0;

out_queue:
	blk_cleanup_queue(rq);
out:
	atomic_dec(&nr_devices);
	return ret;
}

void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
{
	tasklet_kill(&bdev->tasklet);
	del_gendisk(bdev->gendisk);
	blk_cleanup_queue(bdev->gendisk->queue);
	put_disk(bdev->gendisk);
}

void scm_blk_set_available(struct scm_blk_dev *bdev)
{
	unsigned long flags;

	spin_lock_irqsave(&bdev->lock, flags);
	if (bdev->state == SCM_WR_PROHIBIT)
		pr_info("%lx: Write access to the SCM increment is restored\n",
			(unsigned long) bdev->scmdev->address);
	bdev->state = SCM_OPER;
	spin_unlock_irqrestore(&bdev->lock, flags);
}

static int __init scm_blk_init(void)
{
	int ret = -EINVAL;

	if (!scm_cluster_size_valid())
		goto out;

	ret = register_blkdev(0, "scm");
	if (ret < 0)
		goto out;

	scm_major = ret;
	ret = scm_alloc_rqs(nr_requests);
	if (ret)
		goto out_free;

	scm_debug = debug_register("scm_log", 16, 1, 16);
	if (!scm_debug) {
		ret = -ENOMEM;
		goto out_free;
	}

	debug_register_view(scm_debug, &debug_hex_ascii_view);
	debug_set_level(scm_debug, 2);

	ret = scm_drv_init();
	if (ret)
		goto out_dbf;

	return ret;

out_dbf:
	debug_unregister(scm_debug);
out_free:
	scm_free_rqs();
	unregister_blkdev(scm_major, "scm");
out:
	return ret;
}
module_init(scm_blk_init);

static void __exit scm_blk_cleanup(void)
{
	scm_drv_cleanup();
	debug_unregister(scm_debug);
	scm_free_rqs();
	unregister_blkdev(scm_major, "scm");
}
module_exit(scm_blk_cleanup);
Commit	Line	Data
f30664e2 SO	1	/*
	2	* Block driver for s390 storage class memory.
	3	*
	4	* Copyright IBM Corp. 2012
	5	* Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
	6	*/
	7
	8	#define KMSG_COMPONENT "scm_block"
	9	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	10
	11	#include <linux/interrupt.h>
	12	#include <linux/spinlock.h>
9d4df77f	13	#include <linux/mempool.h>
f30664e2 SO	14	#include <linux/module.h>
	15	#include <linux/blkdev.h>
	16	#include <linux/genhd.h>
	17	#include <linux/slab.h>
	18	#include <linux/list.h>
	19	#include <asm/eadm.h>
	20	#include "scm_blk.h"
	21
	22	debug_info_t *scm_debug;
	23	static int scm_major;
9d4df77f	24	static mempool_t *aidaw_pool;
f30664e2 SO	25	static DEFINE_SPINLOCK(list_lock);
	26	static LIST_HEAD(inactive_requests);
	27	static unsigned int nr_requests = 64;
	28	static atomic_t nr_devices = ATOMIC_INIT(0);
	29	module_param(nr_requests, uint, S_IRUGO);
	30	MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
	31
	32	MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
	33	MODULE_LICENSE("GPL");
	34	MODULE_ALIAS("scm:scmdev*");
	35
	36	static void __scm_free_rq(struct scm_request *scmrq)
	37	{
	38	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	39
	40	free_page((unsigned long) scmrq->aob);
0d804b20	41	__scm_free_rq_cluster(scmrq);
f30664e2 SO	42	kfree(aobrq);
	43	}
	44
	45	static void scm_free_rqs(void)
	46	{
	47	struct list_head iter, safe;
	48	struct scm_request *scmrq;
	49
	50	spin_lock_irq(&list_lock);
	51	list_for_each_safe(iter, safe, &inactive_requests) {
	52	scmrq = list_entry(iter, struct scm_request, list);
	53	list_del(&scmrq->list);
	54	__scm_free_rq(scmrq);
	55	}
	56	spin_unlock_irq(&list_lock);
9d4df77f SO	57
9d4df77f SO	58	mempool_destroy(aidaw_pool);
f30664e2 SO	59	}
	60
	61	static int __scm_alloc_rq(void)
	62	{
	63	struct aob_rq_header *aobrq;
	64	struct scm_request *scmrq;
	65
	66	aobrq = kzalloc(sizeof(aobrq) + sizeof(scmrq), GFP_KERNEL);
	67	if (!aobrq)
	68	return -ENOMEM;
	69
	70	scmrq = (void *) aobrq->data;
f30664e2	71	scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
9d4df77f	72	if (!scmrq->aob) {
f30664e2 SO	73	__scm_free_rq(scmrq);
	74	return -ENOMEM;
	75	}
0d804b20 SO	76
	77	if (__scm_alloc_rq_cluster(scmrq)) {
	78	__scm_free_rq(scmrq);
	79	return -ENOMEM;
	80	}
	81
f30664e2 SO	82	INIT_LIST_HEAD(&scmrq->list);
	83	spin_lock_irq(&list_lock);
	84	list_add(&scmrq->list, &inactive_requests);
	85	spin_unlock_irq(&list_lock);
	86
	87	return 0;
	88	}
	89
	90	static int scm_alloc_rqs(unsigned int nrqs)
	91	{
	92	int ret = 0;
	93
9d4df77f SO	94	aidaw_pool = mempool_create_page_pool(max(nrqs/8, 1U), 0);
	95	if (!aidaw_pool)
	96	return -ENOMEM;
	97
f30664e2 SO	98	while (nrqs-- && !ret)
	99	ret = __scm_alloc_rq();
	100
	101	return ret;
	102	}
	103
	104	static struct scm_request *scm_request_fetch(void)
	105	{
	106	struct scm_request *scmrq = NULL;
	107
	108	spin_lock(&list_lock);
	109	if (list_empty(&inactive_requests))
	110	goto out;
	111	scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
	112	list_del(&scmrq->list);
	113	out:
	114	spin_unlock(&list_lock);
	115	return scmrq;
	116	}
	117
	118	static void scm_request_done(struct scm_request *scmrq)
	119	{
	120	unsigned long flags;
bbc610a9 SO	121	struct msb *msb;
	122	u64 aidaw;
	123	int i;
f30664e2	124
bbc610a9 SO	125	for (i = 0; i < SCM_RQ_PER_IO && scmrq->request[i]; i++) {
	126	msb = &scmrq->aob->msb[i];
	127	aidaw = msb->data_addr;
	128
	129	if ((msb->flags & MSB_FLAG_IDA) && aidaw &&
	130	IS_ALIGNED(aidaw, PAGE_SIZE))
	131	mempool_free(virt_to_page(aidaw), aidaw_pool);
	132	}
9d4df77f	133
f30664e2 SO	134	spin_lock_irqsave(&list_lock, flags);
	135	list_add(&scmrq->list, &inactive_requests);
	136	spin_unlock_irqrestore(&list_lock, flags);
	137	}
	138
4fa3c019 SO	139	static bool scm_permit_request(struct scm_blk_dev bdev, struct request req)
	140	{
	141	return rq_data_dir(req) != WRITE \|\| bdev->state != SCM_WR_PROHIBIT;
	142	}
	143
de88d0d2	144	static inline struct aidaw *scm_aidaw_alloc(void)
9d4df77f SO	145	{
	146	struct page *page = mempool_alloc(aidaw_pool, GFP_ATOMIC);
	147
	148	return page ? page_address(page) : NULL;
	149	}
	150
de88d0d2 SO	151	static inline unsigned long scm_aidaw_bytes(struct aidaw *aidaw)
	152	{
	153	unsigned long _aidaw = (unsigned long) aidaw;
	154	unsigned long bytes = ALIGN(_aidaw, PAGE_SIZE) - _aidaw;
	155
	156	return (bytes / sizeof(aidaw)) PAGE_SIZE;
	157	}
	158
	159	struct aidaw scm_aidaw_fetch(struct scm_request scmrq, unsigned int bytes)
	160	{
	161	struct aidaw *aidaw;
	162
	163	if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)
	164	return scmrq->next_aidaw;
	165
	166	aidaw = scm_aidaw_alloc();
	167	if (aidaw)
	168	memset(aidaw, 0, PAGE_SIZE);
	169	return aidaw;
	170	}
	171
9d4df77f	172	static int scm_request_prepare(struct scm_request *scmrq)
f30664e2 SO	173	{
	174	struct scm_blk_dev *bdev = scmrq->bdev;
	175	struct scm_device *scmdev = bdev->gendisk->private_data;
bbc610a9 SO	176	int pos = scmrq->aob->request.msb_count;
	177	struct msb *msb = &scmrq->aob->msb[pos];
	178	struct request *req = scmrq->request[pos];
f30664e2	179	struct req_iterator iter;
de88d0d2	180	struct aidaw *aidaw;
7988613b	181	struct bio_vec bv;
f30664e2	182
bbc610a9	183	aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));
9d4df77f SO	184	if (!aidaw)
	185	return -ENOMEM;
	186
f30664e2	187	msb->bs = MSB_BS_4K;
bbc610a9 SO	188	scmrq->aob->request.msb_count++;
	189	msb->scm_addr = scmdev->address + ((u64) blk_rq_pos(req) << 9);
	190	msb->oc = (rq_data_dir(req) == READ) ? MSB_OC_READ : MSB_OC_WRITE;
f30664e2 SO	191	msb->flags \|= MSB_FLAG_IDA;
	192	msb->data_addr = (u64) aidaw;
	193
bbc610a9	194	rq_for_each_segment(bv, req, iter) {
7988613b KO	195	WARN_ON(bv.bv_offset);
	196	msb->blk_count += bv.bv_len >> 12;
	197	aidaw->data_addr = (u64) page_address(bv.bv_page);
f30664e2 SO	198	aidaw++;
f30664e2 SO	199	}
9d4df77f	200
bbc610a9	201	scmrq->next_aidaw = aidaw;
9d4df77f	202	return 0;
f30664e2 SO	203	}
f30664e2 SO	204
bbc610a9 SO	205	static inline void scm_request_set(struct scm_request *scmrq,
	206	struct request *req)
	207	{
	208	scmrq->request[scmrq->aob->request.msb_count] = req;
	209	}
	210
f30664e2	211	static inline void scm_request_init(struct scm_blk_dev *bdev,
bbc610a9	212	struct scm_request *scmrq)
f30664e2 SO	213	{
	214	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	215	struct aob *aob = scmrq->aob;
	216
bbc610a9	217	memset(scmrq->request, 0, sizeof(scmrq->request));
f30664e2	218	memset(aob, 0, sizeof(*aob));
f30664e2 SO	219	aobrq->scmdev = bdev->scmdev;
	220	aob->request.cmd_code = ARQB_CMD_MOVE;
	221	aob->request.data = (u64) aobrq;
f30664e2 SO	222	scmrq->bdev = bdev;
	223	scmrq->retries = 4;
	224	scmrq->error = 0;
de88d0d2	225	/* We don't use all msbs - place aidaws at the end of the aob page. */
bbc610a9	226	scmrq->next_aidaw = (void *) &aob->msb[SCM_RQ_PER_IO];
0d804b20	227	scm_request_cluster_init(scmrq);
f30664e2 SO	228	}
	229
	230	static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
	231	{
	232	if (atomic_read(&bdev->queued_reqs)) {
	233	/* Queue restart is triggered by the next interrupt. */
	234	return;
	235	}
	236	blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
	237	}
	238
0d804b20	239	void scm_request_requeue(struct scm_request *scmrq)
f30664e2 SO	240	{
f30664e2 SO	241	struct scm_blk_dev *bdev = scmrq->bdev;
bbc610a9	242	int i;
f30664e2	243
0d804b20	244	scm_release_cluster(scmrq);
bbc610a9 SO	245	for (i = 0; i < SCM_RQ_PER_IO && scmrq->request[i]; i++)
	246	blk_requeue_request(bdev->rq, scmrq->request[i]);
	247
8360cb5f	248	atomic_dec(&bdev->queued_reqs);
f30664e2 SO	249	scm_request_done(scmrq);
	250	scm_ensure_queue_restart(bdev);
	251	}
	252
0d804b20	253	void scm_request_finish(struct scm_request *scmrq)
f30664e2	254	{
8360cb5f	255	struct scm_blk_dev *bdev = scmrq->bdev;
bbc610a9	256	int i;
8360cb5f	257
0d804b20	258	scm_release_cluster(scmrq);
bbc610a9 SO	259	for (i = 0; i < SCM_RQ_PER_IO && scmrq->request[i]; i++)
	260	blk_end_request_all(scmrq->request[i], scmrq->error);
	261
8360cb5f	262	atomic_dec(&bdev->queued_reqs);
f30664e2 SO	263	scm_request_done(scmrq);
	264	}
	265
bbc610a9 SO	266	static int scm_request_start(struct scm_request *scmrq)
	267	{
	268	struct scm_blk_dev *bdev = scmrq->bdev;
	269	int ret;
	270
	271	atomic_inc(&bdev->queued_reqs);
	272	if (!scmrq->aob->request.msb_count) {
	273	scm_request_requeue(scmrq);
	274	return -EINVAL;
	275	}
	276
	277	ret = eadm_start_aob(scmrq->aob);
	278	if (ret) {
	279	SCM_LOG(5, "no subchannel");
	280	scm_request_requeue(scmrq);
	281	}
	282	return ret;
	283	}
	284
f30664e2 SO	285	static void scm_blk_request(struct request_queue *rq)
	286	{
	287	struct scm_device *scmdev = rq->queuedata;
	288	struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
bbc610a9	289	struct scm_request *scmrq = NULL;
f30664e2	290	struct request *req;
f30664e2 SO	291
f30664e2 SO	292	while ((req = blk_peek_request(rq))) {
de9587a2 SM	293	if (req->cmd_type != REQ_TYPE_FS) {
	294	blk_start_request(req);
	295	blk_dump_rq_flags(req, KMSG_COMPONENT " bad request");
	296	blk_end_request_all(req, -EIO);
f30664e2	297	continue;
de9587a2	298	}
f30664e2	299
bbc610a9 SO	300	if (!scm_permit_request(bdev, req))
	301	goto out;
	302
f30664e2	303	if (!scmrq) {
bbc610a9 SO	304	scmrq = scm_request_fetch();
	305	if (!scmrq) {
	306	SCM_LOG(5, "no request");
	307	goto out;
	308	}
	309	scm_request_init(bdev, scmrq);
f30664e2	310	}
bbc610a9 SO	311	scm_request_set(scmrq, req);
bbc610a9 SO	312
0d804b20 SO	313	if (!scm_reserve_cluster(scmrq)) {
0d804b20 SO	314	SCM_LOG(5, "cluster busy");
bbc610a9 SO	315	scm_request_set(scmrq, NULL);
	316	if (scmrq->aob->request.msb_count)
	317	goto out;
	318
0d804b20 SO	319	scm_request_done(scmrq);
	320	return;
	321	}
bbc610a9	322
0d804b20	323	if (scm_need_cluster_request(scmrq)) {
bbc610a9 SO	324	if (scmrq->aob->request.msb_count) {
	325	/* Start cluster requests separately. */
	326	scm_request_set(scmrq, NULL);
	327	if (scm_request_start(scmrq))
	328	return;
	329	} else {
	330	atomic_inc(&bdev->queued_reqs);
	331	blk_start_request(req);
	332	scm_initiate_cluster_request(scmrq);
	333	}
	334	scmrq = NULL;
	335	continue;
0d804b20	336	}
9d4df77f SO	337
9d4df77f SO	338	if (scm_request_prepare(scmrq)) {
bbc610a9 SO	339	SCM_LOG(5, "aidaw alloc failed");
	340	scm_request_set(scmrq, NULL);
	341	goto out;
9d4df77f	342	}
f30664e2 SO	343	blk_start_request(req);
f30664e2 SO	344
bbc610a9 SO	345	if (scmrq->aob->request.msb_count < SCM_RQ_PER_IO)
	346	continue;
	347
	348	if (scm_request_start(scmrq))
f30664e2	349	return;
bbc610a9 SO	350
bbc610a9 SO	351	scmrq = NULL;
f30664e2	352	}
bbc610a9 SO	353	out:
	354	if (scmrq)
	355	scm_request_start(scmrq);
	356	else
	357	scm_ensure_queue_restart(bdev);
f30664e2 SO	358	}
	359
	360	static void __scmrq_log_error(struct scm_request *scmrq)
	361	{
	362	struct aob *aob = scmrq->aob;
	363
	364	if (scmrq->error == -ETIMEDOUT)
	365	SCM_LOG(1, "Request timeout");
	366	else {
	367	SCM_LOG(1, "Request error");
	368	SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
	369	}
	370	if (scmrq->retries)
	371	SCM_LOG(1, "Retry request");
	372	else
	373	pr_err("An I/O operation to SCM failed with rc=%d\n",
	374	scmrq->error);
	375	}
	376
	377	void scm_blk_irq(struct scm_device scmdev, void data, int error)
	378	{
	379	struct scm_request *scmrq = data;
	380	struct scm_blk_dev *bdev = scmrq->bdev;
	381
	382	scmrq->error = error;
	383	if (error)
	384	__scmrq_log_error(scmrq);
	385
	386	spin_lock(&bdev->lock);
	387	list_add_tail(&scmrq->list, &bdev->finished_requests);
	388	spin_unlock(&bdev->lock);
	389	tasklet_hi_schedule(&bdev->tasklet);
	390	}
	391
4fa3c019 SO	392	static void scm_blk_handle_error(struct scm_request *scmrq)
	393	{
	394	struct scm_blk_dev *bdev = scmrq->bdev;
	395	unsigned long flags;
	396
	397	if (scmrq->error != -EIO)
	398	goto restart;
	399
	400	/* For -EIO the response block is valid. */
	401	switch (scmrq->aob->response.eqc) {
	402	case EQC_WR_PROHIBIT:
	403	spin_lock_irqsave(&bdev->lock, flags);
	404	if (bdev->state != SCM_WR_PROHIBIT)
3bff6038	405	pr_info("%lx: Write access to the SCM increment is suspended\n",
4fa3c019 SO	406	(unsigned long) bdev->scmdev->address);
	407	bdev->state = SCM_WR_PROHIBIT;
	408	spin_unlock_irqrestore(&bdev->lock, flags);
	409	goto requeue;
	410	default:
	411	break;
	412	}
	413
	414	restart:
605c3698	415	if (!eadm_start_aob(scmrq->aob))
4fa3c019 SO	416	return;
	417
	418	requeue:
	419	spin_lock_irqsave(&bdev->rq_lock, flags);
	420	scm_request_requeue(scmrq);
	421	spin_unlock_irqrestore(&bdev->rq_lock, flags);
	422	}
	423
f30664e2 SO	424	static void scm_blk_tasklet(struct scm_blk_dev *bdev)
	425	{
	426	struct scm_request *scmrq;
	427	unsigned long flags;
	428
	429	spin_lock_irqsave(&bdev->lock, flags);
	430	while (!list_empty(&bdev->finished_requests)) {
	431	scmrq = list_first_entry(&bdev->finished_requests,
	432	struct scm_request, list);
	433	list_del(&scmrq->list);
	434	spin_unlock_irqrestore(&bdev->lock, flags);
	435
	436	if (scmrq->error && scmrq->retries-- > 0) {
4fa3c019 SO	437	scm_blk_handle_error(scmrq);
4fa3c019 SO	438
f30664e2 SO	439	/* Request restarted or requeued, handle next. */
	440	spin_lock_irqsave(&bdev->lock, flags);
	441	continue;
	442	}
0d804b20 SO	443
	444	if (scm_test_cluster_request(scmrq)) {
	445	scm_cluster_request_irq(scmrq);
	446	spin_lock_irqsave(&bdev->lock, flags);
	447	continue;
	448	}
	449
f30664e2	450	scm_request_finish(scmrq);
f30664e2 SO	451	spin_lock_irqsave(&bdev->lock, flags);
	452	}
	453	spin_unlock_irqrestore(&bdev->lock, flags);
	454	/* Look out for more requests. */
	455	blk_run_queue(bdev->rq);
	456	}
	457
605c3698 SO	458	static const struct block_device_operations scm_blk_devops = {
	459	.owner = THIS_MODULE,
	460	};
	461
f30664e2 SO	462	int scm_blk_dev_setup(struct scm_blk_dev bdev, struct scm_device scmdev)
	463	{
	464	struct request_queue *rq;
	465	int len, ret = -ENOMEM;
	466	unsigned int devindex, nr_max_blk;
	467
	468	devindex = atomic_inc_return(&nr_devices) - 1;
	469	/* scma..scmz + scmaa..scmzz */
	470	if (devindex > 701) {
	471	ret = -ENODEV;
	472	goto out;
	473	}
	474
	475	bdev->scmdev = scmdev;
4fa3c019	476	bdev->state = SCM_OPER;
f30664e2 SO	477	spin_lock_init(&bdev->rq_lock);
	478	spin_lock_init(&bdev->lock);
	479	INIT_LIST_HEAD(&bdev->finished_requests);
	480	atomic_set(&bdev->queued_reqs, 0);
	481	tasklet_init(&bdev->tasklet,
	482	(void (*)(unsigned long)) scm_blk_tasklet,
	483	(unsigned long) bdev);
	484
	485	rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
	486	if (!rq)
	487	goto out;
	488
	489	bdev->rq = rq;
	490	nr_max_blk = min(scmdev->nr_max_block,
	491	(unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
	492
	493	blk_queue_logical_block_size(rq, 1 << 12);
	494	blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
	495	blk_queue_max_segments(rq, nr_max_blk);
	496	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
b277da0a	497	queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, rq);
0d804b20	498	scm_blk_dev_cluster_setup(bdev);
f30664e2 SO	499
	500	bdev->gendisk = alloc_disk(SCM_NR_PARTS);
	501	if (!bdev->gendisk)
	502	goto out_queue;
	503
	504	rq->queuedata = scmdev;
	505	bdev->gendisk->driverfs_dev = &scmdev->dev;
	506	bdev->gendisk->private_data = scmdev;
	507	bdev->gendisk->fops = &scm_blk_devops;
	508	bdev->gendisk->queue = rq;
	509	bdev->gendisk->major = scm_major;
	510	bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
	511
	512	len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
	513	if (devindex > 25) {
	514	len += snprintf(bdev->gendisk->disk_name + len,
	515	DISK_NAME_LEN - len, "%c",
	516	'a' + (devindex / 26) - 1);
	517	devindex = devindex % 26;
	518	}
	519	snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
	520	'a' + devindex);
	521
	522	/* 512 byte sectors */
	523	set_capacity(bdev->gendisk, scmdev->size >> 9);
	524	add_disk(bdev->gendisk);
	525	return 0;
	526
	527	out_queue:
	528	blk_cleanup_queue(rq);
	529	out:
	530	atomic_dec(&nr_devices);
	531	return ret;
	532	}
	533
	534	void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
	535	{
	536	tasklet_kill(&bdev->tasklet);
	537	del_gendisk(bdev->gendisk);
	538	blk_cleanup_queue(bdev->gendisk->queue);
	539	put_disk(bdev->gendisk);
	540	}
	541
4fa3c019 SO	542	void scm_blk_set_available(struct scm_blk_dev *bdev)
	543	{
	544	unsigned long flags;
	545
	546	spin_lock_irqsave(&bdev->lock, flags);
	547	if (bdev->state == SCM_WR_PROHIBIT)
3bff6038	548	pr_info("%lx: Write access to the SCM increment is restored\n",
4fa3c019 SO	549	(unsigned long) bdev->scmdev->address);
	550	bdev->state = SCM_OPER;
	551	spin_unlock_irqrestore(&bdev->lock, flags);
	552	}
	553
f30664e2 SO	554	static int __init scm_blk_init(void)
f30664e2 SO	555	{
0d804b20 SO	556	int ret = -EINVAL;
	557
	558	if (!scm_cluster_size_valid())
	559	goto out;
f30664e2 SO	560
	561	ret = register_blkdev(0, "scm");
	562	if (ret < 0)
	563	goto out;
	564
	565	scm_major = ret;
94f9852d WY	566	ret = scm_alloc_rqs(nr_requests);
94f9852d WY	567	if (ret)
fff60fab	568	goto out_free;
f30664e2 SO	569
f30664e2 SO	570	scm_debug = debug_register("scm_log", 16, 1, 16);
94f9852d WY	571	if (!scm_debug) {
94f9852d WY	572	ret = -ENOMEM;
f30664e2	573	goto out_free;
94f9852d	574	}
f30664e2 SO	575
	576	debug_register_view(scm_debug, &debug_hex_ascii_view);
	577	debug_set_level(scm_debug, 2);
	578
	579	ret = scm_drv_init();
	580	if (ret)
	581	goto out_dbf;
	582
	583	return ret;
	584
	585	out_dbf:
	586	debug_unregister(scm_debug);
	587	out_free:
	588	scm_free_rqs();
f30664e2 SO	589	unregister_blkdev(scm_major, "scm");
	590	out:
	591	return ret;
	592	}
	593	module_init(scm_blk_init);
	594
	595	static void __exit scm_blk_cleanup(void)
	596	{
	597	scm_drv_cleanup();
	598	debug_unregister(scm_debug);
	599	scm_free_rqs();
	600	unregister_blkdev(scm_major, "scm");
	601	}
	602	module_exit(scm_blk_cleanup);