[deliverable/linux.git] / drivers / staging / lustre / lustre / ptlrpc / sec_bulk.c

/*
 * GPL HEADER START
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 only,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License version 2 for more details (a copy is included
 * in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; If not, see
 * http://www.gnu.org/licenses/gpl-2.0.html
 *
 * GPL HEADER END
 */
/*
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright (c) 2011, 2015, Intel Corporation.
 */
/*
 * This file is part of Lustre, http://www.lustre.org/
 * Lustre is a trademark of Sun Microsystems, Inc.
 *
 * lustre/ptlrpc/sec_bulk.c
 *
 * Author: Eric Mei <ericm@clusterfs.com>
 */

#define DEBUG_SUBSYSTEM S_SEC

#include "../../include/linux/libcfs/libcfs.h"

#include "../include/obd.h"
#include "../include/obd_cksum.h"
#include "../include/obd_class.h"
#include "../include/obd_support.h"
#include "../include/lustre_net.h"
#include "../include/lustre_import.h"
#include "../include/lustre_dlm.h"
#include "../include/lustre_sec.h"

#include "ptlrpc_internal.h"

/****************************************
 * bulk encryption page pools	   *
 ****************************************/

#define POINTERS_PER_PAGE	(PAGE_SIZE / sizeof(void *))
#define PAGES_PER_POOL		(POINTERS_PER_PAGE)

#define IDLE_IDX_MAX	 (100)
#define IDLE_IDX_WEIGHT	 (3)

#define CACHE_QUIESCENT_PERIOD  (20)

static struct ptlrpc_enc_page_pool {
	/*
	 * constants
	 */
	unsigned long    epp_max_pages;   /* maximum pages can hold, const */
	unsigned int     epp_max_pools;   /* number of pools, const */

	/*
	 * wait queue in case of not enough free pages.
	 */
	wait_queue_head_t      epp_waitq;       /* waiting threads */
	unsigned int     epp_waitqlen;    /* wait queue length */
	unsigned long    epp_pages_short; /* # of pages wanted of in-q users */
	unsigned int     epp_growing:1;   /* during adding pages */

	/*
	 * indicating how idle the pools are, from 0 to MAX_IDLE_IDX
	 * this is counted based on each time when getting pages from
	 * the pools, not based on time. which means in case that system
	 * is idled for a while but the idle_idx might still be low if no
	 * activities happened in the pools.
	 */
	unsigned long    epp_idle_idx;

	/* last shrink time due to mem tight */
	time64_t         epp_last_shrink;
	time64_t         epp_last_access;

	/*
	 * in-pool pages bookkeeping
	 */
	spinlock_t	 epp_lock;	   /* protect following fields */
	unsigned long    epp_total_pages; /* total pages in pools */
	unsigned long    epp_free_pages;  /* current pages available */

	/*
	 * statistics
	 */
	unsigned long    epp_st_max_pages;      /* # of pages ever reached */
	unsigned int     epp_st_grows;	  /* # of grows */
	unsigned int     epp_st_grow_fails;     /* # of add pages failures */
	unsigned int     epp_st_shrinks;	/* # of shrinks */
	unsigned long    epp_st_access;	 /* # of access */
	unsigned long    epp_st_missings;       /* # of cache missing */
	unsigned long    epp_st_lowfree;	/* lowest free pages reached */
	unsigned int     epp_st_max_wqlen;      /* highest waitqueue length */
	unsigned long       epp_st_max_wait;       /* in jiffies */
	/*
	 * pointers to pools
	 */
	struct page    ***epp_pools;
} page_pools;

/*
 * /sys/kernel/debug/lustre/sptlrpc/encrypt_page_pools
 */
int sptlrpc_proc_enc_pool_seq_show(struct seq_file *m, void *v)
{
	spin_lock(&page_pools.epp_lock);

	seq_printf(m,
		   "physical pages:	  %lu\n"
		   "pages per pool:	  %lu\n"
		   "max pages:	       %lu\n"
		   "max pools:	       %u\n"
		   "total pages:	     %lu\n"
		   "total free:	      %lu\n"
		   "idle index:	      %lu/100\n"
		   "last shrink:	     %lds\n"
		   "last access:	     %lds\n"
		   "max pages reached:       %lu\n"
		   "grows:		   %u\n"
		   "grows failure:	   %u\n"
		   "shrinks:		 %u\n"
		   "cache access:	    %lu\n"
		   "cache missing:	   %lu\n"
		   "low free mark:	   %lu\n"
		   "max waitqueue depth:     %u\n"
		   "max wait time:	   %ld/%u\n",
		   totalram_pages,
		   PAGES_PER_POOL,
		   page_pools.epp_max_pages,
		   page_pools.epp_max_pools,
		   page_pools.epp_total_pages,
		   page_pools.epp_free_pages,
		   page_pools.epp_idle_idx,
		   (long)(ktime_get_seconds() - page_pools.epp_last_shrink),
		   (long)(ktime_get_seconds() - page_pools.epp_last_access),
		   page_pools.epp_st_max_pages,
		   page_pools.epp_st_grows,
		   page_pools.epp_st_grow_fails,
		   page_pools.epp_st_shrinks,
		   page_pools.epp_st_access,
		   page_pools.epp_st_missings,
		   page_pools.epp_st_lowfree,
		   page_pools.epp_st_max_wqlen,
		   page_pools.epp_st_max_wait,
		   HZ);

	spin_unlock(&page_pools.epp_lock);

	return 0;
}

static void enc_pools_release_free_pages(long npages)
{
	int p_idx, g_idx;
	int p_idx_max1, p_idx_max2;

	LASSERT(npages > 0);
	LASSERT(npages <= page_pools.epp_free_pages);
	LASSERT(page_pools.epp_free_pages <= page_pools.epp_total_pages);

	/* max pool index before the release */
	p_idx_max2 = (page_pools.epp_total_pages - 1) / PAGES_PER_POOL;

	page_pools.epp_free_pages -= npages;
	page_pools.epp_total_pages -= npages;

	/* max pool index after the release */
	p_idx_max1 = page_pools.epp_total_pages == 0 ? -1 :
		     ((page_pools.epp_total_pages - 1) / PAGES_PER_POOL);

	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	LASSERT(page_pools.epp_pools[p_idx]);

	while (npages--) {
		LASSERT(page_pools.epp_pools[p_idx]);
		LASSERT(page_pools.epp_pools[p_idx][g_idx]);

		__free_page(page_pools.epp_pools[p_idx][g_idx]);
		page_pools.epp_pools[p_idx][g_idx] = NULL;

		if (++g_idx == PAGES_PER_POOL) {
			p_idx++;
			g_idx = 0;
		}
	}

	/* free unused pools */
	while (p_idx_max1 < p_idx_max2) {
		LASSERT(page_pools.epp_pools[p_idx_max2]);
		kfree(page_pools.epp_pools[p_idx_max2]);
		page_pools.epp_pools[p_idx_max2] = NULL;
		p_idx_max2--;
	}
}

/*
 * we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
 */
static unsigned long enc_pools_shrink_count(struct shrinker *s,
					    struct shrink_control *sc)
{
	/*
	 * if no pool access for a long time, we consider it's fully idle.
	 * a little race here is fine.
	 */
	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
		     CACHE_QUIESCENT_PERIOD)) {
		spin_lock(&page_pools.epp_lock);
		page_pools.epp_idle_idx = IDLE_IDX_MAX;
		spin_unlock(&page_pools.epp_lock);
	}

	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	return max((int)page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES, 0) *
		(IDLE_IDX_MAX - page_pools.epp_idle_idx) / IDLE_IDX_MAX;
}

/*
 * we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
 */
static unsigned long enc_pools_shrink_scan(struct shrinker *s,
					   struct shrink_control *sc)
{
	spin_lock(&page_pools.epp_lock);
	sc->nr_to_scan = min_t(unsigned long, sc->nr_to_scan,
			      page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES);
	if (sc->nr_to_scan > 0) {
		enc_pools_release_free_pages(sc->nr_to_scan);
		CDEBUG(D_SEC, "released %ld pages, %ld left\n",
		       (long)sc->nr_to_scan, page_pools.epp_free_pages);

		page_pools.epp_st_shrinks++;
		page_pools.epp_last_shrink = ktime_get_seconds();
	}
	spin_unlock(&page_pools.epp_lock);

	/*
	 * if no pool access for a long time, we consider it's fully idle.
	 * a little race here is fine.
	 */
	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
		     CACHE_QUIESCENT_PERIOD)) {
		spin_lock(&page_pools.epp_lock);
		page_pools.epp_idle_idx = IDLE_IDX_MAX;
		spin_unlock(&page_pools.epp_lock);
	}

	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	return sc->nr_to_scan;
}

static inline
int npages_to_npools(unsigned long npages)
{
	return (int)((npages + PAGES_PER_POOL - 1) / PAGES_PER_POOL);
}

/*
 * return how many pages cleaned up.
 */
static unsigned long enc_pools_cleanup(struct page ***pools, int npools)
{
	unsigned long cleaned = 0;
	int i, j;

	for (i = 0; i < npools; i++) {
		if (pools[i]) {
			for (j = 0; j < PAGES_PER_POOL; j++) {
				if (pools[i][j]) {
					__free_page(pools[i][j]);
					cleaned++;
				}
			}
			kfree(pools[i]);
			pools[i] = NULL;
		}
	}

	return cleaned;
}

static inline void enc_pools_wakeup(void)
{
	assert_spin_locked(&page_pools.epp_lock);

	if (unlikely(page_pools.epp_waitqlen)) {
		LASSERT(waitqueue_active(&page_pools.epp_waitq));
		wake_up_all(&page_pools.epp_waitq);
	}
}

void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
{
	int p_idx, g_idx;
	int i;

	if (!desc->bd_enc_iov)
		return;

	LASSERT(desc->bd_iov_count > 0);

	spin_lock(&page_pools.epp_lock);

	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;

	LASSERT(page_pools.epp_free_pages + desc->bd_iov_count <=
		page_pools.epp_total_pages);
	LASSERT(page_pools.epp_pools[p_idx]);

	for (i = 0; i < desc->bd_iov_count; i++) {
		LASSERT(desc->bd_enc_iov[i].bv_page);
		LASSERT(g_idx != 0 || page_pools.epp_pools[p_idx]);
		LASSERT(!page_pools.epp_pools[p_idx][g_idx]);

		page_pools.epp_pools[p_idx][g_idx] =
					desc->bd_enc_iov[i].bv_page;

		if (++g_idx == PAGES_PER_POOL) {
			p_idx++;
			g_idx = 0;
		}
	}

	page_pools.epp_free_pages += desc->bd_iov_count;

	enc_pools_wakeup();

	spin_unlock(&page_pools.epp_lock);

	kfree(desc->bd_enc_iov);
	desc->bd_enc_iov = NULL;
}
EXPORT_SYMBOL(sptlrpc_enc_pool_put_pages);

static inline void enc_pools_alloc(void)
{
	LASSERT(page_pools.epp_max_pools);
	page_pools.epp_pools =
		libcfs_kvzalloc(page_pools.epp_max_pools *
				sizeof(*page_pools.epp_pools),
				GFP_NOFS);
}

static inline void enc_pools_free(void)
{
	LASSERT(page_pools.epp_max_pools);
	LASSERT(page_pools.epp_pools);

	kvfree(page_pools.epp_pools);
}

static struct shrinker pools_shrinker = {
	.count_objects	= enc_pools_shrink_count,
	.scan_objects	= enc_pools_shrink_scan,
	.seeks		= DEFAULT_SEEKS,
};

int sptlrpc_enc_pool_init(void)
{
	/*
	 * maximum capacity is 1/8 of total physical memory.
	 * is the 1/8 a good number?
	 */
	page_pools.epp_max_pages = totalram_pages / 8;
	page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);

	init_waitqueue_head(&page_pools.epp_waitq);
	page_pools.epp_waitqlen = 0;
	page_pools.epp_pages_short = 0;

	page_pools.epp_growing = 0;

	page_pools.epp_idle_idx = 0;
	page_pools.epp_last_shrink = ktime_get_seconds();
	page_pools.epp_last_access = ktime_get_seconds();

	spin_lock_init(&page_pools.epp_lock);
	page_pools.epp_total_pages = 0;
	page_pools.epp_free_pages = 0;

	page_pools.epp_st_max_pages = 0;
	page_pools.epp_st_grows = 0;
	page_pools.epp_st_grow_fails = 0;
	page_pools.epp_st_shrinks = 0;
	page_pools.epp_st_access = 0;
	page_pools.epp_st_missings = 0;
	page_pools.epp_st_lowfree = 0;
	page_pools.epp_st_max_wqlen = 0;
	page_pools.epp_st_max_wait = 0;

	enc_pools_alloc();
	if (!page_pools.epp_pools)
		return -ENOMEM;

	register_shrinker(&pools_shrinker);

	return 0;
}

void sptlrpc_enc_pool_fini(void)
{
	unsigned long cleaned, npools;

	LASSERT(page_pools.epp_pools);
	LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);

	unregister_shrinker(&pools_shrinker);

	npools = npages_to_npools(page_pools.epp_total_pages);
	cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
	LASSERT(cleaned == page_pools.epp_total_pages);

	enc_pools_free();

	if (page_pools.epp_st_access > 0) {
		CDEBUG(D_SEC,
		       "max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%d\n",
		       page_pools.epp_st_max_pages, page_pools.epp_st_grows,
		       page_pools.epp_st_grow_fails,
		       page_pools.epp_st_shrinks, page_pools.epp_st_access,
		       page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
		       page_pools.epp_st_max_wait, HZ);
	}
}

static int cfs_hash_alg_id[] = {
	[BULK_HASH_ALG_NULL]	= CFS_HASH_ALG_NULL,
	[BULK_HASH_ALG_ADLER32]	= CFS_HASH_ALG_ADLER32,
	[BULK_HASH_ALG_CRC32]	= CFS_HASH_ALG_CRC32,
	[BULK_HASH_ALG_MD5]	= CFS_HASH_ALG_MD5,
	[BULK_HASH_ALG_SHA1]	= CFS_HASH_ALG_SHA1,
	[BULK_HASH_ALG_SHA256]	= CFS_HASH_ALG_SHA256,
	[BULK_HASH_ALG_SHA384]	= CFS_HASH_ALG_SHA384,
	[BULK_HASH_ALG_SHA512]	= CFS_HASH_ALG_SHA512,
};

const char *sptlrpc_get_hash_name(__u8 hash_alg)
{
	return cfs_crypto_hash_name(cfs_hash_alg_id[hash_alg]);
}
EXPORT_SYMBOL(sptlrpc_get_hash_name);

__u8 sptlrpc_get_hash_alg(const char *algname)
{
	return cfs_crypto_hash_alg(algname);
}
EXPORT_SYMBOL(sptlrpc_get_hash_alg);

int bulk_sec_desc_unpack(struct lustre_msg *msg, int offset, int swabbed)
{
	struct ptlrpc_bulk_sec_desc *bsd;
	int			  size = msg->lm_buflens[offset];

	bsd = lustre_msg_buf(msg, offset, sizeof(*bsd));
	if (!bsd) {
		CERROR("Invalid bulk sec desc: size %d\n", size);
		return -EINVAL;
	}

	if (swabbed)
		__swab32s(&bsd->bsd_nob);

	if (unlikely(bsd->bsd_version != 0)) {
		CERROR("Unexpected version %u\n", bsd->bsd_version);
		return -EPROTO;
	}

	if (unlikely(bsd->bsd_type >= SPTLRPC_BULK_MAX)) {
		CERROR("Invalid type %u\n", bsd->bsd_type);
		return -EPROTO;
	}

	/* FIXME more sanity check here */

	if (unlikely(bsd->bsd_svc != SPTLRPC_BULK_SVC_NULL &&
		     bsd->bsd_svc != SPTLRPC_BULK_SVC_INTG &&
		     bsd->bsd_svc != SPTLRPC_BULK_SVC_PRIV)) {
		CERROR("Invalid svc %u\n", bsd->bsd_svc);
		return -EPROTO;
	}

	return 0;
}
EXPORT_SYMBOL(bulk_sec_desc_unpack);

int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
			      void *buf, int buflen)
{
	struct cfs_crypto_hash_desc *hdesc;
	int hashsize;
	unsigned int bufsize;
	int i, err;

	LASSERT(alg > BULK_HASH_ALG_NULL && alg < BULK_HASH_ALG_MAX);
	LASSERT(buflen >= 4);

	hdesc = cfs_crypto_hash_init(cfs_hash_alg_id[alg], NULL, 0);
	if (IS_ERR(hdesc)) {
		CERROR("Unable to initialize checksum hash %s\n",
		       cfs_crypto_hash_name(cfs_hash_alg_id[alg]));
		return PTR_ERR(hdesc);
	}

	hashsize = cfs_crypto_hash_digestsize(cfs_hash_alg_id[alg]);

	for (i = 0; i < desc->bd_iov_count; i++) {
		cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].bv_page,
					    desc->bd_iov[i].bv_offset &
					    ~PAGE_MASK,
					    desc->bd_iov[i].bv_len);
	}

	if (hashsize > buflen) {
		unsigned char hashbuf[CFS_CRYPTO_HASH_DIGESTSIZE_MAX];

		bufsize = sizeof(hashbuf);
		LASSERTF(bufsize >= hashsize, "bufsize = %u < hashsize %u\n",
			 bufsize, hashsize);
		err = cfs_crypto_hash_final(hdesc, hashbuf, &bufsize);
		memcpy(buf, hashbuf, buflen);
	} else {
		bufsize = buflen;
		err = cfs_crypto_hash_final(hdesc, buf, &bufsize);
	}

	return err;
}
EXPORT_SYMBOL(sptlrpc_get_bulk_checksum);
Commit	Line	Data
d7e09d03 PT	1	/*
	2	* GPL HEADER START
	3	*
	4	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 only,
	8	* as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful, but
	11	* WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	* General Public License version 2 for more details (a copy is included
	14	* in the LICENSE file that accompanied this code).
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* version 2 along with this program; If not, see
6a5b99a4	18	* http://www.gnu.org/licenses/gpl-2.0.html
d7e09d03	19	*
d7e09d03 PT	20	* GPL HEADER END
	21	*/
	22	/*
	23	* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
	24	* Use is subject to license terms.
	25	*
1dc563a6	26	* Copyright (c) 2011, 2015, Intel Corporation.
d7e09d03 PT	27	*/
	28	/*
	29	* This file is part of Lustre, http://www.lustre.org/
	30	* Lustre is a trademark of Sun Microsystems, Inc.
	31	*
	32	* lustre/ptlrpc/sec_bulk.c
	33	*
	34	* Author: Eric Mei <ericm@clusterfs.com>
	35	*/
	36
	37	#define DEBUG_SUBSYSTEM S_SEC
	38
9fdaf8c0	39	#include "../../include/linux/libcfs/libcfs.h"
d7e09d03	40
e27db149 GKH	41	#include "../include/obd.h"
	42	#include "../include/obd_cksum.h"
	43	#include "../include/obd_class.h"
	44	#include "../include/obd_support.h"
	45	#include "../include/lustre_net.h"
	46	#include "../include/lustre_import.h"
	47	#include "../include/lustre_dlm.h"
	48	#include "../include/lustre_sec.h"
d7e09d03 PT	49
	50	#include "ptlrpc_internal.h"
	51
	52	/****************************************
	53	* bulk encryption page pools *
	54	****************************************/
	55
09cbfeaf	56	#define POINTERS_PER_PAGE (PAGE_SIZE / sizeof(void *))
ae18c5c6	57	#define PAGES_PER_POOL (POINTERS_PER_PAGE)
d7e09d03	58
d0bfef31	59	#define IDLE_IDX_MAX (100)
d7e09d03 PT	60	#define IDLE_IDX_WEIGHT (3)
	61
	62	#define CACHE_QUIESCENT_PERIOD (20)
	63
	64	static struct ptlrpc_enc_page_pool {
	65	/*
	66	* constants
	67	*/
	68	unsigned long epp_max_pages; /* maximum pages can hold, const */
	69	unsigned int epp_max_pools; /* number of pools, const */
	70
	71	/*
	72	* wait queue in case of not enough free pages.
	73	*/
	74	wait_queue_head_t epp_waitq; /* waiting threads */
	75	unsigned int epp_waitqlen; /* wait queue length */
	76	unsigned long epp_pages_short; /* # of pages wanted of in-q users */
	77	unsigned int epp_growing:1; /* during adding pages */
	78
	79	/*
	80	* indicating how idle the pools are, from 0 to MAX_IDLE_IDX
	81	* this is counted based on each time when getting pages from
	82	* the pools, not based on time. which means in case that system
	83	* is idled for a while but the idle_idx might still be low if no
	84	* activities happened in the pools.
	85	*/
	86	unsigned long epp_idle_idx;
	87
	88	/* last shrink time due to mem tight */
80018a9e AB	89	time64_t epp_last_shrink;
80018a9e AB	90	time64_t epp_last_access;
d7e09d03 PT	91
	92	/*
	93	* in-pool pages bookkeeping
	94	*/
	95	spinlock_t epp_lock; /* protect following fields */
	96	unsigned long epp_total_pages; /* total pages in pools */
	97	unsigned long epp_free_pages; /* current pages available */
	98
	99	/*
	100	* statistics
	101	*/
	102	unsigned long epp_st_max_pages; /* # of pages ever reached */
	103	unsigned int epp_st_grows; /* # of grows */
	104	unsigned int epp_st_grow_fails; /* # of add pages failures */
	105	unsigned int epp_st_shrinks; /* # of shrinks */
	106	unsigned long epp_st_access; /* # of access */
	107	unsigned long epp_st_missings; /* # of cache missing */
	108	unsigned long epp_st_lowfree; /* lowest free pages reached */
	109	unsigned int epp_st_max_wqlen; /* highest waitqueue length */
a649ad1d	110	unsigned long epp_st_max_wait; /* in jiffies */
d7e09d03 PT	111	/*
	112	* pointers to pools
	113	*/
	114	struct page ***epp_pools;
	115	} page_pools;
	116
d7e09d03	117	/*
f6e42a40	118	* /sys/kernel/debug/lustre/sptlrpc/encrypt_page_pools
d7e09d03	119	*/
73bb1da6	120	int sptlrpc_proc_enc_pool_seq_show(struct seq_file m, void v)
d7e09d03	121	{
d7e09d03 PT	122	spin_lock(&page_pools.epp_lock);
d7e09d03 PT	123
91b3a685 JP	124	seq_printf(m,
	125	"physical pages: %lu\n"
	126	"pages per pool: %lu\n"
	127	"max pages: %lu\n"
	128	"max pools: %u\n"
	129	"total pages: %lu\n"
	130	"total free: %lu\n"
	131	"idle index: %lu/100\n"
	132	"last shrink: %lds\n"
	133	"last access: %lds\n"
	134	"max pages reached: %lu\n"
	135	"grows: %u\n"
	136	"grows failure: %u\n"
	137	"shrinks: %u\n"
	138	"cache access: %lu\n"
	139	"cache missing: %lu\n"
	140	"low free mark: %lu\n"
	141	"max waitqueue depth: %u\n"
93d3a405	142	"max wait time: %ld/%u\n",
91b3a685 JP	143	totalram_pages,
	144	PAGES_PER_POOL,
	145	page_pools.epp_max_pages,
	146	page_pools.epp_max_pools,
	147	page_pools.epp_total_pages,
	148	page_pools.epp_free_pages,
	149	page_pools.epp_idle_idx,
80018a9e AB	150	(long)(ktime_get_seconds() - page_pools.epp_last_shrink),
80018a9e AB	151	(long)(ktime_get_seconds() - page_pools.epp_last_access),
91b3a685 JP	152	page_pools.epp_st_max_pages,
	153	page_pools.epp_st_grows,
	154	page_pools.epp_st_grow_fails,
	155	page_pools.epp_st_shrinks,
	156	page_pools.epp_st_access,
	157	page_pools.epp_st_missings,
	158	page_pools.epp_st_lowfree,
	159	page_pools.epp_st_max_wqlen,
	160	page_pools.epp_st_max_wait,
	161	HZ);
d7e09d03 PT	162
d7e09d03 PT	163	spin_unlock(&page_pools.epp_lock);
91b3a685 JP	164
91b3a685 JP	165	return 0;
d7e09d03 PT	166	}
	167
	168	static void enc_pools_release_free_pages(long npages)
	169	{
d0bfef31 CH	170	int p_idx, g_idx;
d0bfef31 CH	171	int p_idx_max1, p_idx_max2;
d7e09d03 PT	172
	173	LASSERT(npages > 0);
	174	LASSERT(npages <= page_pools.epp_free_pages);
	175	LASSERT(page_pools.epp_free_pages <= page_pools.epp_total_pages);
	176
	177	/* max pool index before the release */
	178	p_idx_max2 = (page_pools.epp_total_pages - 1) / PAGES_PER_POOL;
	179
	180	page_pools.epp_free_pages -= npages;
	181	page_pools.epp_total_pages -= npages;
	182
	183	/* max pool index after the release */
	184	p_idx_max1 = page_pools.epp_total_pages == 0 ? -1 :
	185	((page_pools.epp_total_pages - 1) / PAGES_PER_POOL);
	186
	187	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	188	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	189	LASSERT(page_pools.epp_pools[p_idx]);
	190
	191	while (npages--) {
	192	LASSERT(page_pools.epp_pools[p_idx]);
8b382089	193	LASSERT(page_pools.epp_pools[p_idx][g_idx]);
d7e09d03 PT	194
	195	__free_page(page_pools.epp_pools[p_idx][g_idx]);
	196	page_pools.epp_pools[p_idx][g_idx] = NULL;
	197
	198	if (++g_idx == PAGES_PER_POOL) {
	199	p_idx++;
	200	g_idx = 0;
	201	}
9076b09e	202	}
d7e09d03 PT	203
	204	/* free unused pools */
	205	while (p_idx_max1 < p_idx_max2) {
	206	LASSERT(page_pools.epp_pools[p_idx_max2]);
9ae10597	207	kfree(page_pools.epp_pools[p_idx_max2]);
d7e09d03 PT	208	page_pools.epp_pools[p_idx_max2] = NULL;
	209	p_idx_max2--;
	210	}
	211	}
	212
	213	/*
d7e09d03 PT	214	* we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
d7e09d03 PT	215	*/
3bb22ec5 PT	216	static unsigned long enc_pools_shrink_count(struct shrinker *s,
3bb22ec5 PT	217	struct shrink_control *sc)
d7e09d03	218	{
3bb22ec5 PT	219	/*
	220	* if no pool access for a long time, we consider it's fully idle.
	221	* a little race here is fine.
	222	*/
80018a9e	223	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
3bb22ec5	224	CACHE_QUIESCENT_PERIOD)) {
d7e09d03	225	spin_lock(&page_pools.epp_lock);
3bb22ec5	226	page_pools.epp_idle_idx = IDLE_IDX_MAX;
d7e09d03 PT	227	spin_unlock(&page_pools.epp_lock);
	228	}
	229
3bb22ec5 PT	230	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	231	return max((int)page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES, 0) *
	232	(IDLE_IDX_MAX - page_pools.epp_idle_idx) / IDLE_IDX_MAX;
	233	}
	234
	235	/*
	236	* we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
	237	*/
	238	static unsigned long enc_pools_shrink_scan(struct shrinker *s,
	239	struct shrink_control *sc)
	240	{
	241	spin_lock(&page_pools.epp_lock);
	242	sc->nr_to_scan = min_t(unsigned long, sc->nr_to_scan,
	243	page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES);
	244	if (sc->nr_to_scan > 0) {
	245	enc_pools_release_free_pages(sc->nr_to_scan);
	246	CDEBUG(D_SEC, "released %ld pages, %ld left\n",
	247	(long)sc->nr_to_scan, page_pools.epp_free_pages);
	248
	249	page_pools.epp_st_shrinks++;
80018a9e	250	page_pools.epp_last_shrink = ktime_get_seconds();
3bb22ec5 PT	251	}
	252	spin_unlock(&page_pools.epp_lock);
	253
d7e09d03 PT	254	/*
	255	* if no pool access for a long time, we consider it's fully idle.
	256	* a little race here is fine.
	257	*/
80018a9e	258	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
d7e09d03 PT	259	CACHE_QUIESCENT_PERIOD)) {
	260	spin_lock(&page_pools.epp_lock);
	261	page_pools.epp_idle_idx = IDLE_IDX_MAX;
	262	spin_unlock(&page_pools.epp_lock);
	263	}
	264
	265	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
3bb22ec5	266	return sc->nr_to_scan;
d7e09d03 PT	267	}
	268
	269	static inline
	270	int npages_to_npools(unsigned long npages)
	271	{
9797fb0e	272	return (int)((npages + PAGES_PER_POOL - 1) / PAGES_PER_POOL);
d7e09d03 PT	273	}
	274
	275	/*
	276	* return how many pages cleaned up.
	277	*/
	278	static unsigned long enc_pools_cleanup(struct page ***pools, int npools)
	279	{
	280	unsigned long cleaned = 0;
d0bfef31	281	int i, j;
d7e09d03 PT	282
	283	for (i = 0; i < npools; i++) {
	284	if (pools[i]) {
	285	for (j = 0; j < PAGES_PER_POOL; j++) {
	286	if (pools[i][j]) {
	287	__free_page(pools[i][j]);
	288	cleaned++;
	289	}
	290	}
9ae10597	291	kfree(pools[i]);
d7e09d03 PT	292	pools[i] = NULL;
	293	}
	294	}
	295
	296	return cleaned;
	297	}
	298
d7e09d03 PT	299	static inline void enc_pools_wakeup(void)
d7e09d03 PT	300	{
5e42bc9d	301	assert_spin_locked(&page_pools.epp_lock);
d7e09d03 PT	302
	303	if (unlikely(page_pools.epp_waitqlen)) {
	304	LASSERT(waitqueue_active(&page_pools.epp_waitq));
	305	wake_up_all(&page_pools.epp_waitq);
	306	}
	307	}
	308
d7e09d03 PT	309	void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
d7e09d03 PT	310	{
d0bfef31 CH	311	int p_idx, g_idx;
d0bfef31 CH	312	int i;
d7e09d03	313
8b382089	314	if (!desc->bd_enc_iov)
d7e09d03 PT	315	return;
	316
	317	LASSERT(desc->bd_iov_count > 0);
	318
	319	spin_lock(&page_pools.epp_lock);
	320
	321	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	322	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	323
	324	LASSERT(page_pools.epp_free_pages + desc->bd_iov_count <=
	325	page_pools.epp_total_pages);
	326	LASSERT(page_pools.epp_pools[p_idx]);
	327
	328	for (i = 0; i < desc->bd_iov_count; i++) {
65ffc679	329	LASSERT(desc->bd_enc_iov[i].bv_page);
d7e09d03	330	LASSERT(g_idx != 0 \|\| page_pools.epp_pools[p_idx]);
8b382089	331	LASSERT(!page_pools.epp_pools[p_idx][g_idx]);
d7e09d03 PT	332
d7e09d03 PT	333	page_pools.epp_pools[p_idx][g_idx] =
65ffc679	334	desc->bd_enc_iov[i].bv_page;
d7e09d03 PT	335
	336	if (++g_idx == PAGES_PER_POOL) {
	337	p_idx++;
	338	g_idx = 0;
	339	}
	340	}
	341
	342	page_pools.epp_free_pages += desc->bd_iov_count;
	343
	344	enc_pools_wakeup();
	345
	346	spin_unlock(&page_pools.epp_lock);
	347
9ae10597	348	kfree(desc->bd_enc_iov);
d7e09d03 PT	349	desc->bd_enc_iov = NULL;
	350	}
	351	EXPORT_SYMBOL(sptlrpc_enc_pool_put_pages);
	352
d7e09d03 PT	353	static inline void enc_pools_alloc(void)
	354	{
	355	LASSERT(page_pools.epp_max_pools);
ee0ec194 JL	356	page_pools.epp_pools =
	357	libcfs_kvzalloc(page_pools.epp_max_pools *
	358	sizeof(*page_pools.epp_pools),
	359	GFP_NOFS);
d7e09d03 PT	360	}
	361
	362	static inline void enc_pools_free(void)
	363	{
	364	LASSERT(page_pools.epp_max_pools);
	365	LASSERT(page_pools.epp_pools);
	366
ee0ec194	367	kvfree(page_pools.epp_pools);
d7e09d03 PT	368	}
d7e09d03 PT	369
3bb22ec5 PT	370	static struct shrinker pools_shrinker = {
	371	.count_objects = enc_pools_shrink_count,
	372	.scan_objects = enc_pools_shrink_scan,
	373	.seeks = DEFAULT_SEEKS,
	374	};
	375
d7e09d03 PT	376	int sptlrpc_enc_pool_init(void)
	377	{
	378	/*
	379	* maximum capacity is 1/8 of total physical memory.
	380	* is the 1/8 a good number?
	381	*/
4f6cc9ab	382	page_pools.epp_max_pages = totalram_pages / 8;
d7e09d03 PT	383	page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);
	384
	385	init_waitqueue_head(&page_pools.epp_waitq);
	386	page_pools.epp_waitqlen = 0;
	387	page_pools.epp_pages_short = 0;
	388
	389	page_pools.epp_growing = 0;
	390
	391	page_pools.epp_idle_idx = 0;
80018a9e AB	392	page_pools.epp_last_shrink = ktime_get_seconds();
80018a9e AB	393	page_pools.epp_last_access = ktime_get_seconds();
d7e09d03 PT	394
	395	spin_lock_init(&page_pools.epp_lock);
	396	page_pools.epp_total_pages = 0;
	397	page_pools.epp_free_pages = 0;
	398
	399	page_pools.epp_st_max_pages = 0;
	400	page_pools.epp_st_grows = 0;
	401	page_pools.epp_st_grow_fails = 0;
	402	page_pools.epp_st_shrinks = 0;
	403	page_pools.epp_st_access = 0;
	404	page_pools.epp_st_missings = 0;
	405	page_pools.epp_st_lowfree = 0;
	406	page_pools.epp_st_max_wqlen = 0;
	407	page_pools.epp_st_max_wait = 0;
	408
	409	enc_pools_alloc();
8b382089	410	if (!page_pools.epp_pools)
d7e09d03 PT	411	return -ENOMEM;
d7e09d03 PT	412
3bb22ec5	413	register_shrinker(&pools_shrinker);
d7e09d03 PT	414
	415	return 0;
	416	}
	417
	418	void sptlrpc_enc_pool_fini(void)
	419	{
	420	unsigned long cleaned, npools;
	421
d7e09d03 PT	422	LASSERT(page_pools.epp_pools);
	423	LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);
	424
3bb22ec5	425	unregister_shrinker(&pools_shrinker);
d7e09d03 PT	426
	427	npools = npages_to_npools(page_pools.epp_total_pages);
	428	cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
	429	LASSERT(cleaned == page_pools.epp_total_pages);
	430
	431	enc_pools_free();
	432
	433	if (page_pools.epp_st_access > 0) {
	434	CDEBUG(D_SEC,
93d3a405	435	"max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%d\n",
d7e09d03 PT	436	page_pools.epp_st_max_pages, page_pools.epp_st_grows,
	437	page_pools.epp_st_grow_fails,
	438	page_pools.epp_st_shrinks, page_pools.epp_st_access,
	439	page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
	440	page_pools.epp_st_max_wait, HZ);
	441	}
	442	}
	443
d7e09d03 PT	444	static int cfs_hash_alg_id[] = {
	445	[BULK_HASH_ALG_NULL] = CFS_HASH_ALG_NULL,
	446	[BULK_HASH_ALG_ADLER32] = CFS_HASH_ALG_ADLER32,
	447	[BULK_HASH_ALG_CRC32] = CFS_HASH_ALG_CRC32,
	448	[BULK_HASH_ALG_MD5] = CFS_HASH_ALG_MD5,
	449	[BULK_HASH_ALG_SHA1] = CFS_HASH_ALG_SHA1,
	450	[BULK_HASH_ALG_SHA256] = CFS_HASH_ALG_SHA256,
	451	[BULK_HASH_ALG_SHA384] = CFS_HASH_ALG_SHA384,
	452	[BULK_HASH_ALG_SHA512] = CFS_HASH_ALG_SHA512,
	453	};
c9f6bb96	454
aff9d8e8	455	const char *sptlrpc_get_hash_name(__u8 hash_alg)
d7e09d03 PT	456	{
	457	return cfs_crypto_hash_name(cfs_hash_alg_id[hash_alg]);
	458	}
	459	EXPORT_SYMBOL(sptlrpc_get_hash_name);
	460
	461	__u8 sptlrpc_get_hash_alg(const char *algname)
	462	{
	463	return cfs_crypto_hash_alg(algname);
	464	}
	465	EXPORT_SYMBOL(sptlrpc_get_hash_alg);
	466
	467	int bulk_sec_desc_unpack(struct lustre_msg *msg, int offset, int swabbed)
	468	{
	469	struct ptlrpc_bulk_sec_desc *bsd;
	470	int size = msg->lm_buflens[offset];
	471
	472	bsd = lustre_msg_buf(msg, offset, sizeof(*bsd));
8b382089	473	if (!bsd) {
d7e09d03 PT	474	CERROR("Invalid bulk sec desc: size %d\n", size);
	475	return -EINVAL;
	476	}
	477
acf60c3d	478	if (swabbed)
d7e09d03	479	__swab32s(&bsd->bsd_nob);
d7e09d03 PT	480
	481	if (unlikely(bsd->bsd_version != 0)) {
	482	CERROR("Unexpected version %u\n", bsd->bsd_version);
	483	return -EPROTO;
	484	}
	485
	486	if (unlikely(bsd->bsd_type >= SPTLRPC_BULK_MAX)) {
	487	CERROR("Invalid type %u\n", bsd->bsd_type);
	488	return -EPROTO;
	489	}
	490
	491	/* FIXME more sanity check here */
	492
	493	if (unlikely(bsd->bsd_svc != SPTLRPC_BULK_SVC_NULL &&
	494	bsd->bsd_svc != SPTLRPC_BULK_SVC_INTG &&
	495	bsd->bsd_svc != SPTLRPC_BULK_SVC_PRIV)) {
	496	CERROR("Invalid svc %u\n", bsd->bsd_svc);
	497	return -EPROTO;
	498	}
	499
	500	return 0;
	501	}
	502	EXPORT_SYMBOL(bulk_sec_desc_unpack);
	503
	504	int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
	505	void *buf, int buflen)
	506	{
d0bfef31 CH	507	struct cfs_crypto_hash_desc *hdesc;
d0bfef31 CH	508	int hashsize;
d0bfef31 CH	509	unsigned int bufsize;
d0bfef31 CH	510	int i, err;
d7e09d03 PT	511
	512	LASSERT(alg > BULK_HASH_ALG_NULL && alg < BULK_HASH_ALG_MAX);
	513	LASSERT(buflen >= 4);
	514
	515	hdesc = cfs_crypto_hash_init(cfs_hash_alg_id[alg], NULL, 0);
	516	if (IS_ERR(hdesc)) {
	517	CERROR("Unable to initialize checksum hash %s\n",
	518	cfs_crypto_hash_name(cfs_hash_alg_id[alg]));
	519	return PTR_ERR(hdesc);
	520	}
	521
	522	hashsize = cfs_crypto_hash_digestsize(cfs_hash_alg_id[alg]);
	523
	524	for (i = 0; i < desc->bd_iov_count; i++) {
65ffc679	525	cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].bv_page,
24c198e9 OD	526	desc->bd_iov[i].bv_offset &
	527	~PAGE_MASK,
	528	desc->bd_iov[i].bv_len);
d7e09d03	529	}
c11e27a4	530
d7e09d03	531	if (hashsize > buflen) {
c11e27a4 AD	532	unsigned char hashbuf[CFS_CRYPTO_HASH_DIGESTSIZE_MAX];
c11e27a4 AD	533
d7e09d03	534	bufsize = sizeof(hashbuf);
c11e27a4 AD	535	LASSERTF(bufsize >= hashsize, "bufsize = %u < hashsize %u\n",
	536	bufsize, hashsize);
	537	err = cfs_crypto_hash_final(hdesc, hashbuf, &bufsize);
d7e09d03 PT	538	memcpy(buf, hashbuf, buflen);
	539	} else {
	540	bufsize = buflen;
3df82ddc	541	err = cfs_crypto_hash_final(hdesc, buf, &bufsize);
d7e09d03 PT	542	}
d7e09d03 PT	543
d7e09d03 PT	544	return err;
	545	}
	546	EXPORT_SYMBOL(sptlrpc_get_bulk_checksum);