[deliverable/linux.git] / net / sched / sch_codel.c

/*
 * Codel - The Controlled-Delay Active Queue Management algorithm
 *
 *  Copyright (C) 2011-2012 Kathleen Nichols <nichols@pollere.com>
 *  Copyright (C) 2011-2012 Van Jacobson <van@pollere.net>
 *
 *  Implemented on linux by :
 *  Copyright (C) 2012 Michael D. Taht <dave.taht@bufferbloat.net>
 *  Copyright (C) 2012,2015 Eric Dumazet <edumazet@google.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions, and the following disclaimer,
 *    without modification.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The names of the authors may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * Alternatively, provided that this notice is retained in full, this
 * software may be distributed under the terms of the GNU General
 * Public License ("GPL") version 2, in which case the provisions of the
 * GPL apply INSTEAD OF those given above.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 *
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/prefetch.h>
#include <net/pkt_sched.h>
#include <net/codel.h>


#define DEFAULT_CODEL_LIMIT 1000

struct codel_sched_data {
	struct codel_params	params;
	struct codel_vars	vars;
	struct codel_stats	stats;
	u32			drop_overlimit;
};

/* This is the specific function called from codel_dequeue()
 * to dequeue a packet from queue. Note: backlog is handled in
 * codel, we dont need to reduce it here.
 */
static struct sk_buff *dequeue(struct codel_vars *vars, struct Qdisc *sch)
{
	struct sk_buff *skb = __skb_dequeue(&sch->q);

	prefetch(&skb->end); /* we'll need skb_shinfo() */
	return skb;
}

static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch)
{
	struct codel_sched_data *q = qdisc_priv(sch);
	struct sk_buff *skb;

	skb = codel_dequeue(sch, &q->params, &q->vars, &q->stats, dequeue);

	/* We cant call qdisc_tree_decrease_qlen() if our qlen is 0,
	 * or HTB crashes. Defer it for next round.
	 */
	if (q->stats.drop_count && sch->q.qlen) {
		qdisc_tree_decrease_qlen(sch, q->stats.drop_count);
		q->stats.drop_count = 0;
	}
	if (skb)
		qdisc_bstats_update(sch, skb);
	return skb;
}

static int codel_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
{
	struct codel_sched_data *q;

	if (likely(qdisc_qlen(sch) < sch->limit)) {
		codel_set_enqueue_time(skb);
		return qdisc_enqueue_tail(skb, sch);
	}
	q = qdisc_priv(sch);
	q->drop_overlimit++;
	return qdisc_drop(skb, sch);
}

static const struct nla_policy codel_policy[TCA_CODEL_MAX + 1] = {
	[TCA_CODEL_TARGET]	= { .type = NLA_U32 },
	[TCA_CODEL_LIMIT]	= { .type = NLA_U32 },
	[TCA_CODEL_INTERVAL]	= { .type = NLA_U32 },
	[TCA_CODEL_ECN]		= { .type = NLA_U32 },
	[TCA_CODEL_CE_THRESHOLD]= { .type = NLA_U32 },
};

static int codel_change(struct Qdisc *sch, struct nlattr *opt)
{
	struct codel_sched_data *q = qdisc_priv(sch);
	struct nlattr *tb[TCA_CODEL_MAX + 1];
	unsigned int qlen;
	int err;

	if (!opt)
		return -EINVAL;

	err = nla_parse_nested(tb, TCA_CODEL_MAX, opt, codel_policy);
	if (err < 0)
		return err;

	sch_tree_lock(sch);

	if (tb[TCA_CODEL_TARGET]) {
		u32 target = nla_get_u32(tb[TCA_CODEL_TARGET]);

		q->params.target = ((u64)target * NSEC_PER_USEC) >> CODEL_SHIFT;
	}

	if (tb[TCA_CODEL_CE_THRESHOLD]) {
		u64 val = nla_get_u32(tb[TCA_CODEL_CE_THRESHOLD]);

		q->params.ce_threshold = (val * NSEC_PER_USEC) >> CODEL_SHIFT;
	}

	if (tb[TCA_CODEL_INTERVAL]) {
		u32 interval = nla_get_u32(tb[TCA_CODEL_INTERVAL]);

		q->params.interval = ((u64)interval * NSEC_PER_USEC) >> CODEL_SHIFT;
	}

	if (tb[TCA_CODEL_LIMIT])
		sch->limit = nla_get_u32(tb[TCA_CODEL_LIMIT]);

	if (tb[TCA_CODEL_ECN])
		q->params.ecn = !!nla_get_u32(tb[TCA_CODEL_ECN]);

	qlen = sch->q.qlen;
	while (sch->q.qlen > sch->limit) {
		struct sk_buff *skb = __skb_dequeue(&sch->q);

		qdisc_qstats_backlog_dec(sch, skb);
		qdisc_drop(skb, sch);
	}
	qdisc_tree_decrease_qlen(sch, qlen - sch->q.qlen);

	sch_tree_unlock(sch);
	return 0;
}

static int codel_init(struct Qdisc *sch, struct nlattr *opt)
{
	struct codel_sched_data *q = qdisc_priv(sch);

	sch->limit = DEFAULT_CODEL_LIMIT;

	codel_params_init(&q->params, sch);
	codel_vars_init(&q->vars);
	codel_stats_init(&q->stats);

	if (opt) {
		int err = codel_change(sch, opt);

		if (err)
			return err;
	}

	if (sch->limit >= 1)
		sch->flags |= TCQ_F_CAN_BYPASS;
	else
		sch->flags &= ~TCQ_F_CAN_BYPASS;

	return 0;
}

static int codel_dump(struct Qdisc *sch, struct sk_buff *skb)
{
	struct codel_sched_data *q = qdisc_priv(sch);
	struct nlattr *opts;

	opts = nla_nest_start(skb, TCA_OPTIONS);
	if (opts == NULL)
		goto nla_put_failure;

	if (nla_put_u32(skb, TCA_CODEL_TARGET,
			codel_time_to_us(q->params.target)) ||
	    nla_put_u32(skb, TCA_CODEL_LIMIT,
			sch->limit) ||
	    nla_put_u32(skb, TCA_CODEL_INTERVAL,
			codel_time_to_us(q->params.interval)) ||
	    nla_put_u32(skb, TCA_CODEL_ECN,
			q->params.ecn))
		goto nla_put_failure;
	if (q->params.ce_threshold != CODEL_DISABLED_THRESHOLD &&
	    nla_put_u32(skb, TCA_CODEL_CE_THRESHOLD,
			codel_time_to_us(q->params.ce_threshold)))
		goto nla_put_failure;
	return nla_nest_end(skb, opts);

nla_put_failure:
	nla_nest_cancel(skb, opts);
	return -1;
}

static int codel_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
{
	const struct codel_sched_data *q = qdisc_priv(sch);
	struct tc_codel_xstats st = {
		.maxpacket	= q->stats.maxpacket,
		.count		= q->vars.count,
		.lastcount	= q->vars.lastcount,
		.drop_overlimit = q->drop_overlimit,
		.ldelay		= codel_time_to_us(q->vars.ldelay),
		.dropping	= q->vars.dropping,
		.ecn_mark	= q->stats.ecn_mark,
		.ce_mark	= q->stats.ce_mark,
	};

	if (q->vars.dropping) {
		codel_tdiff_t delta = q->vars.drop_next - codel_get_time();

		if (delta >= 0)
			st.drop_next = codel_time_to_us(delta);
		else
			st.drop_next = -codel_time_to_us(-delta);
	}

	return gnet_stats_copy_app(d, &st, sizeof(st));
}

static void codel_reset(struct Qdisc *sch)
{
	struct codel_sched_data *q = qdisc_priv(sch);

	qdisc_reset_queue(sch);
	codel_vars_init(&q->vars);
}

static struct Qdisc_ops codel_qdisc_ops __read_mostly = {
	.id		=	"codel",
	.priv_size	=	sizeof(struct codel_sched_data),

	.enqueue	=	codel_qdisc_enqueue,
	.dequeue	=	codel_qdisc_dequeue,
	.peek		=	qdisc_peek_dequeued,
	.init		=	codel_init,
	.reset		=	codel_reset,
	.change 	=	codel_change,
	.dump		=	codel_dump,
	.dump_stats	=	codel_dump_stats,
	.owner		=	THIS_MODULE,
};

static int __init codel_module_init(void)
{
	return register_qdisc(&codel_qdisc_ops);
}

static void __exit codel_module_exit(void)
{
	unregister_qdisc(&codel_qdisc_ops);
}

module_init(codel_module_init)
module_exit(codel_module_exit)

MODULE_DESCRIPTION("Controlled Delay queue discipline");
MODULE_AUTHOR("Dave Taht");
MODULE_AUTHOR("Eric Dumazet");
MODULE_LICENSE("Dual BSD/GPL");
Commit	Line	Data
76e3cc12 ED	1	/*
	2	* Codel - The Controlled-Delay Active Queue Management algorithm
	3	*
	4	* Copyright (C) 2011-2012 Kathleen Nichols <nichols@pollere.com>
	5	* Copyright (C) 2011-2012 Van Jacobson <van@pollere.net>
	6	*
	7	* Implemented on linux by :
	8	* Copyright (C) 2012 Michael D. Taht <dave.taht@bufferbloat.net>
80ba92fa	9	* Copyright (C) 2012,2015 Eric Dumazet <edumazet@google.com>
76e3cc12 ED	10	*
	11	* Redistribution and use in source and binary forms, with or without
	12	* modification, are permitted provided that the following conditions
	13	* are met:
	14	* 1. Redistributions of source code must retain the above copyright
	15	* notice, this list of conditions, and the following disclaimer,
	16	* without modification.
	17	* 2. Redistributions in binary form must reproduce the above copyright
	18	* notice, this list of conditions and the following disclaimer in the
	19	* documentation and/or other materials provided with the distribution.
	20	* 3. The names of the authors may not be used to endorse or promote products
	21	* derived from this software without specific prior written permission.
	22	*
	23	* Alternatively, provided that this notice is retained in full, this
	24	* software may be distributed under the terms of the GNU General
	25	* Public License ("GPL") version 2, in which case the provisions of the
	26	* GPL apply INSTEAD OF those given above.
	27	*
	28	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	29	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	30	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	31	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	32	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	33	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	34	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	35	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	36	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	37	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	38	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
	39	* DAMAGE.
	40	*
	41	*/
	42
	43	#include <linux/module.h>
	44	#include <linux/slab.h>
	45	#include <linux/types.h>
	46	#include <linux/kernel.h>
	47	#include <linux/errno.h>
	48	#include <linux/skbuff.h>
ce5b4b97	49	#include <linux/prefetch.h>
76e3cc12 ED	50	#include <net/pkt_sched.h>
	51	#include <net/codel.h>
	52
	53
	54	#define DEFAULT_CODEL_LIMIT 1000
	55
	56	struct codel_sched_data {
	57	struct codel_params params;
	58	struct codel_vars vars;
	59	struct codel_stats stats;
	60	u32 drop_overlimit;
	61	};
	62
	63	/* This is the specific function called from codel_dequeue()
	64	* to dequeue a packet from queue. Note: backlog is handled in
	65	* codel, we dont need to reduce it here.
	66	*/
	67	static struct sk_buff dequeue(struct codel_vars vars, struct Qdisc *sch)
	68	{
	69	struct sk_buff *skb = __skb_dequeue(&sch->q);
	70
	71	prefetch(&skb->end); /* we'll need skb_shinfo() */
	72	return skb;
	73	}
	74
	75	static struct sk_buff codel_qdisc_dequeue(struct Qdisc sch)
	76	{
	77	struct codel_sched_data *q = qdisc_priv(sch);
	78	struct sk_buff *skb;
	79
865ec552 ED	80	skb = codel_dequeue(sch, &q->params, &q->vars, &q->stats, dequeue);
865ec552 ED	81
76e3cc12 ED	82	/* We cant call qdisc_tree_decrease_qlen() if our qlen is 0,
	83	* or HTB crashes. Defer it for next round.
	84	*/
	85	if (q->stats.drop_count && sch->q.qlen) {
	86	qdisc_tree_decrease_qlen(sch, q->stats.drop_count);
	87	q->stats.drop_count = 0;
	88	}
	89	if (skb)
	90	qdisc_bstats_update(sch, skb);
	91	return skb;
	92	}
	93
	94	static int codel_qdisc_enqueue(struct sk_buff skb, struct Qdisc sch)
	95	{
	96	struct codel_sched_data *q;
	97
	98	if (likely(qdisc_qlen(sch) < sch->limit)) {
	99	codel_set_enqueue_time(skb);
	100	return qdisc_enqueue_tail(skb, sch);
	101	}
	102	q = qdisc_priv(sch);
	103	q->drop_overlimit++;
	104	return qdisc_drop(skb, sch);
	105	}
	106
	107	static const struct nla_policy codel_policy[TCA_CODEL_MAX + 1] = {
	108	[TCA_CODEL_TARGET] = { .type = NLA_U32 },
	109	[TCA_CODEL_LIMIT] = { .type = NLA_U32 },
	110	[TCA_CODEL_INTERVAL] = { .type = NLA_U32 },
	111	[TCA_CODEL_ECN] = { .type = NLA_U32 },
80ba92fa	112	[TCA_CODEL_CE_THRESHOLD]= { .type = NLA_U32 },
76e3cc12 ED	113	};
	114
	115	static int codel_change(struct Qdisc sch, struct nlattr opt)
	116	{
	117	struct codel_sched_data *q = qdisc_priv(sch);
	118	struct nlattr *tb[TCA_CODEL_MAX + 1];
	119	unsigned int qlen;
	120	int err;
	121
	122	if (!opt)
	123	return -EINVAL;
	124
	125	err = nla_parse_nested(tb, TCA_CODEL_MAX, opt, codel_policy);
	126	if (err < 0)
	127	return err;
	128
	129	sch_tree_lock(sch);
	130
	131	if (tb[TCA_CODEL_TARGET]) {
	132	u32 target = nla_get_u32(tb[TCA_CODEL_TARGET]);
	133
	134	q->params.target = ((u64)target * NSEC_PER_USEC) >> CODEL_SHIFT;
	135	}
	136
80ba92fa ED	137	if (tb[TCA_CODEL_CE_THRESHOLD]) {
	138	u64 val = nla_get_u32(tb[TCA_CODEL_CE_THRESHOLD]);
	139
	140	q->params.ce_threshold = (val * NSEC_PER_USEC) >> CODEL_SHIFT;
	141	}
	142
76e3cc12 ED	143	if (tb[TCA_CODEL_INTERVAL]) {
	144	u32 interval = nla_get_u32(tb[TCA_CODEL_INTERVAL]);
	145
	146	q->params.interval = ((u64)interval * NSEC_PER_USEC) >> CODEL_SHIFT;
	147	}
	148
	149	if (tb[TCA_CODEL_LIMIT])
	150	sch->limit = nla_get_u32(tb[TCA_CODEL_LIMIT]);
	151
	152	if (tb[TCA_CODEL_ECN])
	153	q->params.ecn = !!nla_get_u32(tb[TCA_CODEL_ECN]);
	154
	155	qlen = sch->q.qlen;
	156	while (sch->q.qlen > sch->limit) {
	157	struct sk_buff *skb = __skb_dequeue(&sch->q);
	158
25331d6c	159	qdisc_qstats_backlog_dec(sch, skb);
76e3cc12 ED	160	qdisc_drop(skb, sch);
	161	}
	162	qdisc_tree_decrease_qlen(sch, qlen - sch->q.qlen);
	163
	164	sch_tree_unlock(sch);
	165	return 0;
	166	}
	167
	168	static int codel_init(struct Qdisc sch, struct nlattr opt)
	169	{
	170	struct codel_sched_data *q = qdisc_priv(sch);
	171
	172	sch->limit = DEFAULT_CODEL_LIMIT;
	173
a5d28090	174	codel_params_init(&q->params, sch);
76e3cc12 ED	175	codel_vars_init(&q->vars);
	176	codel_stats_init(&q->stats);
	177
	178	if (opt) {
	179	int err = codel_change(sch, opt);
	180
	181	if (err)
	182	return err;
	183	}
	184
	185	if (sch->limit >= 1)
	186	sch->flags \|= TCQ_F_CAN_BYPASS;
	187	else
	188	sch->flags &= ~TCQ_F_CAN_BYPASS;
	189
	190	return 0;
	191	}
	192
	193	static int codel_dump(struct Qdisc sch, struct sk_buff skb)
	194	{
	195	struct codel_sched_data *q = qdisc_priv(sch);
	196	struct nlattr *opts;
	197
	198	opts = nla_nest_start(skb, TCA_OPTIONS);
	199	if (opts == NULL)
	200	goto nla_put_failure;
	201
	202	if (nla_put_u32(skb, TCA_CODEL_TARGET,
	203	codel_time_to_us(q->params.target)) \|\|
	204	nla_put_u32(skb, TCA_CODEL_LIMIT,
	205	sch->limit) \|\|
	206	nla_put_u32(skb, TCA_CODEL_INTERVAL,
	207	codel_time_to_us(q->params.interval)) \|\|
	208	nla_put_u32(skb, TCA_CODEL_ECN,
	209	q->params.ecn))
	210	goto nla_put_failure;
80ba92fa ED	211	if (q->params.ce_threshold != CODEL_DISABLED_THRESHOLD &&
	212	nla_put_u32(skb, TCA_CODEL_CE_THRESHOLD,
	213	codel_time_to_us(q->params.ce_threshold)))
	214	goto nla_put_failure;
76e3cc12 ED	215	return nla_nest_end(skb, opts);
	216
	217	nla_put_failure:
	218	nla_nest_cancel(skb, opts);
	219	return -1;
	220	}
	221
	222	static int codel_dump_stats(struct Qdisc sch, struct gnet_dump d)
	223	{
	224	const struct codel_sched_data *q = qdisc_priv(sch);
	225	struct tc_codel_xstats st = {
	226	.maxpacket = q->stats.maxpacket,
	227	.count = q->vars.count,
	228	.lastcount = q->vars.lastcount,
	229	.drop_overlimit = q->drop_overlimit,
	230	.ldelay = codel_time_to_us(q->vars.ldelay),
	231	.dropping = q->vars.dropping,
	232	.ecn_mark = q->stats.ecn_mark,
80ba92fa	233	.ce_mark = q->stats.ce_mark,
76e3cc12 ED	234	};
	235
	236	if (q->vars.dropping) {
	237	codel_tdiff_t delta = q->vars.drop_next - codel_get_time();
	238
	239	if (delta >= 0)
	240	st.drop_next = codel_time_to_us(delta);
	241	else
	242	st.drop_next = -codel_time_to_us(-delta);
	243	}
	244
	245	return gnet_stats_copy_app(d, &st, sizeof(st));
	246	}
	247
	248	static void codel_reset(struct Qdisc *sch)
	249	{
	250	struct codel_sched_data *q = qdisc_priv(sch);
	251
	252	qdisc_reset_queue(sch);
	253	codel_vars_init(&q->vars);
	254	}
	255
	256	static struct Qdisc_ops codel_qdisc_ops __read_mostly = {
	257	.id = "codel",
	258	.priv_size = sizeof(struct codel_sched_data),
	259
	260	.enqueue = codel_qdisc_enqueue,
	261	.dequeue = codel_qdisc_dequeue,
	262	.peek = qdisc_peek_dequeued,
	263	.init = codel_init,
	264	.reset = codel_reset,
	265	.change = codel_change,
	266	.dump = codel_dump,
	267	.dump_stats = codel_dump_stats,
	268	.owner = THIS_MODULE,
	269	};
	270
	271	static int __init codel_module_init(void)
	272	{
	273	return register_qdisc(&codel_qdisc_ops);
	274	}
	275
	276	static void __exit codel_module_exit(void)
	277	{
	278	unregister_qdisc(&codel_qdisc_ops);
	279	}
	280
	281	module_init(codel_module_init)
	282	module_exit(codel_module_exit)
	283
	284	MODULE_DESCRIPTION("Controlled Delay queue discipline");
	285	MODULE_AUTHOR("Dave Taht");
	286	MODULE_AUTHOR("Eric Dumazet");
	287	MODULE_LICENSE("Dual BSD/GPL");