[deliverable/linux.git] / drivers / cpufreq / cpufreq_governor.c

/*
 * drivers/cpufreq/cpufreq_governor.c
 *
 * CPUFREQ governors common code
 *
 * Copyright	(C) 2001 Russell King
 *		(C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
 *		(C) 2003 Jun Nakajima <jun.nakajima@intel.com>
 *		(C) 2009 Alexander Clouter <alex@digriz.org.uk>
 *		(c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/export.h>
#include <linux/kernel_stat.h>
#include <linux/slab.h>

#include "cpufreq_governor.h"

static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data)
{
	if (have_governor_per_policy())
		return dbs_data->cdata->attr_group_gov_pol;
	else
		return dbs_data->cdata->attr_group_gov_sys;
}

void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
{
	struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
	struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
	struct cpufreq_policy *policy;
	unsigned int sampling_rate;
	unsigned int max_load = 0;
	unsigned int ignore_nice;
	unsigned int j;

	if (dbs_data->cdata->governor == GOV_ONDEMAND) {
		struct od_cpu_dbs_info_s *od_dbs_info =
				dbs_data->cdata->get_cpu_dbs_info_s(cpu);

		/*
		 * Sometimes, the ondemand governor uses an additional
		 * multiplier to give long delays. So apply this multiplier to
		 * the 'sampling_rate', so as to keep the wake-up-from-idle
		 * detection logic a bit conservative.
		 */
		sampling_rate = od_tuners->sampling_rate;
		sampling_rate *= od_dbs_info->rate_mult;

		ignore_nice = od_tuners->ignore_nice_load;
	} else {
		sampling_rate = cs_tuners->sampling_rate;
		ignore_nice = cs_tuners->ignore_nice_load;
	}

	policy = cdbs->cur_policy;

	/* Get Absolute Load */
	for_each_cpu(j, policy->cpus) {
		struct cpu_dbs_common_info *j_cdbs;
		u64 cur_wall_time, cur_idle_time;
		unsigned int idle_time, wall_time;
		unsigned int load;
		int io_busy = 0;

		j_cdbs = dbs_data->cdata->get_cpu_cdbs(j);

		/*
		 * For the purpose of ondemand, waiting for disk IO is
		 * an indication that you're performance critical, and
		 * not that the system is actually idle. So do not add
		 * the iowait time to the cpu idle time.
		 */
		if (dbs_data->cdata->governor == GOV_ONDEMAND)
			io_busy = od_tuners->io_is_busy;
		cur_idle_time = get_cpu_idle_time(j, &cur_wall_time, io_busy);

		wall_time = (unsigned int)
			(cur_wall_time - j_cdbs->prev_cpu_wall);
		j_cdbs->prev_cpu_wall = cur_wall_time;

		idle_time = (unsigned int)
			(cur_idle_time - j_cdbs->prev_cpu_idle);
		j_cdbs->prev_cpu_idle = cur_idle_time;

		if (ignore_nice) {
			u64 cur_nice;
			unsigned long cur_nice_jiffies;

			cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE] -
					 cdbs->prev_cpu_nice;
			/*
			 * Assumption: nice time between sampling periods will
			 * be less than 2^32 jiffies for 32 bit sys
			 */
			cur_nice_jiffies = (unsigned long)
					cputime64_to_jiffies64(cur_nice);

			cdbs->prev_cpu_nice =
				kcpustat_cpu(j).cpustat[CPUTIME_NICE];
			idle_time += jiffies_to_usecs(cur_nice_jiffies);
		}

		if (unlikely(!wall_time || wall_time < idle_time))
			continue;

		/*
		 * If the CPU had gone completely idle, and a task just woke up
		 * on this CPU now, it would be unfair to calculate 'load' the
		 * usual way for this elapsed time-window, because it will show
		 * near-zero load, irrespective of how CPU intensive that task
		 * actually is. This is undesirable for latency-sensitive bursty
		 * workloads.
		 *
		 * To avoid this, we reuse the 'load' from the previous
		 * time-window and give this task a chance to start with a
		 * reasonably high CPU frequency. (However, we shouldn't over-do
		 * this copy, lest we get stuck at a high load (high frequency)
		 * for too long, even when the current system load has actually
		 * dropped down. So we perform the copy only once, upon the
		 * first wake-up from idle.)
		 *
		 * Detecting this situation is easy: the governor's deferrable
		 * timer would not have fired during CPU-idle periods. Hence
		 * an unusually large 'wall_time' (as compared to the sampling
		 * rate) indicates this scenario.
		 *
		 * prev_load can be zero in two cases and we must recalculate it
		 * for both cases:
		 * - during long idle intervals
		 * - explicitly set to zero
		 */
		if (unlikely(wall_time > (2 * sampling_rate) &&
			     j_cdbs->prev_load)) {
			load = j_cdbs->prev_load;

			/*
			 * Perform a destructive copy, to ensure that we copy
			 * the previous load only once, upon the first wake-up
			 * from idle.
			 */
			j_cdbs->prev_load = 0;
		} else {
			load = 100 * (wall_time - idle_time) / wall_time;
			j_cdbs->prev_load = load;
		}

		if (load > max_load)
			max_load = load;
	}

	dbs_data->cdata->gov_check_cpu(cpu, max_load);
}
EXPORT_SYMBOL_GPL(dbs_check_cpu);

static inline void __gov_queue_work(int cpu, struct dbs_data *dbs_data,
		unsigned int delay)
{
	struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);

	mod_delayed_work_on(cpu, system_wq, &cdbs->work, delay);
}

void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy,
		unsigned int delay, bool all_cpus)
{
	int i;

	mutex_lock(&cpufreq_governor_lock);
	if (!policy->governor_enabled)
		goto out_unlock;

	if (!all_cpus) {
		/*
		 * Use raw_smp_processor_id() to avoid preemptible warnings.
		 * We know that this is only called with all_cpus == false from
		 * works that have been queued with *_work_on() functions and
		 * those works are canceled during CPU_DOWN_PREPARE so they
		 * can't possibly run on any other CPU.
		 */
		__gov_queue_work(raw_smp_processor_id(), dbs_data, delay);
	} else {
		for_each_cpu(i, policy->cpus)
			__gov_queue_work(i, dbs_data, delay);
	}

out_unlock:
	mutex_unlock(&cpufreq_governor_lock);
}
EXPORT_SYMBOL_GPL(gov_queue_work);

static inline void gov_cancel_work(struct dbs_data *dbs_data,
		struct cpufreq_policy *policy)
{
	struct cpu_dbs_common_info *cdbs;
	int i;

	for_each_cpu(i, policy->cpus) {
		cdbs = dbs_data->cdata->get_cpu_cdbs(i);
		cancel_delayed_work_sync(&cdbs->work);
	}
}

/* Will return if we need to evaluate cpu load again or not */
bool need_load_eval(struct cpu_dbs_common_info *cdbs,
		unsigned int sampling_rate)
{
	if (policy_is_shared(cdbs->cur_policy)) {
		ktime_t time_now = ktime_get();
		s64 delta_us = ktime_us_delta(time_now, cdbs->time_stamp);

		/* Do nothing if we recently have sampled */
		if (delta_us < (s64)(sampling_rate / 2))
			return false;
		else
			cdbs->time_stamp = time_now;
	}

	return true;
}
EXPORT_SYMBOL_GPL(need_load_eval);

static void set_sampling_rate(struct dbs_data *dbs_data,
		unsigned int sampling_rate)
{
	if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
		struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
		cs_tuners->sampling_rate = sampling_rate;
	} else {
		struct od_dbs_tuners *od_tuners = dbs_data->tuners;
		od_tuners->sampling_rate = sampling_rate;
	}
}

static int cpufreq_governor_init(struct cpufreq_policy *policy,
				 struct dbs_data *dbs_data,
				 struct common_dbs_data *cdata)
{
	unsigned int latency;
	int ret;

	if (dbs_data) {
		if (WARN_ON(have_governor_per_policy()))
			return -EINVAL;
		dbs_data->usage_count++;
		policy->governor_data = dbs_data;
		return 0;
	}

	dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
	if (!dbs_data)
		return -ENOMEM;

	dbs_data->cdata = cdata;
	dbs_data->usage_count = 1;

	ret = cdata->init(dbs_data, !policy->governor->initialized);
	if (ret)
		goto free_dbs_data;

	/* policy latency is in ns. Convert it to us first */
	latency = policy->cpuinfo.transition_latency / 1000;
	if (latency == 0)
		latency = 1;

	/* Bring kernel and HW constraints together */
	dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
					  MIN_LATENCY_MULTIPLIER * latency);
	set_sampling_rate(dbs_data, max(dbs_data->min_sampling_rate,
					latency * LATENCY_MULTIPLIER));

	if (!have_governor_per_policy()) {
		if (WARN_ON(cpufreq_get_global_kobject())) {
			ret = -EINVAL;
			goto cdata_exit;
		}
		cdata->gdbs_data = dbs_data;
	}

	ret = sysfs_create_group(get_governor_parent_kobj(policy),
				 get_sysfs_attr(dbs_data));
	if (ret)
		goto put_kobj;

	policy->governor_data = dbs_data;

	return 0;

put_kobj:
	if (!have_governor_per_policy()) {
		cdata->gdbs_data = NULL;
		cpufreq_put_global_kobject();
	}
cdata_exit:
	cdata->exit(dbs_data, !policy->governor->initialized);
free_dbs_data:
	kfree(dbs_data);
	return ret;
}

static void cpufreq_governor_exit(struct cpufreq_policy *policy,
				  struct dbs_data *dbs_data)
{
	struct common_dbs_data *cdata = dbs_data->cdata;

	policy->governor_data = NULL;
	if (!--dbs_data->usage_count) {
		sysfs_remove_group(get_governor_parent_kobj(policy),
				   get_sysfs_attr(dbs_data));

		if (!have_governor_per_policy()) {
			cdata->gdbs_data = NULL;
			cpufreq_put_global_kobject();
		}

		cdata->exit(dbs_data, policy->governor->initialized == 1);
		kfree(dbs_data);
	}
}

static int cpufreq_governor_start(struct cpufreq_policy *policy,
				  struct dbs_data *dbs_data)
{
	struct common_dbs_data *cdata = dbs_data->cdata;
	unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu;
	struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu);
	int io_busy = 0;

	if (!policy->cur)
		return -EINVAL;

	if (cdata->governor == GOV_CONSERVATIVE) {
		struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;

		sampling_rate = cs_tuners->sampling_rate;
		ignore_nice = cs_tuners->ignore_nice_load;
	} else {
		struct od_dbs_tuners *od_tuners = dbs_data->tuners;

		sampling_rate = od_tuners->sampling_rate;
		ignore_nice = od_tuners->ignore_nice_load;
		io_busy = od_tuners->io_is_busy;
	}

	mutex_lock(&dbs_data->mutex);

	for_each_cpu(j, policy->cpus) {
		struct cpu_dbs_common_info *j_cdbs = cdata->get_cpu_cdbs(j);
		unsigned int prev_load;

		j_cdbs->cpu = j;
		j_cdbs->cur_policy = policy;
		j_cdbs->prev_cpu_idle =
			get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy);

		prev_load = (unsigned int)(j_cdbs->prev_cpu_wall -
					    j_cdbs->prev_cpu_idle);
		j_cdbs->prev_load = 100 * prev_load /
				    (unsigned int)j_cdbs->prev_cpu_wall;

		if (ignore_nice)
			j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];

		mutex_init(&j_cdbs->timer_mutex);
		INIT_DEFERRABLE_WORK(&j_cdbs->work, cdata->gov_dbs_timer);
	}

	if (cdata->governor == GOV_CONSERVATIVE) {
		struct cs_cpu_dbs_info_s *cs_dbs_info =
			cdata->get_cpu_dbs_info_s(cpu);

		cs_dbs_info->down_skip = 0;
		cs_dbs_info->enable = 1;
		cs_dbs_info->requested_freq = policy->cur;
	} else {
		struct od_ops *od_ops = cdata->gov_ops;
		struct od_cpu_dbs_info_s *od_dbs_info = cdata->get_cpu_dbs_info_s(cpu);

		od_dbs_info->rate_mult = 1;
		od_dbs_info->sample_type = OD_NORMAL_SAMPLE;
		od_ops->powersave_bias_init_cpu(cpu);
	}

	mutex_unlock(&dbs_data->mutex);

	/* Initiate timer time stamp */
	cpu_cdbs->time_stamp = ktime_get();

	gov_queue_work(dbs_data, policy, delay_for_sampling_rate(sampling_rate),
		       true);
	return 0;
}

static void cpufreq_governor_stop(struct cpufreq_policy *policy,
				  struct dbs_data *dbs_data)
{
	struct common_dbs_data *cdata = dbs_data->cdata;
	unsigned int cpu = policy->cpu;
	struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu);

	if (cdata->governor == GOV_CONSERVATIVE) {
		struct cs_cpu_dbs_info_s *cs_dbs_info =
			cdata->get_cpu_dbs_info_s(cpu);

		cs_dbs_info->enable = 0;
	}

	gov_cancel_work(dbs_data, policy);

	mutex_lock(&dbs_data->mutex);
	mutex_destroy(&cpu_cdbs->timer_mutex);
	cpu_cdbs->cur_policy = NULL;
	mutex_unlock(&dbs_data->mutex);
}

static void cpufreq_governor_limits(struct cpufreq_policy *policy,
				    struct dbs_data *dbs_data)
{
	struct common_dbs_data *cdata = dbs_data->cdata;
	unsigned int cpu = policy->cpu;
	struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu);

	mutex_lock(&dbs_data->mutex);
	if (!cpu_cdbs->cur_policy) {
		mutex_unlock(&dbs_data->mutex);
		return;
	}

	mutex_lock(&cpu_cdbs->timer_mutex);
	if (policy->max < cpu_cdbs->cur_policy->cur)
		__cpufreq_driver_target(cpu_cdbs->cur_policy, policy->max,
					CPUFREQ_RELATION_H);
	else if (policy->min > cpu_cdbs->cur_policy->cur)
		__cpufreq_driver_target(cpu_cdbs->cur_policy, policy->min,
					CPUFREQ_RELATION_L);
	dbs_check_cpu(dbs_data, cpu);
	mutex_unlock(&cpu_cdbs->timer_mutex);

	mutex_unlock(&dbs_data->mutex);
}

int cpufreq_governor_dbs(struct cpufreq_policy *policy,
			 struct common_dbs_data *cdata, unsigned int event)
{
	struct dbs_data *dbs_data;
	int ret = 0;

	if (have_governor_per_policy())
		dbs_data = policy->governor_data;
	else
		dbs_data = cdata->gdbs_data;

	WARN_ON(!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT));

	switch (event) {
	case CPUFREQ_GOV_POLICY_INIT:
		ret = cpufreq_governor_init(policy, dbs_data, cdata);
		break;
	case CPUFREQ_GOV_POLICY_EXIT:
		cpufreq_governor_exit(policy, dbs_data);
		break;
	case CPUFREQ_GOV_START:
		ret = cpufreq_governor_start(policy, dbs_data);
		break;
	case CPUFREQ_GOV_STOP:
		cpufreq_governor_stop(policy, dbs_data);
		break;
	case CPUFREQ_GOV_LIMITS:
		cpufreq_governor_limits(policy, dbs_data);
		break;
	}

	return ret;
}
EXPORT_SYMBOL_GPL(cpufreq_governor_dbs);
Commit	Line	Data
2aacdfff	1	/*
	2	* drivers/cpufreq/cpufreq_governor.c
	3	*
	4	* CPUFREQ governors common code
	5	*
4471a34f VK	6	* Copyright (C) 2001 Russell King
	7	* (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
	8	* (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
	9	* (C) 2009 Alexander Clouter <alex@digriz.org.uk>
	10	* (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
	11	*
2aacdfff	12	* This program is free software; you can redistribute it and/or modify
	13	* it under the terms of the GNU General Public License version 2 as
	14	* published by the Free Software Foundation.
	15	*/
	16
4471a34f VK	17	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
4471a34f VK	18
2aacdfff	19	#include <linux/export.h>
2aacdfff	20	#include <linux/kernel_stat.h>
4d5dcc42	21	#include <linux/slab.h>
4471a34f VK	22
	23	#include "cpufreq_governor.h"
	24
4d5dcc42 VK	25	static struct attribute_group get_sysfs_attr(struct dbs_data dbs_data)
	26	{
	27	if (have_governor_per_policy())
	28	return dbs_data->cdata->attr_group_gov_pol;
	29	else
	30	return dbs_data->cdata->attr_group_gov_sys;
	31	}
	32
4471a34f VK	33	void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)
4471a34f VK	34	{
4d5dcc42	35	struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
4471a34f VK	36	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
	37	struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
	38	struct cpufreq_policy *policy;
18b46abd	39	unsigned int sampling_rate;
4471a34f VK	40	unsigned int max_load = 0;
	41	unsigned int ignore_nice;
	42	unsigned int j;
	43
18b46abd SB	44	if (dbs_data->cdata->governor == GOV_ONDEMAND) {
	45	struct od_cpu_dbs_info_s *od_dbs_info =
	46	dbs_data->cdata->get_cpu_dbs_info_s(cpu);
	47
	48	/*
	49	* Sometimes, the ondemand governor uses an additional
	50	* multiplier to give long delays. So apply this multiplier to
	51	* the 'sampling_rate', so as to keep the wake-up-from-idle
	52	* detection logic a bit conservative.
	53	*/
	54	sampling_rate = od_tuners->sampling_rate;
	55	sampling_rate *= od_dbs_info->rate_mult;
	56
6c4640c3	57	ignore_nice = od_tuners->ignore_nice_load;
18b46abd SB	58	} else {
18b46abd SB	59	sampling_rate = cs_tuners->sampling_rate;
6c4640c3	60	ignore_nice = cs_tuners->ignore_nice_load;
18b46abd	61	}
4471a34f VK	62
	63	policy = cdbs->cur_policy;
	64
dfa5bb62	65	/* Get Absolute Load */
4471a34f VK	66	for_each_cpu(j, policy->cpus) {
4471a34f VK	67	struct cpu_dbs_common_info *j_cdbs;
9366d840 SK	68	u64 cur_wall_time, cur_idle_time;
9366d840 SK	69	unsigned int idle_time, wall_time;
4471a34f	70	unsigned int load;
9366d840	71	int io_busy = 0;
4471a34f	72
4d5dcc42	73	j_cdbs = dbs_data->cdata->get_cpu_cdbs(j);
4471a34f	74
9366d840 SK	75	/*
	76	* For the purpose of ondemand, waiting for disk IO is
	77	* an indication that you're performance critical, and
	78	* not that the system is actually idle. So do not add
	79	* the iowait time to the cpu idle time.
	80	*/
	81	if (dbs_data->cdata->governor == GOV_ONDEMAND)
	82	io_busy = od_tuners->io_is_busy;
	83	cur_idle_time = get_cpu_idle_time(j, &cur_wall_time, io_busy);
4471a34f VK	84
	85	wall_time = (unsigned int)
	86	(cur_wall_time - j_cdbs->prev_cpu_wall);
	87	j_cdbs->prev_cpu_wall = cur_wall_time;
	88
	89	idle_time = (unsigned int)
	90	(cur_idle_time - j_cdbs->prev_cpu_idle);
	91	j_cdbs->prev_cpu_idle = cur_idle_time;
	92
	93	if (ignore_nice) {
	94	u64 cur_nice;
	95	unsigned long cur_nice_jiffies;
	96
	97	cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE] -
	98	cdbs->prev_cpu_nice;
	99	/*
	100	* Assumption: nice time between sampling periods will
	101	* be less than 2^32 jiffies for 32 bit sys
	102	*/
	103	cur_nice_jiffies = (unsigned long)
	104	cputime64_to_jiffies64(cur_nice);
	105
	106	cdbs->prev_cpu_nice =
	107	kcpustat_cpu(j).cpustat[CPUTIME_NICE];
	108	idle_time += jiffies_to_usecs(cur_nice_jiffies);
	109	}
	110
4471a34f VK	111	if (unlikely(!wall_time \|\| wall_time < idle_time))
	112	continue;
	113
18b46abd SB	114	/*
	115	* If the CPU had gone completely idle, and a task just woke up
	116	* on this CPU now, it would be unfair to calculate 'load' the
	117	* usual way for this elapsed time-window, because it will show
	118	* near-zero load, irrespective of how CPU intensive that task
	119	* actually is. This is undesirable for latency-sensitive bursty
	120	* workloads.
	121	*
	122	* To avoid this, we reuse the 'load' from the previous
	123	* time-window and give this task a chance to start with a
	124	* reasonably high CPU frequency. (However, we shouldn't over-do
	125	* this copy, lest we get stuck at a high load (high frequency)
	126	* for too long, even when the current system load has actually
	127	* dropped down. So we perform the copy only once, upon the
	128	* first wake-up from idle.)
	129	*
	130	* Detecting this situation is easy: the governor's deferrable
	131	* timer would not have fired during CPU-idle periods. Hence
	132	* an unusually large 'wall_time' (as compared to the sampling
	133	* rate) indicates this scenario.
c8ae481b VK	134	*
	135	* prev_load can be zero in two cases and we must recalculate it
	136	* for both cases:
	137	* - during long idle intervals
	138	* - explicitly set to zero
18b46abd	139	*/
c8ae481b VK	140	if (unlikely(wall_time > (2 * sampling_rate) &&
c8ae481b VK	141	j_cdbs->prev_load)) {
18b46abd	142	load = j_cdbs->prev_load;
c8ae481b VK	143
	144	/*
	145	* Perform a destructive copy, to ensure that we copy
	146	* the previous load only once, upon the first wake-up
	147	* from idle.
	148	*/
	149	j_cdbs->prev_load = 0;
18b46abd SB	150	} else {
	151	load = 100 * (wall_time - idle_time) / wall_time;
	152	j_cdbs->prev_load = load;
18b46abd	153	}
4471a34f	154
4471a34f VK	155	if (load > max_load)
	156	max_load = load;
	157	}
	158
4d5dcc42	159	dbs_data->cdata->gov_check_cpu(cpu, max_load);
4471a34f VK	160	}
	161	EXPORT_SYMBOL_GPL(dbs_check_cpu);
	162
031299b3 VK	163	static inline void __gov_queue_work(int cpu, struct dbs_data *dbs_data,
031299b3 VK	164	unsigned int delay)
4471a34f	165	{
4d5dcc42	166	struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);
4471a34f	167
031299b3	168	mod_delayed_work_on(cpu, system_wq, &cdbs->work, delay);
4471a34f VK	169	}
4471a34f VK	170
031299b3 VK	171	void gov_queue_work(struct dbs_data dbs_data, struct cpufreq_policy policy,
031299b3 VK	172	unsigned int delay, bool all_cpus)
4471a34f	173	{
031299b3 VK	174	int i;
031299b3 VK	175
6f1e4efd	176	mutex_lock(&cpufreq_governor_lock);
3617f2ca	177	if (!policy->governor_enabled)
6f1e4efd	178	goto out_unlock;
3617f2ca	179
031299b3	180	if (!all_cpus) {
69320783 SB	181	/*
	182	* Use raw_smp_processor_id() to avoid preemptible warnings.
	183	* We know that this is only called with all_cpus == false from
	184	* works that have been queued with *_work_on() functions and
	185	* those works are canceled during CPU_DOWN_PREPARE so they
	186	* can't possibly run on any other CPU.
	187	*/
	188	__gov_queue_work(raw_smp_processor_id(), dbs_data, delay);
031299b3 VK	189	} else {
	190	for_each_cpu(i, policy->cpus)
	191	__gov_queue_work(i, dbs_data, delay);
	192	}
6f1e4efd JL	193
	194	out_unlock:
	195	mutex_unlock(&cpufreq_governor_lock);
031299b3 VK	196	}
	197	EXPORT_SYMBOL_GPL(gov_queue_work);
	198
	199	static inline void gov_cancel_work(struct dbs_data *dbs_data,
	200	struct cpufreq_policy *policy)
	201	{
	202	struct cpu_dbs_common_info *cdbs;
	203	int i;
58ddcead	204
031299b3 VK	205	for_each_cpu(i, policy->cpus) {
	206	cdbs = dbs_data->cdata->get_cpu_cdbs(i);
	207	cancel_delayed_work_sync(&cdbs->work);
	208	}
4471a34f VK	209	}
4471a34f VK	210
4447266b VK	211	/* Will return if we need to evaluate cpu load again or not */
	212	bool need_load_eval(struct cpu_dbs_common_info *cdbs,
	213	unsigned int sampling_rate)
	214	{
	215	if (policy_is_shared(cdbs->cur_policy)) {
	216	ktime_t time_now = ktime_get();
	217	s64 delta_us = ktime_us_delta(time_now, cdbs->time_stamp);
	218
	219	/* Do nothing if we recently have sampled */
	220	if (delta_us < (s64)(sampling_rate / 2))
	221	return false;
	222	else
	223	cdbs->time_stamp = time_now;
	224	}
	225
	226	return true;
	227	}
	228	EXPORT_SYMBOL_GPL(need_load_eval);
	229
4d5dcc42 VK	230	static void set_sampling_rate(struct dbs_data *dbs_data,
	231	unsigned int sampling_rate)
	232	{
	233	if (dbs_data->cdata->governor == GOV_CONSERVATIVE) {
	234	struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
	235	cs_tuners->sampling_rate = sampling_rate;
	236	} else {
	237	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
	238	od_tuners->sampling_rate = sampling_rate;
	239	}
	240	}
	241
714a2d9c VK	242	static int cpufreq_governor_init(struct cpufreq_policy *policy,
	243	struct dbs_data *dbs_data,
	244	struct common_dbs_data *cdata)
4471a34f	245	{
714a2d9c VK	246	unsigned int latency;
714a2d9c VK	247	int ret;
4471a34f	248
714a2d9c VK	249	if (dbs_data) {
	250	if (WARN_ON(have_governor_per_policy()))
	251	return -EINVAL;
	252	dbs_data->usage_count++;
	253	policy->governor_data = dbs_data;
	254	return 0;
	255	}
4d5dcc42	256
714a2d9c VK	257	dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
	258	if (!dbs_data)
	259	return -ENOMEM;
4d5dcc42	260
714a2d9c VK	261	dbs_data->cdata = cdata;
714a2d9c VK	262	dbs_data->usage_count = 1;
4d5dcc42	263
714a2d9c VK	264	ret = cdata->init(dbs_data, !policy->governor->initialized);
	265	if (ret)
	266	goto free_dbs_data;
4d5dcc42	267
714a2d9c VK	268	/* policy latency is in ns. Convert it to us first */
	269	latency = policy->cpuinfo.transition_latency / 1000;
	270	if (latency == 0)
	271	latency = 1;
4d5dcc42	272
714a2d9c VK	273	/* Bring kernel and HW constraints together */
	274	dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
	275	MIN_LATENCY_MULTIPLIER * latency);
	276	set_sampling_rate(dbs_data, max(dbs_data->min_sampling_rate,
	277	latency * LATENCY_MULTIPLIER));
2361be23	278
714a2d9c VK	279	if (!have_governor_per_policy()) {
	280	if (WARN_ON(cpufreq_get_global_kobject())) {
	281	ret = -EINVAL;
	282	goto cdata_exit;
4d5dcc42	283	}
714a2d9c VK	284	cdata->gdbs_data = dbs_data;
714a2d9c VK	285	}
4d5dcc42	286
714a2d9c VK	287	ret = sysfs_create_group(get_governor_parent_kobj(policy),
	288	get_sysfs_attr(dbs_data));
	289	if (ret)
	290	goto put_kobj;
4d5dcc42	291
714a2d9c	292	policy->governor_data = dbs_data;
4d5dcc42	293
714a2d9c	294	return 0;
4d5dcc42	295
714a2d9c VK	296	put_kobj:
	297	if (!have_governor_per_policy()) {
	298	cdata->gdbs_data = NULL;
	299	cpufreq_put_global_kobject();
	300	}
	301	cdata_exit:
	302	cdata->exit(dbs_data, !policy->governor->initialized);
	303	free_dbs_data:
	304	kfree(dbs_data);
	305	return ret;
	306	}
4d5dcc42	307
714a2d9c VK	308	static void cpufreq_governor_exit(struct cpufreq_policy *policy,
	309	struct dbs_data *dbs_data)
	310	{
	311	struct common_dbs_data *cdata = dbs_data->cdata;
4d5dcc42	312
714a2d9c VK	313	policy->governor_data = NULL;
	314	if (!--dbs_data->usage_count) {
	315	sysfs_remove_group(get_governor_parent_kobj(policy),
	316	get_sysfs_attr(dbs_data));
2361be23	317
714a2d9c	318	if (!have_governor_per_policy()) {
4d5dcc42	319	cdata->gdbs_data = NULL;
714a2d9c	320	cpufreq_put_global_kobject();
4d5dcc42	321	}
4471a34f	322
714a2d9c VK	323	cdata->exit(dbs_data, policy->governor->initialized == 1);
714a2d9c VK	324	kfree(dbs_data);
4d5dcc42	325	}
714a2d9c	326	}
4d5dcc42	327
714a2d9c VK	328	static int cpufreq_governor_start(struct cpufreq_policy *policy,
	329	struct dbs_data *dbs_data)
	330	{
	331	struct common_dbs_data *cdata = dbs_data->cdata;
	332	unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu;
	333	struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu);
	334	int io_busy = 0;
	335
	336	if (!policy->cur)
	337	return -EINVAL;
	338
	339	if (cdata->governor == GOV_CONSERVATIVE) {
	340	struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
4d5dcc42	341
4d5dcc42	342	sampling_rate = cs_tuners->sampling_rate;
6c4640c3	343	ignore_nice = cs_tuners->ignore_nice_load;
4471a34f	344	} else {
714a2d9c VK	345	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
714a2d9c VK	346
4d5dcc42	347	sampling_rate = od_tuners->sampling_rate;
6c4640c3	348	ignore_nice = od_tuners->ignore_nice_load;
9366d840	349	io_busy = od_tuners->io_is_busy;
4471a34f VK	350	}
4471a34f VK	351
714a2d9c	352	mutex_lock(&dbs_data->mutex);
4471a34f	353
714a2d9c VK	354	for_each_cpu(j, policy->cpus) {
	355	struct cpu_dbs_common_info *j_cdbs = cdata->get_cpu_cdbs(j);
	356	unsigned int prev_load;
4471a34f	357
714a2d9c VK	358	j_cdbs->cpu = j;
	359	j_cdbs->cur_policy = policy;
	360	j_cdbs->prev_cpu_idle =
	361	get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy);
4471a34f	362
714a2d9c VK	363	prev_load = (unsigned int)(j_cdbs->prev_cpu_wall -
	364	j_cdbs->prev_cpu_idle);
	365	j_cdbs->prev_load = 100 * prev_load /
	366	(unsigned int)j_cdbs->prev_cpu_wall;
18b46abd	367
714a2d9c VK	368	if (ignore_nice)
714a2d9c VK	369	j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
18b46abd	370
714a2d9c VK	371	mutex_init(&j_cdbs->timer_mutex);
	372	INIT_DEFERRABLE_WORK(&j_cdbs->work, cdata->gov_dbs_timer);
	373	}
2abfa876	374
714a2d9c VK	375	if (cdata->governor == GOV_CONSERVATIVE) {
	376	struct cs_cpu_dbs_info_s *cs_dbs_info =
	377	cdata->get_cpu_dbs_info_s(cpu);
4471a34f	378
714a2d9c VK	379	cs_dbs_info->down_skip = 0;
	380	cs_dbs_info->enable = 1;
	381	cs_dbs_info->requested_freq = policy->cur;
	382	} else {
	383	struct od_ops *od_ops = cdata->gov_ops;
	384	struct od_cpu_dbs_info_s *od_dbs_info = cdata->get_cpu_dbs_info_s(cpu);
4471a34f	385
714a2d9c VK	386	od_dbs_info->rate_mult = 1;
	387	od_dbs_info->sample_type = OD_NORMAL_SAMPLE;
	388	od_ops->powersave_bias_init_cpu(cpu);
	389	}
4471a34f	390
714a2d9c	391	mutex_unlock(&dbs_data->mutex);
da53d61e	392
714a2d9c VK	393	/* Initiate timer time stamp */
714a2d9c VK	394	cpu_cdbs->time_stamp = ktime_get();
4471a34f	395
714a2d9c VK	396	gov_queue_work(dbs_data, policy, delay_for_sampling_rate(sampling_rate),
	397	true);
	398	return 0;
	399	}
	400
	401	static void cpufreq_governor_stop(struct cpufreq_policy *policy,
	402	struct dbs_data *dbs_data)
	403	{
	404	struct common_dbs_data *cdata = dbs_data->cdata;
	405	unsigned int cpu = policy->cpu;
	406	struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu);
	407
	408	if (cdata->governor == GOV_CONSERVATIVE) {
	409	struct cs_cpu_dbs_info_s *cs_dbs_info =
	410	cdata->get_cpu_dbs_info_s(cpu);
4471a34f	411
714a2d9c VK	412	cs_dbs_info->enable = 0;
	413	}
	414
	415	gov_cancel_work(dbs_data, policy);
	416
	417	mutex_lock(&dbs_data->mutex);
	418	mutex_destroy(&cpu_cdbs->timer_mutex);
	419	cpu_cdbs->cur_policy = NULL;
	420	mutex_unlock(&dbs_data->mutex);
	421	}
4471a34f	422
714a2d9c VK	423	static void cpufreq_governor_limits(struct cpufreq_policy *policy,
	424	struct dbs_data *dbs_data)
	425	{
	426	struct common_dbs_data *cdata = dbs_data->cdata;
	427	unsigned int cpu = policy->cpu;
	428	struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu);
8eeed095	429
714a2d9c VK	430	mutex_lock(&dbs_data->mutex);
714a2d9c VK	431	if (!cpu_cdbs->cur_policy) {
4471a34f	432	mutex_unlock(&dbs_data->mutex);
714a2d9c VK	433	return;
714a2d9c VK	434	}
4471a34f	435
714a2d9c VK	436	mutex_lock(&cpu_cdbs->timer_mutex);
	437	if (policy->max < cpu_cdbs->cur_policy->cur)
	438	__cpufreq_driver_target(cpu_cdbs->cur_policy, policy->max,
	439	CPUFREQ_RELATION_H);
	440	else if (policy->min > cpu_cdbs->cur_policy->cur)
	441	__cpufreq_driver_target(cpu_cdbs->cur_policy, policy->min,
	442	CPUFREQ_RELATION_L);
	443	dbs_check_cpu(dbs_data, cpu);
	444	mutex_unlock(&cpu_cdbs->timer_mutex);
	445
	446	mutex_unlock(&dbs_data->mutex);
	447	}
4471a34f	448
714a2d9c VK	449	int cpufreq_governor_dbs(struct cpufreq_policy *policy,
	450	struct common_dbs_data *cdata, unsigned int event)
	451	{
	452	struct dbs_data *dbs_data;
	453	int ret = 0;
	454
	455	if (have_governor_per_policy())
	456	dbs_data = policy->governor_data;
	457	else
	458	dbs_data = cdata->gdbs_data;
	459
	460	WARN_ON(!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT));
	461
	462	switch (event) {
	463	case CPUFREQ_GOV_POLICY_INIT:
	464	ret = cpufreq_governor_init(policy, dbs_data, cdata);
	465	break;
	466	case CPUFREQ_GOV_POLICY_EXIT:
	467	cpufreq_governor_exit(policy, dbs_data);
	468	break;
	469	case CPUFREQ_GOV_START:
	470	ret = cpufreq_governor_start(policy, dbs_data);
	471	break;
	472	case CPUFREQ_GOV_STOP:
	473	cpufreq_governor_stop(policy, dbs_data);
	474	break;
4471a34f	475	case CPUFREQ_GOV_LIMITS:
714a2d9c	476	cpufreq_governor_limits(policy, dbs_data);
4471a34f VK	477	break;
4471a34f VK	478	}
714a2d9c VK	479
714a2d9c VK	480	return ret;
4471a34f VK	481	}
4471a34f VK	482	EXPORT_SYMBOL_GPL(cpufreq_governor_dbs);