2 * drivers/cpufreq/cpufreq_governor.c
4 * CPUFREQ governors common code
6 * Copyright (C) 2001 Russell King
7 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
8 * (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
9 * (C) 2009 Alexander Clouter <alex@digriz.org.uk>
10 * (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 #include <asm/cputime.h>
20 #include <linux/cpufreq.h>
21 #include <linux/cpumask.h>
22 #include <linux/export.h>
23 #include <linux/kernel_stat.h>
24 #include <linux/mutex.h>
25 #include <linux/slab.h>
26 #include <linux/tick.h>
27 #include <linux/types.h>
28 #include <linux/workqueue.h>
30 #include "cpufreq_governor.h"
32 static struct kobject
*get_governor_parent_kobj(struct cpufreq_policy
*policy
)
34 if (have_governor_per_policy())
37 return cpufreq_global_kobject
;
40 static struct attribute_group
*get_sysfs_attr(struct dbs_data
*dbs_data
)
42 if (have_governor_per_policy())
43 return dbs_data
->cdata
->attr_group_gov_pol
;
45 return dbs_data
->cdata
->attr_group_gov_sys
;
48 static inline u64
get_cpu_idle_time_jiffy(unsigned int cpu
, u64
*wall
)
54 cur_wall_time
= jiffies64_to_cputime64(get_jiffies_64());
56 busy_time
= kcpustat_cpu(cpu
).cpustat
[CPUTIME_USER
];
57 busy_time
+= kcpustat_cpu(cpu
).cpustat
[CPUTIME_SYSTEM
];
58 busy_time
+= kcpustat_cpu(cpu
).cpustat
[CPUTIME_IRQ
];
59 busy_time
+= kcpustat_cpu(cpu
).cpustat
[CPUTIME_SOFTIRQ
];
60 busy_time
+= kcpustat_cpu(cpu
).cpustat
[CPUTIME_STEAL
];
61 busy_time
+= kcpustat_cpu(cpu
).cpustat
[CPUTIME_NICE
];
63 idle_time
= cur_wall_time
- busy_time
;
65 *wall
= cputime_to_usecs(cur_wall_time
);
67 return cputime_to_usecs(idle_time
);
70 u64
get_cpu_idle_time(unsigned int cpu
, u64
*wall
)
72 u64 idle_time
= get_cpu_idle_time_us(cpu
, NULL
);
74 if (idle_time
== -1ULL)
75 return get_cpu_idle_time_jiffy(cpu
, wall
);
77 idle_time
+= get_cpu_iowait_time_us(cpu
, wall
);
81 EXPORT_SYMBOL_GPL(get_cpu_idle_time
);
83 void dbs_check_cpu(struct dbs_data
*dbs_data
, int cpu
)
85 struct cpu_dbs_common_info
*cdbs
= dbs_data
->cdata
->get_cpu_cdbs(cpu
);
86 struct od_dbs_tuners
*od_tuners
= dbs_data
->tuners
;
87 struct cs_dbs_tuners
*cs_tuners
= dbs_data
->tuners
;
88 struct cpufreq_policy
*policy
;
89 unsigned int max_load
= 0;
90 unsigned int ignore_nice
;
93 if (dbs_data
->cdata
->governor
== GOV_ONDEMAND
)
94 ignore_nice
= od_tuners
->ignore_nice
;
96 ignore_nice
= cs_tuners
->ignore_nice
;
98 policy
= cdbs
->cur_policy
;
100 /* Get Absolute Load (in terms of freq for ondemand gov) */
101 for_each_cpu(j
, policy
->cpus
) {
102 struct cpu_dbs_common_info
*j_cdbs
;
103 u64 cur_wall_time
, cur_idle_time
, cur_iowait_time
;
104 unsigned int idle_time
, wall_time
, iowait_time
;
107 j_cdbs
= dbs_data
->cdata
->get_cpu_cdbs(j
);
109 cur_idle_time
= get_cpu_idle_time(j
, &cur_wall_time
);
111 wall_time
= (unsigned int)
112 (cur_wall_time
- j_cdbs
->prev_cpu_wall
);
113 j_cdbs
->prev_cpu_wall
= cur_wall_time
;
115 idle_time
= (unsigned int)
116 (cur_idle_time
- j_cdbs
->prev_cpu_idle
);
117 j_cdbs
->prev_cpu_idle
= cur_idle_time
;
121 unsigned long cur_nice_jiffies
;
123 cur_nice
= kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
] -
126 * Assumption: nice time between sampling periods will
127 * be less than 2^32 jiffies for 32 bit sys
129 cur_nice_jiffies
= (unsigned long)
130 cputime64_to_jiffies64(cur_nice
);
132 cdbs
->prev_cpu_nice
=
133 kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
];
134 idle_time
+= jiffies_to_usecs(cur_nice_jiffies
);
137 if (dbs_data
->cdata
->governor
== GOV_ONDEMAND
) {
138 struct od_cpu_dbs_info_s
*od_j_dbs_info
=
139 dbs_data
->cdata
->get_cpu_dbs_info_s(cpu
);
141 cur_iowait_time
= get_cpu_iowait_time_us(j
,
143 if (cur_iowait_time
== -1ULL)
146 iowait_time
= (unsigned int) (cur_iowait_time
-
147 od_j_dbs_info
->prev_cpu_iowait
);
148 od_j_dbs_info
->prev_cpu_iowait
= cur_iowait_time
;
151 * For the purpose of ondemand, waiting for disk IO is
152 * an indication that you're performance critical, and
153 * not that the system is actually idle. So subtract the
154 * iowait time from the cpu idle time.
156 if (od_tuners
->io_is_busy
&& idle_time
>= iowait_time
)
157 idle_time
-= iowait_time
;
160 if (unlikely(!wall_time
|| wall_time
< idle_time
))
163 load
= 100 * (wall_time
- idle_time
) / wall_time
;
165 if (dbs_data
->cdata
->governor
== GOV_ONDEMAND
) {
166 int freq_avg
= __cpufreq_driver_getavg(policy
, j
);
168 freq_avg
= policy
->cur
;
177 dbs_data
->cdata
->gov_check_cpu(cpu
, max_load
);
179 EXPORT_SYMBOL_GPL(dbs_check_cpu
);
181 static inline void __gov_queue_work(int cpu
, struct dbs_data
*dbs_data
,
184 struct cpu_dbs_common_info
*cdbs
= dbs_data
->cdata
->get_cpu_cdbs(cpu
);
186 mod_delayed_work_on(cpu
, system_wq
, &cdbs
->work
, delay
);
189 void gov_queue_work(struct dbs_data
*dbs_data
, struct cpufreq_policy
*policy
,
190 unsigned int delay
, bool all_cpus
)
195 __gov_queue_work(smp_processor_id(), dbs_data
, delay
);
197 for_each_cpu(i
, policy
->cpus
)
198 __gov_queue_work(i
, dbs_data
, delay
);
201 EXPORT_SYMBOL_GPL(gov_queue_work
);
203 static inline void gov_cancel_work(struct dbs_data
*dbs_data
,
204 struct cpufreq_policy
*policy
)
206 struct cpu_dbs_common_info
*cdbs
;
209 for_each_cpu(i
, policy
->cpus
) {
210 cdbs
= dbs_data
->cdata
->get_cpu_cdbs(i
);
211 cancel_delayed_work_sync(&cdbs
->work
);
215 /* Will return if we need to evaluate cpu load again or not */
216 bool need_load_eval(struct cpu_dbs_common_info
*cdbs
,
217 unsigned int sampling_rate
)
219 if (policy_is_shared(cdbs
->cur_policy
)) {
220 ktime_t time_now
= ktime_get();
221 s64 delta_us
= ktime_us_delta(time_now
, cdbs
->time_stamp
);
223 /* Do nothing if we recently have sampled */
224 if (delta_us
< (s64
)(sampling_rate
/ 2))
227 cdbs
->time_stamp
= time_now
;
232 EXPORT_SYMBOL_GPL(need_load_eval
);
234 static void set_sampling_rate(struct dbs_data
*dbs_data
,
235 unsigned int sampling_rate
)
237 if (dbs_data
->cdata
->governor
== GOV_CONSERVATIVE
) {
238 struct cs_dbs_tuners
*cs_tuners
= dbs_data
->tuners
;
239 cs_tuners
->sampling_rate
= sampling_rate
;
241 struct od_dbs_tuners
*od_tuners
= dbs_data
->tuners
;
242 od_tuners
->sampling_rate
= sampling_rate
;
246 int cpufreq_governor_dbs(struct cpufreq_policy
*policy
,
247 struct common_dbs_data
*cdata
, unsigned int event
)
249 struct dbs_data
*dbs_data
;
250 struct od_cpu_dbs_info_s
*od_dbs_info
= NULL
;
251 struct cs_cpu_dbs_info_s
*cs_dbs_info
= NULL
;
252 struct od_ops
*od_ops
= NULL
;
253 struct od_dbs_tuners
*od_tuners
= NULL
;
254 struct cs_dbs_tuners
*cs_tuners
= NULL
;
255 struct cpu_dbs_common_info
*cpu_cdbs
;
256 unsigned int sampling_rate
, latency
, ignore_nice
, j
, cpu
= policy
->cpu
;
259 if (have_governor_per_policy())
260 dbs_data
= policy
->governor_data
;
262 dbs_data
= cdata
->gdbs_data
;
264 WARN_ON(!dbs_data
&& (event
!= CPUFREQ_GOV_POLICY_INIT
));
267 case CPUFREQ_GOV_POLICY_INIT
:
268 if (have_governor_per_policy()) {
270 } else if (dbs_data
) {
271 policy
->governor_data
= dbs_data
;
275 dbs_data
= kzalloc(sizeof(*dbs_data
), GFP_KERNEL
);
277 pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__
);
281 dbs_data
->cdata
= cdata
;
282 rc
= cdata
->init(dbs_data
);
284 pr_err("%s: POLICY_INIT: init() failed\n", __func__
);
289 rc
= sysfs_create_group(get_governor_parent_kobj(policy
),
290 get_sysfs_attr(dbs_data
));
292 cdata
->exit(dbs_data
);
297 policy
->governor_data
= dbs_data
;
299 /* policy latency is in nS. Convert it to uS first */
300 latency
= policy
->cpuinfo
.transition_latency
/ 1000;
304 /* Bring kernel and HW constraints together */
305 dbs_data
->min_sampling_rate
= max(dbs_data
->min_sampling_rate
,
306 MIN_LATENCY_MULTIPLIER
* latency
);
307 set_sampling_rate(dbs_data
, max(dbs_data
->min_sampling_rate
,
308 latency
* LATENCY_MULTIPLIER
));
310 if (dbs_data
->cdata
->governor
== GOV_CONSERVATIVE
) {
311 struct cs_ops
*cs_ops
= dbs_data
->cdata
->gov_ops
;
313 cpufreq_register_notifier(cs_ops
->notifier_block
,
314 CPUFREQ_TRANSITION_NOTIFIER
);
317 if (!have_governor_per_policy())
318 cdata
->gdbs_data
= dbs_data
;
321 case CPUFREQ_GOV_POLICY_EXIT
:
322 if ((policy
->governor
->initialized
== 1) ||
323 have_governor_per_policy()) {
324 sysfs_remove_group(get_governor_parent_kobj(policy
),
325 get_sysfs_attr(dbs_data
));
327 if (dbs_data
->cdata
->governor
== GOV_CONSERVATIVE
) {
328 struct cs_ops
*cs_ops
= dbs_data
->cdata
->gov_ops
;
330 cpufreq_unregister_notifier(cs_ops
->notifier_block
,
331 CPUFREQ_TRANSITION_NOTIFIER
);
334 cdata
->exit(dbs_data
);
336 cdata
->gdbs_data
= NULL
;
339 policy
->governor_data
= NULL
;
343 cpu_cdbs
= dbs_data
->cdata
->get_cpu_cdbs(cpu
);
345 if (dbs_data
->cdata
->governor
== GOV_CONSERVATIVE
) {
346 cs_tuners
= dbs_data
->tuners
;
347 cs_dbs_info
= dbs_data
->cdata
->get_cpu_dbs_info_s(cpu
);
348 sampling_rate
= cs_tuners
->sampling_rate
;
349 ignore_nice
= cs_tuners
->ignore_nice
;
351 od_tuners
= dbs_data
->tuners
;
352 od_dbs_info
= dbs_data
->cdata
->get_cpu_dbs_info_s(cpu
);
353 sampling_rate
= od_tuners
->sampling_rate
;
354 ignore_nice
= od_tuners
->ignore_nice
;
355 od_ops
= dbs_data
->cdata
->gov_ops
;
359 case CPUFREQ_GOV_START
:
363 mutex_lock(&dbs_data
->mutex
);
365 for_each_cpu(j
, policy
->cpus
) {
366 struct cpu_dbs_common_info
*j_cdbs
=
367 dbs_data
->cdata
->get_cpu_cdbs(j
);
370 j_cdbs
->cur_policy
= policy
;
371 j_cdbs
->prev_cpu_idle
= get_cpu_idle_time(j
,
372 &j_cdbs
->prev_cpu_wall
);
374 j_cdbs
->prev_cpu_nice
=
375 kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
];
377 mutex_init(&j_cdbs
->timer_mutex
);
378 INIT_DEFERRABLE_WORK(&j_cdbs
->work
,
379 dbs_data
->cdata
->gov_dbs_timer
);
383 * conservative does not implement micro like ondemand
384 * governor, thus we are bound to jiffes/HZ
386 if (dbs_data
->cdata
->governor
== GOV_CONSERVATIVE
) {
387 cs_dbs_info
->down_skip
= 0;
388 cs_dbs_info
->enable
= 1;
389 cs_dbs_info
->requested_freq
= policy
->cur
;
391 od_dbs_info
->rate_mult
= 1;
392 od_dbs_info
->sample_type
= OD_NORMAL_SAMPLE
;
393 od_ops
->powersave_bias_init_cpu(cpu
);
396 mutex_unlock(&dbs_data
->mutex
);
398 /* Initiate timer time stamp */
399 cpu_cdbs
->time_stamp
= ktime_get();
401 gov_queue_work(dbs_data
, policy
,
402 delay_for_sampling_rate(sampling_rate
), true);
405 case CPUFREQ_GOV_STOP
:
406 if (dbs_data
->cdata
->governor
== GOV_CONSERVATIVE
)
407 cs_dbs_info
->enable
= 0;
409 gov_cancel_work(dbs_data
, policy
);
411 mutex_lock(&dbs_data
->mutex
);
412 mutex_destroy(&cpu_cdbs
->timer_mutex
);
414 mutex_unlock(&dbs_data
->mutex
);
418 case CPUFREQ_GOV_LIMITS
:
419 mutex_lock(&cpu_cdbs
->timer_mutex
);
420 if (policy
->max
< cpu_cdbs
->cur_policy
->cur
)
421 __cpufreq_driver_target(cpu_cdbs
->cur_policy
,
422 policy
->max
, CPUFREQ_RELATION_H
);
423 else if (policy
->min
> cpu_cdbs
->cur_policy
->cur
)
424 __cpufreq_driver_target(cpu_cdbs
->cur_policy
,
425 policy
->min
, CPUFREQ_RELATION_L
);
426 dbs_check_cpu(dbs_data
, cpu
);
427 mutex_unlock(&cpu_cdbs
->timer_mutex
);
432 EXPORT_SYMBOL_GPL(cpufreq_governor_dbs
);