From 5f98ced1c95e7706af6895f7b7b0d2216f075d59 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 9 Mar 2017 16:30:38 +0100 Subject: [PATCH 01/37] cpufreq: intel_pstate: Drop redundant wrapper function intel_pstate_hwp_set_policy() is a wrapper around intel_pstate_hwp_set(), but the only value it adds is to check hwp_active before calling the latter and one of its two callers has already checked hwp_active before that happens, so in that code path the additional check is redundant and using the wrapper is rather pointless. For this reason, drop intel_pstate_hwp_set_policy() and make its callers invoke intel_pstate_hwp_set() directly (after checking hwp_active). Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- drivers/cpufreq/intel_pstate.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 3d37219a0dd7..162657228c15 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -930,14 +930,6 @@ skip_epp: } } -static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy) -{ - if (hwp_active) - intel_pstate_hwp_set(policy); - - return 0; -} - static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) { struct cpudata *cpu_data = all_cpu_data[policy->cpu]; @@ -952,20 +944,17 @@ static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) static int intel_pstate_resume(struct cpufreq_policy *policy) { - int ret; - if (!hwp_active) return 0; mutex_lock(&intel_pstate_limits_lock); all_cpu_data[policy->cpu]->epp_policy = 0; - - ret = intel_pstate_hwp_set_policy(policy); + intel_pstate_hwp_set(policy); mutex_unlock(&intel_pstate_limits_lock); - return ret; + return 0; } static void intel_pstate_update_policies(void) @@ -2169,7 +2158,8 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) intel_pstate_set_update_util_hook(policy->cpu); - intel_pstate_hwp_set_policy(policy); + if (hwp_active) + intel_pstate_hwp_set(policy); mutex_unlock(&intel_pstate_limits_lock); From 08a74cbb1ba337fca6add5674506440c044b2c03 Mon Sep 17 00:00:00 2001 From: Daniel Kurtz Date: Thu, 2 Mar 2017 19:08:58 +0800 Subject: [PATCH 02/37] cpufreq: mt8173: Mark mt8173_cpufreq_driver_init as __init This function is only called once at boot by device_initcall(), so mark it as __init. Signed-off-by: Daniel Kurtz Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/mt8173-cpufreq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/mt8173-cpufreq.c b/drivers/cpufreq/mt8173-cpufreq.c index ab25b1235a5e..72bc1192bd30 100644 --- a/drivers/cpufreq/mt8173-cpufreq.c +++ b/drivers/cpufreq/mt8173-cpufreq.c @@ -573,7 +573,7 @@ static struct platform_driver mt8173_cpufreq_platdrv = { .probe = mt8173_cpufreq_probe, }; -static int mt8173_cpufreq_driver_init(void) +static int __init mt8173_cpufreq_driver_init(void) { struct platform_device *pdev; int err; From cf9a2438257da2cbc55f82085b2e0add7583cf79 Mon Sep 17 00:00:00 2001 From: Daniel Kurtz Date: Thu, 2 Mar 2017 19:03:45 +0800 Subject: [PATCH 03/37] cpufreq: mediatek: Add support for MT8176 and MT817x The Mediatek MT8173 is just one of several SOCs from the same MT817x family, including the 6-core (4-little/2-big) MT8176. The mt8173-cpufreq driver supports all of these SOCs, however, machines using them may use a different machine compatible. Since this driver checks explicitly for the machine compatible string, add support for the whole family. Signed-off-by: Daniel Kurtz Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/mt8173-cpufreq.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/mt8173-cpufreq.c b/drivers/cpufreq/mt8173-cpufreq.c index 72bc1192bd30..fd1886faf33a 100644 --- a/drivers/cpufreq/mt8173-cpufreq.c +++ b/drivers/cpufreq/mt8173-cpufreq.c @@ -573,14 +573,33 @@ static struct platform_driver mt8173_cpufreq_platdrv = { .probe = mt8173_cpufreq_probe, }; +/* List of machines supported by this driver */ +static const struct of_device_id mt8173_cpufreq_machines[] __initconst = { + { .compatible = "mediatek,mt817x", }, + { .compatible = "mediatek,mt8173", }, + { .compatible = "mediatek,mt8176", }, + + { } +}; + static int __init mt8173_cpufreq_driver_init(void) { + struct device_node *np; + const struct of_device_id *match; struct platform_device *pdev; int err; - if (!of_machine_is_compatible("mediatek,mt8173")) + np = of_find_node_by_path("/"); + if (!np) return -ENODEV; + match = of_match_node(mt8173_cpufreq_machines, np); + of_node_put(np); + if (!match) { + pr_warn("Machine is not compatible with mt8173-cpufreq\n"); + return -ENODEV; + } + err = platform_driver_register(&mt8173_cpufreq_platdrv); if (err) return err; From b51d3388e2350138f6acc5cacda009a67f6f6352 Mon Sep 17 00:00:00 2001 From: YuanTian Tang Date: Fri, 10 Mar 2017 09:28:43 +0800 Subject: [PATCH 04/37] cpufreq: qoriq: enhance bus frequency calculation On some platforms, property device-type may be missed in soc node in dts which caused the bus-frequency can not be obtained correctly. This patch enhanced the bus-frequency calculation. When property device-type is missed in dts, bus-frequency will be obtained by looking up clock table to get platform clock and hence get its frequency. Signed-off-by: Tang Yuantian Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/qoriq-cpufreq.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c index bfec1bcd3835..e2ea433a5f9c 100644 --- a/drivers/cpufreq/qoriq-cpufreq.c +++ b/drivers/cpufreq/qoriq-cpufreq.c @@ -52,17 +52,27 @@ static u32 get_bus_freq(void) { struct device_node *soc; u32 sysfreq; + struct clk *pltclk; + int ret; + /* get platform freq by searching bus-frequency property */ soc = of_find_node_by_type(NULL, "soc"); - if (!soc) - return 0; + if (soc) { + ret = of_property_read_u32(soc, "bus-frequency", &sysfreq); + of_node_put(soc); + if (!ret) + return sysfreq; + } - if (of_property_read_u32(soc, "bus-frequency", &sysfreq)) - sysfreq = 0; + /* get platform freq by its clock name */ + pltclk = clk_get(NULL, "cg-pll0-div1"); + if (IS_ERR(pltclk)) { + pr_err("%s: can't get bus frequency %ld\n", + __func__, PTR_ERR(pltclk)); + return PTR_ERR(pltclk); + } - of_node_put(soc); - - return sysfreq; + return clk_get_rate(pltclk); } static struct clk *cpu_to_clk(int cpu) From 6f19363503541ee6020d35b468a998c213bace36 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Sun, 12 Mar 2017 14:16:47 +0100 Subject: [PATCH 05/37] MAINTAINERS: Add file patterns for cpufreq device tree bindings Submitters of device tree binding documentation may forget to CC the subsystem maintainer if this is missing. Signed-off-by: Geert Uytterhoeven Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index c776906f67a9..fabed4c65542 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3450,6 +3450,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git T: git git://git.linaro.org/people/vireshk/linux.git (For ARM Updates) B: https://bugzilla.kernel.org F: Documentation/cpu-freq/ +F: Documentation/devicetree/bindings/cpufreq/ F: drivers/cpufreq/ F: include/linux/cpufreq.h F: tools/testing/selftests/cpufreq/ From 994a8f2514e91c16616c4a1b53e9eb2b24de97b7 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 21 Feb 2017 10:15:18 +0530 Subject: [PATCH 06/37] cpufreq: schedutil: Redefine the rate_limit_us tunable The rate_limit_us tunable is intended to reduce the possible overhead from running the schedutil governor. However, that overhead can be divided into two separate parts: the governor computations and the invocation of the scaling driver to set the CPU frequency. The latter is where the real overhead comes from. The former is much less expensive in terms of execution time and running it every time the governor callback is invoked by the scheduler, after rate_limit_us interval has passed since the last frequency update, would not be a problem. For this reason, redefine the rate_limit_us tunable so that it means the minimum time that has to pass between two consecutive invocations of the scaling driver by the schedutil governor (to set the CPU frequency). Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- kernel/sched/cpufreq_schedutil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index cd7cd489f739..78468aa051ab 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -93,14 +93,13 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time, { struct cpufreq_policy *policy = sg_policy->policy; - sg_policy->last_freq_update_time = time; - if (policy->fast_switch_enabled) { if (sg_policy->next_freq == next_freq) { trace_cpu_frequency(policy->cur, smp_processor_id()); return; } sg_policy->next_freq = next_freq; + sg_policy->last_freq_update_time = time; next_freq = cpufreq_driver_fast_switch(policy, next_freq); if (next_freq == CPUFREQ_ENTRY_INVALID) return; @@ -109,6 +108,7 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time, trace_cpu_frequency(next_freq, smp_processor_id()); } else if (sg_policy->next_freq != next_freq) { sg_policy->next_freq = next_freq; + sg_policy->last_freq_update_time = time; sg_policy->work_in_progress = true; irq_work_queue(&sg_policy->irq_work); } From cba1dfb57b94c234728b689d9b00d4267fa1a879 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 9 Mar 2017 09:34:54 +0530 Subject: [PATCH 07/37] cpufreq: schedutil: Refactor sugov_next_freq_shared() The loop in sugov_next_freq_shared() contains an if block to skip the loop for the current CPU. This turns out to be an unnecessary conditional in the scheduler's hot-path for every CPU in the policy. It would be better to drop the conditional and make the loop treat all the CPUs in the same way. That would eliminate the need of calling sugov_iowait_boost() at the top of the routine. To keep the code optimized to return early if the current CPU has RT/DL flags set, move the flags check to sugov_update_shared() instead in order to avoid the function call entirely. Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- kernel/sched/cpufreq_schedutil.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 78468aa051ab..f5ffe241812e 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -217,30 +217,19 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, sugov_update_commit(sg_policy, time, next_f); } -static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, - unsigned long util, unsigned long max, - unsigned int flags) +static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu) { struct sugov_policy *sg_policy = sg_cpu->sg_policy; struct cpufreq_policy *policy = sg_policy->policy; - unsigned int max_f = policy->cpuinfo.max_freq; u64 last_freq_update_time = sg_policy->last_freq_update_time; + unsigned long util = 0, max = 1; unsigned int j; - if (flags & SCHED_CPUFREQ_RT_DL) - return max_f; - - sugov_iowait_boost(sg_cpu, &util, &max); - for_each_cpu(j, policy->cpus) { - struct sugov_cpu *j_sg_cpu; + struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j); unsigned long j_util, j_max; s64 delta_ns; - if (j == smp_processor_id()) - continue; - - j_sg_cpu = &per_cpu(sugov_cpu, j); /* * If the CPU utilization was last updated before the previous * frequency update and the time elapsed between the last update @@ -254,7 +243,7 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, continue; } if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL) - return max_f; + return policy->cpuinfo.max_freq; j_util = j_sg_cpu->util; j_max = j_sg_cpu->max; @@ -289,7 +278,11 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time, sg_cpu->last_update = time; if (sugov_should_update_freq(sg_policy, time)) { - next_f = sugov_next_freq_shared(sg_cpu, util, max, flags); + if (flags & SCHED_CPUFREQ_RT_DL) + next_f = sg_policy->policy->cpuinfo.max_freq; + else + next_f = sugov_next_freq_shared(sg_cpu); + sugov_update_commit(sg_policy, time, next_f); } From 19678ffb9fd6f216f530714e62fb469a961874db Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 14 Mar 2017 10:48:31 +0530 Subject: [PATCH 08/37] cpufreq: dbx500: Manage cooling device from cpufreq driver The best place to register the CPU cooling device is from the cpufreq driver as we would know if all the resources are already available or not. That's what is done for the cpufreq-dt.c driver as well. The cpu-cooling driver for dbx500 platform was just (un)registering with the thermal framework and that can be handled easily by the cpufreq driver as well and in proper sequence as well. Get rid of the cooling driver and its its users and manage everything from the cpufreq driver instead. Signed-off-by: Viresh Kumar Tested-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- arch/arm/boot/dts/ste-dbx5x0.dtsi | 5 -- drivers/cpufreq/dbx500-cpufreq.c | 20 +++++ drivers/thermal/Kconfig | 12 --- drivers/thermal/Makefile | 1 - drivers/thermal/db8500_cpufreq_cooling.c | 105 ----------------------- 5 files changed, 20 insertions(+), 123 deletions(-) delete mode 100644 drivers/thermal/db8500_cpufreq_cooling.c diff --git a/arch/arm/boot/dts/ste-dbx5x0.dtsi b/arch/arm/boot/dts/ste-dbx5x0.dtsi index 82d8c4771293..9eea52013914 100644 --- a/arch/arm/boot/dts/ste-dbx5x0.dtsi +++ b/arch/arm/boot/dts/ste-dbx5x0.dtsi @@ -1170,11 +1170,6 @@ status = "disabled"; }; - cpufreq-cooling { - compatible = "stericsson,db8500-cpufreq-cooling"; - status = "disabled"; - }; - mcde@a0350000 { compatible = "stericsson,mcde"; reg = <0xa0350000 0x1000>, /* MCDE */ diff --git a/drivers/cpufreq/dbx500-cpufreq.c b/drivers/cpufreq/dbx500-cpufreq.c index 5c3ec1dd4921..3575b82210ba 100644 --- a/drivers/cpufreq/dbx500-cpufreq.c +++ b/drivers/cpufreq/dbx500-cpufreq.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ static struct cpufreq_frequency_table *freq_table; static struct clk *armss_clk; +static struct thermal_cooling_device *cdev; static int dbx500_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) @@ -32,6 +34,22 @@ static int dbx500_cpufreq_init(struct cpufreq_policy *policy) return cpufreq_generic_init(policy, freq_table, 20 * 1000); } +static int dbx500_cpufreq_exit(struct cpufreq_policy *policy) +{ + if (!IS_ERR(cdev)) + cpufreq_cooling_unregister(cdev); + return 0; +} + +static void dbx500_cpufreq_ready(struct cpufreq_policy *policy) +{ + cdev = cpufreq_cooling_register(policy->cpus); + if (IS_ERR(cdev)) + pr_err("Failed to register cooling device %ld\n", PTR_ERR(cdev)); + else + pr_info("Cooling device registered: %s\n", cdev->type); +} + static struct cpufreq_driver dbx500_cpufreq_driver = { .flags = CPUFREQ_STICKY | CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_INITIAL_FREQ_CHECK, @@ -39,6 +57,8 @@ static struct cpufreq_driver dbx500_cpufreq_driver = { .target_index = dbx500_cpufreq_target, .get = cpufreq_generic_get, .init = dbx500_cpufreq_init, + .exit = dbx500_cpufreq_exit, + .ready = dbx500_cpufreq_ready, .name = "DBX500", .attr = cpufreq_generic_attr, }; diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 776b34396144..0a16cf4bed39 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -291,18 +291,6 @@ config ARMADA_THERMAL Enable this option if you want to have support for thermal management controller present in Armada 370 and Armada XP SoC. -config DB8500_CPUFREQ_COOLING - tristate "DB8500 cpufreq cooling" - depends on ARCH_U8500 || COMPILE_TEST - depends on HAS_IOMEM - depends on CPU_THERMAL - default y - help - Adds DB8500 cpufreq cooling devices, and these cooling devices can be - bound to thermal zone trip points. When a trip point reached, the - bound cpufreq cooling device turns active to set CPU frequency low to - cool down the CPU. - config INTEL_POWERCLAMP tristate "Intel PowerClamp idle injection driver" depends on THERMAL diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index 7adae2029355..c2372f10dae5 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -41,7 +41,6 @@ obj-$(CONFIG_TANGO_THERMAL) += tango_thermal.o obj-$(CONFIG_IMX_THERMAL) += imx_thermal.o obj-$(CONFIG_MAX77620_THERMAL) += max77620_thermal.o obj-$(CONFIG_QORIQ_THERMAL) += qoriq_thermal.o -obj-$(CONFIG_DB8500_CPUFREQ_COOLING) += db8500_cpufreq_cooling.o obj-$(CONFIG_INTEL_POWERCLAMP) += intel_powerclamp.o obj-$(CONFIG_X86_PKG_TEMP_THERMAL) += x86_pkg_temp_thermal.o obj-$(CONFIG_INTEL_SOC_DTS_IOSF_CORE) += intel_soc_dts_iosf.o diff --git a/drivers/thermal/db8500_cpufreq_cooling.c b/drivers/thermal/db8500_cpufreq_cooling.c deleted file mode 100644 index e58bd0b658b5..000000000000 --- a/drivers/thermal/db8500_cpufreq_cooling.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * db8500_cpufreq_cooling.c - DB8500 cpufreq works as cooling device. - * - * Copyright (C) 2012 ST-Ericsson - * Copyright (C) 2012 Linaro Ltd. - * - * Author: Hongbo Zhang - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#include -#include -#include -#include -#include -#include - -static int db8500_cpufreq_cooling_probe(struct platform_device *pdev) -{ - struct thermal_cooling_device *cdev; - - cdev = cpufreq_cooling_register(cpu_present_mask); - if (IS_ERR(cdev)) { - int ret = PTR_ERR(cdev); - - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, - "Failed to register cooling device %d\n", - ret); - - return ret; - } - - platform_set_drvdata(pdev, cdev); - - dev_info(&pdev->dev, "Cooling device registered: %s\n", cdev->type); - - return 0; -} - -static int db8500_cpufreq_cooling_remove(struct platform_device *pdev) -{ - struct thermal_cooling_device *cdev = platform_get_drvdata(pdev); - - cpufreq_cooling_unregister(cdev); - - return 0; -} - -static int db8500_cpufreq_cooling_suspend(struct platform_device *pdev, - pm_message_t state) -{ - return -ENOSYS; -} - -static int db8500_cpufreq_cooling_resume(struct platform_device *pdev) -{ - return -ENOSYS; -} - -#ifdef CONFIG_OF -static const struct of_device_id db8500_cpufreq_cooling_match[] = { - { .compatible = "stericsson,db8500-cpufreq-cooling" }, - {}, -}; -MODULE_DEVICE_TABLE(of, db8500_cpufreq_cooling_match); -#endif - -static struct platform_driver db8500_cpufreq_cooling_driver = { - .driver = { - .name = "db8500-cpufreq-cooling", - .of_match_table = of_match_ptr(db8500_cpufreq_cooling_match), - }, - .probe = db8500_cpufreq_cooling_probe, - .suspend = db8500_cpufreq_cooling_suspend, - .resume = db8500_cpufreq_cooling_resume, - .remove = db8500_cpufreq_cooling_remove, -}; - -static int __init db8500_cpufreq_cooling_init(void) -{ - return platform_driver_register(&db8500_cpufreq_cooling_driver); -} - -static void __exit db8500_cpufreq_cooling_exit(void) -{ - platform_driver_unregister(&db8500_cpufreq_cooling_driver); -} - -/* Should be later than db8500_cpufreq_register */ -late_initcall(db8500_cpufreq_cooling_init); -module_exit(db8500_cpufreq_cooling_exit); - -MODULE_AUTHOR("Hongbo Zhang "); -MODULE_DESCRIPTION("DB8500 cpufreq cooling driver"); -MODULE_LICENSE("GPL"); From b7eaf1aab9f8bd2e49fceed77ebc66c1b5800718 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 22 Mar 2017 00:08:50 +0100 Subject: [PATCH 09/37] cpufreq: schedutil: Avoid reducing frequency of busy CPUs prematurely The way the schedutil governor uses the PELT metric causes it to underestimate the CPU utilization in some cases. That can be easily demonstrated by running kernel compilation on a Sandy Bridge Intel processor, running turbostat in parallel with it and looking at the values written to the MSR_IA32_PERF_CTL register. Namely, the expected result would be that when all CPUs were 100% busy, all of them would be requested to run in the maximum P-state, but observation shows that this clearly isn't the case. The CPUs run in the maximum P-state for a while and then are requested to run slower and go back to the maximum P-state after a while again. That causes the actual frequency of the processor to visibly oscillate below the sustainable maximum in a jittery fashion which clearly is not desirable. That has been attributed to CPU utilization metric updates on task migration that cause the total utilization value for the CPU to be reduced by the utilization of the migrated task. If that happens, the schedutil governor may see a CPU utilization reduction and will attempt to reduce the CPU frequency accordingly right away. That may be premature, though, for example if the system is generally busy and there are other runnable tasks waiting to be run on that CPU already. This is unlikely to be an issue on systems where cpufreq policies are shared between multiple CPUs, because in those cases the policy utilization is computed as the maximum of the CPU utilization values over the whole policy and if that turns out to be low, reducing the frequency for the policy most likely is a good idea anyway. On systems with one CPU per policy, however, it may affect performance adversely and even lead to increased energy consumption in some cases. On those systems it may be addressed by taking another utilization metric into consideration, like whether or not the CPU whose frequency is about to be reduced has been idle recently, because if that's not the case, the CPU is likely to be busy in the near future and its frequency should not be reduced. To that end, use the counter of idle calls in the timekeeping code. Namely, make the schedutil governor look at that counter for the current CPU every time before its frequency is about to be reduced. If the counter has not changed since the previous iteration of the governor computations for that CPU, the CPU has been busy for all that time and its frequency should not be decreased, so if the new frequency would be lower than the one set previously, the governor will skip the frequency update. Signed-off-by: Rafael J. Wysocki Acked-by: Peter Zijlstra (Intel) Acked-by: Viresh Kumar Reviewed-by: Joel Fernandes --- include/linux/tick.h | 1 + kernel/sched/cpufreq_schedutil.c | 27 +++++++++++++++++++++++++++ kernel/time/tick-sched.c | 12 ++++++++++++ 3 files changed, 40 insertions(+) diff --git a/include/linux/tick.h b/include/linux/tick.h index a04fea19676f..fe01e68bf520 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -117,6 +117,7 @@ extern void tick_nohz_idle_enter(void); extern void tick_nohz_idle_exit(void); extern void tick_nohz_irq_exit(void); extern ktime_t tick_nohz_get_sleep_length(void); +extern unsigned long tick_nohz_get_idle_calls(void); extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); #else /* !CONFIG_NO_HZ_COMMON */ diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index f5ffe241812e..c1ffb5dc8af6 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -61,6 +61,11 @@ struct sugov_cpu { unsigned long util; unsigned long max; unsigned int flags; + + /* The field below is for single-CPU policies only. */ +#ifdef CONFIG_NO_HZ_COMMON + unsigned long saved_idle_calls; +#endif }; static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu); @@ -192,6 +197,19 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util, sg_cpu->iowait_boost >>= 1; } +#ifdef CONFIG_NO_HZ_COMMON +static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) +{ + unsigned long idle_calls = tick_nohz_get_idle_calls(); + bool ret = idle_calls == sg_cpu->saved_idle_calls; + + sg_cpu->saved_idle_calls = idle_calls; + return ret; +} +#else +static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; } +#endif /* CONFIG_NO_HZ_COMMON */ + static void sugov_update_single(struct update_util_data *hook, u64 time, unsigned int flags) { @@ -200,6 +218,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, struct cpufreq_policy *policy = sg_policy->policy; unsigned long util, max; unsigned int next_f; + bool busy; sugov_set_iowait_boost(sg_cpu, time, flags); sg_cpu->last_update = time; @@ -207,12 +226,20 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, if (!sugov_should_update_freq(sg_policy, time)) return; + busy = sugov_cpu_is_busy(sg_cpu); + if (flags & SCHED_CPUFREQ_RT_DL) { next_f = policy->cpuinfo.max_freq; } else { sugov_get_util(&util, &max); sugov_iowait_boost(sg_cpu, &util, &max); next_f = get_next_freq(sg_policy, util, max); + /* + * Do not reduce the frequency if the CPU has not been idle + * recently, as the reduction is likely to be premature then. + */ + if (busy && next_f < sg_policy->next_freq) + next_f = sg_policy->next_freq; } sugov_update_commit(sg_policy, time, next_f); } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 7fe53be86077..64c97fc130c4 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -993,6 +993,18 @@ ktime_t tick_nohz_get_sleep_length(void) return ts->sleep_length; } +/** + * tick_nohz_get_idle_calls - return the current idle calls counter value + * + * Called from the schedutil frequency scaling governor in scheduler context. + */ +unsigned long tick_nohz_get_idle_calls(void) +{ + struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); + + return ts->idle_calls; +} + static void tick_nohz_account_idle_ticks(struct tick_sched *ts) { #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE From 38d4ea229d25d30be6bf41bcd6cd663a587866ca Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 22 Mar 2017 18:32:47 +0100 Subject: [PATCH 10/37] cpufreq: schedutil: Trace frequency only if it has changed sugov_update_commit() calls trace_cpu_frequency() to record the current CPU frequency if it has not changed in the fast switch case to prevent utilities from getting confused (they may report that the CPU is idle if the frequency has not been recorded for too long, for example). However, that may cause the tracepoint to be triggered quite often for no real reason (if the frequency doesn't change, we will not modify the last update time stamp and governor computations may run again shortly when that happens), so don't do that (arguably, it is done to work around a utilities bug anyway). That allows code duplication in sugov_update_commit() to be reduced somewhat too. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- kernel/sched/cpufreq_schedutil.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index c1ffb5dc8af6..1054f868d95c 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -98,22 +98,20 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time, { struct cpufreq_policy *policy = sg_policy->policy; + if (sg_policy->next_freq == next_freq) + return; + + sg_policy->next_freq = next_freq; + sg_policy->last_freq_update_time = time; + if (policy->fast_switch_enabled) { - if (sg_policy->next_freq == next_freq) { - trace_cpu_frequency(policy->cur, smp_processor_id()); - return; - } - sg_policy->next_freq = next_freq; - sg_policy->last_freq_update_time = time; next_freq = cpufreq_driver_fast_switch(policy, next_freq); if (next_freq == CPUFREQ_ENTRY_INVALID) return; policy->cur = next_freq; trace_cpu_frequency(next_freq, smp_processor_id()); - } else if (sg_policy->next_freq != next_freq) { - sg_policy->next_freq = next_freq; - sg_policy->last_freq_update_time = time; + } else { sg_policy->work_in_progress = true; irq_work_queue(&sg_policy->irq_work); } From eb5139d1a2272487b223c4879ecd2a1b48c2250e Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 22 Mar 2017 23:52:18 +0100 Subject: [PATCH 11/37] cpufreq: intel_pstate: Support HWP processors in all operation modes Currently, some processors supporting HWP are only supported by intel_pstate if HWP is actually going to be used and not supported otherwise which is confusing. Specifically, they are not supported if "intel_pstate=no_hwp" is passed to the kernel in the command line or if the driver is started in the passive mode ("intel_pstate=passive"). There is no real reason for that, because everything about those processor is known anyway and the driver can work with them in all modes, so make that happen, but use the load-based P-state selection algorithm for the active mode "powersave" policy with them. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 37 +++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 815f5577b32a..87725e2ac3ac 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2627,29 +2627,34 @@ static const struct x86_cpu_id hwp_support_ids[] __initconst = { static int __init intel_pstate_init(void) { - const struct x86_cpu_id *id; - struct cpu_defaults *cpu_def; - int rc = 0; + int rc; if (no_load) return -ENODEV; - if (x86_match_cpu(hwp_support_ids) && !no_hwp) { + if (x86_match_cpu(hwp_support_ids)) { copy_cpu_funcs(&core_params.funcs); - hwp_active++; - intel_pstate.attr = hwp_cpufreq_attrs; - goto hwp_cpu_matched; + if (no_hwp) { + pstate_funcs.get_target_pstate = get_target_pstate_use_cpu_load; + } else { + hwp_active++; + intel_pstate.attr = hwp_cpufreq_attrs; + goto hwp_cpu_matched; + } + } else { + const struct x86_cpu_id *id; + struct cpu_defaults *cpu_def; + + id = x86_match_cpu(intel_pstate_cpu_ids); + if (!id) + return -ENODEV; + + cpu_def = (struct cpu_defaults *)id->driver_data; + + copy_pid_params(&cpu_def->pid_policy); + copy_cpu_funcs(&cpu_def->funcs); } - id = x86_match_cpu(intel_pstate_cpu_ids); - if (!id) - return -ENODEV; - - cpu_def = (struct cpu_defaults *)id->driver_data; - - copy_pid_params(&cpu_def->pid_policy); - copy_cpu_funcs(&cpu_def->funcs); - if (intel_pstate_msrs_not_valid()) return -ENODEV; From 553953453b4b64fbccba31691257d006cee36613 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 22 Mar 2017 23:53:54 +0100 Subject: [PATCH 12/37] cpufreq: intel_pstate: Use load-based P-state selection more widely Extend the set of systems for which intel_pstate will use the "powersave" P-state selection algorithm based on CPU load in the active mode by systems with ACPI preferred profile set to "tablet", "appliance PC", "desktop", or "workstation" (ie. everything with a specified preferred profile that is not a "server"). Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 87725e2ac3ac..2ef02fd568a6 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2469,9 +2469,15 @@ static void __init copy_pid_params(struct pstate_adjust_policy *policy) #ifdef CONFIG_ACPI static void intel_pstate_use_acpi_profile(void) { - if (acpi_gbl_FADT.preferred_profile == PM_MOBILE) + switch (acpi_gbl_FADT.preferred_profile) { + case PM_MOBILE: + case PM_TABLET: + case PM_APPLIANCE_PC: + case PM_DESKTOP: + case PM_WORKSTATION: pstate_funcs.get_target_pstate = get_target_pstate_use_cpu_load; + } } #else static void intel_pstate_use_acpi_profile(void) From c5a2ee7dde893e0a06044e75c16711f08d5c011d Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 22 Mar 2017 23:58:57 +0100 Subject: [PATCH 13/37] cpufreq: intel_pstate: Active mode P-state limits rework The coordination of P-state limits used by intel_pstate in the active mode (ie. by default) is problematic, because it synchronizes all of the limits (ie. the global ones and the per-policy ones) so as to use one common pair of P-state limits (min and max) across all CPUs in the system. The drawbacks of that are as follows: - If P-states are coordinated in hardware, it is not necessary to coordinate them in software on top of that, so in that case all of the above activity is in vain. - If P-states are not coordinated in hardware, then the processor is actually capable of setting different P-states for different CPUs and coordinating them at the software level simply doesn't allow that capability to be utilized. - The coordination works in such a way that setting a per-policy limit (eg. scaling_max_freq) for one CPU causes the common effective limit to change (and it will affect all of the other CPUs too), but subsequent reads from the corresponding sysfs attributes for the other CPUs will return stale values (which is confusing). - Reads from the global P-state limit attributes, min_perf_pct and max_perf_pct, return the effective common values and not the last values set through these attributes. However, the last values set through these attributes become hard limits that cannot be exceeded by writes to scaling_min_freq and scaling_max_freq, respectively, and they are not exposed, so essentially users have to remember what they are. All of that is painful enough to warrant a change of the management of P-state limits in the active mode. To that end, redesign the active mode P-state limits management in intel_pstate in accordance with the following rules: (1) All CPUs are affected by the global limits (that is, none of them can be requested to run faster than the global max and none of them can be requested to run slower than the global min). (2) Each individual CPU is affected by its own per-policy limits (that is, it cannot be requested to run faster than its own per-policy max and it cannot be requested to run slower than its own per-policy min). (3) The global and per-policy limits can be set independently. Also, the global maximum and minimum P-state limits will be always expressed as percentages of the maximum supported turbo P-state. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 185 +++++++++++++++------------------ 1 file changed, 85 insertions(+), 100 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 2ef02fd568a6..c0afa78624a1 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -187,44 +187,35 @@ struct _pid { /** * struct perf_limits - Store user and policy limits - * @no_turbo: User requested turbo state from intel_pstate sysfs - * @turbo_disabled: Platform turbo status either from msr - * MSR_IA32_MISC_ENABLE or when maximum available pstate - * matches the maximum turbo pstate - * @max_perf_pct: Effective maximum performance limit in percentage, this - * is minimum of either limits enforced by cpufreq policy - * or limits from user set limits via intel_pstate sysfs - * @min_perf_pct: Effective minimum performance limit in percentage, this - * is maximum of either limits enforced by cpufreq policy - * or limits from user set limits via intel_pstate sysfs * @max_perf: This is a scaled value between 0 to 255 for max_perf_pct * This value is used to limit max pstate * @min_perf: This is a scaled value between 0 to 255 for min_perf_pct * This value is used to limit min pstate - * @max_policy_pct: The maximum performance in percentage enforced by - * cpufreq setpolicy interface - * @max_sysfs_pct: The maximum performance in percentage enforced by - * intel pstate sysfs interface, unused when per cpu - * controls are enforced - * @min_policy_pct: The minimum performance in percentage enforced by - * cpufreq setpolicy interface - * @min_sysfs_pct: The minimum performance in percentage enforced by - * intel pstate sysfs interface, unused when per cpu - * controls are enforced * - * Storage for user and policy defined limits. + * Storage for policy defined limits. */ struct perf_limits { - int no_turbo; - int turbo_disabled; - int max_perf_pct; - int min_perf_pct; int32_t max_perf; int32_t min_perf; - int max_policy_pct; - int max_sysfs_pct; - int min_policy_pct; - int min_sysfs_pct; +}; + +/** + * struct global_params - Global parameters, mostly tunable via sysfs. + * @no_turbo: Whether or not to use turbo P-states. + * @turbo_disabled: Whethet or not turbo P-states are available at all, + * based on the MSR_IA32_MISC_ENABLE value and whether or + * not the maximum reported turbo P-state is different from + * the maximum reported non-turbo one. + * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo + * P-state capacity. + * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo + * P-state capacity. + */ +struct global_params { + bool no_turbo; + bool turbo_disabled; + int max_perf_pct; + int min_perf_pct; }; /** @@ -245,9 +236,7 @@ struct perf_limits { * @prev_cummulative_iowait: IO Wait time difference from last and * current sample * @sample: Storage for storing last Sample data - * @perf_limits: Pointer to perf_limit unique to this CPU - * Not all field in the structure are applicable - * when per cpu controls are enforced + * @perf_limits: Capacity limits unique to this CPU * @acpi_perf_data: Stores ACPI perf information read from _PSS * @valid_pss_table: Set to true for valid ACPI _PSS entries found * @epp_powersave: Last saved HWP energy performance preference @@ -279,7 +268,7 @@ struct cpudata { u64 prev_tsc; u64 prev_cummulative_iowait; struct sample sample; - struct perf_limits *perf_limits; + struct perf_limits perf_limits; #ifdef CONFIG_ACPI struct acpi_processor_performance acpi_perf_data; bool valid_pss_table; @@ -364,16 +353,7 @@ static bool driver_registered __read_mostly; static bool acpi_ppc; #endif -static struct perf_limits global; - -static void intel_pstate_init_limits(struct perf_limits *limits) -{ - memset(limits, 0, sizeof(*limits)); - limits->max_perf_pct = 100; - limits->max_perf = int_ext_tofp(1); - limits->max_policy_pct = 100; - limits->max_sysfs_pct = 100; -} +static struct global_params global; static DEFINE_MUTEX(intel_pstate_driver_lock); static DEFINE_MUTEX(intel_pstate_limits_lock); @@ -621,6 +601,14 @@ static inline void update_turbo_state(void) cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); } +static int min_perf_pct_min(void) +{ + struct cpudata *cpu = all_cpu_data[0]; + + return DIV_ROUND_UP(cpu->pstate.min_pstate * 100, + cpu->pstate.turbo_pstate); +} + static s16 intel_pstate_get_epb(struct cpudata *cpu_data) { u64 epb; @@ -841,16 +829,13 @@ static struct freq_attr *hwp_cpufreq_attrs[] = { static void intel_pstate_hwp_set(struct cpufreq_policy *policy) { int min, hw_min, max, hw_max, cpu; - struct perf_limits *perf_limits = &global; u64 value, cap; for_each_cpu(cpu, policy->cpus) { struct cpudata *cpu_data = all_cpu_data[cpu]; + struct perf_limits *perf_limits = &cpu_data->perf_limits; s16 epp; - if (per_cpu_limits) - perf_limits = all_cpu_data[cpu]->perf_limits; - rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap); hw_min = HWP_LOWEST_PERF(cap); if (global.no_turbo) @@ -1163,6 +1148,15 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, global.no_turbo = clamp_t(int, input, 0, 1); + if (global.no_turbo) { + struct cpudata *cpu = all_cpu_data[0]; + int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate; + + /* Squash the global minimum into the permitted range. */ + if (global.min_perf_pct > pct) + global.min_perf_pct = pct; + } + mutex_unlock(&intel_pstate_limits_lock); intel_pstate_update_policies(); @@ -1191,11 +1185,7 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, mutex_lock(&intel_pstate_limits_lock); - global.max_sysfs_pct = clamp_t(int, input, 0 , 100); - global.max_perf_pct = min(global.max_policy_pct, global.max_sysfs_pct); - global.max_perf_pct = max(global.min_policy_pct, global.max_perf_pct); - global.max_perf_pct = max(global.min_perf_pct, global.max_perf_pct); - global.max_perf = percent_ext_fp(global.max_perf_pct); + global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100); mutex_unlock(&intel_pstate_limits_lock); @@ -1225,11 +1215,8 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, mutex_lock(&intel_pstate_limits_lock); - global.min_sysfs_pct = clamp_t(int, input, 0 , 100); - global.min_perf_pct = max(global.min_policy_pct, global.min_sysfs_pct); - global.min_perf_pct = min(global.max_policy_pct, global.min_perf_pct); - global.min_perf_pct = min(global.max_perf_pct, global.min_perf_pct); - global.min_perf = percent_ext_fp(global.min_perf_pct); + global.min_perf_pct = clamp_t(int, input, + min_perf_pct_min(), global.max_perf_pct); mutex_unlock(&intel_pstate_limits_lock); @@ -1650,14 +1637,11 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) int max_perf = cpu->pstate.turbo_pstate; int max_perf_adj; int min_perf; - struct perf_limits *perf_limits = &global; + struct perf_limits *perf_limits = &cpu->perf_limits; if (global.no_turbo || global.turbo_disabled) max_perf = cpu->pstate.max_pstate; - if (per_cpu_limits) - perf_limits = cpu->perf_limits; - /* * performance can be limited by user through sysfs, by cpufreq * policy, or by cpu specific default values determined through @@ -1968,18 +1952,11 @@ static int intel_pstate_init_cpu(unsigned int cpunum) cpu = all_cpu_data[cpunum]; if (!cpu) { - unsigned int size = sizeof(struct cpudata); - - if (per_cpu_limits) - size += sizeof(struct perf_limits); - - cpu = kzalloc(size, GFP_KERNEL); + cpu = kzalloc(sizeof(*cpu), GFP_KERNEL); if (!cpu) return -ENOMEM; all_cpu_data[cpunum] = cpu; - if (per_cpu_limits) - cpu->perf_limits = (struct perf_limits *)(cpu + 1); cpu->epp_default = -EINVAL; cpu->epp_powersave = -EINVAL; @@ -2045,8 +2022,9 @@ static void intel_pstate_clear_update_util_hook(unsigned int cpu) } static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, - struct perf_limits *limits) + struct cpudata *cpu) { + struct perf_limits *limits = &cpu->perf_limits; int32_t max_policy_perf, min_policy_perf; max_policy_perf = div_ext_fp(policy->max, policy->cpuinfo.max_freq); @@ -2061,29 +2039,45 @@ static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, } /* Normalize user input to [min_perf, max_perf] */ - limits->min_perf = max(min_policy_perf, - percent_ext_fp(limits->min_sysfs_pct)); - limits->min_perf = min(limits->min_perf, max_policy_perf); - limits->max_perf = min(max_policy_perf, - percent_ext_fp(limits->max_sysfs_pct)); - limits->max_perf = max(min_policy_perf, limits->max_perf); + if (per_cpu_limits) { + limits->min_perf = min_policy_perf; + limits->max_perf = max_policy_perf; + } else { + int32_t global_min, global_max; - /* Make sure min_perf <= max_perf */ - limits->min_perf = min(limits->min_perf, limits->max_perf); + /* Global limits are in percent of the maximum turbo P-state. */ + global_max = percent_ext_fp(global.max_perf_pct); + global_min = percent_ext_fp(global.min_perf_pct); + if (policy->cpuinfo.max_freq != cpu->pstate.turbo_freq) { + int32_t turbo_factor; + + turbo_factor = div_ext_fp(cpu->pstate.turbo_pstate, + cpu->pstate.max_pstate); + global_min = mul_ext_fp(global_min, turbo_factor); + global_max = mul_ext_fp(global_max, turbo_factor); + } + global_min = clamp_t(int32_t, global_min, 0, global_max); + + limits->min_perf = max(min_policy_perf, global_min); + limits->min_perf = min(limits->min_perf, max_policy_perf); + limits->max_perf = min(max_policy_perf, global_max); + limits->max_perf = max(min_policy_perf, limits->max_perf); + + /* Make sure min_perf <= max_perf */ + limits->min_perf = min(limits->min_perf, limits->max_perf); + } limits->max_perf = round_up(limits->max_perf, EXT_FRAC_BITS); limits->min_perf = round_up(limits->min_perf, EXT_FRAC_BITS); - limits->max_perf_pct = fp_ext_toint(limits->max_perf * 100); - limits->min_perf_pct = fp_ext_toint(limits->min_perf * 100); pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu, - limits->max_perf_pct, limits->min_perf_pct); + fp_ext_toint(limits->max_perf * 100), + fp_ext_toint(limits->min_perf * 100)); } static int intel_pstate_set_policy(struct cpufreq_policy *policy) { struct cpudata *cpu; - struct perf_limits *perf_limits = &global; if (!policy->cpuinfo.max_freq) return -ENODEV; @@ -2101,12 +2095,9 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) policy->max = policy->cpuinfo.max_freq; } - if (per_cpu_limits) - perf_limits = cpu->perf_limits; - mutex_lock(&intel_pstate_limits_lock); - intel_pstate_update_perf_limits(policy, perf_limits); + intel_pstate_update_perf_limits(policy, cpu); if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { /* @@ -2142,17 +2133,6 @@ static int intel_pstate_verify_policy(struct cpufreq_policy *policy) policy->policy != CPUFREQ_POLICY_PERFORMANCE) return -EINVAL; - /* When per-CPU limits are used, sysfs limits are not used */ - if (!per_cpu_limits) { - unsigned int max_freq, min_freq; - - max_freq = policy->cpuinfo.max_freq * - global.max_sysfs_pct / 100; - min_freq = policy->cpuinfo.max_freq * - global.min_sysfs_pct / 100; - cpufreq_verify_within_limits(policy, min_freq, max_freq); - } - return 0; } @@ -2192,8 +2172,8 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) cpu = all_cpu_data[policy->cpu]; - if (per_cpu_limits) - intel_pstate_init_limits(cpu->perf_limits); + cpu->perf_limits.max_perf = int_ext_tofp(1); + cpu->perf_limits.min_perf = 0; policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling; policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling; @@ -2252,6 +2232,8 @@ static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy) cpufreq_verify_within_cpu_limits(policy); + intel_pstate_update_perf_limits(policy, cpu); + return 0; } @@ -2354,7 +2336,8 @@ static int intel_pstate_register_driver(void) { int ret; - intel_pstate_init_limits(&global); + memset(&global, 0, sizeof(global)); + global.max_perf_pct = 100; ret = cpufreq_register_driver(intel_pstate_driver); if (ret) { @@ -2362,6 +2345,8 @@ static int intel_pstate_register_driver(void) return ret; } + global.min_perf_pct = min_perf_pct_min(); + mutex_lock(&intel_pstate_limits_lock); driver_registered = true; mutex_unlock(&intel_pstate_limits_lock); From 80b120ca1a75c2df093d15936ab0591d90c99de9 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 23 Mar 2017 00:00:47 +0100 Subject: [PATCH 14/37] cpufreq: intel_pstate: Avoid transient updates of cpuinfo.max_freq Both intel_pstate_verify_policy() and intel_cpufreq_verify_policy() set policy->cpuinfo.max_freq depending on the turbo status, but the updates made by them are discarded by the core, because the policy object passed to them by the core is temporary and cpuinfo.max_freq from that object is not copied to the final policy object in cpufreq_set_policy(). However, cpufreq_set_policy() passes the temporary policy object to the ->setpolicy callback of the driver, so intel_pstate_set_policy() actually sees the policy->cpuinfo.max_freq value updated by intel_pstate_verify_policy() and not the final one. It also updates policy->max sometimes which basically has no effect after it returns, because the core discards that update. To avoid confusion, eliminate policy->cpuinfo.max_freq updates from intel_pstate_verify_policy() and intel_cpufreq_verify_policy() entirely and check the maximum frequency explicitly in intel_pstate_update_perf_limits() instead of relying on the transiently updated policy->cpuinfo.max_freq value. Moreover, move the max->policy adjustment carried out in intel_pstate_set_policy() to a separate function and call that function from the ->verify driver callbacks to ensure that it will actually be effective. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 47 ++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index c0afa78624a1..60544c210d75 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2021,19 +2021,25 @@ static void intel_pstate_clear_update_util_hook(unsigned int cpu) synchronize_sched(); } +static int intel_pstate_get_max_freq(struct cpudata *cpu) +{ + return global.turbo_disabled || global.no_turbo ? + cpu->pstate.max_freq : cpu->pstate.turbo_freq; +} + static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, struct cpudata *cpu) { struct perf_limits *limits = &cpu->perf_limits; + int max_freq = intel_pstate_get_max_freq(cpu); int32_t max_policy_perf, min_policy_perf; - max_policy_perf = div_ext_fp(policy->max, policy->cpuinfo.max_freq); + max_policy_perf = div_ext_fp(policy->max, max_freq); max_policy_perf = clamp_t(int32_t, max_policy_perf, 0, int_ext_tofp(1)); if (policy->max == policy->min) { min_policy_perf = max_policy_perf; } else { - min_policy_perf = div_ext_fp(policy->min, - policy->cpuinfo.max_freq); + min_policy_perf = div_ext_fp(policy->min, max_freq); min_policy_perf = clamp_t(int32_t, min_policy_perf, 0, max_policy_perf); } @@ -2048,7 +2054,7 @@ static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, /* Global limits are in percent of the maximum turbo P-state. */ global_max = percent_ext_fp(global.max_perf_pct); global_min = percent_ext_fp(global.min_perf_pct); - if (policy->cpuinfo.max_freq != cpu->pstate.turbo_freq) { + if (max_freq != cpu->pstate.turbo_freq) { int32_t turbo_factor; turbo_factor = div_ext_fp(cpu->pstate.turbo_pstate, @@ -2088,13 +2094,6 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) cpu = all_cpu_data[policy->cpu]; cpu->policy = policy->policy; - if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate && - policy->max < policy->cpuinfo.max_freq && - policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) { - pr_debug("policy->max > max non turbo frequency\n"); - policy->max = policy->cpuinfo.max_freq; - } - mutex_lock(&intel_pstate_limits_lock); intel_pstate_update_perf_limits(policy, cpu); @@ -2118,21 +2117,31 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) return 0; } +static void intel_pstate_adjust_policy_max(struct cpufreq_policy *policy, + struct cpudata *cpu) +{ + if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate && + policy->max < policy->cpuinfo.max_freq && + policy->max > cpu->pstate.max_freq) { + pr_debug("policy->max > max non turbo frequency\n"); + policy->max = policy->cpuinfo.max_freq; + } +} + static int intel_pstate_verify_policy(struct cpufreq_policy *policy) { struct cpudata *cpu = all_cpu_data[policy->cpu]; update_turbo_state(); - policy->cpuinfo.max_freq = global.turbo_disabled || global.no_turbo ? - cpu->pstate.max_freq : - cpu->pstate.turbo_freq; - - cpufreq_verify_within_cpu_limits(policy); + cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, + intel_pstate_get_max_freq(cpu)); if (policy->policy != CPUFREQ_POLICY_POWERSAVE && policy->policy != CPUFREQ_POLICY_PERFORMANCE) return -EINVAL; + intel_pstate_adjust_policy_max(policy, cpu); + return 0; } @@ -2227,10 +2236,10 @@ static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy) struct cpudata *cpu = all_cpu_data[policy->cpu]; update_turbo_state(); - policy->cpuinfo.max_freq = global.no_turbo || global.turbo_disabled ? - cpu->pstate.max_freq : cpu->pstate.turbo_freq; + cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, + intel_pstate_get_max_freq(cpu)); - cpufreq_verify_within_cpu_limits(policy); + intel_pstate_adjust_policy_max(policy, cpu); intel_pstate_update_perf_limits(policy, cpu); From e14cf8857ebd7486a4e30fa7dad06ba187e6cb04 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:03:20 +0200 Subject: [PATCH 15/37] cpufreq: intel_pstate: Eliminate struct perf_limits After recent changes the purpose of struct perf_limits is not particularly clear any more and the code may be made somewhat easier to follow by eliminating it, so go for that. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 59 +++++++++++++--------------------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 60544c210d75..a7ed42d6f366 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -185,20 +185,6 @@ struct _pid { int32_t last_err; }; -/** - * struct perf_limits - Store user and policy limits - * @max_perf: This is a scaled value between 0 to 255 for max_perf_pct - * This value is used to limit max pstate - * @min_perf: This is a scaled value between 0 to 255 for min_perf_pct - * This value is used to limit min pstate - * - * Storage for policy defined limits. - */ -struct perf_limits { - int32_t max_perf; - int32_t min_perf; -}; - /** * struct global_params - Global parameters, mostly tunable via sysfs. * @no_turbo: Whether or not to use turbo P-states. @@ -236,7 +222,10 @@ struct global_params { * @prev_cummulative_iowait: IO Wait time difference from last and * current sample * @sample: Storage for storing last Sample data - * @perf_limits: Capacity limits unique to this CPU + * @min_perf: Minimum capacity limit as a fraction of the maximum + * turbo P-state capacity. + * @max_perf: Maximum capacity limit as a fraction of the maximum + * turbo P-state capacity. * @acpi_perf_data: Stores ACPI perf information read from _PSS * @valid_pss_table: Set to true for valid ACPI _PSS entries found * @epp_powersave: Last saved HWP energy performance preference @@ -268,7 +257,8 @@ struct cpudata { u64 prev_tsc; u64 prev_cummulative_iowait; struct sample sample; - struct perf_limits perf_limits; + int32_t min_perf; + int32_t max_perf; #ifdef CONFIG_ACPI struct acpi_processor_performance acpi_perf_data; bool valid_pss_table; @@ -833,7 +823,6 @@ static void intel_pstate_hwp_set(struct cpufreq_policy *policy) for_each_cpu(cpu, policy->cpus) { struct cpudata *cpu_data = all_cpu_data[cpu]; - struct perf_limits *perf_limits = &cpu_data->perf_limits; s16 epp; rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap); @@ -843,11 +832,11 @@ static void intel_pstate_hwp_set(struct cpufreq_policy *policy) else hw_max = HWP_HIGHEST_PERF(cap); - max = fp_ext_toint(hw_max * perf_limits->max_perf); + max = fp_ext_toint(hw_max * cpu_data->max_perf); if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) min = max; else - min = fp_ext_toint(hw_max * perf_limits->min_perf); + min = fp_ext_toint(hw_max * cpu_data->min_perf); rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); @@ -1637,7 +1626,6 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) int max_perf = cpu->pstate.turbo_pstate; int max_perf_adj; int min_perf; - struct perf_limits *perf_limits = &cpu->perf_limits; if (global.no_turbo || global.turbo_disabled) max_perf = cpu->pstate.max_pstate; @@ -1647,11 +1635,11 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) * policy, or by cpu specific default values determined through * experimentation. */ - max_perf_adj = fp_ext_toint(max_perf * perf_limits->max_perf); + max_perf_adj = fp_ext_toint(max_perf * cpu->max_perf); *max = clamp_t(int, max_perf_adj, cpu->pstate.min_pstate, cpu->pstate.turbo_pstate); - min_perf = fp_ext_toint(max_perf * perf_limits->min_perf); + min_perf = fp_ext_toint(max_perf * cpu->min_perf); *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf); } @@ -2030,7 +2018,6 @@ static int intel_pstate_get_max_freq(struct cpudata *cpu) static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, struct cpudata *cpu) { - struct perf_limits *limits = &cpu->perf_limits; int max_freq = intel_pstate_get_max_freq(cpu); int32_t max_policy_perf, min_policy_perf; @@ -2046,8 +2033,8 @@ static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, /* Normalize user input to [min_perf, max_perf] */ if (per_cpu_limits) { - limits->min_perf = min_policy_perf; - limits->max_perf = max_policy_perf; + cpu->min_perf = min_policy_perf; + cpu->max_perf = max_policy_perf; } else { int32_t global_min, global_max; @@ -2064,21 +2051,21 @@ static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, } global_min = clamp_t(int32_t, global_min, 0, global_max); - limits->min_perf = max(min_policy_perf, global_min); - limits->min_perf = min(limits->min_perf, max_policy_perf); - limits->max_perf = min(max_policy_perf, global_max); - limits->max_perf = max(min_policy_perf, limits->max_perf); + cpu->min_perf = max(min_policy_perf, global_min); + cpu->min_perf = min(cpu->min_perf, max_policy_perf); + cpu->max_perf = min(max_policy_perf, global_max); + cpu->max_perf = max(min_policy_perf, cpu->max_perf); /* Make sure min_perf <= max_perf */ - limits->min_perf = min(limits->min_perf, limits->max_perf); + cpu->min_perf = min(cpu->min_perf, cpu->max_perf); } - limits->max_perf = round_up(limits->max_perf, EXT_FRAC_BITS); - limits->min_perf = round_up(limits->min_perf, EXT_FRAC_BITS); + cpu->max_perf = round_up(cpu->max_perf, EXT_FRAC_BITS); + cpu->min_perf = round_up(cpu->min_perf, EXT_FRAC_BITS); pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu, - fp_ext_toint(limits->max_perf * 100), - fp_ext_toint(limits->min_perf * 100)); + fp_ext_toint(cpu->max_perf * 100), + fp_ext_toint(cpu->min_perf * 100)); } static int intel_pstate_set_policy(struct cpufreq_policy *policy) @@ -2181,8 +2168,8 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) cpu = all_cpu_data[policy->cpu]; - cpu->perf_limits.max_perf = int_ext_tofp(1); - cpu->perf_limits.min_perf = 0; + cpu->max_perf = int_ext_tofp(1); + cpu->min_perf = 0; policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling; policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling; From 6404367862bb25730e373cb9d443757b76f6abcc Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:04:30 +0200 Subject: [PATCH 16/37] cpufreq: intel_pstate: Drop pointless initialization of PID parameters The P-state selection algorithm used by intel_pstate for Atom processors is not based on the PID controller and the initialization of PID parametrs for those processors is pointless and confusing, so drop it. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index a7ed42d6f366..efce4e7eeeca 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1540,14 +1540,6 @@ static struct cpu_defaults core_params = { }; static const struct cpu_defaults silvermont_params = { - .pid_policy = { - .sample_rate_ms = 10, - .deadband = 0, - .setpoint = 60, - .p_gain_pct = 14, - .d_gain_pct = 0, - .i_gain_pct = 4, - }, .funcs = { .get_max = atom_get_max_pstate, .get_max_physical = atom_get_max_pstate, @@ -1561,14 +1553,6 @@ static const struct cpu_defaults silvermont_params = { }; static const struct cpu_defaults airmont_params = { - .pid_policy = { - .sample_rate_ms = 10, - .deadband = 0, - .setpoint = 60, - .p_gain_pct = 14, - .d_gain_pct = 0, - .i_gain_pct = 4, - }, .funcs = { .get_max = atom_get_max_pstate, .get_max_physical = atom_get_max_pstate, @@ -1602,14 +1586,6 @@ static const struct cpu_defaults knl_params = { }; static const struct cpu_defaults bxt_params = { - .pid_policy = { - .sample_rate_ms = 10, - .deadband = 0, - .setpoint = 60, - .p_gain_pct = 14, - .d_gain_pct = 0, - .i_gain_pct = 4, - }, .funcs = { .get_max = core_get_max_pstate, .get_max_physical = core_get_max_pstate_physical, @@ -2637,9 +2613,9 @@ static int __init intel_pstate_init(void) return -ENODEV; cpu_def = (struct cpu_defaults *)id->driver_data; - - copy_pid_params(&cpu_def->pid_policy); copy_cpu_funcs(&cpu_def->funcs); + if (pstate_funcs.get_target_pstate == get_target_pstate_use_performance) + copy_pid_params(&cpu_def->pid_policy); } if (intel_pstate_msrs_not_valid()) From 5c43905369bb85fd518363e743b68e2407d83f7c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:05:44 +0200 Subject: [PATCH 17/37] cpufreq: intel_pstate: Initialize pid_params statically Notice that both the existing struct cpu_defaults instances in which PID parameters are actually initialized use the same values of those parameters, so it is not really necessary to copy them over to pid_params dynamically. Instead, initialize pid_params statically with those values and drop the unused pid_policy member from struct cpu_defaults along with copy_pid_params() used for initializing it. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 42 ++++++++-------------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index efce4e7eeeca..01f8f289b882 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -321,19 +321,26 @@ struct pstate_funcs { /** * struct cpu_defaults- Per CPU model default config data - * @pid_policy: PID config data * @funcs: Callback function data */ struct cpu_defaults { - struct pstate_adjust_policy pid_policy; struct pstate_funcs funcs; }; static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu); static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu); -static struct pstate_adjust_policy pid_params __read_mostly; static struct pstate_funcs pstate_funcs __read_mostly; +static struct pstate_adjust_policy pid_params __read_mostly = { + .sample_rate_ms = 10, + .sample_rate_ns = 10 * NSEC_PER_MSEC, + .deadband = 0, + .setpoint = 97, + .p_gain_pct = 20, + .d_gain_pct = 0, + .i_gain_pct = 0, +}; + static int hwp_active __read_mostly; static bool per_cpu_limits __read_mostly; @@ -1520,14 +1527,6 @@ static int knl_get_turbo_pstate(void) } static struct cpu_defaults core_params = { - .pid_policy = { - .sample_rate_ms = 10, - .deadband = 0, - .setpoint = 97, - .p_gain_pct = 20, - .d_gain_pct = 0, - .i_gain_pct = 0, - }, .funcs = { .get_max = core_get_max_pstate, .get_max_physical = core_get_max_pstate_physical, @@ -1566,14 +1565,6 @@ static const struct cpu_defaults airmont_params = { }; static const struct cpu_defaults knl_params = { - .pid_policy = { - .sample_rate_ms = 10, - .deadband = 0, - .setpoint = 97, - .p_gain_pct = 20, - .d_gain_pct = 0, - .i_gain_pct = 0, - }, .funcs = { .get_max = core_get_max_pstate, .get_max_physical = core_get_max_pstate_physical, @@ -2412,17 +2403,6 @@ static int __init intel_pstate_msrs_not_valid(void) return 0; } -static void __init copy_pid_params(struct pstate_adjust_policy *policy) -{ - pid_params.sample_rate_ms = policy->sample_rate_ms; - pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; - pid_params.p_gain_pct = policy->p_gain_pct; - pid_params.i_gain_pct = policy->i_gain_pct; - pid_params.d_gain_pct = policy->d_gain_pct; - pid_params.deadband = policy->deadband; - pid_params.setpoint = policy->setpoint; -} - #ifdef CONFIG_ACPI static void intel_pstate_use_acpi_profile(void) { @@ -2614,8 +2594,6 @@ static int __init intel_pstate_init(void) cpu_def = (struct cpu_defaults *)id->driver_data; copy_cpu_funcs(&cpu_def->funcs); - if (pstate_funcs.get_target_pstate == get_target_pstate_use_performance) - copy_pid_params(&cpu_def->pid_policy); } if (intel_pstate_msrs_not_valid()) From 4ddd0146c790e647a05ee5c734b82cb40ef26296 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:07:15 +0200 Subject: [PATCH 18/37] cpufreq: intel_pstate: Fold intel_pstate_reset_all_pid() into the caller There is only one caller of intel_pstate_reset_all_pid(), which is pid_param_set() used in the debugfs interface only, and having that code split does not make it particularly convenient to follow. For this reason, move the body of intel_pstate_reset_all_pid() into its caller and drop that function. Also change the loop from for_each_online_cpu() (which is obviously racy with respect to CPU offline/online) to for_each_possible_cpu(), so that all PID parameters are reset for all CPUs regardless of their online/offline status (to prevent, for example, a previously offline CPU from going online with a stale set of PID parameters). Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 01f8f289b882..a5af890827eb 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -576,16 +576,6 @@ static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu) pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0); } -static inline void intel_pstate_reset_all_pid(void) -{ - unsigned int cpu; - - for_each_online_cpu(cpu) { - if (all_cpu_data[cpu]) - intel_pstate_busy_pid_reset(all_cpu_data[cpu]); - } -} - static inline void update_turbo_state(void) { u64 misc_en; @@ -941,9 +931,14 @@ static void intel_pstate_update_policies(void) /************************** debugfs begin ************************/ static int pid_param_set(void *data, u64 val) { + unsigned int cpu; + *(u32 *)data = val; pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; - intel_pstate_reset_all_pid(); + for_each_possible_cpu(cpu) + if (all_cpu_data[cpu]) + intel_pstate_busy_pid_reset(all_cpu_data[cpu]); + return 0; } From ff35f02ea1e3ac4e774f2784c1444fba4cf8e16a Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:09:18 +0200 Subject: [PATCH 19/37] cpufreq: intel_pstate: Clean up intel_pstate_busy_pid_reset() intel_pstate_busy_pid_reset() is the only caller of pid_reset(), pid_p_gain_set(), pid_i_gain_set(), and pid_d_gain_set(). Moreover, it passes constants as two parameters of pid_reset() and all of the other routines above essentially contain the same code, so fold all of them into the caller and drop unnecessary computations. Introduce percent_fp() for converting integer values in percent to fixed-point fractions and use it in the above code cleanup. Finally, rename intel_pstate_busy_pid_reset() to intel_pstate_pid_reset() as it also is used for the initialization of PID parameters for every CPU and the meaning of the "busy" part of the name is not particularly clear. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 46 ++++++++++++---------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index a5af890827eb..5585a2d101a7 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -74,6 +74,11 @@ static inline int ceiling_fp(int32_t x) return ret; } +static inline int32_t percent_fp(int percent) +{ + return div_fp(percent, 100); +} + static inline u64 mul_ext_fp(u64 x, u64 y) { return (x * y) >> EXT_FRAC_BITS; @@ -507,29 +512,6 @@ static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) } #endif -static inline void pid_reset(struct _pid *pid, int setpoint, int busy, - int deadband, int integral) { - pid->setpoint = int_tofp(setpoint); - pid->deadband = int_tofp(deadband); - pid->integral = int_tofp(integral); - pid->last_err = int_tofp(setpoint) - int_tofp(busy); -} - -static inline void pid_p_gain_set(struct _pid *pid, int percent) -{ - pid->p_gain = div_fp(percent, 100); -} - -static inline void pid_i_gain_set(struct _pid *pid, int percent) -{ - pid->i_gain = div_fp(percent, 100); -} - -static inline void pid_d_gain_set(struct _pid *pid, int percent) -{ - pid->d_gain = div_fp(percent, 100); -} - static signed int pid_calc(struct _pid *pid, int32_t busy) { signed int result; @@ -567,13 +549,17 @@ static signed int pid_calc(struct _pid *pid, int32_t busy) return (signed int)fp_toint(result); } -static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu) +static inline void intel_pstate_pid_reset(struct cpudata *cpu) { - pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct); - pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct); - pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct); + struct _pid *pid = &cpu->pid; - pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0); + pid->p_gain = percent_fp(pid_params.p_gain_pct); + pid->d_gain = percent_fp(pid_params.d_gain_pct); + pid->i_gain = percent_fp(pid_params.i_gain_pct); + pid->setpoint = int_tofp(pid_params.setpoint); + pid->last_err = pid->setpoint - int_tofp(100); + pid->deadband = int_tofp(pid_params.deadband); + pid->integral = 0; } static inline void update_turbo_state(void) @@ -937,7 +923,7 @@ static int pid_param_set(void *data, u64 val) pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; for_each_possible_cpu(cpu) if (all_cpu_data[cpu]) - intel_pstate_busy_pid_reset(all_cpu_data[cpu]); + intel_pstate_pid_reset(all_cpu_data[cpu]); return 0; } @@ -1931,7 +1917,7 @@ static int intel_pstate_init_cpu(unsigned int cpunum) intel_pstate_get_cpu_pstates(cpu); - intel_pstate_busy_pid_reset(cpu); + intel_pstate_pid_reset(cpu); pr_debug("controlling: cpu %d\n", cpunum); From 7aec5b50e97dd3e2e6ad8a87ccfd62ba8f49a105 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:10:47 +0200 Subject: [PATCH 20/37] cpufreq: intel_pstate: Set HWP sampling interval once In the HWP enabled case pid_params.sample_rate_ns only needs to be updated once, because it is global, so do that when setting hwp_active instead of doing it during the initialization of every CPU. Moreover, pid_params.sample_rate_ms is never used if HWP is enabled, so do not update it at all then. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 5585a2d101a7..b631ab02f170 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1911,8 +1911,6 @@ static int intel_pstate_init_cpu(unsigned int cpunum) intel_pstate_disable_ee(cpunum); intel_pstate_hwp_enable(cpu); - pid_params.sample_rate_ms = 50; - pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC; } intel_pstate_get_cpu_pstates(cpu); @@ -2563,6 +2561,7 @@ static int __init intel_pstate_init(void) } else { hwp_active++; intel_pstate.attr = hwp_cpufreq_attrs; + pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC; goto hwp_cpu_matched; } } else { From 694cb173475a048a05daebf27cc8fdb7865c158b Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:11:53 +0200 Subject: [PATCH 21/37] cpufreq: intel_pstate: Skip unnecessary PID resets on init PID controller parameters only need to be initialized if the get_target_pstate_use_performance() P-state selection routine is going to be used. It is not necessary to initialize them otherwise, so don't do that. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index b631ab02f170..ee61db93163c 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1911,12 +1911,12 @@ static int intel_pstate_init_cpu(unsigned int cpunum) intel_pstate_disable_ee(cpunum); intel_pstate_hwp_enable(cpu); + } else if (pstate_funcs.get_target_pstate == get_target_pstate_use_performance) { + intel_pstate_pid_reset(cpu); } intel_pstate_get_cpu_pstates(cpu); - intel_pstate_pid_reset(cpu); - pr_debug("controlling: cpu %d\n", cpunum); return 0; From ee8df89a68f27a7484f1fc71d3d69149dd4dd267 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:13:00 +0200 Subject: [PATCH 22/37] cpufreq: intel_pstate: Drop driver_registered variable The driver_registered variable in intel_pstate is used for checking whether or not the driver has been registered, but intel_pstate_driver can be used for that too (with the rule that the driver is not registered as long as it is NULL). That is a bit more straightforward and the code may be simplified a bit this way, so modify the driver accordingly. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 46 ++++++++++++++-------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ee61db93163c..73ccddf94cf4 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -349,7 +349,7 @@ static struct pstate_adjust_policy pid_params __read_mostly = { static int hwp_active __read_mostly; static bool per_cpu_limits __read_mostly; -static bool driver_registered __read_mostly; +static struct cpufreq_driver *intel_pstate_driver __read_mostly; #ifdef CONFIG_ACPI static bool acpi_ppc; @@ -1035,7 +1035,7 @@ static ssize_t show_turbo_pct(struct kobject *kobj, mutex_lock(&intel_pstate_driver_lock); - if (!driver_registered) { + if (!intel_pstate_driver) { mutex_unlock(&intel_pstate_driver_lock); return -EAGAIN; } @@ -1060,7 +1060,7 @@ static ssize_t show_num_pstates(struct kobject *kobj, mutex_lock(&intel_pstate_driver_lock); - if (!driver_registered) { + if (!intel_pstate_driver) { mutex_unlock(&intel_pstate_driver_lock); return -EAGAIN; } @@ -1080,7 +1080,7 @@ static ssize_t show_no_turbo(struct kobject *kobj, mutex_lock(&intel_pstate_driver_lock); - if (!driver_registered) { + if (!intel_pstate_driver) { mutex_unlock(&intel_pstate_driver_lock); return -EAGAIN; } @@ -1108,7 +1108,7 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, mutex_lock(&intel_pstate_driver_lock); - if (!driver_registered) { + if (!intel_pstate_driver) { mutex_unlock(&intel_pstate_driver_lock); return -EAGAIN; } @@ -1155,7 +1155,7 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, mutex_lock(&intel_pstate_driver_lock); - if (!driver_registered) { + if (!intel_pstate_driver) { mutex_unlock(&intel_pstate_driver_lock); return -EAGAIN; } @@ -1185,7 +1185,7 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, mutex_lock(&intel_pstate_driver_lock); - if (!driver_registered) { + if (!intel_pstate_driver) { mutex_unlock(&intel_pstate_driver_lock); return -EAGAIN; } @@ -2255,7 +2255,7 @@ static struct cpufreq_driver intel_cpufreq = { .name = "intel_cpufreq", }; -static struct cpufreq_driver *intel_pstate_driver = &intel_pstate; +static struct cpufreq_driver *default_driver = &intel_pstate; static void intel_pstate_driver_cleanup(void) { @@ -2272,15 +2272,17 @@ static void intel_pstate_driver_cleanup(void) } } put_online_cpus(); + intel_pstate_driver = NULL; } -static int intel_pstate_register_driver(void) +static int intel_pstate_register_driver(struct cpufreq_driver *driver) { int ret; memset(&global, 0, sizeof(global)); global.max_perf_pct = 100; + intel_pstate_driver = driver; ret = cpufreq_register_driver(intel_pstate_driver); if (ret) { intel_pstate_driver_cleanup(); @@ -2289,10 +2291,6 @@ static int intel_pstate_register_driver(void) global.min_perf_pct = min_perf_pct_min(); - mutex_lock(&intel_pstate_limits_lock); - driver_registered = true; - mutex_unlock(&intel_pstate_limits_lock); - if (intel_pstate_driver == &intel_pstate && !hwp_active && pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load) intel_pstate_debug_expose_params(); @@ -2309,10 +2307,6 @@ static int intel_pstate_unregister_driver(void) pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load) intel_pstate_debug_hide_params(); - mutex_lock(&intel_pstate_limits_lock); - driver_registered = false; - mutex_unlock(&intel_pstate_limits_lock); - cpufreq_unregister_driver(intel_pstate_driver); intel_pstate_driver_cleanup(); @@ -2321,7 +2315,7 @@ static int intel_pstate_unregister_driver(void) static ssize_t intel_pstate_show_status(char *buf) { - if (!driver_registered) + if (!intel_pstate_driver) return sprintf(buf, "off\n"); return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ? @@ -2333,11 +2327,11 @@ static int intel_pstate_update_status(const char *buf, size_t size) int ret; if (size == 3 && !strncmp(buf, "off", size)) - return driver_registered ? + return intel_pstate_driver ? intel_pstate_unregister_driver() : -EINVAL; if (size == 6 && !strncmp(buf, "active", size)) { - if (driver_registered) { + if (intel_pstate_driver) { if (intel_pstate_driver == &intel_pstate) return 0; @@ -2346,12 +2340,11 @@ static int intel_pstate_update_status(const char *buf, size_t size) return ret; } - intel_pstate_driver = &intel_pstate; - return intel_pstate_register_driver(); + return intel_pstate_register_driver(&intel_pstate); } if (size == 7 && !strncmp(buf, "passive", size)) { - if (driver_registered) { + if (intel_pstate_driver) { if (intel_pstate_driver != &intel_pstate) return 0; @@ -2360,8 +2353,7 @@ static int intel_pstate_update_status(const char *buf, size_t size) return ret; } - intel_pstate_driver = &intel_cpufreq; - return intel_pstate_register_driver(); + return intel_pstate_register_driver(&intel_cpufreq); } return -EINVAL; @@ -2601,7 +2593,7 @@ hwp_cpu_matched: intel_pstate_sysfs_expose_params(); mutex_lock(&intel_pstate_driver_lock); - rc = intel_pstate_register_driver(); + rc = intel_pstate_register_driver(default_driver); mutex_unlock(&intel_pstate_driver_lock); if (rc) return rc; @@ -2622,7 +2614,7 @@ static int __init intel_pstate_setup(char *str) no_load = 1; } else if (!strcmp(str, "passive")) { pr_info("Passive mode enabled\n"); - intel_pstate_driver = &intel_cpufreq; + default_driver = &intel_cpufreq; no_hwp = 1; } if (!strcmp(str, "no_hwp")) { From 0042b2c0692ade097ef3a6bbffa491da5dc89273 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:14:08 +0200 Subject: [PATCH 23/37] cpufreq: intel_pstate: Modify check in intel_pstate_update_status() One of the checks in intel_pstate_update_status() implicitly relies on the information that there are only two struct cpufreq_driver objects available, but it is better to do it directly against the value it really is about (to make the code easier to follow if nothing else). Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 73ccddf94cf4..185006e9fbdb 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2345,7 +2345,7 @@ static int intel_pstate_update_status(const char *buf, size_t size) if (size == 7 && !strncmp(buf, "passive", size)) { if (intel_pstate_driver) { - if (intel_pstate_driver != &intel_pstate) + if (intel_pstate_driver == &intel_cpufreq) return 0; ret = intel_pstate_unregister_driver(); From eabd22c657f1d23c714f536b859a22a0f22ac7f5 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:15:37 +0200 Subject: [PATCH 24/37] cpufreq: intel_pstate: Use different utilization update callbacks Notice that some overhead in the utilization update callbacks registered by intel_pstate in the active mode can be avoided if those callbacks are tailored to specific configurations of the driver. For example, the utilization update callback for the HWP enabled case only needs to update the average CPU performance periodically whereas the utilization update callback for the PID-based algorithm does not need to take IO-wait boosting into account and so on. With that in mind, define three utilization update callbacks for three different use cases: HWP enabled, the CPU load "powersave" P-state selection algorithm and the PID-based "powersave" P-state selection algorithm and modify the driver initialization to choose the callback matching its current configuration. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 79 +++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 185006e9fbdb..ca7bc19bf10b 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -37,6 +37,9 @@ #include #include +#define INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC) +#define INTEL_PSTATE_HWP_SAMPLING_INTERVAL (50 * NSEC_PER_MSEC) + #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 #ifdef CONFIG_ACPI @@ -1676,7 +1679,11 @@ static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time) * that sample.time will always be reset before setting the utilization * update hook and make the caller skip the sample then. */ - return !!cpu->last_sample_time; + if (cpu->last_sample_time) { + intel_pstate_calc_avg_perf(cpu); + return true; + } + return false; } static inline int32_t get_avg_frequency(struct cpudata *cpu) @@ -1783,7 +1790,7 @@ static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); } -static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) +static void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) { int from, target_pstate; struct sample *sample; @@ -1811,36 +1818,56 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) fp_toint(cpu->iowait_boost * 100)); } +static void intel_pstate_update_util_hwp(struct update_util_data *data, + u64 time, unsigned int flags) +{ + struct cpudata *cpu = container_of(data, struct cpudata, update_util); + u64 delta_ns = time - cpu->sample.time; + + if ((s64)delta_ns >= INTEL_PSTATE_HWP_SAMPLING_INTERVAL) + intel_pstate_sample(cpu, time); +} + +static void intel_pstate_update_util_pid(struct update_util_data *data, + u64 time, unsigned int flags) +{ + struct cpudata *cpu = container_of(data, struct cpudata, update_util); + u64 delta_ns = time - cpu->sample.time; + + if ((s64)delta_ns < pid_params.sample_rate_ns) + return; + + if (intel_pstate_sample(cpu, time)) + intel_pstate_adjust_busy_pstate(cpu); +} + static void intel_pstate_update_util(struct update_util_data *data, u64 time, unsigned int flags) { struct cpudata *cpu = container_of(data, struct cpudata, update_util); u64 delta_ns; - if (pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) { - if (flags & SCHED_CPUFREQ_IOWAIT) { - cpu->iowait_boost = int_tofp(1); - } else if (cpu->iowait_boost) { - /* Clear iowait_boost if the CPU may have been idle. */ - delta_ns = time - cpu->last_update; - if (delta_ns > TICK_NSEC) - cpu->iowait_boost = 0; - } - cpu->last_update = time; + if (flags & SCHED_CPUFREQ_IOWAIT) { + cpu->iowait_boost = int_tofp(1); + } else if (cpu->iowait_boost) { + /* Clear iowait_boost if the CPU may have been idle. */ + delta_ns = time - cpu->last_update; + if (delta_ns > TICK_NSEC) + cpu->iowait_boost = 0; } - + cpu->last_update = time; delta_ns = time - cpu->sample.time; - if ((s64)delta_ns >= pid_params.sample_rate_ns) { - bool sample_taken = intel_pstate_sample(cpu, time); + if ((s64)delta_ns < INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL) + return; - if (sample_taken) { - intel_pstate_calc_avg_perf(cpu); - if (!hwp_active) - intel_pstate_adjust_busy_pstate(cpu); - } - } + if (intel_pstate_sample(cpu, time)) + intel_pstate_adjust_busy_pstate(cpu); } +/* Utilization update callback to register in the active mode. */ +static void (*update_util_cb)(struct update_util_data *data, u64 time, + unsigned int flags) = intel_pstate_update_util; + #define ICPU(model, policy) \ { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ (unsigned long)&policy } @@ -1938,8 +1965,7 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num) /* Prevent intel_pstate_update_util() from using stale data. */ cpu->sample.time = 0; - cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, - intel_pstate_update_util); + cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, update_util_cb); cpu->update_util_set = true; } @@ -2405,6 +2431,9 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs) pstate_funcs.get_target_pstate = funcs->get_target_pstate; intel_pstate_use_acpi_profile(); + + if (pstate_funcs.get_target_pstate == get_target_pstate_use_performance) + update_util_cb = intel_pstate_update_util_pid; } #ifdef CONFIG_ACPI @@ -2549,11 +2578,11 @@ static int __init intel_pstate_init(void) if (x86_match_cpu(hwp_support_ids)) { copy_cpu_funcs(&core_params.funcs); if (no_hwp) { - pstate_funcs.get_target_pstate = get_target_pstate_use_cpu_load; + update_util_cb = intel_pstate_update_util; } else { hwp_active++; intel_pstate.attr = hwp_cpufreq_attrs; - pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC; + update_util_cb = intel_pstate_update_util_hwp; goto hwp_cpu_matched; } } else { From 67dd9bf4416305811d58280dbe108d78ab573d56 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:17:10 +0200 Subject: [PATCH 25/37] cpufreq: intel_pstate: Add update_util callback to pstate_funcs Avoid using extra function pointers during P-state selection by dropping the get_target_pstate member from struct pstate_funcs, adding a new update_util callback to it (to be registered with the CPU scheduler as the utilization update callback in the active mode) and reworking the utilization update callback routines to invoke specific P-state selection functions directly. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 81 ++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ca7bc19bf10b..68ede1006b07 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -311,7 +311,7 @@ struct pstate_adjust_policy { * @get_scaling: Callback to get frequency scaling factor * @get_val: Callback to convert P state to actual MSR write value * @get_vid: Callback to get VID data for Atom platforms - * @get_target_pstate: Callback to a function to calculate next P state to use + * @update_util: Active mode utilization update callback. * * Core and Atom CPU models have different way to get P State limits. This * structure is used to store those callbacks. @@ -324,7 +324,8 @@ struct pstate_funcs { int (*get_scaling)(void); u64 (*get_val)(struct cpudata*, int pstate); void (*get_vid)(struct cpudata *); - int32_t (*get_target_pstate)(struct cpudata *); + void (*update_util)(struct update_util_data *data, u64 time, + unsigned int flags); }; /** @@ -335,9 +336,6 @@ struct cpu_defaults { struct pstate_funcs funcs; }; -static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu); -static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu); - static struct pstate_funcs pstate_funcs __read_mostly; static struct pstate_adjust_policy pid_params __read_mostly = { .sample_rate_ms = 10, @@ -1510,6 +1508,11 @@ static int knl_get_turbo_pstate(void) return ret; } +static void intel_pstate_update_util_pid(struct update_util_data *data, + u64 time, unsigned int flags); +static void intel_pstate_update_util(struct update_util_data *data, u64 time, + unsigned int flags); + static struct cpu_defaults core_params = { .funcs = { .get_max = core_get_max_pstate, @@ -1518,7 +1521,7 @@ static struct cpu_defaults core_params = { .get_turbo = core_get_turbo_pstate, .get_scaling = core_get_scaling, .get_val = core_get_val, - .get_target_pstate = get_target_pstate_use_performance, + .update_util = intel_pstate_update_util_pid, }, }; @@ -1531,7 +1534,7 @@ static const struct cpu_defaults silvermont_params = { .get_val = atom_get_val, .get_scaling = silvermont_get_scaling, .get_vid = atom_get_vid, - .get_target_pstate = get_target_pstate_use_cpu_load, + .update_util = intel_pstate_update_util, }, }; @@ -1544,7 +1547,7 @@ static const struct cpu_defaults airmont_params = { .get_val = atom_get_val, .get_scaling = airmont_get_scaling, .get_vid = atom_get_vid, - .get_target_pstate = get_target_pstate_use_cpu_load, + .update_util = intel_pstate_update_util, }, }; @@ -1556,7 +1559,7 @@ static const struct cpu_defaults knl_params = { .get_turbo = knl_get_turbo_pstate, .get_scaling = core_get_scaling, .get_val = core_get_val, - .get_target_pstate = get_target_pstate_use_performance, + .update_util = intel_pstate_update_util_pid, }, }; @@ -1568,7 +1571,7 @@ static const struct cpu_defaults bxt_params = { .get_turbo = core_get_turbo_pstate, .get_scaling = core_get_scaling, .get_val = core_get_val, - .get_target_pstate = get_target_pstate_use_cpu_load, + .update_util = intel_pstate_update_util, }, }; @@ -1704,6 +1707,9 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu) int32_t busy_frac, boost; int target, avg_pstate; + if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) + return cpu->pstate.turbo_pstate; + busy_frac = div_fp(sample->mperf, sample->tsc); boost = cpu->iowait_boost; @@ -1740,6 +1746,9 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu) int32_t perf_scaled, max_pstate, current_pstate, sample_ratio; u64 duration_ns; + if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) + return cpu->pstate.turbo_pstate; + /* * perf_scaled is the ratio of the average P-state during the last * sampling period to the P-state requested last time (in percent). @@ -1790,16 +1799,11 @@ static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); } -static void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) +static void intel_pstate_adjust_pstate(struct cpudata *cpu, int target_pstate) { - int from, target_pstate; + int from = cpu->pstate.current_pstate; struct sample *sample; - from = cpu->pstate.current_pstate; - - target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ? - cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu); - update_turbo_state(); target_pstate = intel_pstate_prepare_request(cpu, target_pstate); @@ -1837,8 +1841,12 @@ static void intel_pstate_update_util_pid(struct update_util_data *data, if ((s64)delta_ns < pid_params.sample_rate_ns) return; - if (intel_pstate_sample(cpu, time)) - intel_pstate_adjust_busy_pstate(cpu); + if (intel_pstate_sample(cpu, time)) { + int target_pstate; + + target_pstate = get_target_pstate_use_performance(cpu); + intel_pstate_adjust_pstate(cpu, target_pstate); + } } static void intel_pstate_update_util(struct update_util_data *data, u64 time, @@ -1860,13 +1868,13 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time, if ((s64)delta_ns < INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL) return; - if (intel_pstate_sample(cpu, time)) - intel_pstate_adjust_busy_pstate(cpu); -} + if (intel_pstate_sample(cpu, time)) { + int target_pstate; -/* Utilization update callback to register in the active mode. */ -static void (*update_util_cb)(struct update_util_data *data, u64 time, - unsigned int flags) = intel_pstate_update_util; + target_pstate = get_target_pstate_use_cpu_load(cpu); + intel_pstate_adjust_pstate(cpu, target_pstate); + } +} #define ICPU(model, policy) \ { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ @@ -1938,7 +1946,7 @@ static int intel_pstate_init_cpu(unsigned int cpunum) intel_pstate_disable_ee(cpunum); intel_pstate_hwp_enable(cpu); - } else if (pstate_funcs.get_target_pstate == get_target_pstate_use_performance) { + } else if (pstate_funcs.update_util == intel_pstate_update_util_pid) { intel_pstate_pid_reset(cpu); } @@ -1965,7 +1973,8 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num) /* Prevent intel_pstate_update_util() from using stale data. */ cpu->sample.time = 0; - cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, update_util_cb); + cpufreq_add_update_util_hook(cpu_num, &cpu->update_util, + pstate_funcs.update_util); cpu->update_util_set = true; } @@ -2318,7 +2327,7 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver) global.min_perf_pct = min_perf_pct_min(); if (intel_pstate_driver == &intel_pstate && !hwp_active && - pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load) + pstate_funcs.update_util == intel_pstate_update_util_pid) intel_pstate_debug_expose_params(); return 0; @@ -2329,8 +2338,8 @@ static int intel_pstate_unregister_driver(void) if (hwp_active) return -EBUSY; - if (intel_pstate_driver == &intel_pstate && !hwp_active && - pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load) + if (intel_pstate_driver == &intel_pstate && + pstate_funcs.update_util == intel_pstate_update_util_pid) intel_pstate_debug_hide_params(); cpufreq_unregister_driver(intel_pstate_driver); @@ -2409,8 +2418,7 @@ static void intel_pstate_use_acpi_profile(void) case PM_APPLIANCE_PC: case PM_DESKTOP: case PM_WORKSTATION: - pstate_funcs.get_target_pstate = - get_target_pstate_use_cpu_load; + pstate_funcs.update_util = intel_pstate_update_util; } } #else @@ -2428,12 +2436,9 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs) pstate_funcs.get_scaling = funcs->get_scaling; pstate_funcs.get_val = funcs->get_val; pstate_funcs.get_vid = funcs->get_vid; - pstate_funcs.get_target_pstate = funcs->get_target_pstate; + pstate_funcs.update_util = funcs->update_util; intel_pstate_use_acpi_profile(); - - if (pstate_funcs.get_target_pstate == get_target_pstate_use_performance) - update_util_cb = intel_pstate_update_util_pid; } #ifdef CONFIG_ACPI @@ -2578,11 +2583,11 @@ static int __init intel_pstate_init(void) if (x86_match_cpu(hwp_support_ids)) { copy_cpu_funcs(&core_params.funcs); if (no_hwp) { - update_util_cb = intel_pstate_update_util; + pstate_funcs.update_util = intel_pstate_update_util; } else { hwp_active++; intel_pstate.attr = hwp_cpufreq_attrs; - update_util_cb = intel_pstate_update_util_hwp; + pstate_funcs.update_util = intel_pstate_update_util_hwp; goto hwp_cpu_matched; } } else { From de4a76cb585da13f8b0c9f2161ed6fcf5f2379fc Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:18:02 +0200 Subject: [PATCH 26/37] cpufreq: intel_pstate: Move cpu_defaults definitions Move the definitions of the cpu_defaults structures after the definitions of utilization update callback routines to avoid extra declarations of the latter. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 129 ++++++++++++++++----------------- 1 file changed, 62 insertions(+), 67 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 68ede1006b07..59312dc4c401 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1508,73 +1508,6 @@ static int knl_get_turbo_pstate(void) return ret; } -static void intel_pstate_update_util_pid(struct update_util_data *data, - u64 time, unsigned int flags); -static void intel_pstate_update_util(struct update_util_data *data, u64 time, - unsigned int flags); - -static struct cpu_defaults core_params = { - .funcs = { - .get_max = core_get_max_pstate, - .get_max_physical = core_get_max_pstate_physical, - .get_min = core_get_min_pstate, - .get_turbo = core_get_turbo_pstate, - .get_scaling = core_get_scaling, - .get_val = core_get_val, - .update_util = intel_pstate_update_util_pid, - }, -}; - -static const struct cpu_defaults silvermont_params = { - .funcs = { - .get_max = atom_get_max_pstate, - .get_max_physical = atom_get_max_pstate, - .get_min = atom_get_min_pstate, - .get_turbo = atom_get_turbo_pstate, - .get_val = atom_get_val, - .get_scaling = silvermont_get_scaling, - .get_vid = atom_get_vid, - .update_util = intel_pstate_update_util, - }, -}; - -static const struct cpu_defaults airmont_params = { - .funcs = { - .get_max = atom_get_max_pstate, - .get_max_physical = atom_get_max_pstate, - .get_min = atom_get_min_pstate, - .get_turbo = atom_get_turbo_pstate, - .get_val = atom_get_val, - .get_scaling = airmont_get_scaling, - .get_vid = atom_get_vid, - .update_util = intel_pstate_update_util, - }, -}; - -static const struct cpu_defaults knl_params = { - .funcs = { - .get_max = core_get_max_pstate, - .get_max_physical = core_get_max_pstate_physical, - .get_min = core_get_min_pstate, - .get_turbo = knl_get_turbo_pstate, - .get_scaling = core_get_scaling, - .get_val = core_get_val, - .update_util = intel_pstate_update_util_pid, - }, -}; - -static const struct cpu_defaults bxt_params = { - .funcs = { - .get_max = core_get_max_pstate, - .get_max_physical = core_get_max_pstate_physical, - .get_min = core_get_min_pstate, - .get_turbo = core_get_turbo_pstate, - .get_scaling = core_get_scaling, - .get_val = core_get_val, - .update_util = intel_pstate_update_util, - }, -}; - static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) { int max_perf = cpu->pstate.turbo_pstate; @@ -1876,6 +1809,68 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time, } } +static struct cpu_defaults core_params = { + .funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = core_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, + .update_util = intel_pstate_update_util_pid, + }, +}; + +static const struct cpu_defaults silvermont_params = { + .funcs = { + .get_max = atom_get_max_pstate, + .get_max_physical = atom_get_max_pstate, + .get_min = atom_get_min_pstate, + .get_turbo = atom_get_turbo_pstate, + .get_val = atom_get_val, + .get_scaling = silvermont_get_scaling, + .get_vid = atom_get_vid, + .update_util = intel_pstate_update_util, + }, +}; + +static const struct cpu_defaults airmont_params = { + .funcs = { + .get_max = atom_get_max_pstate, + .get_max_physical = atom_get_max_pstate, + .get_min = atom_get_min_pstate, + .get_turbo = atom_get_turbo_pstate, + .get_val = atom_get_val, + .get_scaling = airmont_get_scaling, + .get_vid = atom_get_vid, + .update_util = intel_pstate_update_util, + }, +}; + +static const struct cpu_defaults knl_params = { + .funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = knl_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, + .update_util = intel_pstate_update_util_pid, + }, +}; + +static const struct cpu_defaults bxt_params = { + .funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = core_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, + .update_util = intel_pstate_update_util, + }, +}; + #define ICPU(model, policy) \ { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\ (unsigned long)&policy } From 2f49afc2a6f6ec4bacb2e50d5482ecc111b41ab5 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:19:03 +0200 Subject: [PATCH 27/37] cpufreq: intel_pstate: Drop struct cpu_defaults The cpu_defaults structure is redundant, because it only contains one member of type struct pstate_funcs which can be used directly instead of struct cpu_defaults. For this reason, drop struct cpu_defaults, use struct pstate_funcs directly instead of it where applicable and rename all of the variables of that type accordingly. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 154 ++++++++++++++------------------- 1 file changed, 67 insertions(+), 87 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 59312dc4c401..f8496faa1085 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -328,14 +328,6 @@ struct pstate_funcs { unsigned int flags); }; -/** - * struct cpu_defaults- Per CPU model default config data - * @funcs: Callback function data - */ -struct cpu_defaults { - struct pstate_funcs funcs; -}; - static struct pstate_funcs pstate_funcs __read_mostly; static struct pstate_adjust_policy pid_params __read_mostly = { .sample_rate_ms = 10, @@ -1809,66 +1801,56 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time, } } -static struct cpu_defaults core_params = { - .funcs = { - .get_max = core_get_max_pstate, - .get_max_physical = core_get_max_pstate_physical, - .get_min = core_get_min_pstate, - .get_turbo = core_get_turbo_pstate, - .get_scaling = core_get_scaling, - .get_val = core_get_val, - .update_util = intel_pstate_update_util_pid, - }, +static struct pstate_funcs core_funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = core_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, + .update_util = intel_pstate_update_util_pid, }; -static const struct cpu_defaults silvermont_params = { - .funcs = { - .get_max = atom_get_max_pstate, - .get_max_physical = atom_get_max_pstate, - .get_min = atom_get_min_pstate, - .get_turbo = atom_get_turbo_pstate, - .get_val = atom_get_val, - .get_scaling = silvermont_get_scaling, - .get_vid = atom_get_vid, - .update_util = intel_pstate_update_util, - }, +static const struct pstate_funcs silvermont_funcs = { + .get_max = atom_get_max_pstate, + .get_max_physical = atom_get_max_pstate, + .get_min = atom_get_min_pstate, + .get_turbo = atom_get_turbo_pstate, + .get_val = atom_get_val, + .get_scaling = silvermont_get_scaling, + .get_vid = atom_get_vid, + .update_util = intel_pstate_update_util, }; -static const struct cpu_defaults airmont_params = { - .funcs = { - .get_max = atom_get_max_pstate, - .get_max_physical = atom_get_max_pstate, - .get_min = atom_get_min_pstate, - .get_turbo = atom_get_turbo_pstate, - .get_val = atom_get_val, - .get_scaling = airmont_get_scaling, - .get_vid = atom_get_vid, - .update_util = intel_pstate_update_util, - }, +static const struct pstate_funcs airmont_funcs = { + .get_max = atom_get_max_pstate, + .get_max_physical = atom_get_max_pstate, + .get_min = atom_get_min_pstate, + .get_turbo = atom_get_turbo_pstate, + .get_val = atom_get_val, + .get_scaling = airmont_get_scaling, + .get_vid = atom_get_vid, + .update_util = intel_pstate_update_util, }; -static const struct cpu_defaults knl_params = { - .funcs = { - .get_max = core_get_max_pstate, - .get_max_physical = core_get_max_pstate_physical, - .get_min = core_get_min_pstate, - .get_turbo = knl_get_turbo_pstate, - .get_scaling = core_get_scaling, - .get_val = core_get_val, - .update_util = intel_pstate_update_util_pid, - }, +static const struct pstate_funcs knl_funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = knl_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, + .update_util = intel_pstate_update_util_pid, }; -static const struct cpu_defaults bxt_params = { - .funcs = { - .get_max = core_get_max_pstate, - .get_max_physical = core_get_max_pstate_physical, - .get_min = core_get_min_pstate, - .get_turbo = core_get_turbo_pstate, - .get_scaling = core_get_scaling, - .get_val = core_get_val, - .update_util = intel_pstate_update_util, - }, +static const struct pstate_funcs bxt_funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = core_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, + .update_util = intel_pstate_update_util, }; #define ICPU(model, policy) \ @@ -1876,38 +1858,38 @@ static const struct cpu_defaults bxt_params = { (unsigned long)&policy } static const struct x86_cpu_id intel_pstate_cpu_ids[] = { - ICPU(INTEL_FAM6_SANDYBRIDGE, core_params), - ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_params), - ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_params), - ICPU(INTEL_FAM6_IVYBRIDGE, core_params), - ICPU(INTEL_FAM6_HASWELL_CORE, core_params), - ICPU(INTEL_FAM6_BROADWELL_CORE, core_params), - ICPU(INTEL_FAM6_IVYBRIDGE_X, core_params), - ICPU(INTEL_FAM6_HASWELL_X, core_params), - ICPU(INTEL_FAM6_HASWELL_ULT, core_params), - ICPU(INTEL_FAM6_HASWELL_GT3E, core_params), - ICPU(INTEL_FAM6_BROADWELL_GT3E, core_params), - ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_params), - ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_params), - ICPU(INTEL_FAM6_BROADWELL_X, core_params), - ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params), - ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), - ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params), - ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_params), - ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params), + ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs), + ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_funcs), + ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_funcs), + ICPU(INTEL_FAM6_IVYBRIDGE, core_funcs), + ICPU(INTEL_FAM6_HASWELL_CORE, core_funcs), + ICPU(INTEL_FAM6_BROADWELL_CORE, core_funcs), + ICPU(INTEL_FAM6_IVYBRIDGE_X, core_funcs), + ICPU(INTEL_FAM6_HASWELL_X, core_funcs), + ICPU(INTEL_FAM6_HASWELL_ULT, core_funcs), + ICPU(INTEL_FAM6_HASWELL_GT3E, core_funcs), + ICPU(INTEL_FAM6_BROADWELL_GT3E, core_funcs), + ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_funcs), + ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_funcs), + ICPU(INTEL_FAM6_BROADWELL_X, core_funcs), + ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs), + ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs), + ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs), + ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs), + ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_funcs), {} }; MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { - ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), - ICPU(INTEL_FAM6_BROADWELL_X, core_params), - ICPU(INTEL_FAM6_SKYLAKE_X, core_params), + ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs), + ICPU(INTEL_FAM6_BROADWELL_X, core_funcs), + ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs), {} }; static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { - ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_params), + ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_funcs), {} }; @@ -2576,7 +2558,7 @@ static int __init intel_pstate_init(void) return -ENODEV; if (x86_match_cpu(hwp_support_ids)) { - copy_cpu_funcs(&core_params.funcs); + copy_cpu_funcs(&core_funcs); if (no_hwp) { pstate_funcs.update_util = intel_pstate_update_util; } else { @@ -2587,14 +2569,12 @@ static int __init intel_pstate_init(void) } } else { const struct x86_cpu_id *id; - struct cpu_defaults *cpu_def; id = x86_match_cpu(intel_pstate_cpu_ids); if (!id) return -ENODEV; - cpu_def = (struct cpu_defaults *)id->driver_data; - copy_cpu_funcs(&cpu_def->funcs); + copy_cpu_funcs((struct pstate_funcs *)id->driver_data); } if (intel_pstate_msrs_not_valid()) From 8ca6ce37014e5a9b127fc076448eb95e2b366d05 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:20:13 +0200 Subject: [PATCH 28/37] cpufreq: intel_pstate: Introduce pid_in_use() Add a new function pid_in_use() to return the information on whether or not the PID-based P-state selection algorithm is in use. That allows a couple of complicated conditions in the code to be reduced to simple checks against the new function's return value. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index f8496faa1085..6384557cea69 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1893,6 +1893,8 @@ static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { {} }; +static bool pid_in_use(void); + static int intel_pstate_init_cpu(unsigned int cpunum) { struct cpudata *cpu; @@ -1923,7 +1925,7 @@ static int intel_pstate_init_cpu(unsigned int cpunum) intel_pstate_disable_ee(cpunum); intel_pstate_hwp_enable(cpu); - } else if (pstate_funcs.update_util == intel_pstate_update_util_pid) { + } else if (pid_in_use()) { intel_pstate_pid_reset(cpu); } @@ -2269,6 +2271,12 @@ static struct cpufreq_driver intel_cpufreq = { static struct cpufreq_driver *default_driver = &intel_pstate; +static bool pid_in_use(void) +{ + return intel_pstate_driver == &intel_pstate && + pstate_funcs.update_util == intel_pstate_update_util_pid; +} + static void intel_pstate_driver_cleanup(void) { unsigned int cpu; @@ -2303,8 +2311,7 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver) global.min_perf_pct = min_perf_pct_min(); - if (intel_pstate_driver == &intel_pstate && !hwp_active && - pstate_funcs.update_util == intel_pstate_update_util_pid) + if (pid_in_use()) intel_pstate_debug_expose_params(); return 0; @@ -2315,8 +2322,7 @@ static int intel_pstate_unregister_driver(void) if (hwp_active) return -EBUSY; - if (intel_pstate_driver == &intel_pstate && - pstate_funcs.update_util == intel_pstate_update_util_pid) + if (pid_in_use()) intel_pstate_debug_hide_params(); cpufreq_unregister_driver(intel_pstate_driver); From 2bfc4cbb5fd3848669f1b95fea793f63d8e77fa0 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:22:16 +0200 Subject: [PATCH 29/37] cpufreq: intel_pstate: Do not walk policy->cpus intel_pstate_hwp_set() is the only function walking policy->cpus in intel_pstate. The rest of the code simply assumes one CPU per policy, including the initialization code. Therefore it doesn't make sense for intel_pstate_hwp_set() to walk policy->cpus as it is guaranteed to have only one bit set for policy->cpu. For this reason, rearrange intel_pstate_hwp_set() to take the CPU number as the argument and drop the loop over policy->cpus from it. Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 128 ++++++++++++++++----------------- 1 file changed, 62 insertions(+), 66 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 6384557cea69..5236701958d0 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -792,84 +792,80 @@ static struct freq_attr *hwp_cpufreq_attrs[] = { NULL, }; -static void intel_pstate_hwp_set(struct cpufreq_policy *policy) +static void intel_pstate_hwp_set(unsigned int cpu) { - int min, hw_min, max, hw_max, cpu; + struct cpudata *cpu_data = all_cpu_data[cpu]; + int min, hw_min, max, hw_max; u64 value, cap; + s16 epp; - for_each_cpu(cpu, policy->cpus) { - struct cpudata *cpu_data = all_cpu_data[cpu]; - s16 epp; + rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap); + hw_min = HWP_LOWEST_PERF(cap); + if (global.no_turbo) + hw_max = HWP_GUARANTEED_PERF(cap); + else + hw_max = HWP_HIGHEST_PERF(cap); - rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap); - hw_min = HWP_LOWEST_PERF(cap); - if (global.no_turbo) - hw_max = HWP_GUARANTEED_PERF(cap); - else - hw_max = HWP_HIGHEST_PERF(cap); + max = fp_ext_toint(hw_max * cpu_data->max_perf); + if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) + min = max; + else + min = fp_ext_toint(hw_max * cpu_data->min_perf); - max = fp_ext_toint(hw_max * cpu_data->max_perf); - if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) - min = max; - else - min = fp_ext_toint(hw_max * cpu_data->min_perf); + rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); - rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); + value &= ~HWP_MIN_PERF(~0L); + value |= HWP_MIN_PERF(min); - value &= ~HWP_MIN_PERF(~0L); - value |= HWP_MIN_PERF(min); + value &= ~HWP_MAX_PERF(~0L); + value |= HWP_MAX_PERF(max); - value &= ~HWP_MAX_PERF(~0L); - value |= HWP_MAX_PERF(max); + if (cpu_data->epp_policy == cpu_data->policy) + goto skip_epp; - if (cpu_data->epp_policy == cpu_data->policy) + cpu_data->epp_policy = cpu_data->policy; + + if (cpu_data->epp_saved >= 0) { + epp = cpu_data->epp_saved; + cpu_data->epp_saved = -EINVAL; + goto update_epp; + } + + if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) { + epp = intel_pstate_get_epp(cpu_data, value); + cpu_data->epp_powersave = epp; + /* If EPP read was failed, then don't try to write */ + if (epp < 0) goto skip_epp; - cpu_data->epp_policy = cpu_data->policy; + epp = 0; + } else { + /* skip setting EPP, when saved value is invalid */ + if (cpu_data->epp_powersave < 0) + goto skip_epp; - if (cpu_data->epp_saved >= 0) { - epp = cpu_data->epp_saved; - cpu_data->epp_saved = -EINVAL; - goto update_epp; - } + /* + * No need to restore EPP when it is not zero. This + * means: + * - Policy is not changed + * - user has manually changed + * - Error reading EPB + */ + epp = intel_pstate_get_epp(cpu_data, value); + if (epp) + goto skip_epp; - if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) { - epp = intel_pstate_get_epp(cpu_data, value); - cpu_data->epp_powersave = epp; - /* If EPP read was failed, then don't try to write */ - if (epp < 0) - goto skip_epp; - - - epp = 0; - } else { - /* skip setting EPP, when saved value is invalid */ - if (cpu_data->epp_powersave < 0) - goto skip_epp; - - /* - * No need to restore EPP when it is not zero. This - * means: - * - Policy is not changed - * - user has manually changed - * - Error reading EPB - */ - epp = intel_pstate_get_epp(cpu_data, value); - if (epp) - goto skip_epp; - - epp = cpu_data->epp_powersave; - } -update_epp: - if (static_cpu_has(X86_FEATURE_HWP_EPP)) { - value &= ~GENMASK_ULL(31, 24); - value |= (u64)epp << 24; - } else { - intel_pstate_set_epb(cpu, epp); - } -skip_epp: - wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value); + epp = cpu_data->epp_powersave; } +update_epp: + if (static_cpu_has(X86_FEATURE_HWP_EPP)) { + value &= ~GENMASK_ULL(31, 24); + value |= (u64)epp << 24; + } else { + intel_pstate_set_epb(cpu, epp); + } +skip_epp: + wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value); } static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) @@ -892,7 +888,7 @@ static int intel_pstate_resume(struct cpufreq_policy *policy) mutex_lock(&intel_pstate_limits_lock); all_cpu_data[policy->cpu]->epp_policy = 0; - intel_pstate_hwp_set(policy); + intel_pstate_hwp_set(policy->cpu); mutex_unlock(&intel_pstate_limits_lock); @@ -2057,7 +2053,7 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) intel_pstate_set_update_util_hook(policy->cpu); if (hwp_active) - intel_pstate_hwp_set(policy); + intel_pstate_hwp_set(policy->cpu); mutex_unlock(&intel_pstate_limits_lock); From b02aabe8ab9757a7dd5aa50e201a6d970f7e7a2f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 28 Mar 2017 00:24:26 +0200 Subject: [PATCH 30/37] cpufreq: intel_pstate: Eliminate intel_pstate_get_min_max() Some computations in intel_pstate_get_min_max() are not necessary and one of its two callers doesn't even use the full result. First off, the fixed-point value of cpu->max_perf represents a non-negative number between 0 and 1 inclusive and cpu->min_perf cannot be greater than cpu->max_perf. It is not necessary to check those conditions every time the numbers in question are used. Moreover, since intel_pstate_max_within_limits() only needs the upper boundary, it doesn't make sense to compute the lower one in there and returning min and max from intel_pstate_get_min_max() via pointers doesn't look particularly nice. For the above reasons, drop intel_pstate_get_min_max(), add a helper to get the base P-state for min/max computations and carry out them directly in the previous callers of intel_pstate_get_min_max(). Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 40 ++++++++++++---------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 5236701958d0..b62daf5a4ee8 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1496,26 +1496,10 @@ static int knl_get_turbo_pstate(void) return ret; } -static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) +static int intel_pstate_get_base_pstate(struct cpudata *cpu) { - int max_perf = cpu->pstate.turbo_pstate; - int max_perf_adj; - int min_perf; - - if (global.no_turbo || global.turbo_disabled) - max_perf = cpu->pstate.max_pstate; - - /* - * performance can be limited by user through sysfs, by cpufreq - * policy, or by cpu specific default values determined through - * experimentation. - */ - max_perf_adj = fp_ext_toint(max_perf * cpu->max_perf); - *max = clamp_t(int, max_perf_adj, - cpu->pstate.min_pstate, cpu->pstate.turbo_pstate); - - min_perf = fp_ext_toint(max_perf * cpu->min_perf); - *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf); + return global.no_turbo || global.turbo_disabled ? + cpu->pstate.max_pstate : cpu->pstate.turbo_pstate; } static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) @@ -1538,11 +1522,13 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu) static void intel_pstate_max_within_limits(struct cpudata *cpu) { - int min_pstate, max_pstate; + int pstate; update_turbo_state(); - intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate); - intel_pstate_set_pstate(cpu, max_pstate); + pstate = intel_pstate_get_base_pstate(cpu); + pstate = max(cpu->pstate.min_pstate, + fp_ext_toint(pstate * cpu->max_perf)); + intel_pstate_set_pstate(cpu, pstate); } static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) @@ -1704,11 +1690,13 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu) static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate) { - int max_perf, min_perf; + int max_pstate = intel_pstate_get_base_pstate(cpu); + int min_pstate; - intel_pstate_get_min_max(cpu, &min_perf, &max_perf); - pstate = clamp_t(int, pstate, min_perf, max_perf); - return pstate; + min_pstate = max(cpu->pstate.min_pstate, + fp_ext_toint(max_pstate * cpu->min_perf)); + max_pstate = max(min_pstate, fp_ext_toint(max_pstate * cpu->max_perf)); + return clamp_t(int, pstate, min_pstate, max_pstate); } static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) From 630e57573efa20b586c808400005d0ebfb93fc6a Mon Sep 17 00:00:00 2001 From: "Box, David E" Date: Wed, 29 Mar 2017 09:45:57 -0700 Subject: [PATCH 31/37] cpufreq: intel_pstate: Add support for Gemini Lake Use same parameters as INTEL_FAM6_ATOM_GOLDMONT to enable Gemini Lake. Signed-off-by: Box, David E Signed-off-by: Srinivas Pandruvada Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index b62daf5a4ee8..c31b72b16c2b 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1861,6 +1861,7 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = { ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs), ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs), ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_funcs), + ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE, bxt_funcs), {} }; MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); From 39b64aa1c007b98727db9f501266454fa403166c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 30 Mar 2017 23:36:41 +0200 Subject: [PATCH 32/37] cpufreq: schedutil: Reduce frequencies slower The schedutil governor reduces frequencies too fast in some situations which cases undesirable performance drops to appear. To address that issue, make schedutil reduce the frequency slower by setting it to the average of the value chosen during the previous iteration of governor computations and the new one coming from its frequency selection formula. Link: https://bugzilla.kernel.org/show_bug.cgi?id=194963 Reported-by: John Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- kernel/sched/cpufreq_schedutil.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 848cb47094cd..b1fedf9932d6 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -101,6 +101,9 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time, if (sg_policy->next_freq == next_freq) return; + if (sg_policy->next_freq > next_freq) + next_freq = (sg_policy->next_freq + next_freq) >> 1; + sg_policy->next_freq = next_freq; sg_policy->last_freq_update_time = time; From 1b72e7fd304639f1cd49d1e11955c4974936d88c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 11 Apr 2017 00:20:41 +0200 Subject: [PATCH 33/37] cpufreq: schedutil: Use policy-dependent transition delays Make the schedutil governor take the initial (default) value of the rate_limit_us sysfs attribute from the (new) transition_delay_us policy parameter (to be set by the scaling driver). That will allow scaling drivers to make schedutil use smaller default values of rate_limit_us and reduce the default average time interval between consecutive frequency changes. Make intel_pstate set transition_delay_us to 500. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar --- drivers/cpufreq/intel_pstate.c | 2 ++ include/linux/cpufreq.h | 7 +++++++ kernel/sched/cpufreq_schedutil.c | 15 ++++++++++----- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index c31b72b16c2b..b7de5bd76a31 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -41,6 +41,7 @@ #define INTEL_PSTATE_HWP_SAMPLING_INTERVAL (50 * NSEC_PER_MSEC) #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 +#define INTEL_CPUFREQ_TRANSITION_DELAY 500 #ifdef CONFIG_ACPI #include @@ -2237,6 +2238,7 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) return ret; policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; + policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; /* This reflects the intel_pstate_get_cpu_pstates() setting. */ policy->cur = policy->cpuinfo.min_freq; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 87165f06a307..a5ce0bbeadb5 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -120,6 +120,13 @@ struct cpufreq_policy { bool fast_switch_possible; bool fast_switch_enabled; + /* + * Preferred average time interval between consecutive invocations of + * the driver to set the frequency for this policy. To be set by the + * scaling driver (0, which is the default, means no preference). + */ + unsigned int transition_delay_us; + /* Cached frequency lookup from cpufreq_driver_resolve_freq. */ unsigned int cached_target_freq; int cached_resolved_idx; diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index b1fedf9932d6..76877a62b5fa 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -494,7 +494,6 @@ static int sugov_init(struct cpufreq_policy *policy) { struct sugov_policy *sg_policy; struct sugov_tunables *tunables; - unsigned int lat; int ret = 0; /* State should be equivalent to EXIT */ @@ -533,10 +532,16 @@ static int sugov_init(struct cpufreq_policy *policy) goto stop_kthread; } - tunables->rate_limit_us = LATENCY_MULTIPLIER; - lat = policy->cpuinfo.transition_latency / NSEC_PER_USEC; - if (lat) - tunables->rate_limit_us *= lat; + if (policy->transition_delay_us) { + tunables->rate_limit_us = policy->transition_delay_us; + } else { + unsigned int lat; + + tunables->rate_limit_us = LATENCY_MULTIPLIER; + lat = policy->cpuinfo.transition_latency / NSEC_PER_USEC; + if (lat) + tunables->rate_limit_us *= lat; + } policy->governor_data = sg_policy; sg_policy->tunables = tunables; From 54cad2fce71f3ed2995bfc6d17d4ea5c898f20b1 Mon Sep 17 00:00:00 2001 From: Irina Tirdea Date: Tue, 4 Apr 2017 20:04:11 +0300 Subject: [PATCH 34/37] cpufreq: imx6q: Fix handling EPROBE_DEFER from regulator If there are any errors in getting the cpu0 regulators, the driver returns -ENOENT. In case the regulators are not yet available, the devm_regulator_get calls will return -EPROBE_DEFER, so that the driver can be probed later. If we return -ENOENT, the driver will fail its initialization and will not try to probe again (when the regulators become available). Return the actual error received from regulator_get in probe. Print a differentiated message in case we need to probe the device later and in case we actually failed. Also add a message to inform when the driver has been successfully registered. Signed-off-by: Irina Tirdea Signed-off-by: Leonard Crestez Reviewed-by: Lucas Stach Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/imx6q-cpufreq.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index 7719b02e04f5..be90ee3810bf 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -222,6 +222,13 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) arm_reg = regulator_get(cpu_dev, "arm"); pu_reg = regulator_get_optional(cpu_dev, "pu"); soc_reg = regulator_get(cpu_dev, "soc"); + if (PTR_ERR(arm_reg) == -EPROBE_DEFER || + PTR_ERR(soc_reg) == -EPROBE_DEFER || + PTR_ERR(pu_reg) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + dev_dbg(cpu_dev, "regulators not ready, defer\n"); + goto put_reg; + } if (IS_ERR(arm_reg) || IS_ERR(soc_reg)) { dev_err(cpu_dev, "failed to get regulators\n"); ret = -ENOENT; From 5aa1599ff039a68a5c43e9aa74973f40b1065746 Mon Sep 17 00:00:00 2001 From: Leonard Crestez Date: Tue, 4 Apr 2017 20:04:12 +0300 Subject: [PATCH 35/37] cpufreq: imx6q: Set max suspend_freq to avoid changes during suspend If the cpufreq driver tries to modify voltage/freq during suspend/resume it might need to control an external PMIC via I2C or SPI but those devices might be already suspended. This issue is likely to happen whenever the LDOs have their vin-supply set. To avoid this scenario we just increase cpufreq to the maximum before suspend. Signed-off-by: Leonard Crestez Reviewed-by: Lucas Stach Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/imx6q-cpufreq.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index be90ee3810bf..786122e0455e 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -161,8 +161,13 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index) static int imx6q_cpufreq_init(struct cpufreq_policy *policy) { + int ret; + policy->clk = arm_clk; - return cpufreq_generic_init(policy, freq_table, transition_latency); + ret = cpufreq_generic_init(policy, freq_table, transition_latency); + policy->suspend_freq = policy->max; + + return ret; } static struct cpufreq_driver imx6q_cpufreq_driver = { @@ -173,6 +178,7 @@ static struct cpufreq_driver imx6q_cpufreq_driver = { .init = imx6q_cpufreq_init, .name = "imx6q-cpufreq", .attr = cpufreq_generic_attr, + .suspend = cpufreq_generic_suspend, }; static int imx6q_cpufreq_probe(struct platform_device *pdev) From eafca851639038a7863910e7fac869f5c8bdfb9d Mon Sep 17 00:00:00 2001 From: Christophe Jaillet Date: Sun, 9 Apr 2017 09:33:52 +0200 Subject: [PATCH 36/37] cpufreq: imx6q: Fix error handling code According to the previous error handling code, it is likely that 'goto out_free_opp' is expected here in order to avoid a memory leak in error handling path. Signed-off-by: Christophe JAILLET Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/imx6q-cpufreq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index 786122e0455e..9c13f097fd8c 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -268,7 +268,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); if (ret) { dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); - goto put_reg; + goto out_free_opp; } /* Make imx6_soc_volt array's size same as arm opp number */ From 939dc6f51e90c95a7d88034da48b747f01873bce Mon Sep 17 00:00:00 2001 From: Mikko Perttunen Date: Tue, 11 Apr 2017 11:09:15 +0300 Subject: [PATCH 37/37] cpufreq: Add Tegra186 cpufreq driver Add a new cpufreq driver for Tegra186 (and likely later). The CPUs are organized into two clusters, Denver and A57, with two and four cores respectively. CPU frequency can be adjusted by writing the desired rate divisor and a voltage hint to a special per-core register. The frequency of each core can be set individually; however, this is just a hint as all CPUs in a cluster will run at the maximum rate of non-idle CPUs in the cluster. Signed-off-by: Mikko Perttunen Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/Kconfig.arm | 6 + drivers/cpufreq/Makefile | 1 + drivers/cpufreq/tegra186-cpufreq.c | 275 +++++++++++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 drivers/cpufreq/tegra186-cpufreq.c diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index 74fa5c5904d3..74ed7e9a7f27 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -247,6 +247,12 @@ config ARM_TEGRA124_CPUFREQ help This adds the CPUFreq driver support for Tegra124 SOCs. +config ARM_TEGRA186_CPUFREQ + tristate "Tegra186 CPUFreq support" + depends on ARCH_TEGRA && TEGRA_BPMP + help + This adds the CPUFreq driver support for Tegra186 SOCs. + config ARM_TI_CPUFREQ bool "Texas Instruments CPUFreq support" depends on ARCH_OMAP2PLUS diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 9f5a8045f36d..b7e78f063c4f 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -77,6 +77,7 @@ obj-$(CONFIG_ARM_SPEAR_CPUFREQ) += spear-cpufreq.o obj-$(CONFIG_ARM_STI_CPUFREQ) += sti-cpufreq.o obj-$(CONFIG_ARM_TEGRA20_CPUFREQ) += tegra20-cpufreq.o obj-$(CONFIG_ARM_TEGRA124_CPUFREQ) += tegra124-cpufreq.o +obj-$(CONFIG_ARM_TEGRA186_CPUFREQ) += tegra186-cpufreq.o obj-$(CONFIG_ARM_TI_CPUFREQ) += ti-cpufreq.o obj-$(CONFIG_ARM_VEXPRESS_SPC_CPUFREQ) += vexpress-spc-cpufreq.o obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c new file mode 100644 index 000000000000..fe7875311d62 --- /dev/null +++ b/drivers/cpufreq/tegra186-cpufreq.c @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#define EDVD_CORE_VOLT_FREQ(core) (0x20 + (core) * 0x4) +#define EDVD_CORE_VOLT_FREQ_F_SHIFT 0 +#define EDVD_CORE_VOLT_FREQ_V_SHIFT 16 + +struct tegra186_cpufreq_cluster_info { + unsigned long offset; + int cpus[4]; + unsigned int bpmp_cluster_id; +}; + +#define NO_CPU -1 +static const struct tegra186_cpufreq_cluster_info tegra186_clusters[] = { + /* Denver cluster */ + { + .offset = SZ_64K * 7, + .cpus = { 1, 2, NO_CPU, NO_CPU }, + .bpmp_cluster_id = 0, + }, + /* A57 cluster */ + { + .offset = SZ_64K * 6, + .cpus = { 0, 3, 4, 5 }, + .bpmp_cluster_id = 1, + }, +}; + +struct tegra186_cpufreq_cluster { + const struct tegra186_cpufreq_cluster_info *info; + struct cpufreq_frequency_table *table; +}; + +struct tegra186_cpufreq_data { + void __iomem *regs; + + size_t num_clusters; + struct tegra186_cpufreq_cluster *clusters; +}; + +static int tegra186_cpufreq_init(struct cpufreq_policy *policy) +{ + struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); + unsigned int i; + + for (i = 0; i < data->num_clusters; i++) { + struct tegra186_cpufreq_cluster *cluster = &data->clusters[i]; + const struct tegra186_cpufreq_cluster_info *info = + cluster->info; + int core; + + for (core = 0; core < ARRAY_SIZE(info->cpus); core++) { + if (info->cpus[core] == policy->cpu) + break; + } + if (core == ARRAY_SIZE(info->cpus)) + continue; + + policy->driver_data = + data->regs + info->offset + EDVD_CORE_VOLT_FREQ(core); + cpufreq_table_validate_and_show(policy, cluster->table); + } + + policy->cpuinfo.transition_latency = 300 * 1000; + + return 0; +} + +static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + struct cpufreq_frequency_table *tbl = policy->freq_table + index; + void __iomem *edvd_reg = policy->driver_data; + u32 edvd_val = tbl->driver_data; + + writel(edvd_val, edvd_reg); + + return 0; +} + +static struct cpufreq_driver tegra186_cpufreq_driver = { + .name = "tegra186", + .flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = tegra186_cpufreq_set_target, + .init = tegra186_cpufreq_init, + .attr = cpufreq_generic_attr, +}; + +static struct cpufreq_frequency_table *init_vhint_table( + struct platform_device *pdev, struct tegra_bpmp *bpmp, + unsigned int cluster_id) +{ + struct cpufreq_frequency_table *table; + struct mrq_cpu_vhint_request req; + struct tegra_bpmp_message msg; + struct cpu_vhint_data *data; + int err, i, j, num_rates = 0; + dma_addr_t phys; + void *virt; + + virt = dma_alloc_coherent(bpmp->dev, sizeof(*data), &phys, + GFP_KERNEL | GFP_DMA32); + if (!virt) + return ERR_PTR(-ENOMEM); + + data = (struct cpu_vhint_data *)virt; + + memset(&req, 0, sizeof(req)); + req.addr = phys; + req.cluster_id = cluster_id; + + memset(&msg, 0, sizeof(msg)); + msg.mrq = MRQ_CPU_VHINT; + msg.tx.data = &req; + msg.tx.size = sizeof(req); + + err = tegra_bpmp_transfer(bpmp, &msg); + if (err) { + table = ERR_PTR(err); + goto free; + } + + for (i = data->vfloor; i <= data->vceil; i++) { + u16 ndiv = data->ndiv[i]; + + if (ndiv < data->ndiv_min || ndiv > data->ndiv_max) + continue; + + /* Only store lowest voltage index for each rate */ + if (i > 0 && ndiv == data->ndiv[i - 1]) + continue; + + num_rates++; + } + + table = devm_kcalloc(&pdev->dev, num_rates + 1, sizeof(*table), + GFP_KERNEL); + if (!table) { + table = ERR_PTR(-ENOMEM); + goto free; + } + + for (i = data->vfloor, j = 0; i <= data->vceil; i++) { + struct cpufreq_frequency_table *point; + u16 ndiv = data->ndiv[i]; + u32 edvd_val = 0; + + if (ndiv < data->ndiv_min || ndiv > data->ndiv_max) + continue; + + /* Only store lowest voltage index for each rate */ + if (i > 0 && ndiv == data->ndiv[i - 1]) + continue; + + edvd_val |= i << EDVD_CORE_VOLT_FREQ_V_SHIFT; + edvd_val |= ndiv << EDVD_CORE_VOLT_FREQ_F_SHIFT; + + point = &table[j++]; + point->driver_data = edvd_val; + point->frequency = data->ref_clk_hz * ndiv / data->pdiv / + data->mdiv / 1000; + } + + table[j].frequency = CPUFREQ_TABLE_END; + +free: + dma_free_coherent(bpmp->dev, sizeof(*data), virt, phys); + + return table; +} + +static int tegra186_cpufreq_probe(struct platform_device *pdev) +{ + struct tegra186_cpufreq_data *data; + struct tegra_bpmp *bpmp; + struct resource *res; + unsigned int i = 0, err; + + data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->clusters = devm_kcalloc(&pdev->dev, ARRAY_SIZE(tegra186_clusters), + sizeof(*data->clusters), GFP_KERNEL); + if (!data->clusters) + return -ENOMEM; + + data->num_clusters = ARRAY_SIZE(tegra186_clusters); + + bpmp = tegra_bpmp_get(&pdev->dev); + if (IS_ERR(bpmp)) + return PTR_ERR(bpmp); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(data->regs)) { + err = PTR_ERR(data->regs); + goto put_bpmp; + } + + for (i = 0; i < data->num_clusters; i++) { + struct tegra186_cpufreq_cluster *cluster = &data->clusters[i]; + + cluster->info = &tegra186_clusters[i]; + cluster->table = init_vhint_table( + pdev, bpmp, cluster->info->bpmp_cluster_id); + if (IS_ERR(cluster->table)) { + err = PTR_ERR(cluster->table); + goto put_bpmp; + } + } + + tegra_bpmp_put(bpmp); + + tegra186_cpufreq_driver.driver_data = data; + + err = cpufreq_register_driver(&tegra186_cpufreq_driver); + if (err) + return err; + + return 0; + +put_bpmp: + tegra_bpmp_put(bpmp); + + return err; +} + +static int tegra186_cpufreq_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&tegra186_cpufreq_driver); + + return 0; +} + +static const struct of_device_id tegra186_cpufreq_of_match[] = { + { .compatible = "nvidia,tegra186-ccplex-cluster", }, + { } +}; +MODULE_DEVICE_TABLE(of, tegra186_cpufreq_of_match); + +static struct platform_driver tegra186_cpufreq_platform_driver = { + .driver = { + .name = "tegra186-cpufreq", + .of_match_table = tegra186_cpufreq_of_match, + }, + .probe = tegra186_cpufreq_probe, + .remove = tegra186_cpufreq_remove, +}; +module_platform_driver(tegra186_cpufreq_platform_driver); + +MODULE_AUTHOR("Mikko Perttunen "); +MODULE_DESCRIPTION("NVIDIA Tegra186 cpufreq driver"); +MODULE_LICENSE("GPL v2");