2017-06-18 16:38:11 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/*
|
|
|
|
* System Control and Power Interface (SCMI) based CPUFreq Interface driver
|
|
|
|
*
|
|
|
|
* Copyright (C) 2018 ARM Ltd.
|
|
|
|
* Sudeep Holla <sudeep.holla@arm.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
2020-11-10 19:10:40 +08:00
|
|
|
#include <linux/clk-provider.h>
|
2017-06-18 16:38:11 +08:00
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/cpufreq.h>
|
|
|
|
#include <linux/cpumask.h>
|
2019-02-04 19:09:52 +08:00
|
|
|
#include <linux/energy_model.h>
|
2017-06-18 16:38:11 +08:00
|
|
|
#include <linux/export.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/pm_opp.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/scmi_protocol.h>
|
|
|
|
#include <linux/types.h>
|
|
|
|
|
|
|
|
struct scmi_data {
|
|
|
|
int domain_id;
|
|
|
|
struct device *cpu_dev;
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct scmi_handle *handle;
|
|
|
|
|
|
|
|
static unsigned int scmi_cpufreq_get_rate(unsigned int cpu)
|
|
|
|
{
|
|
|
|
struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
|
2020-09-07 07:04:50 +08:00
|
|
|
const struct scmi_perf_ops *perf_ops = handle->perf_ops;
|
2017-06-18 16:38:11 +08:00
|
|
|
struct scmi_data *priv = policy->driver_data;
|
|
|
|
unsigned long rate;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = perf_ops->freq_get(handle, priv->domain_id, &rate, false);
|
|
|
|
if (ret)
|
|
|
|
return 0;
|
|
|
|
return rate / 1000;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* perf_ops->freq_set is not a synchronous, the actual OPP change will
|
|
|
|
* happen asynchronously and can get notified if the events are
|
|
|
|
* subscribed for by the SCMI firmware
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
scmi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index)
|
|
|
|
{
|
|
|
|
struct scmi_data *priv = policy->driver_data;
|
2020-09-07 07:04:50 +08:00
|
|
|
const struct scmi_perf_ops *perf_ops = handle->perf_ops;
|
2019-01-09 18:42:36 +08:00
|
|
|
u64 freq = policy->freq_table[index].frequency;
|
2017-06-18 16:38:11 +08:00
|
|
|
|
cpufreq: move invariance setter calls in cpufreq core
To properly scale its per-entity load-tracking signals, the task scheduler
needs to be given a frequency scale factor, i.e. some image of the current
frequency the CPU is running at. Currently, this scale can be computed
either by using counters (APERF/MPERF on x86, AMU on arm64), or by
piggy-backing on the frequency selection done by cpufreq.
For the latter, drivers have to explicitly set the scale factor
themselves, despite it being purely boiler-plate code: the required
information depends entirely on the kind of frequency switch callback
implemented by the driver, i.e. either of: target_index(), target(),
fast_switch() and setpolicy().
The fitness of those callbacks with regard to driving the Frequency
Invariance Engine (FIE) is studied below:
target_index()
==============
Documentation states that the chosen frequency "must be determined by
freq_table[index].frequency". It isn't clear if it *has* to be that
frequency, or if it can use that frequency value to do some computation
that ultimately leads to a different frequency selection. All drivers
go for the former, while the vexpress-spc-cpufreq has an atypical
implementation which is handled separately.
Therefore, the hook works on the assumption the core can use
freq_table[index].frequency.
target()
=======
This has been flagged as deprecated since:
commit 9c0ebcf78fde ("cpufreq: Implement light weight ->target_index() routine")
It also doesn't have that many users:
gx-suspmod.c:439: .target = cpufreq_gx_target,
s3c24xx-cpufreq.c:428: .target = s3c_cpufreq_target,
intel_pstate.c:2528: .target = intel_cpufreq_target,
cppc_cpufreq.c:401: .target = cppc_cpufreq_set_target,
cpufreq-nforce2.c:371: .target = nforce2_target,
sh-cpufreq.c:163: .target = sh_cpufreq_target,
pcc-cpufreq.c:573: .target = pcc_cpufreq_target,
Similarly to the path taken for target_index() calls in the cpufreq core
during a frequency change, all of the drivers above will mark the end of a
frequency change by a call to cpufreq_freq_transition_end().
Therefore, cpufreq_freq_transition_end() can be used as the location for
the arch_set_freq_scale() call to potentially inform the scheduler of the
frequency change.
This change maintains the previous functionality for the drivers that
implement the target_index() callback, while also adding support for the
few drivers that implement the deprecated target() callback.
fast_switch()
=============
This callback *has* to return the frequency that was selected.
setpolicy()
===========
This callback does not have any designated way of informing what was the
end choice. But there are only two drivers using setpolicy(), and none
of them have current FIE support:
drivers/cpufreq/longrun.c:281: .setpolicy = longrun_set_policy,
drivers/cpufreq/intel_pstate.c:2215: .setpolicy = intel_pstate_set_policy,
The intel_pstate is known to use counter-driven frequency invariance.
Conclusion
==========
Given that the significant majority of current FIE enabled drivers use
callbacks that lend themselves to triggering the setting of the FIE scale
factor in a generic way, move the invariance setter calls to cpufreq core.
As a result of setting the frequency scale factor in cpufreq core, after
callbacks that lend themselves to trigger it, remove this functionality
from the driver side.
To be noted that despite marking a successful frequency change, many
cpufreq drivers will consider the new frequency as the requested
frequency, although this is might not be the one granted by the hardware.
Therefore, the call to arch_set_freq_scale() is a "best effort" one, and
it is up to the architecture if the new frequency is used in the new
frequency scale factor setting (determined by the implementation of
arch_set_freq_scale()) or eventually used by the scheduler (determined
by the implementation of arch_scale_freq_capacity()). The architecture
is in a better position to decide if it has better methods to obtain
more accurate information regarding the current frequency and use that
information instead (for example, the use of counters).
Also, the implementation to arch_set_freq_scale() will now have to handle
error conditions (current frequency == 0) in order to prevent the
overhead in cpufreq core when the default arch_set_freq_scale()
implementation is used.
Signed-off-by: Ionela Voinescu <ionela.voinescu@arm.com>
Suggested-by: Valentin Schneider <valentin.schneider@arm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Acked-by: Sudeep Holla <sudeep.holla@arm.com>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2020-09-02 04:55:46 +08:00
|
|
|
return perf_ops->freq_set(handle, priv->domain_id, freq * 1000, false);
|
2017-06-18 16:38:11 +08:00
|
|
|
}
|
|
|
|
|
2017-07-20 21:43:07 +08:00
|
|
|
static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy,
|
|
|
|
unsigned int target_freq)
|
|
|
|
{
|
|
|
|
struct scmi_data *priv = policy->driver_data;
|
2020-09-07 07:04:50 +08:00
|
|
|
const struct scmi_perf_ops *perf_ops = handle->perf_ops;
|
2017-07-20 21:43:07 +08:00
|
|
|
|
|
|
|
if (!perf_ops->freq_set(handle, priv->domain_id,
|
cpufreq: move invariance setter calls in cpufreq core
To properly scale its per-entity load-tracking signals, the task scheduler
needs to be given a frequency scale factor, i.e. some image of the current
frequency the CPU is running at. Currently, this scale can be computed
either by using counters (APERF/MPERF on x86, AMU on arm64), or by
piggy-backing on the frequency selection done by cpufreq.
For the latter, drivers have to explicitly set the scale factor
themselves, despite it being purely boiler-plate code: the required
information depends entirely on the kind of frequency switch callback
implemented by the driver, i.e. either of: target_index(), target(),
fast_switch() and setpolicy().
The fitness of those callbacks with regard to driving the Frequency
Invariance Engine (FIE) is studied below:
target_index()
==============
Documentation states that the chosen frequency "must be determined by
freq_table[index].frequency". It isn't clear if it *has* to be that
frequency, or if it can use that frequency value to do some computation
that ultimately leads to a different frequency selection. All drivers
go for the former, while the vexpress-spc-cpufreq has an atypical
implementation which is handled separately.
Therefore, the hook works on the assumption the core can use
freq_table[index].frequency.
target()
=======
This has been flagged as deprecated since:
commit 9c0ebcf78fde ("cpufreq: Implement light weight ->target_index() routine")
It also doesn't have that many users:
gx-suspmod.c:439: .target = cpufreq_gx_target,
s3c24xx-cpufreq.c:428: .target = s3c_cpufreq_target,
intel_pstate.c:2528: .target = intel_cpufreq_target,
cppc_cpufreq.c:401: .target = cppc_cpufreq_set_target,
cpufreq-nforce2.c:371: .target = nforce2_target,
sh-cpufreq.c:163: .target = sh_cpufreq_target,
pcc-cpufreq.c:573: .target = pcc_cpufreq_target,
Similarly to the path taken for target_index() calls in the cpufreq core
during a frequency change, all of the drivers above will mark the end of a
frequency change by a call to cpufreq_freq_transition_end().
Therefore, cpufreq_freq_transition_end() can be used as the location for
the arch_set_freq_scale() call to potentially inform the scheduler of the
frequency change.
This change maintains the previous functionality for the drivers that
implement the target_index() callback, while also adding support for the
few drivers that implement the deprecated target() callback.
fast_switch()
=============
This callback *has* to return the frequency that was selected.
setpolicy()
===========
This callback does not have any designated way of informing what was the
end choice. But there are only two drivers using setpolicy(), and none
of them have current FIE support:
drivers/cpufreq/longrun.c:281: .setpolicy = longrun_set_policy,
drivers/cpufreq/intel_pstate.c:2215: .setpolicy = intel_pstate_set_policy,
The intel_pstate is known to use counter-driven frequency invariance.
Conclusion
==========
Given that the significant majority of current FIE enabled drivers use
callbacks that lend themselves to triggering the setting of the FIE scale
factor in a generic way, move the invariance setter calls to cpufreq core.
As a result of setting the frequency scale factor in cpufreq core, after
callbacks that lend themselves to trigger it, remove this functionality
from the driver side.
To be noted that despite marking a successful frequency change, many
cpufreq drivers will consider the new frequency as the requested
frequency, although this is might not be the one granted by the hardware.
Therefore, the call to arch_set_freq_scale() is a "best effort" one, and
it is up to the architecture if the new frequency is used in the new
frequency scale factor setting (determined by the implementation of
arch_set_freq_scale()) or eventually used by the scheduler (determined
by the implementation of arch_scale_freq_capacity()). The architecture
is in a better position to decide if it has better methods to obtain
more accurate information regarding the current frequency and use that
information instead (for example, the use of counters).
Also, the implementation to arch_set_freq_scale() will now have to handle
error conditions (current frequency == 0) in order to prevent the
overhead in cpufreq core when the default arch_set_freq_scale()
implementation is used.
Signed-off-by: Ionela Voinescu <ionela.voinescu@arm.com>
Suggested-by: Valentin Schneider <valentin.schneider@arm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Acked-by: Sudeep Holla <sudeep.holla@arm.com>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2020-09-02 04:55:46 +08:00
|
|
|
target_freq * 1000, true))
|
2017-07-20 21:43:07 +08:00
|
|
|
return target_freq;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-06-18 16:38:11 +08:00
|
|
|
static int
|
|
|
|
scmi_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask)
|
|
|
|
{
|
|
|
|
int cpu, domain, tdomain;
|
|
|
|
struct device *tcpu_dev;
|
|
|
|
|
|
|
|
domain = handle->perf_ops->device_domain_id(cpu_dev);
|
|
|
|
if (domain < 0)
|
|
|
|
return domain;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
if (cpu == cpu_dev->id)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
tcpu_dev = get_cpu_device(cpu);
|
|
|
|
if (!tcpu_dev)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
tdomain = handle->perf_ops->device_domain_id(tcpu_dev);
|
|
|
|
if (tdomain == domain)
|
|
|
|
cpumask_set_cpu(cpu, cpumask);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-02-04 19:09:52 +08:00
|
|
|
static int __maybe_unused
|
2020-05-27 17:58:49 +08:00
|
|
|
scmi_get_cpu_power(unsigned long *power, unsigned long *KHz,
|
|
|
|
struct device *cpu_dev)
|
2019-02-04 19:09:52 +08:00
|
|
|
{
|
|
|
|
unsigned long Hz;
|
|
|
|
int ret, domain;
|
|
|
|
|
|
|
|
domain = handle->perf_ops->device_domain_id(cpu_dev);
|
|
|
|
if (domain < 0)
|
|
|
|
return domain;
|
|
|
|
|
|
|
|
/* Get the power cost of the performance domain. */
|
|
|
|
Hz = *KHz * 1000;
|
|
|
|
ret = handle->perf_ops->est_power_get(handle, domain, &Hz, power);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* The EM framework specifies the frequency in KHz. */
|
|
|
|
*KHz = Hz / 1000;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-06-18 16:38:11 +08:00
|
|
|
static int scmi_cpufreq_init(struct cpufreq_policy *policy)
|
|
|
|
{
|
2019-02-04 19:09:52 +08:00
|
|
|
int ret, nr_opp;
|
2017-06-18 16:38:11 +08:00
|
|
|
unsigned int latency;
|
|
|
|
struct device *cpu_dev;
|
|
|
|
struct scmi_data *priv;
|
|
|
|
struct cpufreq_frequency_table *freq_table;
|
2019-02-04 19:09:52 +08:00
|
|
|
struct em_data_callback em_cb = EM_DATA_CB(scmi_get_cpu_power);
|
2020-11-24 18:43:46 +08:00
|
|
|
bool power_scale_mw;
|
2017-06-18 16:38:11 +08:00
|
|
|
|
|
|
|
cpu_dev = get_cpu_device(policy->cpu);
|
|
|
|
if (!cpu_dev) {
|
|
|
|
pr_err("failed to get cpu%d device\n", policy->cpu);
|
|
|
|
return -ENODEV;
|
|
|
|
}
|
|
|
|
|
2018-05-10 00:52:06 +08:00
|
|
|
ret = handle->perf_ops->device_opps_add(handle, cpu_dev);
|
2017-06-18 16:38:11 +08:00
|
|
|
if (ret) {
|
|
|
|
dev_warn(cpu_dev, "failed to add opps to the device\n");
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = scmi_get_sharing_cpus(cpu_dev, policy->cpus);
|
|
|
|
if (ret) {
|
|
|
|
dev_warn(cpu_dev, "failed to get sharing cpumask\n");
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
|
|
|
|
__func__, ret);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2019-02-04 19:09:52 +08:00
|
|
|
nr_opp = dev_pm_opp_get_opp_count(cpu_dev);
|
|
|
|
if (nr_opp <= 0) {
|
2017-06-18 16:38:11 +08:00
|
|
|
dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
|
|
|
|
ret = -EPROBE_DEFER;
|
|
|
|
goto out_free_opp;
|
|
|
|
}
|
|
|
|
|
|
|
|
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
|
|
|
|
if (!priv) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out_free_opp;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
|
|
|
|
goto out_free_priv;
|
|
|
|
}
|
|
|
|
|
|
|
|
priv->cpu_dev = cpu_dev;
|
|
|
|
priv->domain_id = handle->perf_ops->device_domain_id(cpu_dev);
|
|
|
|
|
|
|
|
policy->driver_data = priv;
|
2018-04-03 18:07:38 +08:00
|
|
|
policy->freq_table = freq_table;
|
2017-06-18 16:38:11 +08:00
|
|
|
|
|
|
|
/* SCMI allows DVFS request for any domain from any CPU */
|
|
|
|
policy->dvfs_possible_from_any_cpu = true;
|
|
|
|
|
2018-05-10 00:52:06 +08:00
|
|
|
latency = handle->perf_ops->transition_latency_get(handle, cpu_dev);
|
2017-06-18 16:38:11 +08:00
|
|
|
if (!latency)
|
|
|
|
latency = CPUFREQ_ETERNAL;
|
|
|
|
|
|
|
|
policy->cpuinfo.transition_latency = latency;
|
|
|
|
|
2020-06-17 17:43:32 +08:00
|
|
|
policy->fast_switch_possible =
|
|
|
|
handle->perf_ops->fast_switch_possible(handle, cpu_dev);
|
2019-02-04 19:09:52 +08:00
|
|
|
|
2020-11-24 18:43:46 +08:00
|
|
|
power_scale_mw = handle->perf_ops->power_scale_mw_get(handle);
|
2020-11-05 20:50:01 +08:00
|
|
|
em_dev_register_perf_domain(cpu_dev, nr_opp, &em_cb, policy->cpus,
|
2020-11-24 18:43:46 +08:00
|
|
|
power_scale_mw);
|
2019-02-04 19:09:52 +08:00
|
|
|
|
2017-06-18 16:38:11 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
out_free_priv:
|
|
|
|
kfree(priv);
|
|
|
|
out_free_opp:
|
2019-01-04 17:44:33 +08:00
|
|
|
dev_pm_opp_remove_all_dynamic(cpu_dev);
|
2017-06-18 16:38:11 +08:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int scmi_cpufreq_exit(struct cpufreq_policy *policy)
|
|
|
|
{
|
|
|
|
struct scmi_data *priv = policy->driver_data;
|
|
|
|
|
|
|
|
dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
|
2019-01-04 17:44:33 +08:00
|
|
|
dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
|
2019-02-17 00:31:48 +08:00
|
|
|
kfree(priv);
|
2017-06-18 16:38:11 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct cpufreq_driver scmi_cpufreq_driver = {
|
|
|
|
.name = "scmi",
|
2021-02-02 12:55:11 +08:00
|
|
|
.flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
|
2019-01-29 12:55:14 +08:00
|
|
|
CPUFREQ_NEED_INITIAL_FREQ_CHECK |
|
|
|
|
CPUFREQ_IS_COOLING_DEV,
|
2017-06-18 16:38:11 +08:00
|
|
|
.verify = cpufreq_generic_frequency_table_verify,
|
|
|
|
.attr = cpufreq_generic_attr,
|
|
|
|
.target_index = scmi_cpufreq_set_target,
|
2017-07-20 21:43:07 +08:00
|
|
|
.fast_switch = scmi_cpufreq_fast_switch,
|
2017-06-18 16:38:11 +08:00
|
|
|
.get = scmi_cpufreq_get_rate,
|
|
|
|
.init = scmi_cpufreq_init,
|
|
|
|
.exit = scmi_cpufreq_exit,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int scmi_cpufreq_probe(struct scmi_device *sdev)
|
|
|
|
{
|
|
|
|
int ret;
|
2020-11-10 19:10:40 +08:00
|
|
|
struct device *dev = &sdev->dev;
|
2017-06-18 16:38:11 +08:00
|
|
|
|
|
|
|
handle = sdev->handle;
|
|
|
|
|
|
|
|
if (!handle || !handle->perf_ops)
|
|
|
|
return -ENODEV;
|
|
|
|
|
2020-11-20 18:12:52 +08:00
|
|
|
#ifdef CONFIG_COMMON_CLK
|
2020-11-10 19:10:40 +08:00
|
|
|
/* dummy clock provider as needed by OPP if clocks property is used */
|
|
|
|
if (of_find_property(dev->of_node, "#clock-cells", NULL))
|
|
|
|
devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, NULL);
|
2020-11-20 18:12:52 +08:00
|
|
|
#endif
|
2020-11-10 19:10:40 +08:00
|
|
|
|
2017-06-18 16:38:11 +08:00
|
|
|
ret = cpufreq_register_driver(&scmi_cpufreq_driver);
|
|
|
|
if (ret) {
|
2020-11-20 18:12:52 +08:00
|
|
|
dev_err(dev, "%s: registering cpufreq failed, err: %d\n",
|
2017-06-18 16:38:11 +08:00
|
|
|
__func__, ret);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void scmi_cpufreq_remove(struct scmi_device *sdev)
|
|
|
|
{
|
|
|
|
cpufreq_unregister_driver(&scmi_cpufreq_driver);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct scmi_device_id scmi_id_table[] = {
|
2019-11-07 01:56:25 +08:00
|
|
|
{ SCMI_PROTOCOL_PERF, "cpufreq" },
|
2017-06-18 16:38:11 +08:00
|
|
|
{ },
|
|
|
|
};
|
|
|
|
MODULE_DEVICE_TABLE(scmi, scmi_id_table);
|
|
|
|
|
|
|
|
static struct scmi_driver scmi_cpufreq_drv = {
|
|
|
|
.name = "scmi-cpufreq",
|
|
|
|
.probe = scmi_cpufreq_probe,
|
|
|
|
.remove = scmi_cpufreq_remove,
|
|
|
|
.id_table = scmi_id_table,
|
|
|
|
};
|
|
|
|
module_scmi_driver(scmi_cpufreq_drv);
|
|
|
|
|
|
|
|
MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>");
|
|
|
|
MODULE_DESCRIPTION("ARM SCMI CPUFreq interface driver");
|
|
|
|
MODULE_LICENSE("GPL v2");
|