Merge branch 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull core timer updates from Thomas Gleixner: "Timers and timekeeping updates: - A large overhaul of the posix CPU timer code which is a preparation for moving the CPU timer expiry out into task work so it can be properly accounted on the task/process. An update to the bogus permission checks will come later during the merge window as feedback was not complete before heading of for travel. - Switch the timerqueue code to use cached rbtrees and get rid of the homebrewn caching of the leftmost node. - Consolidate hrtimer_init() + hrtimer_init_sleeper() calls into a single function - Implement the separation of hrtimers to be forced to expire in hard interrupt context even when PREEMPT_RT is enabled and mark the affected timers accordingly. - Implement a mechanism for hrtimers and the timer wheel to protect RT against priority inversion and live lock issues when a (hr)timer which should be canceled is currently executing the callback. Instead of infinitely spinning, the task which tries to cancel the timer blocks on a per cpu base expiry lock which is held and released by the (hr)timer expiry code. - Enable the Hyper-V TSC page based sched_clock for Hyper-V guests resulting in faster access to timekeeping functions. - Updates to various clocksource/clockevent drivers and their device tree bindings. - The usual small improvements all over the place" * 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (101 commits) posix-cpu-timers: Fix permission check regression posix-cpu-timers: Always clear head pointer on dequeue hrtimer: Add a missing bracket and hide `migration_base' on !SMP posix-cpu-timers: Make expiry_active check actually work correctly posix-timers: Unbreak CONFIG_POSIX_TIMERS=n build tick: Mark sched_timer to expire in hard interrupt context hrtimer: Add kernel doc annotation for HRTIMER_MODE_HARD x86/hyperv: Hide pv_ops access for CONFIG_PARAVIRT=n posix-cpu-timers: Utilize timerqueue for storage posix-cpu-timers: Move state tracking to struct posix_cputimers posix-cpu-timers: Deduplicate rlimit handling posix-cpu-timers: Remove pointless comparisons posix-cpu-timers: Get rid of 64bit divisions posix-cpu-timers: Consolidate timer expiry further posix-cpu-timers: Get rid of zero checks rlimit: Rewrite non-sensical RLIMIT_CPU comment posix-cpu-timers: Respect INFINITY for hard RTTIME limit posix-cpu-timers: Switch thread group sampling to array posix-cpu-timers: Restructure expiry array posix-cpu-timers: Remove cputime_expires ...
This commit is contained in:
commit
7f2444d38f
|
@ -0,0 +1,102 @@
|
||||||
|
# SPDX-License-Identifier: GPL-2.0
|
||||||
|
%YAML 1.2
|
||||||
|
---
|
||||||
|
$id: http://devicetree.org/schemas/timer/allwinner,sun4i-a10-timer.yaml#
|
||||||
|
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||||
|
|
||||||
|
title: Allwinner A10 Timer Device Tree Bindings
|
||||||
|
|
||||||
|
maintainers:
|
||||||
|
- Chen-Yu Tsai <wens@csie.org>
|
||||||
|
- Maxime Ripard <maxime.ripard@bootlin.com>
|
||||||
|
|
||||||
|
properties:
|
||||||
|
compatible:
|
||||||
|
enum:
|
||||||
|
- allwinner,sun4i-a10-timer
|
||||||
|
- allwinner,sun8i-a23-timer
|
||||||
|
- allwinner,sun8i-v3s-timer
|
||||||
|
- allwinner,suniv-f1c100s-timer
|
||||||
|
|
||||||
|
reg:
|
||||||
|
maxItems: 1
|
||||||
|
|
||||||
|
interrupts:
|
||||||
|
description:
|
||||||
|
List of timers interrupts
|
||||||
|
|
||||||
|
clocks:
|
||||||
|
maxItems: 1
|
||||||
|
|
||||||
|
allOf:
|
||||||
|
- if:
|
||||||
|
properties:
|
||||||
|
compatible:
|
||||||
|
items:
|
||||||
|
const: allwinner,sun4i-a10-timer
|
||||||
|
|
||||||
|
then:
|
||||||
|
properties:
|
||||||
|
interrupts:
|
||||||
|
minItems: 6
|
||||||
|
maxItems: 6
|
||||||
|
|
||||||
|
- if:
|
||||||
|
properties:
|
||||||
|
compatible:
|
||||||
|
items:
|
||||||
|
const: allwinner,sun8i-a23-timer
|
||||||
|
|
||||||
|
then:
|
||||||
|
properties:
|
||||||
|
interrupts:
|
||||||
|
minItems: 2
|
||||||
|
maxItems: 2
|
||||||
|
|
||||||
|
- if:
|
||||||
|
properties:
|
||||||
|
compatible:
|
||||||
|
items:
|
||||||
|
const: allwinner,sun8i-v3s-timer
|
||||||
|
|
||||||
|
then:
|
||||||
|
properties:
|
||||||
|
interrupts:
|
||||||
|
minItems: 3
|
||||||
|
maxItems: 3
|
||||||
|
|
||||||
|
- if:
|
||||||
|
properties:
|
||||||
|
compatible:
|
||||||
|
items:
|
||||||
|
const: allwinner,suniv-f1c100s-timer
|
||||||
|
|
||||||
|
then:
|
||||||
|
properties:
|
||||||
|
interrupts:
|
||||||
|
minItems: 3
|
||||||
|
maxItems: 3
|
||||||
|
|
||||||
|
required:
|
||||||
|
- compatible
|
||||||
|
- reg
|
||||||
|
- interrupts
|
||||||
|
- clocks
|
||||||
|
|
||||||
|
additionalProperties: false
|
||||||
|
|
||||||
|
examples:
|
||||||
|
- |
|
||||||
|
timer {
|
||||||
|
compatible = "allwinner,sun4i-a10-timer";
|
||||||
|
reg = <0x01c20c00 0x400>;
|
||||||
|
interrupts = <22>,
|
||||||
|
<23>,
|
||||||
|
<24>,
|
||||||
|
<25>,
|
||||||
|
<67>,
|
||||||
|
<68>;
|
||||||
|
clocks = <&osc>;
|
||||||
|
};
|
||||||
|
|
||||||
|
...
|
|
@ -1,19 +0,0 @@
|
||||||
Allwinner A1X SoCs Timer Controller
|
|
||||||
|
|
||||||
Required properties:
|
|
||||||
|
|
||||||
- compatible : should be one of the following:
|
|
||||||
"allwinner,sun4i-a10-timer"
|
|
||||||
"allwinner,suniv-f1c100s-timer"
|
|
||||||
- reg : Specifies base physical address and size of the registers.
|
|
||||||
- interrupts : The interrupt of the first timer
|
|
||||||
- clocks: phandle to the source clock (usually a 24 MHz fixed clock)
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
timer {
|
|
||||||
compatible = "allwinner,sun4i-a10-timer";
|
|
||||||
reg = <0x01c20c00 0x400>;
|
|
||||||
interrupts = <22>;
|
|
||||||
clocks = <&osc>;
|
|
||||||
};
|
|
|
@ -1,26 +0,0 @@
|
||||||
Allwinner SoCs High Speed Timer Controller
|
|
||||||
|
|
||||||
Required properties:
|
|
||||||
|
|
||||||
- compatible : should be "allwinner,sun5i-a13-hstimer" or
|
|
||||||
"allwinner,sun7i-a20-hstimer"
|
|
||||||
- reg : Specifies base physical address and size of the registers.
|
|
||||||
- interrupts : The interrupts of these timers (2 for the sun5i IP, 4 for the sun7i
|
|
||||||
one)
|
|
||||||
- clocks: phandle to the source clock (usually the AHB clock)
|
|
||||||
|
|
||||||
Optional properties:
|
|
||||||
- resets: phandle to a reset controller asserting the timer
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
timer@1c60000 {
|
|
||||||
compatible = "allwinner,sun7i-a20-hstimer";
|
|
||||||
reg = <0x01c60000 0x1000>;
|
|
||||||
interrupts = <0 51 1>,
|
|
||||||
<0 52 1>,
|
|
||||||
<0 53 1>,
|
|
||||||
<0 54 1>;
|
|
||||||
clocks = <&ahb1_gates 19>;
|
|
||||||
resets = <&ahb1rst 19>;
|
|
||||||
};
|
|
|
@ -0,0 +1,79 @@
|
||||||
|
# SPDX-License-Identifier: GPL-2.0
|
||||||
|
%YAML 1.2
|
||||||
|
---
|
||||||
|
$id: http://devicetree.org/schemas/timer/allwinner,sun5i-a13-hstimer.yaml#
|
||||||
|
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||||
|
|
||||||
|
title: Allwinner A13 High-Speed Timer Device Tree Bindings
|
||||||
|
|
||||||
|
maintainers:
|
||||||
|
- Chen-Yu Tsai <wens@csie.org>
|
||||||
|
- Maxime Ripard <maxime.ripard@bootlin.com>
|
||||||
|
|
||||||
|
properties:
|
||||||
|
compatible:
|
||||||
|
oneOf:
|
||||||
|
- const: allwinner,sun5i-a13-hstimer
|
||||||
|
- const: allwinner,sun7i-a20-hstimer
|
||||||
|
- items:
|
||||||
|
- const: allwinner,sun6i-a31-hstimer
|
||||||
|
- const: allwinner,sun7i-a20-hstimer
|
||||||
|
|
||||||
|
reg:
|
||||||
|
maxItems: 1
|
||||||
|
|
||||||
|
interrupts:
|
||||||
|
minItems: 2
|
||||||
|
maxItems: 4
|
||||||
|
items:
|
||||||
|
- description: Timer 0 Interrupt
|
||||||
|
- description: Timer 1 Interrupt
|
||||||
|
- description: Timer 2 Interrupt
|
||||||
|
- description: Timer 3 Interrupt
|
||||||
|
|
||||||
|
clocks:
|
||||||
|
maxItems: 1
|
||||||
|
|
||||||
|
resets:
|
||||||
|
maxItems: 1
|
||||||
|
|
||||||
|
required:
|
||||||
|
- compatible
|
||||||
|
- reg
|
||||||
|
- interrupts
|
||||||
|
- clocks
|
||||||
|
|
||||||
|
if:
|
||||||
|
properties:
|
||||||
|
compatible:
|
||||||
|
items:
|
||||||
|
const: allwinner,sun5i-a13-hstimer
|
||||||
|
|
||||||
|
then:
|
||||||
|
properties:
|
||||||
|
interrupts:
|
||||||
|
minItems: 2
|
||||||
|
maxItems: 2
|
||||||
|
|
||||||
|
else:
|
||||||
|
properties:
|
||||||
|
interrupts:
|
||||||
|
minItems: 4
|
||||||
|
maxItems: 4
|
||||||
|
|
||||||
|
additionalProperties: false
|
||||||
|
|
||||||
|
examples:
|
||||||
|
- |
|
||||||
|
timer@1c60000 {
|
||||||
|
compatible = "allwinner,sun7i-a20-hstimer";
|
||||||
|
reg = <0x01c60000 0x1000>;
|
||||||
|
interrupts = <0 51 1>,
|
||||||
|
<0 52 1>,
|
||||||
|
<0 53 1>,
|
||||||
|
<0 54 1>;
|
||||||
|
clocks = <&ahb1_gates 19>;
|
||||||
|
resets = <&ahb1rst 19>;
|
||||||
|
};
|
||||||
|
|
||||||
|
...
|
|
@ -12,16 +12,13 @@ datasheets.
|
||||||
Required Properties:
|
Required Properties:
|
||||||
|
|
||||||
- compatible: must contain one or more of the following:
|
- compatible: must contain one or more of the following:
|
||||||
- "renesas,cmt-48-sh73a0" for the sh73A0 48-bit CMT
|
|
||||||
(CMT1)
|
|
||||||
- "renesas,cmt-48-r8a7740" for the r8a7740 48-bit CMT
|
|
||||||
(CMT1)
|
|
||||||
- "renesas,cmt-48" for all non-second generation 48-bit CMT
|
|
||||||
(CMT1 on sh73a0 and r8a7740)
|
|
||||||
This is a fallback for the above renesas,cmt-48-* entries.
|
|
||||||
|
|
||||||
- "renesas,r8a73a4-cmt0" for the 32-bit CMT0 device included in r8a73a4.
|
- "renesas,r8a73a4-cmt0" for the 32-bit CMT0 device included in r8a73a4.
|
||||||
- "renesas,r8a73a4-cmt1" for the 48-bit CMT1 device included in r8a73a4.
|
- "renesas,r8a73a4-cmt1" for the 48-bit CMT1 device included in r8a73a4.
|
||||||
|
- "renesas,r8a7740-cmt0" for the 32-bit CMT0 device included in r8a7740.
|
||||||
|
- "renesas,r8a7740-cmt1" for the 48-bit CMT1 device included in r8a7740.
|
||||||
|
- "renesas,r8a7740-cmt2" for the 32-bit CMT2 device included in r8a7740.
|
||||||
|
- "renesas,r8a7740-cmt3" for the 32-bit CMT3 device included in r8a7740.
|
||||||
|
- "renesas,r8a7740-cmt4" for the 32-bit CMT4 device included in r8a7740.
|
||||||
- "renesas,r8a7743-cmt0" for the 32-bit CMT0 device included in r8a7743.
|
- "renesas,r8a7743-cmt0" for the 32-bit CMT0 device included in r8a7743.
|
||||||
- "renesas,r8a7743-cmt1" for the 48-bit CMT1 device included in r8a7743.
|
- "renesas,r8a7743-cmt1" for the 48-bit CMT1 device included in r8a7743.
|
||||||
- "renesas,r8a7744-cmt0" for the 32-bit CMT0 device included in r8a7744.
|
- "renesas,r8a7744-cmt0" for the 32-bit CMT0 device included in r8a7744.
|
||||||
|
@ -31,29 +28,38 @@ Required Properties:
|
||||||
- "renesas,r8a77470-cmt0" for the 32-bit CMT0 device included in r8a77470.
|
- "renesas,r8a77470-cmt0" for the 32-bit CMT0 device included in r8a77470.
|
||||||
- "renesas,r8a77470-cmt1" for the 48-bit CMT1 device included in r8a77470.
|
- "renesas,r8a77470-cmt1" for the 48-bit CMT1 device included in r8a77470.
|
||||||
- "renesas,r8a774a1-cmt0" for the 32-bit CMT0 device included in r8a774a1.
|
- "renesas,r8a774a1-cmt0" for the 32-bit CMT0 device included in r8a774a1.
|
||||||
- "renesas,r8a774a1-cmt1" for the 48-bit CMT1 device included in r8a774a1.
|
- "renesas,r8a774a1-cmt1" for the 48-bit CMT devices included in r8a774a1.
|
||||||
- "renesas,r8a774c0-cmt0" for the 32-bit CMT0 device included in r8a774c0.
|
- "renesas,r8a774c0-cmt0" for the 32-bit CMT0 device included in r8a774c0.
|
||||||
- "renesas,r8a774c0-cmt1" for the 48-bit CMT1 device included in r8a774c0.
|
- "renesas,r8a774c0-cmt1" for the 48-bit CMT devices included in r8a774c0.
|
||||||
- "renesas,r8a7790-cmt0" for the 32-bit CMT0 device included in r8a7790.
|
- "renesas,r8a7790-cmt0" for the 32-bit CMT0 device included in r8a7790.
|
||||||
- "renesas,r8a7790-cmt1" for the 48-bit CMT1 device included in r8a7790.
|
- "renesas,r8a7790-cmt1" for the 48-bit CMT1 device included in r8a7790.
|
||||||
- "renesas,r8a7791-cmt0" for the 32-bit CMT0 device included in r8a7791.
|
- "renesas,r8a7791-cmt0" for the 32-bit CMT0 device included in r8a7791.
|
||||||
- "renesas,r8a7791-cmt1" for the 48-bit CMT1 device included in r8a7791.
|
- "renesas,r8a7791-cmt1" for the 48-bit CMT1 device included in r8a7791.
|
||||||
|
- "renesas,r8a7792-cmt0" for the 32-bit CMT0 device included in r8a7792.
|
||||||
|
- "renesas,r8a7792-cmt1" for the 48-bit CMT1 device included in r8a7792.
|
||||||
- "renesas,r8a7793-cmt0" for the 32-bit CMT0 device included in r8a7793.
|
- "renesas,r8a7793-cmt0" for the 32-bit CMT0 device included in r8a7793.
|
||||||
- "renesas,r8a7793-cmt1" for the 48-bit CMT1 device included in r8a7793.
|
- "renesas,r8a7793-cmt1" for the 48-bit CMT1 device included in r8a7793.
|
||||||
- "renesas,r8a7794-cmt0" for the 32-bit CMT0 device included in r8a7794.
|
- "renesas,r8a7794-cmt0" for the 32-bit CMT0 device included in r8a7794.
|
||||||
- "renesas,r8a7794-cmt1" for the 48-bit CMT1 device included in r8a7794.
|
- "renesas,r8a7794-cmt1" for the 48-bit CMT1 device included in r8a7794.
|
||||||
- "renesas,r8a7795-cmt0" for the 32-bit CMT0 device included in r8a7795.
|
- "renesas,r8a7795-cmt0" for the 32-bit CMT0 device included in r8a7795.
|
||||||
- "renesas,r8a7795-cmt1" for the 48-bit CMT1 device included in r8a7795.
|
- "renesas,r8a7795-cmt1" for the 48-bit CMT devices included in r8a7795.
|
||||||
- "renesas,r8a7796-cmt0" for the 32-bit CMT0 device included in r8a7796.
|
- "renesas,r8a7796-cmt0" for the 32-bit CMT0 device included in r8a7796.
|
||||||
- "renesas,r8a7796-cmt1" for the 48-bit CMT1 device included in r8a7796.
|
- "renesas,r8a7796-cmt1" for the 48-bit CMT devices included in r8a7796.
|
||||||
- "renesas,r8a77965-cmt0" for the 32-bit CMT0 device included in r8a77965.
|
- "renesas,r8a77965-cmt0" for the 32-bit CMT0 device included in r8a77965.
|
||||||
- "renesas,r8a77965-cmt1" for the 48-bit CMT1 device included in r8a77965.
|
- "renesas,r8a77965-cmt1" for the 48-bit CMT devices included in r8a77965.
|
||||||
- "renesas,r8a77970-cmt0" for the 32-bit CMT0 device included in r8a77970.
|
- "renesas,r8a77970-cmt0" for the 32-bit CMT0 device included in r8a77970.
|
||||||
- "renesas,r8a77970-cmt1" for the 48-bit CMT1 device included in r8a77970.
|
- "renesas,r8a77970-cmt1" for the 48-bit CMT devices included in r8a77970.
|
||||||
- "renesas,r8a77980-cmt0" for the 32-bit CMT0 device included in r8a77980.
|
- "renesas,r8a77980-cmt0" for the 32-bit CMT0 device included in r8a77980.
|
||||||
- "renesas,r8a77980-cmt1" for the 48-bit CMT1 device included in r8a77980.
|
- "renesas,r8a77980-cmt1" for the 48-bit CMT devices included in r8a77980.
|
||||||
- "renesas,r8a77990-cmt0" for the 32-bit CMT0 device included in r8a77990.
|
- "renesas,r8a77990-cmt0" for the 32-bit CMT0 device included in r8a77990.
|
||||||
- "renesas,r8a77990-cmt1" for the 48-bit CMT1 device included in r8a77990.
|
- "renesas,r8a77990-cmt1" for the 48-bit CMT devices included in r8a77990.
|
||||||
|
- "renesas,r8a77995-cmt0" for the 32-bit CMT0 device included in r8a77995.
|
||||||
|
- "renesas,r8a77995-cmt1" for the 48-bit CMT devices included in r8a77995.
|
||||||
|
- "renesas,sh73a0-cmt0" for the 32-bit CMT0 device included in sh73a0.
|
||||||
|
- "renesas,sh73a0-cmt1" for the 48-bit CMT1 device included in sh73a0.
|
||||||
|
- "renesas,sh73a0-cmt2" for the 32-bit CMT2 device included in sh73a0.
|
||||||
|
- "renesas,sh73a0-cmt3" for the 32-bit CMT3 device included in sh73a0.
|
||||||
|
- "renesas,sh73a0-cmt4" for the 32-bit CMT4 device included in sh73a0.
|
||||||
|
|
||||||
- "renesas,rcar-gen2-cmt0" for 32-bit CMT0 devices included in R-Car Gen2
|
- "renesas,rcar-gen2-cmt0" for 32-bit CMT0 devices included in R-Car Gen2
|
||||||
and RZ/G1.
|
and RZ/G1.
|
||||||
|
@ -63,7 +69,7 @@ Required Properties:
|
||||||
listed above.
|
listed above.
|
||||||
- "renesas,rcar-gen3-cmt0" for 32-bit CMT0 devices included in R-Car Gen3
|
- "renesas,rcar-gen3-cmt0" for 32-bit CMT0 devices included in R-Car Gen3
|
||||||
and RZ/G2.
|
and RZ/G2.
|
||||||
- "renesas,rcar-gen3-cmt1" for 48-bit CMT1 devices included in R-Car Gen3
|
- "renesas,rcar-gen3-cmt1" for 48-bit CMT devices included in R-Car Gen3
|
||||||
and RZ/G2.
|
and RZ/G2.
|
||||||
These are fallbacks for R-Car Gen3 and RZ/G2 entries listed
|
These are fallbacks for R-Car Gen3 and RZ/G2 entries listed
|
||||||
above.
|
above.
|
||||||
|
|
|
@ -546,6 +546,14 @@
|
||||||
#pwm-cells = <2>;
|
#pwm-cells = <2>;
|
||||||
status = "disabled";
|
status = "disabled";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
system_counter: timer@306a0000 {
|
||||||
|
compatible = "nxp,sysctr-timer";
|
||||||
|
reg = <0x306a0000 0x20000>;
|
||||||
|
interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
|
||||||
|
clocks = <&osc_24m>;
|
||||||
|
clock-names = "per";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
aips3: bus@30800000 {
|
aips3: bus@30800000 {
|
||||||
|
|
|
@ -651,6 +651,14 @@
|
||||||
#pwm-cells = <2>;
|
#pwm-cells = <2>;
|
||||||
status = "disabled";
|
status = "disabled";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
system_counter: timer@306a0000 {
|
||||||
|
compatible = "nxp,sysctr-timer";
|
||||||
|
reg = <0x306a0000 0x20000>;
|
||||||
|
interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
|
||||||
|
clocks = <&osc_25m>;
|
||||||
|
clock-names = "per";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
bus@30800000 { /* AIPS3 */
|
bus@30800000 { /* AIPS3 */
|
||||||
|
|
|
@ -122,7 +122,7 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
|
||||||
|
|
||||||
if (tsc_pg && vclock_was_used(VCLOCK_HVCLOCK))
|
if (tsc_pg && vclock_was_used(VCLOCK_HVCLOCK))
|
||||||
return vmf_insert_pfn(vma, vmf->address,
|
return vmf_insert_pfn(vma, vmf->address,
|
||||||
vmalloc_to_pfn(tsc_pg));
|
virt_to_phys(tsc_pg) >> PAGE_SHIFT);
|
||||||
}
|
}
|
||||||
|
|
||||||
return VM_FAULT_SIGBUS;
|
return VM_FAULT_SIGBUS;
|
||||||
|
|
|
@ -315,8 +315,6 @@ void __init hyperv_init(void)
|
||||||
|
|
||||||
x86_init.pci.arch_init = hv_pci_init;
|
x86_init.pci.arch_init = hv_pci_init;
|
||||||
|
|
||||||
/* Register Hyper-V specific clocksource */
|
|
||||||
hv_init_clocksource();
|
|
||||||
return;
|
return;
|
||||||
|
|
||||||
remove_cpuhp_state:
|
remove_cpuhp_state:
|
||||||
|
|
|
@ -51,7 +51,7 @@ extern struct pvclock_vsyscall_time_info pvclock_page
|
||||||
__attribute__((visibility("hidden")));
|
__attribute__((visibility("hidden")));
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_HYPERV_TSCPAGE
|
#ifdef CONFIG_HYPERV_TIMER
|
||||||
extern struct ms_hyperv_tsc_page hvclock_page
|
extern struct ms_hyperv_tsc_page hvclock_page
|
||||||
__attribute__((visibility("hidden")));
|
__attribute__((visibility("hidden")));
|
||||||
#endif
|
#endif
|
||||||
|
@ -228,7 +228,7 @@ static u64 vread_pvclock(void)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_HYPERV_TSCPAGE
|
#ifdef CONFIG_HYPERV_TIMER
|
||||||
static u64 vread_hvclock(void)
|
static u64 vread_hvclock(void)
|
||||||
{
|
{
|
||||||
return hv_read_tsc_page(&hvclock_page);
|
return hv_read_tsc_page(&hvclock_page);
|
||||||
|
@ -251,7 +251,7 @@ static inline u64 __arch_get_hw_counter(s32 clock_mode)
|
||||||
return vread_pvclock();
|
return vread_pvclock();
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
#ifdef CONFIG_HYPERV_TSCPAGE
|
#ifdef CONFIG_HYPERV_TIMER
|
||||||
if (clock_mode == VCLOCK_HVCLOCK) {
|
if (clock_mode == VCLOCK_HVCLOCK) {
|
||||||
barrier();
|
barrier();
|
||||||
return vread_hvclock();
|
return vread_hvclock();
|
||||||
|
|
|
@ -29,6 +29,7 @@
|
||||||
#include <asm/timer.h>
|
#include <asm/timer.h>
|
||||||
#include <asm/reboot.h>
|
#include <asm/reboot.h>
|
||||||
#include <asm/nmi.h>
|
#include <asm/nmi.h>
|
||||||
|
#include <clocksource/hyperv_timer.h>
|
||||||
|
|
||||||
struct ms_hyperv_info ms_hyperv;
|
struct ms_hyperv_info ms_hyperv;
|
||||||
EXPORT_SYMBOL_GPL(ms_hyperv);
|
EXPORT_SYMBOL_GPL(ms_hyperv);
|
||||||
|
@ -338,6 +339,15 @@ static void __init ms_hyperv_init_platform(void)
|
||||||
x2apic_phys = 1;
|
x2apic_phys = 1;
|
||||||
# endif
|
# endif
|
||||||
|
|
||||||
|
/* Register Hyper-V specific clocksource */
|
||||||
|
hv_init_clocksource();
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
void hv_setup_sched_clock(void *sched_clock)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_PARAVIRT
|
||||||
|
pv_ops.time.sched_clock = sched_clock;
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1598,7 +1598,7 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic)
|
||||||
likely(ns > apic->lapic_timer.timer_advance_ns)) {
|
likely(ns > apic->lapic_timer.timer_advance_ns)) {
|
||||||
expire = ktime_add_ns(now, ns);
|
expire = ktime_add_ns(now, ns);
|
||||||
expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
|
expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
|
||||||
hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS);
|
hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
|
||||||
} else
|
} else
|
||||||
apic_timer_expired(apic);
|
apic_timer_expired(apic);
|
||||||
|
|
||||||
|
@ -2299,7 +2299,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
|
||||||
apic->vcpu = vcpu;
|
apic->vcpu = vcpu;
|
||||||
|
|
||||||
hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
|
hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
|
||||||
HRTIMER_MODE_ABS);
|
HRTIMER_MODE_ABS_HARD);
|
||||||
apic->lapic_timer.timer.function = apic_timer_fn;
|
apic->lapic_timer.timer.function = apic_timer_fn;
|
||||||
if (timer_advance_ns == -1) {
|
if (timer_advance_ns == -1) {
|
||||||
apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
|
apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
|
||||||
|
@ -2484,7 +2484,7 @@ void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
|
||||||
|
|
||||||
timer = &vcpu->arch.apic->lapic_timer.timer;
|
timer = &vcpu->arch.apic->lapic_timer.timer;
|
||||||
if (hrtimer_cancel(timer))
|
if (hrtimer_cancel(timer))
|
||||||
hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
|
hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -3411,15 +3411,14 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
|
||||||
kt = nsecs;
|
kt = nsecs;
|
||||||
|
|
||||||
mode = HRTIMER_MODE_REL;
|
mode = HRTIMER_MODE_REL;
|
||||||
hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
|
hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode);
|
||||||
hrtimer_set_expires(&hs.timer, kt);
|
hrtimer_set_expires(&hs.timer, kt);
|
||||||
|
|
||||||
hrtimer_init_sleeper(&hs, current);
|
|
||||||
do {
|
do {
|
||||||
if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
|
if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
|
||||||
break;
|
break;
|
||||||
set_current_state(TASK_UNINTERRUPTIBLE);
|
set_current_state(TASK_UNINTERRUPTIBLE);
|
||||||
hrtimer_start_expires(&hs.timer, mode);
|
hrtimer_sleeper_start_expires(&hs, mode);
|
||||||
if (hs.task)
|
if (hs.task)
|
||||||
io_schedule();
|
io_schedule();
|
||||||
hrtimer_cancel(&hs.timer);
|
hrtimer_cancel(&hs.timer);
|
||||||
|
|
|
@ -429,7 +429,7 @@ config ATMEL_ST
|
||||||
|
|
||||||
config ATMEL_TCB_CLKSRC
|
config ATMEL_TCB_CLKSRC
|
||||||
bool "Atmel TC Block timer driver" if COMPILE_TEST
|
bool "Atmel TC Block timer driver" if COMPILE_TEST
|
||||||
depends on HAS_IOMEM
|
depends on ARM && HAS_IOMEM
|
||||||
select TIMER_OF if OF
|
select TIMER_OF if OF
|
||||||
help
|
help
|
||||||
Support for Timer Counter Blocks on Atmel SoCs.
|
Support for Timer Counter Blocks on Atmel SoCs.
|
||||||
|
|
|
@ -291,10 +291,8 @@ static int em_sti_probe(struct platform_device *pdev)
|
||||||
platform_set_drvdata(pdev, p);
|
platform_set_drvdata(pdev, p);
|
||||||
|
|
||||||
irq = platform_get_irq(pdev, 0);
|
irq = platform_get_irq(pdev, 0);
|
||||||
if (irq < 0) {
|
if (irq < 0)
|
||||||
dev_err(&pdev->dev, "failed to get irq\n");
|
|
||||||
return irq;
|
return irq;
|
||||||
}
|
|
||||||
|
|
||||||
/* map memory, let base point to the STI instance */
|
/* map memory, let base point to the STI instance */
|
||||||
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
|
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
|
||||||
|
|
|
@ -22,6 +22,7 @@
|
||||||
#include <asm/mshyperv.h>
|
#include <asm/mshyperv.h>
|
||||||
|
|
||||||
static struct clock_event_device __percpu *hv_clock_event;
|
static struct clock_event_device __percpu *hv_clock_event;
|
||||||
|
static u64 hv_sched_clock_offset __ro_after_init;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If false, we're using the old mechanism for stimer0 interrupts
|
* If false, we're using the old mechanism for stimer0 interrupts
|
||||||
|
@ -212,19 +213,17 @@ EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup);
|
||||||
struct clocksource *hyperv_cs;
|
struct clocksource *hyperv_cs;
|
||||||
EXPORT_SYMBOL_GPL(hyperv_cs);
|
EXPORT_SYMBOL_GPL(hyperv_cs);
|
||||||
|
|
||||||
#ifdef CONFIG_HYPERV_TSCPAGE
|
static struct ms_hyperv_tsc_page tsc_pg __aligned(PAGE_SIZE);
|
||||||
|
|
||||||
static struct ms_hyperv_tsc_page *tsc_pg;
|
|
||||||
|
|
||||||
struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
|
struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
|
||||||
{
|
{
|
||||||
return tsc_pg;
|
return &tsc_pg;
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(hv_get_tsc_page);
|
EXPORT_SYMBOL_GPL(hv_get_tsc_page);
|
||||||
|
|
||||||
static u64 notrace read_hv_sched_clock_tsc(void)
|
static u64 notrace read_hv_clock_tsc(struct clocksource *arg)
|
||||||
{
|
{
|
||||||
u64 current_tick = hv_read_tsc_page(tsc_pg);
|
u64 current_tick = hv_read_tsc_page(&tsc_pg);
|
||||||
|
|
||||||
if (current_tick == U64_MAX)
|
if (current_tick == U64_MAX)
|
||||||
hv_get_time_ref_count(current_tick);
|
hv_get_time_ref_count(current_tick);
|
||||||
|
@ -232,9 +231,9 @@ static u64 notrace read_hv_sched_clock_tsc(void)
|
||||||
return current_tick;
|
return current_tick;
|
||||||
}
|
}
|
||||||
|
|
||||||
static u64 read_hv_clock_tsc(struct clocksource *arg)
|
static u64 read_hv_sched_clock_tsc(void)
|
||||||
{
|
{
|
||||||
return read_hv_sched_clock_tsc();
|
return read_hv_clock_tsc(NULL) - hv_sched_clock_offset;
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct clocksource hyperv_cs_tsc = {
|
static struct clocksource hyperv_cs_tsc = {
|
||||||
|
@ -244,9 +243,8 @@ static struct clocksource hyperv_cs_tsc = {
|
||||||
.mask = CLOCKSOURCE_MASK(64),
|
.mask = CLOCKSOURCE_MASK(64),
|
||||||
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
|
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
|
||||||
};
|
};
|
||||||
#endif
|
|
||||||
|
|
||||||
static u64 notrace read_hv_sched_clock_msr(void)
|
static u64 notrace read_hv_clock_msr(struct clocksource *arg)
|
||||||
{
|
{
|
||||||
u64 current_tick;
|
u64 current_tick;
|
||||||
/*
|
/*
|
||||||
|
@ -258,9 +256,9 @@ static u64 notrace read_hv_sched_clock_msr(void)
|
||||||
return current_tick;
|
return current_tick;
|
||||||
}
|
}
|
||||||
|
|
||||||
static u64 read_hv_clock_msr(struct clocksource *arg)
|
static u64 read_hv_sched_clock_msr(void)
|
||||||
{
|
{
|
||||||
return read_hv_sched_clock_msr();
|
return read_hv_clock_msr(NULL) - hv_sched_clock_offset;
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct clocksource hyperv_cs_msr = {
|
static struct clocksource hyperv_cs_msr = {
|
||||||
|
@ -271,7 +269,6 @@ static struct clocksource hyperv_cs_msr = {
|
||||||
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
|
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
|
||||||
};
|
};
|
||||||
|
|
||||||
#ifdef CONFIG_HYPERV_TSCPAGE
|
|
||||||
static bool __init hv_init_tsc_clocksource(void)
|
static bool __init hv_init_tsc_clocksource(void)
|
||||||
{
|
{
|
||||||
u64 tsc_msr;
|
u64 tsc_msr;
|
||||||
|
@ -280,12 +277,8 @@ static bool __init hv_init_tsc_clocksource(void)
|
||||||
if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE))
|
if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
tsc_pg = vmalloc(PAGE_SIZE);
|
|
||||||
if (!tsc_pg)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
hyperv_cs = &hyperv_cs_tsc;
|
hyperv_cs = &hyperv_cs_tsc;
|
||||||
phys_addr = page_to_phys(vmalloc_to_page(tsc_pg));
|
phys_addr = virt_to_phys(&tsc_pg);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The Hyper-V TLFS specifies to preserve the value of reserved
|
* The Hyper-V TLFS specifies to preserve the value of reserved
|
||||||
|
@ -302,17 +295,11 @@ static bool __init hv_init_tsc_clocksource(void)
|
||||||
hv_set_clocksource_vdso(hyperv_cs_tsc);
|
hv_set_clocksource_vdso(hyperv_cs_tsc);
|
||||||
clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100);
|
clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100);
|
||||||
|
|
||||||
/* sched_clock_register is needed on ARM64 but is a no-op on x86 */
|
hv_sched_clock_offset = hyperv_cs->read(hyperv_cs);
|
||||||
sched_clock_register(read_hv_sched_clock_tsc, 64, HV_CLOCK_HZ);
|
hv_setup_sched_clock(read_hv_sched_clock_tsc);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
#else
|
|
||||||
static bool __init hv_init_tsc_clocksource(void)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
void __init hv_init_clocksource(void)
|
void __init hv_init_clocksource(void)
|
||||||
{
|
{
|
||||||
|
@ -333,7 +320,7 @@ void __init hv_init_clocksource(void)
|
||||||
hyperv_cs = &hyperv_cs_msr;
|
hyperv_cs = &hyperv_cs_msr;
|
||||||
clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100);
|
clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100);
|
||||||
|
|
||||||
/* sched_clock_register is needed on ARM64 but is a no-op on x86 */
|
hv_sched_clock_offset = hyperv_cs->read(hyperv_cs);
|
||||||
sched_clock_register(read_hv_sched_clock_msr, 64, HV_CLOCK_HZ);
|
hv_setup_sched_clock(read_hv_sched_clock_msr);
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(hv_init_clocksource);
|
EXPORT_SYMBOL_GPL(hv_init_clocksource);
|
||||||
|
|
|
@ -221,7 +221,7 @@ static int __init ostm_init(struct device_node *np)
|
||||||
}
|
}
|
||||||
|
|
||||||
rate = clk_get_rate(ostm_clk);
|
rate = clk_get_rate(ostm_clk);
|
||||||
ostm->ticks_per_jiffy = (rate + HZ / 2) / HZ;
|
ostm->ticks_per_jiffy = DIV_ROUND_CLOSEST(rate, HZ);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* First probed device will be used as system clocksource. Any
|
* First probed device will be used as system clocksource. Any
|
||||||
|
|
|
@ -776,11 +776,8 @@ static int sh_cmt_register_clockevent(struct sh_cmt_channel *ch,
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
irq = platform_get_irq(ch->cmt->pdev, ch->index);
|
irq = platform_get_irq(ch->cmt->pdev, ch->index);
|
||||||
if (irq < 0) {
|
if (irq < 0)
|
||||||
dev_err(&ch->cmt->pdev->dev, "ch%u: failed to get irq\n",
|
|
||||||
ch->index);
|
|
||||||
return irq;
|
return irq;
|
||||||
}
|
|
||||||
|
|
||||||
ret = request_irq(irq, sh_cmt_interrupt,
|
ret = request_irq(irq, sh_cmt_interrupt,
|
||||||
IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING,
|
IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING,
|
||||||
|
@ -921,12 +918,24 @@ static const struct platform_device_id sh_cmt_id_table[] = {
|
||||||
MODULE_DEVICE_TABLE(platform, sh_cmt_id_table);
|
MODULE_DEVICE_TABLE(platform, sh_cmt_id_table);
|
||||||
|
|
||||||
static const struct of_device_id sh_cmt_of_table[] __maybe_unused = {
|
static const struct of_device_id sh_cmt_of_table[] __maybe_unused = {
|
||||||
{ .compatible = "renesas,cmt-48", .data = &sh_cmt_info[SH_CMT_48BIT] },
|
{
|
||||||
|
/* deprecated, preserved for backward compatibility */
|
||||||
|
.compatible = "renesas,cmt-48",
|
||||||
|
.data = &sh_cmt_info[SH_CMT_48BIT]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
/* deprecated, preserved for backward compatibility */
|
/* deprecated, preserved for backward compatibility */
|
||||||
.compatible = "renesas,cmt-48-gen2",
|
.compatible = "renesas,cmt-48-gen2",
|
||||||
.data = &sh_cmt_info[SH_CMT0_RCAR_GEN2]
|
.data = &sh_cmt_info[SH_CMT0_RCAR_GEN2]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
.compatible = "renesas,r8a7740-cmt1",
|
||||||
|
.data = &sh_cmt_info[SH_CMT_48BIT]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
.compatible = "renesas,sh73a0-cmt1",
|
||||||
|
.data = &sh_cmt_info[SH_CMT_48BIT]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
.compatible = "renesas,rcar-gen2-cmt0",
|
.compatible = "renesas,rcar-gen2-cmt0",
|
||||||
.data = &sh_cmt_info[SH_CMT0_RCAR_GEN2]
|
.data = &sh_cmt_info[SH_CMT0_RCAR_GEN2]
|
||||||
|
|
|
@ -462,11 +462,8 @@ static int sh_tmu_channel_setup(struct sh_tmu_channel *ch, unsigned int index,
|
||||||
ch->base = tmu->mapbase + 8 + ch->index * 12;
|
ch->base = tmu->mapbase + 8 + ch->index * 12;
|
||||||
|
|
||||||
ch->irq = platform_get_irq(tmu->pdev, index);
|
ch->irq = platform_get_irq(tmu->pdev, index);
|
||||||
if (ch->irq < 0) {
|
if (ch->irq < 0)
|
||||||
dev_err(&tmu->pdev->dev, "ch%u: failed to get irq\n",
|
|
||||||
ch->index);
|
|
||||||
return ch->irq;
|
return ch->irq;
|
||||||
}
|
|
||||||
|
|
||||||
ch->cs_enabled = false;
|
ch->cs_enabled = false;
|
||||||
ch->enable_count = 0;
|
ch->enable_count = 0;
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
#include <linux/irq.h>
|
#include <linux/irq.h>
|
||||||
|
|
||||||
#include <linux/clk.h>
|
#include <linux/clk.h>
|
||||||
|
#include <linux/delay.h>
|
||||||
#include <linux/err.h>
|
#include <linux/err.h>
|
||||||
#include <linux/ioport.h>
|
#include <linux/ioport.h>
|
||||||
#include <linux/io.h>
|
#include <linux/io.h>
|
||||||
|
@ -125,6 +126,18 @@ static u64 notrace tc_sched_clock_read32(void)
|
||||||
return tc_get_cycles32(&clksrc);
|
return tc_get_cycles32(&clksrc);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static struct delay_timer tc_delay_timer;
|
||||||
|
|
||||||
|
static unsigned long tc_delay_timer_read(void)
|
||||||
|
{
|
||||||
|
return tc_get_cycles(&clksrc);
|
||||||
|
}
|
||||||
|
|
||||||
|
static unsigned long notrace tc_delay_timer_read32(void)
|
||||||
|
{
|
||||||
|
return tc_get_cycles32(&clksrc);
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_GENERIC_CLOCKEVENTS
|
#ifdef CONFIG_GENERIC_CLOCKEVENTS
|
||||||
|
|
||||||
struct tc_clkevt_device {
|
struct tc_clkevt_device {
|
||||||
|
@ -432,6 +445,7 @@ static int __init tcb_clksrc_init(struct device_node *node)
|
||||||
/* setup ony channel 0 */
|
/* setup ony channel 0 */
|
||||||
tcb_setup_single_chan(&tc, best_divisor_idx);
|
tcb_setup_single_chan(&tc, best_divisor_idx);
|
||||||
tc_sched_clock = tc_sched_clock_read32;
|
tc_sched_clock = tc_sched_clock_read32;
|
||||||
|
tc_delay_timer.read_current_timer = tc_delay_timer_read32;
|
||||||
} else {
|
} else {
|
||||||
/* we have three clocks no matter what the
|
/* we have three clocks no matter what the
|
||||||
* underlying platform supports.
|
* underlying platform supports.
|
||||||
|
@ -444,6 +458,7 @@ static int __init tcb_clksrc_init(struct device_node *node)
|
||||||
/* setup both channel 0 & 1 */
|
/* setup both channel 0 & 1 */
|
||||||
tcb_setup_dual_chan(&tc, best_divisor_idx);
|
tcb_setup_dual_chan(&tc, best_divisor_idx);
|
||||||
tc_sched_clock = tc_sched_clock_read;
|
tc_sched_clock = tc_sched_clock_read;
|
||||||
|
tc_delay_timer.read_current_timer = tc_delay_timer_read;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* and away we go! */
|
/* and away we go! */
|
||||||
|
@ -458,6 +473,9 @@ static int __init tcb_clksrc_init(struct device_node *node)
|
||||||
|
|
||||||
sched_clock_register(tc_sched_clock, 32, divided_rate);
|
sched_clock_register(tc_sched_clock, 32, divided_rate);
|
||||||
|
|
||||||
|
tc_delay_timer.freq = divided_rate;
|
||||||
|
register_current_timer_delay(&tc_delay_timer);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
err_unregister_clksrc:
|
err_unregister_clksrc:
|
||||||
|
|
|
@ -20,6 +20,8 @@
|
||||||
#define SYS_CTR_EN 0x1
|
#define SYS_CTR_EN 0x1
|
||||||
#define SYS_CTR_IRQ_MASK 0x2
|
#define SYS_CTR_IRQ_MASK 0x2
|
||||||
|
|
||||||
|
#define SYS_CTR_CLK_DIV 0x3
|
||||||
|
|
||||||
static void __iomem *sys_ctr_base;
|
static void __iomem *sys_ctr_base;
|
||||||
static u32 cmpcr;
|
static u32 cmpcr;
|
||||||
|
|
||||||
|
@ -134,6 +136,9 @@ static int __init sysctr_timer_init(struct device_node *np)
|
||||||
if (ret)
|
if (ret)
|
||||||
return ret;
|
return ret;
|
||||||
|
|
||||||
|
/* system counter clock is divided by 3 internally */
|
||||||
|
to_sysctr.of_clk.rate /= SYS_CTR_CLK_DIV;
|
||||||
|
|
||||||
sys_ctr_base = timer_of_base(&to_sysctr);
|
sys_ctr_base = timer_of_base(&to_sysctr);
|
||||||
cmpcr = readl(sys_ctr_base + CMPCR);
|
cmpcr = readl(sys_ctr_base + CMPCR);
|
||||||
cmpcr &= ~SYS_CTR_EN;
|
cmpcr &= ~SYS_CTR_EN;
|
||||||
|
|
|
@ -32,7 +32,7 @@
|
||||||
#define NPCM7XX_Tx_INTEN BIT(29)
|
#define NPCM7XX_Tx_INTEN BIT(29)
|
||||||
#define NPCM7XX_Tx_COUNTEN BIT(30)
|
#define NPCM7XX_Tx_COUNTEN BIT(30)
|
||||||
#define NPCM7XX_Tx_ONESHOT 0x0
|
#define NPCM7XX_Tx_ONESHOT 0x0
|
||||||
#define NPCM7XX_Tx_OPER GENMASK(27, 3)
|
#define NPCM7XX_Tx_OPER GENMASK(28, 27)
|
||||||
#define NPCM7XX_Tx_MIN_PRESCALE 0x1
|
#define NPCM7XX_Tx_MIN_PRESCALE 0x1
|
||||||
#define NPCM7XX_Tx_TDR_MASK_BITS 24
|
#define NPCM7XX_Tx_TDR_MASK_BITS 24
|
||||||
#define NPCM7XX_Tx_MAX_CNT 0xFFFFFF
|
#define NPCM7XX_Tx_MAX_CNT 0xFFFFFF
|
||||||
|
@ -84,8 +84,6 @@ static int npcm7xx_timer_oneshot(struct clock_event_device *evt)
|
||||||
|
|
||||||
val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
||||||
val &= ~NPCM7XX_Tx_OPER;
|
val &= ~NPCM7XX_Tx_OPER;
|
||||||
|
|
||||||
val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
|
||||||
val |= NPCM7XX_START_ONESHOT_Tx;
|
val |= NPCM7XX_START_ONESHOT_Tx;
|
||||||
writel(val, timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
writel(val, timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
||||||
|
|
||||||
|
@ -97,12 +95,11 @@ static int npcm7xx_timer_periodic(struct clock_event_device *evt)
|
||||||
struct timer_of *to = to_timer_of(evt);
|
struct timer_of *to = to_timer_of(evt);
|
||||||
u32 val;
|
u32 val;
|
||||||
|
|
||||||
|
writel(timer_of_period(to), timer_of_base(to) + NPCM7XX_REG_TICR0);
|
||||||
|
|
||||||
val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
||||||
val &= ~NPCM7XX_Tx_OPER;
|
val &= ~NPCM7XX_Tx_OPER;
|
||||||
|
|
||||||
writel(timer_of_period(to), timer_of_base(to) + NPCM7XX_REG_TICR0);
|
|
||||||
val |= NPCM7XX_START_PERIODIC_Tx;
|
val |= NPCM7XX_START_PERIODIC_Tx;
|
||||||
|
|
||||||
writel(val, timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
writel(val, timer_of_base(to) + NPCM7XX_REG_TCSR0);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
@ -113,8 +113,10 @@ static __init int timer_of_clk_init(struct device_node *np,
|
||||||
of_clk->clk = of_clk->name ? of_clk_get_by_name(np, of_clk->name) :
|
of_clk->clk = of_clk->name ? of_clk_get_by_name(np, of_clk->name) :
|
||||||
of_clk_get(np, of_clk->index);
|
of_clk_get(np, of_clk->index);
|
||||||
if (IS_ERR(of_clk->clk)) {
|
if (IS_ERR(of_clk->clk)) {
|
||||||
pr_err("Failed to get clock for %pOF\n", np);
|
ret = PTR_ERR(of_clk->clk);
|
||||||
return PTR_ERR(of_clk->clk);
|
if (ret != -EPROBE_DEFER)
|
||||||
|
pr_err("Failed to get clock for %pOF\n", np);
|
||||||
|
goto out;
|
||||||
}
|
}
|
||||||
|
|
||||||
ret = clk_prepare_enable(of_clk->clk);
|
ret = clk_prepare_enable(of_clk->clk);
|
||||||
|
|
|
@ -29,7 +29,9 @@ void __init timer_probe(void)
|
||||||
|
|
||||||
ret = init_func_ret(np);
|
ret = init_func_ret(np);
|
||||||
if (ret) {
|
if (ret) {
|
||||||
pr_err("Failed to initialize '%pOF': %d\n", np, ret);
|
if (ret != -EPROBE_DEFER)
|
||||||
|
pr_err("Failed to initialize '%pOF': %d\n", np,
|
||||||
|
ret);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -219,5 +219,9 @@ static int __init sun4i_timer_init(struct device_node *node)
|
||||||
}
|
}
|
||||||
TIMER_OF_DECLARE(sun4i, "allwinner,sun4i-a10-timer",
|
TIMER_OF_DECLARE(sun4i, "allwinner,sun4i-a10-timer",
|
||||||
sun4i_timer_init);
|
sun4i_timer_init);
|
||||||
|
TIMER_OF_DECLARE(sun8i_a23, "allwinner,sun8i-a23-timer",
|
||||||
|
sun4i_timer_init);
|
||||||
|
TIMER_OF_DECLARE(sun8i_v3s, "allwinner,sun8i-v3s-timer",
|
||||||
|
sun4i_timer_init);
|
||||||
TIMER_OF_DECLARE(suniv, "allwinner,suniv-f1c100s-timer",
|
TIMER_OF_DECLARE(suniv, "allwinner,suniv-f1c100s-timer",
|
||||||
sun4i_timer_init);
|
sun4i_timer_init);
|
||||||
|
|
|
@ -14,9 +14,6 @@ config HYPERV
|
||||||
config HYPERV_TIMER
|
config HYPERV_TIMER
|
||||||
def_bool HYPERV
|
def_bool HYPERV
|
||||||
|
|
||||||
config HYPERV_TSCPAGE
|
|
||||||
def_bool HYPERV && X86_64
|
|
||||||
|
|
||||||
config HYPERV_UTILS
|
config HYPERV_UTILS
|
||||||
tristate "Microsoft Hyper-V Utilities driver"
|
tristate "Microsoft Hyper-V Utilities driver"
|
||||||
depends on HYPERV && CONNECTOR && NLS
|
depends on HYPERV && CONNECTOR && NLS
|
||||||
|
|
|
@ -437,12 +437,10 @@ static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg)
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
wake_time = ktime_set(arg->wake_time_sec, arg->wake_time_nsec);
|
wake_time = ktime_set(arg->wake_time_sec, arg->wake_time_nsec);
|
||||||
|
|
||||||
hrtimer_init_on_stack(&to->timer, CLOCK_MONOTONIC,
|
hrtimer_init_sleeper_on_stack(to, CLOCK_MONOTONIC,
|
||||||
HRTIMER_MODE_ABS);
|
HRTIMER_MODE_ABS);
|
||||||
hrtimer_set_expires_range_ns(&to->timer, wake_time,
|
hrtimer_set_expires_range_ns(&to->timer, wake_time,
|
||||||
current->timer_slack_ns);
|
current->timer_slack_ns);
|
||||||
|
|
||||||
hrtimer_init_sleeper(to, current);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
|
@ -460,7 +458,7 @@ static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg)
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (to) {
|
if (to) {
|
||||||
hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
|
hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
|
||||||
if (likely(to->task))
|
if (likely(to->task))
|
||||||
freezable_schedule();
|
freezable_schedule();
|
||||||
hrtimer_cancel(&to->timer);
|
hrtimer_cancel(&to->timer);
|
||||||
|
|
|
@ -471,7 +471,11 @@ static int do_timerfd_settime(int ufd, int flags,
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
spin_unlock_irq(&ctx->wqh.lock);
|
spin_unlock_irq(&ctx->wqh.lock);
|
||||||
cpu_relax();
|
|
||||||
|
if (isalarm(ctx))
|
||||||
|
hrtimer_cancel_wait_running(&ctx->t.alarm.timer);
|
||||||
|
else
|
||||||
|
hrtimer_cancel_wait_running(&ctx->t.tmr);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -167,6 +167,7 @@ void hyperv_report_panic(struct pt_regs *regs, long err);
|
||||||
void hyperv_report_panic_msg(phys_addr_t pa, size_t size);
|
void hyperv_report_panic_msg(phys_addr_t pa, size_t size);
|
||||||
bool hv_is_hyperv_initialized(void);
|
bool hv_is_hyperv_initialized(void);
|
||||||
void hyperv_cleanup(void);
|
void hyperv_cleanup(void);
|
||||||
|
void hv_setup_sched_clock(void *sched_clock);
|
||||||
#else /* CONFIG_HYPERV */
|
#else /* CONFIG_HYPERV */
|
||||||
static inline bool hv_is_hyperv_initialized(void) { return false; }
|
static inline bool hv_is_hyperv_initialized(void) { return false; }
|
||||||
static inline void hyperv_cleanup(void) {}
|
static inline void hyperv_cleanup(void) {}
|
||||||
|
|
|
@ -28,12 +28,10 @@ extern void hv_stimer_cleanup(unsigned int cpu);
|
||||||
extern void hv_stimer_global_cleanup(void);
|
extern void hv_stimer_global_cleanup(void);
|
||||||
extern void hv_stimer0_isr(void);
|
extern void hv_stimer0_isr(void);
|
||||||
|
|
||||||
#if IS_ENABLED(CONFIG_HYPERV)
|
#ifdef CONFIG_HYPERV_TIMER
|
||||||
extern struct clocksource *hyperv_cs;
|
extern struct clocksource *hyperv_cs;
|
||||||
extern void hv_init_clocksource(void);
|
extern void hv_init_clocksource(void);
|
||||||
#endif /* CONFIG_HYPERV */
|
|
||||||
|
|
||||||
#ifdef CONFIG_HYPERV_TSCPAGE
|
|
||||||
extern struct ms_hyperv_tsc_page *hv_get_tsc_page(void);
|
extern struct ms_hyperv_tsc_page *hv_get_tsc_page(void);
|
||||||
|
|
||||||
static inline notrace u64
|
static inline notrace u64
|
||||||
|
@ -91,7 +89,7 @@ hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg)
|
||||||
return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc);
|
return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc);
|
||||||
}
|
}
|
||||||
|
|
||||||
#else /* CONFIG_HYPERV_TSC_PAGE */
|
#else /* CONFIG_HYPERV_TIMER */
|
||||||
static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
|
static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
|
||||||
{
|
{
|
||||||
return NULL;
|
return NULL;
|
||||||
|
@ -102,6 +100,6 @@ static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg,
|
||||||
{
|
{
|
||||||
return U64_MAX;
|
return U64_MAX;
|
||||||
}
|
}
|
||||||
#endif /* CONFIG_HYPERV_TSCPAGE */
|
#endif /* CONFIG_HYPERV_TIMER */
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -5,7 +5,8 @@
|
||||||
#include <linux/time.h>
|
#include <linux/time.h>
|
||||||
#include <linux/hrtimer.h>
|
#include <linux/hrtimer.h>
|
||||||
#include <linux/timerqueue.h>
|
#include <linux/timerqueue.h>
|
||||||
#include <linux/rtc.h>
|
|
||||||
|
struct rtc_device;
|
||||||
|
|
||||||
enum alarmtimer_type {
|
enum alarmtimer_type {
|
||||||
ALARM_REALTIME,
|
ALARM_REALTIME,
|
||||||
|
|
|
@ -32,12 +32,15 @@ struct hrtimer_cpu_base;
|
||||||
* when starting the timer)
|
* when starting the timer)
|
||||||
* HRTIMER_MODE_SOFT - Timer callback function will be executed in
|
* HRTIMER_MODE_SOFT - Timer callback function will be executed in
|
||||||
* soft irq context
|
* soft irq context
|
||||||
|
* HRTIMER_MODE_HARD - Timer callback function will be executed in
|
||||||
|
* hard irq context even on PREEMPT_RT.
|
||||||
*/
|
*/
|
||||||
enum hrtimer_mode {
|
enum hrtimer_mode {
|
||||||
HRTIMER_MODE_ABS = 0x00,
|
HRTIMER_MODE_ABS = 0x00,
|
||||||
HRTIMER_MODE_REL = 0x01,
|
HRTIMER_MODE_REL = 0x01,
|
||||||
HRTIMER_MODE_PINNED = 0x02,
|
HRTIMER_MODE_PINNED = 0x02,
|
||||||
HRTIMER_MODE_SOFT = 0x04,
|
HRTIMER_MODE_SOFT = 0x04,
|
||||||
|
HRTIMER_MODE_HARD = 0x08,
|
||||||
|
|
||||||
HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
|
HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
|
||||||
HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
|
HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
|
||||||
|
@ -48,6 +51,11 @@ enum hrtimer_mode {
|
||||||
HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
|
HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
|
||||||
HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
|
HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
|
||||||
|
|
||||||
|
HRTIMER_MODE_ABS_HARD = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD,
|
||||||
|
HRTIMER_MODE_REL_HARD = HRTIMER_MODE_REL | HRTIMER_MODE_HARD,
|
||||||
|
|
||||||
|
HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD,
|
||||||
|
HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD,
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -101,6 +109,8 @@ enum hrtimer_restart {
|
||||||
* @state: state information (See bit values above)
|
* @state: state information (See bit values above)
|
||||||
* @is_rel: Set if the timer was armed relative
|
* @is_rel: Set if the timer was armed relative
|
||||||
* @is_soft: Set if hrtimer will be expired in soft interrupt context.
|
* @is_soft: Set if hrtimer will be expired in soft interrupt context.
|
||||||
|
* @is_hard: Set if hrtimer will be expired in hard interrupt context
|
||||||
|
* even on RT.
|
||||||
*
|
*
|
||||||
* The hrtimer structure must be initialized by hrtimer_init()
|
* The hrtimer structure must be initialized by hrtimer_init()
|
||||||
*/
|
*/
|
||||||
|
@ -112,6 +122,7 @@ struct hrtimer {
|
||||||
u8 state;
|
u8 state;
|
||||||
u8 is_rel;
|
u8 is_rel;
|
||||||
u8 is_soft;
|
u8 is_soft;
|
||||||
|
u8 is_hard;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -183,6 +194,10 @@ enum hrtimer_base_type {
|
||||||
* @nr_retries: Total number of hrtimer interrupt retries
|
* @nr_retries: Total number of hrtimer interrupt retries
|
||||||
* @nr_hangs: Total number of hrtimer interrupt hangs
|
* @nr_hangs: Total number of hrtimer interrupt hangs
|
||||||
* @max_hang_time: Maximum time spent in hrtimer_interrupt
|
* @max_hang_time: Maximum time spent in hrtimer_interrupt
|
||||||
|
* @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
|
||||||
|
* expired
|
||||||
|
* @timer_waiters: A hrtimer_cancel() invocation waits for the timer
|
||||||
|
* callback to finish.
|
||||||
* @expires_next: absolute time of the next event, is required for remote
|
* @expires_next: absolute time of the next event, is required for remote
|
||||||
* hrtimer enqueue; it is the total first expiry time (hard
|
* hrtimer enqueue; it is the total first expiry time (hard
|
||||||
* and soft hrtimer are taken into account)
|
* and soft hrtimer are taken into account)
|
||||||
|
@ -209,6 +224,10 @@ struct hrtimer_cpu_base {
|
||||||
unsigned short nr_retries;
|
unsigned short nr_retries;
|
||||||
unsigned short nr_hangs;
|
unsigned short nr_hangs;
|
||||||
unsigned int max_hang_time;
|
unsigned int max_hang_time;
|
||||||
|
#endif
|
||||||
|
#ifdef CONFIG_PREEMPT_RT
|
||||||
|
spinlock_t softirq_expiry_lock;
|
||||||
|
atomic_t timer_waiters;
|
||||||
#endif
|
#endif
|
||||||
ktime_t expires_next;
|
ktime_t expires_next;
|
||||||
struct hrtimer *next_timer;
|
struct hrtimer *next_timer;
|
||||||
|
@ -341,16 +360,29 @@ extern void hrtimers_resume(void);
|
||||||
|
|
||||||
DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
|
DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
|
||||||
|
|
||||||
|
#ifdef CONFIG_PREEMPT_RT
|
||||||
|
void hrtimer_cancel_wait_running(const struct hrtimer *timer);
|
||||||
|
#else
|
||||||
|
static inline void hrtimer_cancel_wait_running(struct hrtimer *timer)
|
||||||
|
{
|
||||||
|
cpu_relax();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
/* Exported timer functions: */
|
/* Exported timer functions: */
|
||||||
|
|
||||||
/* Initialize timers: */
|
/* Initialize timers: */
|
||||||
extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
|
extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
|
||||||
enum hrtimer_mode mode);
|
enum hrtimer_mode mode);
|
||||||
|
extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
|
||||||
|
enum hrtimer_mode mode);
|
||||||
|
|
||||||
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
|
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
|
||||||
extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
|
extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
|
||||||
enum hrtimer_mode mode);
|
enum hrtimer_mode mode);
|
||||||
|
extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
|
||||||
|
clockid_t clock_id,
|
||||||
|
enum hrtimer_mode mode);
|
||||||
|
|
||||||
extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
|
extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
|
||||||
#else
|
#else
|
||||||
|
@ -360,6 +392,14 @@ static inline void hrtimer_init_on_stack(struct hrtimer *timer,
|
||||||
{
|
{
|
||||||
hrtimer_init(timer, which_clock, mode);
|
hrtimer_init(timer, which_clock, mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
|
||||||
|
clockid_t clock_id,
|
||||||
|
enum hrtimer_mode mode)
|
||||||
|
{
|
||||||
|
hrtimer_init_sleeper(sl, clock_id, mode);
|
||||||
|
}
|
||||||
|
|
||||||
static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
|
static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
@ -395,6 +435,9 @@ static inline void hrtimer_start_expires(struct hrtimer *timer,
|
||||||
hrtimer_start_range_ns(timer, soft, delta, mode);
|
hrtimer_start_range_ns(timer, soft, delta, mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
|
||||||
|
enum hrtimer_mode mode);
|
||||||
|
|
||||||
static inline void hrtimer_restart(struct hrtimer *timer)
|
static inline void hrtimer_restart(struct hrtimer *timer)
|
||||||
{
|
{
|
||||||
hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
|
hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
|
||||||
|
@ -463,11 +506,8 @@ extern long hrtimer_nanosleep(const struct timespec64 *rqtp,
|
||||||
const enum hrtimer_mode mode,
|
const enum hrtimer_mode mode,
|
||||||
const clockid_t clockid);
|
const clockid_t clockid);
|
||||||
|
|
||||||
extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
|
|
||||||
struct task_struct *tsk);
|
|
||||||
|
|
||||||
extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
|
extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
|
||||||
const enum hrtimer_mode mode);
|
const enum hrtimer_mode mode);
|
||||||
extern int schedule_hrtimeout_range_clock(ktime_t *expires,
|
extern int schedule_hrtimeout_range_clock(ktime_t *expires,
|
||||||
u64 delta,
|
u64 delta,
|
||||||
const enum hrtimer_mode mode,
|
const enum hrtimer_mode mode,
|
||||||
|
|
|
@ -36,17 +36,6 @@ extern struct cred init_cred;
|
||||||
#define INIT_PREV_CPUTIME(x)
|
#define INIT_PREV_CPUTIME(x)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_POSIX_TIMERS
|
|
||||||
#define INIT_CPU_TIMERS(s) \
|
|
||||||
.cpu_timers = { \
|
|
||||||
LIST_HEAD_INIT(s.cpu_timers[0]), \
|
|
||||||
LIST_HEAD_INIT(s.cpu_timers[1]), \
|
|
||||||
LIST_HEAD_INIT(s.cpu_timers[2]), \
|
|
||||||
},
|
|
||||||
#else
|
|
||||||
#define INIT_CPU_TIMERS(s)
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#define INIT_TASK_COMM "swapper"
|
#define INIT_TASK_COMM "swapper"
|
||||||
|
|
||||||
/* Attach to the init_task data structure for proper alignment */
|
/* Attach to the init_task data structure for proper alignment */
|
||||||
|
|
|
@ -4,18 +4,11 @@
|
||||||
|
|
||||||
#include <linux/spinlock.h>
|
#include <linux/spinlock.h>
|
||||||
#include <linux/list.h>
|
#include <linux/list.h>
|
||||||
#include <linux/sched.h>
|
|
||||||
#include <linux/timex.h>
|
|
||||||
#include <linux/alarmtimer.h>
|
#include <linux/alarmtimer.h>
|
||||||
|
#include <linux/timerqueue.h>
|
||||||
|
|
||||||
struct siginfo;
|
struct kernel_siginfo;
|
||||||
|
struct task_struct;
|
||||||
struct cpu_timer_list {
|
|
||||||
struct list_head entry;
|
|
||||||
u64 expires;
|
|
||||||
struct task_struct *task;
|
|
||||||
int firing;
|
|
||||||
};
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Bit fields within a clockid:
|
* Bit fields within a clockid:
|
||||||
|
@ -63,6 +56,115 @@ static inline int clockid_to_fd(const clockid_t clk)
|
||||||
return ~(clk >> 3);
|
return ~(clk >> 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_POSIX_TIMERS
|
||||||
|
|
||||||
|
/**
|
||||||
|
* cpu_timer - Posix CPU timer representation for k_itimer
|
||||||
|
* @node: timerqueue node to queue in the task/sig
|
||||||
|
* @head: timerqueue head on which this timer is queued
|
||||||
|
* @task: Pointer to target task
|
||||||
|
* @elist: List head for the expiry list
|
||||||
|
* @firing: Timer is currently firing
|
||||||
|
*/
|
||||||
|
struct cpu_timer {
|
||||||
|
struct timerqueue_node node;
|
||||||
|
struct timerqueue_head *head;
|
||||||
|
struct task_struct *task;
|
||||||
|
struct list_head elist;
|
||||||
|
int firing;
|
||||||
|
};
|
||||||
|
|
||||||
|
static inline bool cpu_timer_enqueue(struct timerqueue_head *head,
|
||||||
|
struct cpu_timer *ctmr)
|
||||||
|
{
|
||||||
|
ctmr->head = head;
|
||||||
|
return timerqueue_add(head, &ctmr->node);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void cpu_timer_dequeue(struct cpu_timer *ctmr)
|
||||||
|
{
|
||||||
|
if (ctmr->head) {
|
||||||
|
timerqueue_del(ctmr->head, &ctmr->node);
|
||||||
|
ctmr->head = NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr)
|
||||||
|
{
|
||||||
|
return ctmr->node.expires;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp)
|
||||||
|
{
|
||||||
|
ctmr->node.expires = exp;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* posix_cputimer_base - Container per posix CPU clock
|
||||||
|
* @nextevt: Earliest-expiration cache
|
||||||
|
* @tqhead: timerqueue head for cpu_timers
|
||||||
|
*/
|
||||||
|
struct posix_cputimer_base {
|
||||||
|
u64 nextevt;
|
||||||
|
struct timerqueue_head tqhead;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* posix_cputimers - Container for posix CPU timer related data
|
||||||
|
* @bases: Base container for posix CPU clocks
|
||||||
|
* @timers_active: Timers are queued.
|
||||||
|
* @expiry_active: Timer expiry is active. Used for
|
||||||
|
* process wide timers to avoid multiple
|
||||||
|
* task trying to handle expiry concurrently
|
||||||
|
*
|
||||||
|
* Used in task_struct and signal_struct
|
||||||
|
*/
|
||||||
|
struct posix_cputimers {
|
||||||
|
struct posix_cputimer_base bases[CPUCLOCK_MAX];
|
||||||
|
unsigned int timers_active;
|
||||||
|
unsigned int expiry_active;
|
||||||
|
};
|
||||||
|
|
||||||
|
static inline void posix_cputimers_init(struct posix_cputimers *pct)
|
||||||
|
{
|
||||||
|
memset(pct, 0, sizeof(*pct));
|
||||||
|
pct->bases[0].nextevt = U64_MAX;
|
||||||
|
pct->bases[1].nextevt = U64_MAX;
|
||||||
|
pct->bases[2].nextevt = U64_MAX;
|
||||||
|
}
|
||||||
|
|
||||||
|
void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit);
|
||||||
|
|
||||||
|
static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct,
|
||||||
|
u64 runtime)
|
||||||
|
{
|
||||||
|
pct->bases[CPUCLOCK_SCHED].nextevt = runtime;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Init task static initializer */
|
||||||
|
#define INIT_CPU_TIMERBASE(b) { \
|
||||||
|
.nextevt = U64_MAX, \
|
||||||
|
}
|
||||||
|
|
||||||
|
#define INIT_CPU_TIMERBASES(b) { \
|
||||||
|
INIT_CPU_TIMERBASE(b[0]), \
|
||||||
|
INIT_CPU_TIMERBASE(b[1]), \
|
||||||
|
INIT_CPU_TIMERBASE(b[2]), \
|
||||||
|
}
|
||||||
|
|
||||||
|
#define INIT_CPU_TIMERS(s) \
|
||||||
|
.posix_cputimers = { \
|
||||||
|
.bases = INIT_CPU_TIMERBASES(s.posix_cputimers.bases), \
|
||||||
|
},
|
||||||
|
#else
|
||||||
|
struct posix_cputimers { };
|
||||||
|
struct cpu_timer { };
|
||||||
|
#define INIT_CPU_TIMERS(s)
|
||||||
|
static inline void posix_cputimers_init(struct posix_cputimers *pct) { }
|
||||||
|
static inline void posix_cputimers_group_init(struct posix_cputimers *pct,
|
||||||
|
u64 cpu_limit) { }
|
||||||
|
#endif
|
||||||
|
|
||||||
#define REQUEUE_PENDING 1
|
#define REQUEUE_PENDING 1
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -85,7 +187,8 @@ static inline int clockid_to_fd(const clockid_t clk)
|
||||||
* @it_process: The task to wakeup on clock_nanosleep (CPU timers)
|
* @it_process: The task to wakeup on clock_nanosleep (CPU timers)
|
||||||
* @sigq: Pointer to preallocated sigqueue
|
* @sigq: Pointer to preallocated sigqueue
|
||||||
* @it: Union representing the various posix timer type
|
* @it: Union representing the various posix timer type
|
||||||
* internals. Also used for rcu freeing the timer.
|
* internals.
|
||||||
|
* @rcu: RCU head for freeing the timer.
|
||||||
*/
|
*/
|
||||||
struct k_itimer {
|
struct k_itimer {
|
||||||
struct list_head list;
|
struct list_head list;
|
||||||
|
@ -110,15 +213,15 @@ struct k_itimer {
|
||||||
struct {
|
struct {
|
||||||
struct hrtimer timer;
|
struct hrtimer timer;
|
||||||
} real;
|
} real;
|
||||||
struct cpu_timer_list cpu;
|
struct cpu_timer cpu;
|
||||||
struct {
|
struct {
|
||||||
struct alarm alarmtimer;
|
struct alarm alarmtimer;
|
||||||
} alarm;
|
} alarm;
|
||||||
struct rcu_head rcu;
|
|
||||||
} it;
|
} it;
|
||||||
|
struct rcu_head rcu;
|
||||||
};
|
};
|
||||||
|
|
||||||
void run_posix_cpu_timers(struct task_struct *task);
|
void run_posix_cpu_timers(void);
|
||||||
void posix_cpu_timers_exit(struct task_struct *task);
|
void posix_cpu_timers_exit(struct task_struct *task);
|
||||||
void posix_cpu_timers_exit_group(struct task_struct *task);
|
void posix_cpu_timers_exit_group(struct task_struct *task);
|
||||||
void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx,
|
void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx,
|
||||||
|
|
|
@ -25,9 +25,11 @@
|
||||||
#include <linux/resource.h>
|
#include <linux/resource.h>
|
||||||
#include <linux/latencytop.h>
|
#include <linux/latencytop.h>
|
||||||
#include <linux/sched/prio.h>
|
#include <linux/sched/prio.h>
|
||||||
|
#include <linux/sched/types.h>
|
||||||
#include <linux/signal_types.h>
|
#include <linux/signal_types.h>
|
||||||
#include <linux/mm_types_task.h>
|
#include <linux/mm_types_task.h>
|
||||||
#include <linux/task_io_accounting.h>
|
#include <linux/task_io_accounting.h>
|
||||||
|
#include <linux/posix-timers.h>
|
||||||
#include <linux/rseq.h>
|
#include <linux/rseq.h>
|
||||||
|
|
||||||
/* task_struct member predeclarations (sorted alphabetically): */
|
/* task_struct member predeclarations (sorted alphabetically): */
|
||||||
|
@ -244,27 +246,6 @@ struct prev_cputime {
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* struct task_cputime - collected CPU time counts
|
|
||||||
* @utime: time spent in user mode, in nanoseconds
|
|
||||||
* @stime: time spent in kernel mode, in nanoseconds
|
|
||||||
* @sum_exec_runtime: total time spent on the CPU, in nanoseconds
|
|
||||||
*
|
|
||||||
* This structure groups together three kinds of CPU time that are tracked for
|
|
||||||
* threads and thread groups. Most things considering CPU time want to group
|
|
||||||
* these counts together and treat all three of them in parallel.
|
|
||||||
*/
|
|
||||||
struct task_cputime {
|
|
||||||
u64 utime;
|
|
||||||
u64 stime;
|
|
||||||
unsigned long long sum_exec_runtime;
|
|
||||||
};
|
|
||||||
|
|
||||||
/* Alternate field names when used on cache expirations: */
|
|
||||||
#define virt_exp utime
|
|
||||||
#define prof_exp stime
|
|
||||||
#define sched_exp sum_exec_runtime
|
|
||||||
|
|
||||||
enum vtime_state {
|
enum vtime_state {
|
||||||
/* Task is sleeping or running in a CPU with VTIME inactive: */
|
/* Task is sleeping or running in a CPU with VTIME inactive: */
|
||||||
VTIME_INACTIVE = 0,
|
VTIME_INACTIVE = 0,
|
||||||
|
@ -881,10 +862,8 @@ struct task_struct {
|
||||||
unsigned long min_flt;
|
unsigned long min_flt;
|
||||||
unsigned long maj_flt;
|
unsigned long maj_flt;
|
||||||
|
|
||||||
#ifdef CONFIG_POSIX_TIMERS
|
/* Empty if CONFIG_POSIX_CPUTIMERS=n */
|
||||||
struct task_cputime cputime_expires;
|
struct posix_cputimers posix_cputimers;
|
||||||
struct list_head cpu_timers[3];
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* Process credentials: */
|
/* Process credentials: */
|
||||||
|
|
||||||
|
|
|
@ -61,8 +61,7 @@ extern void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
|
||||||
* Thread group CPU time accounting.
|
* Thread group CPU time accounting.
|
||||||
*/
|
*/
|
||||||
void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
|
void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
|
||||||
void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
|
void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples);
|
||||||
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The following are functions that support scheduler-internal time accounting.
|
* The following are functions that support scheduler-internal time accounting.
|
||||||
|
@ -71,7 +70,7 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* get_running_cputimer - return &tsk->signal->cputimer if cputimer is running
|
* get_running_cputimer - return &tsk->signal->cputimer if cputimers are active
|
||||||
*
|
*
|
||||||
* @tsk: Pointer to target task.
|
* @tsk: Pointer to target task.
|
||||||
*/
|
*/
|
||||||
|
@ -81,8 +80,11 @@ struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
|
||||||
{
|
{
|
||||||
struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
|
struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
|
||||||
|
|
||||||
/* Check if cputimer isn't running. This is accessed without locking. */
|
/*
|
||||||
if (!READ_ONCE(cputimer->running))
|
* Check whether posix CPU timers are active. If not the thread
|
||||||
|
* group accounting is not active either. Lockless check.
|
||||||
|
*/
|
||||||
|
if (!READ_ONCE(tsk->signal->posix_cputimers.timers_active))
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -9,6 +9,7 @@
|
||||||
#include <linux/sched/task.h>
|
#include <linux/sched/task.h>
|
||||||
#include <linux/cred.h>
|
#include <linux/cred.h>
|
||||||
#include <linux/refcount.h>
|
#include <linux/refcount.h>
|
||||||
|
#include <linux/posix-timers.h>
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Types defining task->signal and task->sighand and APIs using them:
|
* Types defining task->signal and task->sighand and APIs using them:
|
||||||
|
@ -56,18 +57,12 @@ struct task_cputime_atomic {
|
||||||
/**
|
/**
|
||||||
* struct thread_group_cputimer - thread group interval timer counts
|
* struct thread_group_cputimer - thread group interval timer counts
|
||||||
* @cputime_atomic: atomic thread group interval timers.
|
* @cputime_atomic: atomic thread group interval timers.
|
||||||
* @running: true when there are timers running and
|
|
||||||
* @cputime_atomic receives updates.
|
|
||||||
* @checking_timer: true when a thread in the group is in the
|
|
||||||
* process of checking for thread group timers.
|
|
||||||
*
|
*
|
||||||
* This structure contains the version of task_cputime, above, that is
|
* This structure contains the version of task_cputime, above, that is
|
||||||
* used for thread group CPU timer calculations.
|
* used for thread group CPU timer calculations.
|
||||||
*/
|
*/
|
||||||
struct thread_group_cputimer {
|
struct thread_group_cputimer {
|
||||||
struct task_cputime_atomic cputime_atomic;
|
struct task_cputime_atomic cputime_atomic;
|
||||||
bool running;
|
|
||||||
bool checking_timer;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct multiprocess_signals {
|
struct multiprocess_signals {
|
||||||
|
@ -148,12 +143,9 @@ struct signal_struct {
|
||||||
*/
|
*/
|
||||||
struct thread_group_cputimer cputimer;
|
struct thread_group_cputimer cputimer;
|
||||||
|
|
||||||
/* Earliest-expiration cache. */
|
|
||||||
struct task_cputime cputime_expires;
|
|
||||||
|
|
||||||
struct list_head cpu_timers[3];
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
/* Empty if CONFIG_POSIX_TIMERS=n */
|
||||||
|
struct posix_cputimers posix_cputimers;
|
||||||
|
|
||||||
/* PID/PID hash table linkage. */
|
/* PID/PID hash table linkage. */
|
||||||
struct pid *pids[PIDTYPE_MAX];
|
struct pid *pids[PIDTYPE_MAX];
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
/* SPDX-License-Identifier: GPL-2.0 */
|
||||||
|
#ifndef _LINUX_SCHED_TYPES_H
|
||||||
|
#define _LINUX_SCHED_TYPES_H
|
||||||
|
|
||||||
|
#include <linux/types.h>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct task_cputime - collected CPU time counts
|
||||||
|
* @stime: time spent in kernel mode, in nanoseconds
|
||||||
|
* @utime: time spent in user mode, in nanoseconds
|
||||||
|
* @sum_exec_runtime: total time spent on the CPU, in nanoseconds
|
||||||
|
*
|
||||||
|
* This structure groups together three kinds of CPU time that are tracked for
|
||||||
|
* threads and thread groups. Most things considering CPU time want to group
|
||||||
|
* these counts together and treat all three of them in parallel.
|
||||||
|
*/
|
||||||
|
struct task_cputime {
|
||||||
|
u64 stime;
|
||||||
|
u64 utime;
|
||||||
|
unsigned long long sum_exec_runtime;
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif
|
|
@ -183,7 +183,7 @@ extern void add_timer(struct timer_list *timer);
|
||||||
|
|
||||||
extern int try_to_del_timer_sync(struct timer_list *timer);
|
extern int try_to_del_timer_sync(struct timer_list *timer);
|
||||||
|
|
||||||
#ifdef CONFIG_SMP
|
#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
|
||||||
extern int del_timer_sync(struct timer_list *timer);
|
extern int del_timer_sync(struct timer_list *timer);
|
||||||
#else
|
#else
|
||||||
# define del_timer_sync(t) del_timer(t)
|
# define del_timer_sync(t) del_timer(t)
|
||||||
|
|
|
@ -12,8 +12,7 @@ struct timerqueue_node {
|
||||||
};
|
};
|
||||||
|
|
||||||
struct timerqueue_head {
|
struct timerqueue_head {
|
||||||
struct rb_root head;
|
struct rb_root_cached rb_root;
|
||||||
struct timerqueue_node *next;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -29,13 +28,14 @@ extern struct timerqueue_node *timerqueue_iterate_next(
|
||||||
*
|
*
|
||||||
* @head: head of timerqueue
|
* @head: head of timerqueue
|
||||||
*
|
*
|
||||||
* Returns a pointer to the timer node that has the
|
* Returns a pointer to the timer node that has the earliest expiration time.
|
||||||
* earliest expiration time.
|
|
||||||
*/
|
*/
|
||||||
static inline
|
static inline
|
||||||
struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head)
|
struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head)
|
||||||
{
|
{
|
||||||
return head->next;
|
struct rb_node *leftmost = rb_first_cached(&head->rb_root);
|
||||||
|
|
||||||
|
return rb_entry(leftmost, struct timerqueue_node, node);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void timerqueue_init(struct timerqueue_node *node)
|
static inline void timerqueue_init(struct timerqueue_node *node)
|
||||||
|
@ -43,9 +43,18 @@ static inline void timerqueue_init(struct timerqueue_node *node)
|
||||||
RB_CLEAR_NODE(&node->node);
|
RB_CLEAR_NODE(&node->node);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline bool timerqueue_node_queued(struct timerqueue_node *node)
|
||||||
|
{
|
||||||
|
return !RB_EMPTY_NODE(&node->node);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline bool timerqueue_node_expires(struct timerqueue_node *node)
|
||||||
|
{
|
||||||
|
return node->expires;
|
||||||
|
}
|
||||||
|
|
||||||
static inline void timerqueue_init_head(struct timerqueue_head *head)
|
static inline void timerqueue_init_head(struct timerqueue_head *head)
|
||||||
{
|
{
|
||||||
head->head = RB_ROOT;
|
head->rb_root = RB_ROOT_CACHED;
|
||||||
head->next = NULL;
|
|
||||||
}
|
}
|
||||||
#endif /* _LINUX_TIMERQUEUE_H */
|
#endif /* _LINUX_TIMERQUEUE_H */
|
||||||
|
|
|
@ -501,8 +501,8 @@ do { \
|
||||||
int __ret = 0; \
|
int __ret = 0; \
|
||||||
struct hrtimer_sleeper __t; \
|
struct hrtimer_sleeper __t; \
|
||||||
\
|
\
|
||||||
hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); \
|
hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
|
||||||
hrtimer_init_sleeper(&__t, current); \
|
HRTIMER_MODE_REL); \
|
||||||
if ((timeout) != KTIME_MAX) \
|
if ((timeout) != KTIME_MAX) \
|
||||||
hrtimer_start_range_ns(&__t.timer, timeout, \
|
hrtimer_start_range_ns(&__t.timer, timeout, \
|
||||||
current->timer_slack_ns, \
|
current->timer_slack_ns, \
|
||||||
|
|
|
@ -30,8 +30,6 @@ static struct signal_struct init_signals = {
|
||||||
.posix_timers = LIST_HEAD_INIT(init_signals.posix_timers),
|
.posix_timers = LIST_HEAD_INIT(init_signals.posix_timers),
|
||||||
.cputimer = {
|
.cputimer = {
|
||||||
.cputime_atomic = INIT_CPUTIME_ATOMIC,
|
.cputime_atomic = INIT_CPUTIME_ATOMIC,
|
||||||
.running = false,
|
|
||||||
.checking_timer = false,
|
|
||||||
},
|
},
|
||||||
#endif
|
#endif
|
||||||
INIT_CPU_TIMERS(init_signals)
|
INIT_CPU_TIMERS(init_signals)
|
||||||
|
|
|
@ -1103,7 +1103,7 @@ static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
|
||||||
cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
|
cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
|
||||||
|
|
||||||
raw_spin_lock_init(&cpuctx->hrtimer_lock);
|
raw_spin_lock_init(&cpuctx->hrtimer_lock);
|
||||||
hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
|
hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
|
||||||
timer->function = perf_mux_hrtimer_handler;
|
timer->function = perf_mux_hrtimer_handler;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1121,7 +1121,7 @@ static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)
|
||||||
if (!cpuctx->hrtimer_active) {
|
if (!cpuctx->hrtimer_active) {
|
||||||
cpuctx->hrtimer_active = 1;
|
cpuctx->hrtimer_active = 1;
|
||||||
hrtimer_forward_now(timer, cpuctx->hrtimer_interval);
|
hrtimer_forward_now(timer, cpuctx->hrtimer_interval);
|
||||||
hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
|
hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
|
||||||
}
|
}
|
||||||
raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);
|
raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);
|
||||||
|
|
||||||
|
@ -9574,7 +9574,7 @@ static void perf_swevent_start_hrtimer(struct perf_event *event)
|
||||||
period = max_t(u64, 10000, hwc->sample_period);
|
period = max_t(u64, 10000, hwc->sample_period);
|
||||||
}
|
}
|
||||||
hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
|
hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
|
||||||
HRTIMER_MODE_REL_PINNED);
|
HRTIMER_MODE_REL_PINNED_HARD);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void perf_swevent_cancel_hrtimer(struct perf_event *event)
|
static void perf_swevent_cancel_hrtimer(struct perf_event *event)
|
||||||
|
@ -9596,7 +9596,7 @@ static void perf_swevent_init_hrtimer(struct perf_event *event)
|
||||||
if (!is_sampling_event(event))
|
if (!is_sampling_event(event))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
|
||||||
hwc->hrtimer.function = perf_swevent_hrtimer;
|
hwc->hrtimer.function = perf_swevent_hrtimer;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -1519,28 +1519,17 @@ void __cleanup_sighand(struct sighand_struct *sighand)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_POSIX_TIMERS
|
|
||||||
/*
|
/*
|
||||||
* Initialize POSIX timer handling for a thread group.
|
* Initialize POSIX timer handling for a thread group.
|
||||||
*/
|
*/
|
||||||
static void posix_cpu_timers_init_group(struct signal_struct *sig)
|
static void posix_cpu_timers_init_group(struct signal_struct *sig)
|
||||||
{
|
{
|
||||||
|
struct posix_cputimers *pct = &sig->posix_cputimers;
|
||||||
unsigned long cpu_limit;
|
unsigned long cpu_limit;
|
||||||
|
|
||||||
cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
|
cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
|
||||||
if (cpu_limit != RLIM_INFINITY) {
|
posix_cputimers_group_init(pct, cpu_limit);
|
||||||
sig->cputime_expires.prof_exp = cpu_limit * NSEC_PER_SEC;
|
|
||||||
sig->cputimer.running = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* The timer lists. */
|
|
||||||
INIT_LIST_HEAD(&sig->cpu_timers[0]);
|
|
||||||
INIT_LIST_HEAD(&sig->cpu_timers[1]);
|
|
||||||
INIT_LIST_HEAD(&sig->cpu_timers[2]);
|
|
||||||
}
|
}
|
||||||
#else
|
|
||||||
static inline void posix_cpu_timers_init_group(struct signal_struct *sig) { }
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
|
static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
|
||||||
{
|
{
|
||||||
|
@ -1642,23 +1631,6 @@ static void rt_mutex_init_task(struct task_struct *p)
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_POSIX_TIMERS
|
|
||||||
/*
|
|
||||||
* Initialize POSIX timer handling for a single task.
|
|
||||||
*/
|
|
||||||
static void posix_cpu_timers_init(struct task_struct *tsk)
|
|
||||||
{
|
|
||||||
tsk->cputime_expires.prof_exp = 0;
|
|
||||||
tsk->cputime_expires.virt_exp = 0;
|
|
||||||
tsk->cputime_expires.sched_exp = 0;
|
|
||||||
INIT_LIST_HEAD(&tsk->cpu_timers[0]);
|
|
||||||
INIT_LIST_HEAD(&tsk->cpu_timers[1]);
|
|
||||||
INIT_LIST_HEAD(&tsk->cpu_timers[2]);
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
static inline void posix_cpu_timers_init(struct task_struct *tsk) { }
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static inline void init_task_pid_links(struct task_struct *task)
|
static inline void init_task_pid_links(struct task_struct *task)
|
||||||
{
|
{
|
||||||
enum pid_type type;
|
enum pid_type type;
|
||||||
|
@ -1945,7 +1917,7 @@ static __latent_entropy struct task_struct *copy_process(
|
||||||
task_io_accounting_init(&p->ioac);
|
task_io_accounting_init(&p->ioac);
|
||||||
acct_clear_integrals(p);
|
acct_clear_integrals(p);
|
||||||
|
|
||||||
posix_cpu_timers_init(p);
|
posix_cputimers_init(&p->posix_cputimers);
|
||||||
|
|
||||||
p->io_context = NULL;
|
p->io_context = NULL;
|
||||||
audit_set_context(p, NULL);
|
audit_set_context(p, NULL);
|
||||||
|
|
|
@ -487,11 +487,9 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
|
||||||
if (!time)
|
if (!time)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
hrtimer_init_on_stack(&timeout->timer, (flags & FLAGS_CLOCKRT) ?
|
hrtimer_init_sleeper_on_stack(timeout, (flags & FLAGS_CLOCKRT) ?
|
||||||
CLOCK_REALTIME : CLOCK_MONOTONIC,
|
CLOCK_REALTIME : CLOCK_MONOTONIC,
|
||||||
HRTIMER_MODE_ABS);
|
HRTIMER_MODE_ABS);
|
||||||
hrtimer_init_sleeper(timeout, current);
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If range_ns is 0, calling hrtimer_set_expires_range_ns() is
|
* If range_ns is 0, calling hrtimer_set_expires_range_ns() is
|
||||||
* effectively the same as calling hrtimer_set_expires().
|
* effectively the same as calling hrtimer_set_expires().
|
||||||
|
@ -2613,7 +2611,7 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
|
||||||
|
|
||||||
/* Arm the timer */
|
/* Arm the timer */
|
||||||
if (timeout)
|
if (timeout)
|
||||||
hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
|
hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If we have been removed from the hash list, then another task
|
* If we have been removed from the hash list, then another task
|
||||||
|
@ -2899,7 +2897,7 @@ retry_private:
|
||||||
}
|
}
|
||||||
|
|
||||||
if (unlikely(to))
|
if (unlikely(to))
|
||||||
hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
|
hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
|
||||||
|
|
||||||
ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
|
ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
|
||||||
|
|
||||||
|
|
|
@ -255,7 +255,7 @@ static void __hrtick_restart(struct rq *rq)
|
||||||
{
|
{
|
||||||
struct hrtimer *timer = &rq->hrtick_timer;
|
struct hrtimer *timer = &rq->hrtick_timer;
|
||||||
|
|
||||||
hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
|
hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -314,7 +314,7 @@ void hrtick_start(struct rq *rq, u64 delay)
|
||||||
*/
|
*/
|
||||||
delay = max_t(u64, delay, 10000LL);
|
delay = max_t(u64, delay, 10000LL);
|
||||||
hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
|
hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
|
||||||
HRTIMER_MODE_REL_PINNED);
|
HRTIMER_MODE_REL_PINNED_HARD);
|
||||||
}
|
}
|
||||||
#endif /* CONFIG_SMP */
|
#endif /* CONFIG_SMP */
|
||||||
|
|
||||||
|
@ -328,7 +328,7 @@ static void hrtick_rq_init(struct rq *rq)
|
||||||
rq->hrtick_csd.info = rq;
|
rq->hrtick_csd.info = rq;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
|
||||||
rq->hrtick_timer.function = hrtick;
|
rq->hrtick_timer.function = hrtick;
|
||||||
}
|
}
|
||||||
#else /* CONFIG_SCHED_HRTICK */
|
#else /* CONFIG_SCHED_HRTICK */
|
||||||
|
|
|
@ -287,7 +287,7 @@ static void task_non_contending(struct task_struct *p)
|
||||||
|
|
||||||
dl_se->dl_non_contending = 1;
|
dl_se->dl_non_contending = 1;
|
||||||
get_task_struct(p);
|
get_task_struct(p);
|
||||||
hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL);
|
hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL_HARD);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void task_contending(struct sched_dl_entity *dl_se, int flags)
|
static void task_contending(struct sched_dl_entity *dl_se, int flags)
|
||||||
|
@ -956,7 +956,7 @@ static int start_dl_timer(struct task_struct *p)
|
||||||
*/
|
*/
|
||||||
if (!hrtimer_is_queued(timer)) {
|
if (!hrtimer_is_queued(timer)) {
|
||||||
get_task_struct(p);
|
get_task_struct(p);
|
||||||
hrtimer_start(timer, act, HRTIMER_MODE_ABS);
|
hrtimer_start(timer, act, HRTIMER_MODE_ABS_HARD);
|
||||||
}
|
}
|
||||||
|
|
||||||
return 1;
|
return 1;
|
||||||
|
@ -1086,7 +1086,7 @@ void init_dl_task_timer(struct sched_dl_entity *dl_se)
|
||||||
{
|
{
|
||||||
struct hrtimer *timer = &dl_se->dl_timer;
|
struct hrtimer *timer = &dl_se->dl_timer;
|
||||||
|
|
||||||
hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
|
||||||
timer->function = dl_task_timer;
|
timer->function = dl_task_timer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1325,7 +1325,7 @@ void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se)
|
||||||
{
|
{
|
||||||
struct hrtimer *timer = &dl_se->inactive_timer;
|
struct hrtimer *timer = &dl_se->inactive_timer;
|
||||||
|
|
||||||
hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
|
||||||
timer->function = inactive_task_timer;
|
timer->function = inactive_task_timer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -45,8 +45,8 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
|
||||||
|
|
||||||
raw_spin_lock_init(&rt_b->rt_runtime_lock);
|
raw_spin_lock_init(&rt_b->rt_runtime_lock);
|
||||||
|
|
||||||
hrtimer_init(&rt_b->rt_period_timer,
|
hrtimer_init(&rt_b->rt_period_timer, CLOCK_MONOTONIC,
|
||||||
CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
HRTIMER_MODE_REL_HARD);
|
||||||
rt_b->rt_period_timer.function = sched_rt_period_timer;
|
rt_b->rt_period_timer.function = sched_rt_period_timer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -67,7 +67,8 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
|
||||||
* to update the period.
|
* to update the period.
|
||||||
*/
|
*/
|
||||||
hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
|
hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
|
||||||
hrtimer_start_expires(&rt_b->rt_period_timer, HRTIMER_MODE_ABS_PINNED);
|
hrtimer_start_expires(&rt_b->rt_period_timer,
|
||||||
|
HRTIMER_MODE_ABS_PINNED_HARD);
|
||||||
}
|
}
|
||||||
raw_spin_unlock(&rt_b->rt_runtime_lock);
|
raw_spin_unlock(&rt_b->rt_runtime_lock);
|
||||||
}
|
}
|
||||||
|
@ -2289,8 +2290,10 @@ static void watchdog(struct rq *rq, struct task_struct *p)
|
||||||
}
|
}
|
||||||
|
|
||||||
next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
|
next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
|
||||||
if (p->rt.timeout > next)
|
if (p->rt.timeout > next) {
|
||||||
p->cputime_expires.sched_exp = p->se.sum_exec_runtime;
|
posix_cputimers_rt_watchdog(&p->posix_cputimers,
|
||||||
|
p->se.sum_exec_runtime);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
|
|
16
kernel/sys.c
16
kernel/sys.c
|
@ -1557,15 +1557,6 @@ int do_prlimit(struct task_struct *tsk, unsigned int resource,
|
||||||
retval = -EPERM;
|
retval = -EPERM;
|
||||||
if (!retval)
|
if (!retval)
|
||||||
retval = security_task_setrlimit(tsk, resource, new_rlim);
|
retval = security_task_setrlimit(tsk, resource, new_rlim);
|
||||||
if (resource == RLIMIT_CPU && new_rlim->rlim_cur == 0) {
|
|
||||||
/*
|
|
||||||
* The caller is asking for an immediate RLIMIT_CPU
|
|
||||||
* expiry. But we use the zero value to mean "it was
|
|
||||||
* never set". So let's cheat and make it one second
|
|
||||||
* instead
|
|
||||||
*/
|
|
||||||
new_rlim->rlim_cur = 1;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (!retval) {
|
if (!retval) {
|
||||||
if (old_rlim)
|
if (old_rlim)
|
||||||
|
@ -1576,10 +1567,9 @@ int do_prlimit(struct task_struct *tsk, unsigned int resource,
|
||||||
task_unlock(tsk->group_leader);
|
task_unlock(tsk->group_leader);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* RLIMIT_CPU handling. Note that the kernel fails to return an error
|
* RLIMIT_CPU handling. Arm the posix CPU timer if the limit is not
|
||||||
* code if it rejected the user's attempt to set RLIMIT_CPU. This is a
|
* infite. In case of RLIM_INFINITY the posix CPU timer code
|
||||||
* very long-standing error, and fixing it now risks breakage of
|
* ignores the rlimit.
|
||||||
* applications, so we live with it
|
|
||||||
*/
|
*/
|
||||||
if (!retval && new_rlim && resource == RLIMIT_CPU &&
|
if (!retval && new_rlim && resource == RLIMIT_CPU &&
|
||||||
new_rlim->rlim_cur != RLIM_INFINITY &&
|
new_rlim->rlim_cur != RLIM_INFINITY &&
|
||||||
|
|
|
@ -432,7 +432,7 @@ int alarm_cancel(struct alarm *alarm)
|
||||||
int ret = alarm_try_to_cancel(alarm);
|
int ret = alarm_try_to_cancel(alarm);
|
||||||
if (ret >= 0)
|
if (ret >= 0)
|
||||||
return ret;
|
return ret;
|
||||||
cpu_relax();
|
hrtimer_cancel_wait_running(&alarm->timer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(alarm_cancel);
|
EXPORT_SYMBOL_GPL(alarm_cancel);
|
||||||
|
@ -605,6 +605,19 @@ static int alarm_timer_try_to_cancel(struct k_itimer *timr)
|
||||||
return alarm_try_to_cancel(&timr->it.alarm.alarmtimer);
|
return alarm_try_to_cancel(&timr->it.alarm.alarmtimer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* alarm_timer_wait_running - Posix timer callback to wait for a timer
|
||||||
|
* @timr: Pointer to the posixtimer data struct
|
||||||
|
*
|
||||||
|
* Called from the core code when timer cancel detected that the callback
|
||||||
|
* is running. @timr is unlocked and rcu read lock is held to prevent it
|
||||||
|
* from being freed.
|
||||||
|
*/
|
||||||
|
static void alarm_timer_wait_running(struct k_itimer *timr)
|
||||||
|
{
|
||||||
|
hrtimer_cancel_wait_running(&timr->it.alarm.alarmtimer.timer);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* alarm_timer_arm - Posix timer callback to arm a timer
|
* alarm_timer_arm - Posix timer callback to arm a timer
|
||||||
* @timr: Pointer to the posixtimer data struct
|
* @timr: Pointer to the posixtimer data struct
|
||||||
|
@ -834,6 +847,7 @@ const struct k_clock alarm_clock = {
|
||||||
.timer_forward = alarm_timer_forward,
|
.timer_forward = alarm_timer_forward,
|
||||||
.timer_remaining = alarm_timer_remaining,
|
.timer_remaining = alarm_timer_remaining,
|
||||||
.timer_try_to_cancel = alarm_timer_try_to_cancel,
|
.timer_try_to_cancel = alarm_timer_try_to_cancel,
|
||||||
|
.timer_wait_running = alarm_timer_wait_running,
|
||||||
.nsleep = alarm_timer_nsleep,
|
.nsleep = alarm_timer_nsleep,
|
||||||
};
|
};
|
||||||
#endif /* CONFIG_POSIX_TIMERS */
|
#endif /* CONFIG_POSIX_TIMERS */
|
||||||
|
|
|
@ -140,6 +140,11 @@ static struct hrtimer_cpu_base migration_cpu_base = {
|
||||||
|
|
||||||
#define migration_base migration_cpu_base.clock_base[0]
|
#define migration_base migration_cpu_base.clock_base[0]
|
||||||
|
|
||||||
|
static inline bool is_migration_base(struct hrtimer_clock_base *base)
|
||||||
|
{
|
||||||
|
return base == &migration_base;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
|
* We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
|
||||||
* means that all timers which are tied to this base via timer->base are
|
* means that all timers which are tied to this base via timer->base are
|
||||||
|
@ -264,6 +269,11 @@ again:
|
||||||
|
|
||||||
#else /* CONFIG_SMP */
|
#else /* CONFIG_SMP */
|
||||||
|
|
||||||
|
static inline bool is_migration_base(struct hrtimer_clock_base *base)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
static inline struct hrtimer_clock_base *
|
static inline struct hrtimer_clock_base *
|
||||||
lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
|
lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
|
||||||
{
|
{
|
||||||
|
@ -427,6 +437,17 @@ void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
|
EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
|
||||||
|
|
||||||
|
static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
|
||||||
|
clockid_t clock_id, enum hrtimer_mode mode);
|
||||||
|
|
||||||
|
void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
|
||||||
|
clockid_t clock_id, enum hrtimer_mode mode)
|
||||||
|
{
|
||||||
|
debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
|
||||||
|
__hrtimer_init_sleeper(sl, clock_id, mode);
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
|
||||||
|
|
||||||
void destroy_hrtimer_on_stack(struct hrtimer *timer)
|
void destroy_hrtimer_on_stack(struct hrtimer *timer)
|
||||||
{
|
{
|
||||||
debug_object_free(timer, &hrtimer_debug_descr);
|
debug_object_free(timer, &hrtimer_debug_descr);
|
||||||
|
@ -1096,9 +1117,13 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
|
* Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
|
||||||
* match.
|
* match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
|
||||||
|
* expiry mode because unmarked timers are moved to softirq expiry.
|
||||||
*/
|
*/
|
||||||
WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
|
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
|
||||||
|
WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
|
||||||
|
else
|
||||||
|
WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
|
||||||
|
|
||||||
base = lock_hrtimer_base(timer, &flags);
|
base = lock_hrtimer_base(timer, &flags);
|
||||||
|
|
||||||
|
@ -1147,6 +1172,93 @@ int hrtimer_try_to_cancel(struct hrtimer *timer)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
|
EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
|
||||||
|
|
||||||
|
#ifdef CONFIG_PREEMPT_RT
|
||||||
|
static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
|
||||||
|
{
|
||||||
|
spin_lock_init(&base->softirq_expiry_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
|
||||||
|
{
|
||||||
|
spin_lock(&base->softirq_expiry_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
|
||||||
|
{
|
||||||
|
spin_unlock(&base->softirq_expiry_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The counterpart to hrtimer_cancel_wait_running().
|
||||||
|
*
|
||||||
|
* If there is a waiter for cpu_base->expiry_lock, then it was waiting for
|
||||||
|
* the timer callback to finish. Drop expiry_lock and reaquire it. That
|
||||||
|
* allows the waiter to acquire the lock and make progress.
|
||||||
|
*/
|
||||||
|
static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,
|
||||||
|
unsigned long flags)
|
||||||
|
{
|
||||||
|
if (atomic_read(&cpu_base->timer_waiters)) {
|
||||||
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
||||||
|
spin_unlock(&cpu_base->softirq_expiry_lock);
|
||||||
|
spin_lock(&cpu_base->softirq_expiry_lock);
|
||||||
|
raw_spin_lock_irq(&cpu_base->lock);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This function is called on PREEMPT_RT kernels when the fast path
|
||||||
|
* deletion of a timer failed because the timer callback function was
|
||||||
|
* running.
|
||||||
|
*
|
||||||
|
* This prevents priority inversion: if the soft irq thread is preempted
|
||||||
|
* in the middle of a timer callback, then calling del_timer_sync() can
|
||||||
|
* lead to two issues:
|
||||||
|
*
|
||||||
|
* - If the caller is on a remote CPU then it has to spin wait for the timer
|
||||||
|
* handler to complete. This can result in unbound priority inversion.
|
||||||
|
*
|
||||||
|
* - If the caller originates from the task which preempted the timer
|
||||||
|
* handler on the same CPU, then spin waiting for the timer handler to
|
||||||
|
* complete is never going to end.
|
||||||
|
*/
|
||||||
|
void hrtimer_cancel_wait_running(const struct hrtimer *timer)
|
||||||
|
{
|
||||||
|
/* Lockless read. Prevent the compiler from reloading it below */
|
||||||
|
struct hrtimer_clock_base *base = READ_ONCE(timer->base);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Just relax if the timer expires in hard interrupt context or if
|
||||||
|
* it is currently on the migration base.
|
||||||
|
*/
|
||||||
|
if (!timer->is_soft || is_migration_base(base)) {
|
||||||
|
cpu_relax();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Mark the base as contended and grab the expiry lock, which is
|
||||||
|
* held by the softirq across the timer callback. Drop the lock
|
||||||
|
* immediately so the softirq can expire the next timer. In theory
|
||||||
|
* the timer could already be running again, but that's more than
|
||||||
|
* unlikely and just causes another wait loop.
|
||||||
|
*/
|
||||||
|
atomic_inc(&base->cpu_base->timer_waiters);
|
||||||
|
spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
|
||||||
|
atomic_dec(&base->cpu_base->timer_waiters);
|
||||||
|
spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
static inline void
|
||||||
|
hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
|
||||||
|
static inline void
|
||||||
|
hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
|
||||||
|
static inline void
|
||||||
|
hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
|
||||||
|
static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base,
|
||||||
|
unsigned long flags) { }
|
||||||
|
#endif
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* hrtimer_cancel - cancel a timer and wait for the handler to finish.
|
* hrtimer_cancel - cancel a timer and wait for the handler to finish.
|
||||||
* @timer: the timer to be cancelled
|
* @timer: the timer to be cancelled
|
||||||
|
@ -1157,13 +1269,15 @@ EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
|
||||||
*/
|
*/
|
||||||
int hrtimer_cancel(struct hrtimer *timer)
|
int hrtimer_cancel(struct hrtimer *timer)
|
||||||
{
|
{
|
||||||
for (;;) {
|
int ret;
|
||||||
int ret = hrtimer_try_to_cancel(timer);
|
|
||||||
|
|
||||||
if (ret >= 0)
|
do {
|
||||||
return ret;
|
ret = hrtimer_try_to_cancel(timer);
|
||||||
cpu_relax();
|
|
||||||
}
|
if (ret < 0)
|
||||||
|
hrtimer_cancel_wait_running(timer);
|
||||||
|
} while (ret < 0);
|
||||||
|
return ret;
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(hrtimer_cancel);
|
EXPORT_SYMBOL_GPL(hrtimer_cancel);
|
||||||
|
|
||||||
|
@ -1260,8 +1374,17 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
||||||
enum hrtimer_mode mode)
|
enum hrtimer_mode mode)
|
||||||
{
|
{
|
||||||
bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
|
bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
|
||||||
int base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
|
|
||||||
struct hrtimer_cpu_base *cpu_base;
|
struct hrtimer_cpu_base *cpu_base;
|
||||||
|
int base;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* On PREEMPT_RT enabled kernels hrtimers which are not explicitely
|
||||||
|
* marked for hard interrupt expiry mode are moved into soft
|
||||||
|
* interrupt context for latency reasons and because the callbacks
|
||||||
|
* can invoke functions which might sleep on RT, e.g. spin_lock().
|
||||||
|
*/
|
||||||
|
if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
|
||||||
|
softtimer = true;
|
||||||
|
|
||||||
memset(timer, 0, sizeof(struct hrtimer));
|
memset(timer, 0, sizeof(struct hrtimer));
|
||||||
|
|
||||||
|
@ -1275,8 +1398,10 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
|
||||||
if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
|
if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
|
||||||
clock_id = CLOCK_MONOTONIC;
|
clock_id = CLOCK_MONOTONIC;
|
||||||
|
|
||||||
|
base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
|
||||||
base += hrtimer_clockid_to_base(clock_id);
|
base += hrtimer_clockid_to_base(clock_id);
|
||||||
timer->is_soft = softtimer;
|
timer->is_soft = softtimer;
|
||||||
|
timer->is_hard = !softtimer;
|
||||||
timer->base = &cpu_base->clock_base[base];
|
timer->base = &cpu_base->clock_base[base];
|
||||||
timerqueue_init(&timer->node);
|
timerqueue_init(&timer->node);
|
||||||
}
|
}
|
||||||
|
@ -1449,6 +1574,8 @@ static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
|
||||||
break;
|
break;
|
||||||
|
|
||||||
__run_hrtimer(cpu_base, base, timer, &basenow, flags);
|
__run_hrtimer(cpu_base, base, timer, &basenow, flags);
|
||||||
|
if (active_mask == HRTIMER_ACTIVE_SOFT)
|
||||||
|
hrtimer_sync_wait_running(cpu_base, flags);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1459,6 +1586,7 @@ static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
ktime_t now;
|
ktime_t now;
|
||||||
|
|
||||||
|
hrtimer_cpu_base_lock_expiry(cpu_base);
|
||||||
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
raw_spin_lock_irqsave(&cpu_base->lock, flags);
|
||||||
|
|
||||||
now = hrtimer_update_base(cpu_base);
|
now = hrtimer_update_base(cpu_base);
|
||||||
|
@ -1468,6 +1596,7 @@ static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
|
||||||
hrtimer_update_softirq_timer(cpu_base, true);
|
hrtimer_update_softirq_timer(cpu_base, true);
|
||||||
|
|
||||||
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
|
||||||
|
hrtimer_cpu_base_unlock_expiry(cpu_base);
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_HIGH_RES_TIMERS
|
#ifdef CONFIG_HIGH_RES_TIMERS
|
||||||
|
@ -1639,10 +1768,75 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
|
||||||
return HRTIMER_NORESTART;
|
return HRTIMER_NORESTART;
|
||||||
}
|
}
|
||||||
|
|
||||||
void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
|
/**
|
||||||
|
* hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
|
||||||
|
* @sl: sleeper to be started
|
||||||
|
* @mode: timer mode abs/rel
|
||||||
|
*
|
||||||
|
* Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
|
||||||
|
* to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
|
||||||
|
*/
|
||||||
|
void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
|
||||||
|
enum hrtimer_mode mode)
|
||||||
{
|
{
|
||||||
|
/*
|
||||||
|
* Make the enqueue delivery mode check work on RT. If the sleeper
|
||||||
|
* was initialized for hard interrupt delivery, force the mode bit.
|
||||||
|
* This is a special case for hrtimer_sleepers because
|
||||||
|
* hrtimer_init_sleeper() determines the delivery mode on RT so the
|
||||||
|
* fiddling with this decision is avoided at the call sites.
|
||||||
|
*/
|
||||||
|
if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
|
||||||
|
mode |= HRTIMER_MODE_HARD;
|
||||||
|
|
||||||
|
hrtimer_start_expires(&sl->timer, mode);
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
|
||||||
|
|
||||||
|
static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
|
||||||
|
clockid_t clock_id, enum hrtimer_mode mode)
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
* On PREEMPT_RT enabled kernels hrtimers which are not explicitely
|
||||||
|
* marked for hard interrupt expiry mode are moved into soft
|
||||||
|
* interrupt context either for latency reasons or because the
|
||||||
|
* hrtimer callback takes regular spinlocks or invokes other
|
||||||
|
* functions which are not suitable for hard interrupt context on
|
||||||
|
* PREEMPT_RT.
|
||||||
|
*
|
||||||
|
* The hrtimer_sleeper callback is RT compatible in hard interrupt
|
||||||
|
* context, but there is a latency concern: Untrusted userspace can
|
||||||
|
* spawn many threads which arm timers for the same expiry time on
|
||||||
|
* the same CPU. That causes a latency spike due to the wakeup of
|
||||||
|
* a gazillion threads.
|
||||||
|
*
|
||||||
|
* OTOH, priviledged real-time user space applications rely on the
|
||||||
|
* low latency of hard interrupt wakeups. If the current task is in
|
||||||
|
* a real-time scheduling class, mark the mode for hard interrupt
|
||||||
|
* expiry.
|
||||||
|
*/
|
||||||
|
if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
|
||||||
|
if (task_is_realtime(current) && !(mode & HRTIMER_MODE_SOFT))
|
||||||
|
mode |= HRTIMER_MODE_HARD;
|
||||||
|
}
|
||||||
|
|
||||||
|
__hrtimer_init(&sl->timer, clock_id, mode);
|
||||||
sl->timer.function = hrtimer_wakeup;
|
sl->timer.function = hrtimer_wakeup;
|
||||||
sl->task = task;
|
sl->task = current;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* hrtimer_init_sleeper - initialize sleeper to the given clock
|
||||||
|
* @sl: sleeper to be initialized
|
||||||
|
* @clock_id: the clock to be used
|
||||||
|
* @mode: timer mode abs/rel
|
||||||
|
*/
|
||||||
|
void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
|
||||||
|
enum hrtimer_mode mode)
|
||||||
|
{
|
||||||
|
debug_init(&sl->timer, clock_id, mode);
|
||||||
|
__hrtimer_init_sleeper(sl, clock_id, mode);
|
||||||
|
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
|
EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
|
||||||
|
|
||||||
|
@ -1669,11 +1863,9 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod
|
||||||
{
|
{
|
||||||
struct restart_block *restart;
|
struct restart_block *restart;
|
||||||
|
|
||||||
hrtimer_init_sleeper(t, current);
|
|
||||||
|
|
||||||
do {
|
do {
|
||||||
set_current_state(TASK_INTERRUPTIBLE);
|
set_current_state(TASK_INTERRUPTIBLE);
|
||||||
hrtimer_start_expires(&t->timer, mode);
|
hrtimer_sleeper_start_expires(t, mode);
|
||||||
|
|
||||||
if (likely(t->task))
|
if (likely(t->task))
|
||||||
freezable_schedule();
|
freezable_schedule();
|
||||||
|
@ -1707,10 +1899,9 @@ static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
|
||||||
struct hrtimer_sleeper t;
|
struct hrtimer_sleeper t;
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,
|
hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid,
|
||||||
HRTIMER_MODE_ABS);
|
HRTIMER_MODE_ABS);
|
||||||
hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
|
hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
|
||||||
|
|
||||||
ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
|
ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
|
||||||
destroy_hrtimer_on_stack(&t.timer);
|
destroy_hrtimer_on_stack(&t.timer);
|
||||||
return ret;
|
return ret;
|
||||||
|
@ -1728,7 +1919,7 @@ long hrtimer_nanosleep(const struct timespec64 *rqtp,
|
||||||
if (dl_task(current) || rt_task(current))
|
if (dl_task(current) || rt_task(current))
|
||||||
slack = 0;
|
slack = 0;
|
||||||
|
|
||||||
hrtimer_init_on_stack(&t.timer, clockid, mode);
|
hrtimer_init_sleeper_on_stack(&t, clockid, mode);
|
||||||
hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);
|
hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);
|
||||||
ret = do_nanosleep(&t, mode);
|
ret = do_nanosleep(&t, mode);
|
||||||
if (ret != -ERESTART_RESTARTBLOCK)
|
if (ret != -ERESTART_RESTARTBLOCK)
|
||||||
|
@ -1809,6 +2000,7 @@ int hrtimers_prepare_cpu(unsigned int cpu)
|
||||||
cpu_base->softirq_next_timer = NULL;
|
cpu_base->softirq_next_timer = NULL;
|
||||||
cpu_base->expires_next = KTIME_MAX;
|
cpu_base->expires_next = KTIME_MAX;
|
||||||
cpu_base->softirq_expires_next = KTIME_MAX;
|
cpu_base->softirq_expires_next = KTIME_MAX;
|
||||||
|
hrtimer_cpu_base_init_expiry_lock(cpu_base);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1927,12 +2119,9 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
|
||||||
return -EINTR;
|
return -EINTR;
|
||||||
}
|
}
|
||||||
|
|
||||||
hrtimer_init_on_stack(&t.timer, clock_id, mode);
|
hrtimer_init_sleeper_on_stack(&t, clock_id, mode);
|
||||||
hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
|
hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
|
||||||
|
hrtimer_sleeper_start_expires(&t, mode);
|
||||||
hrtimer_init_sleeper(&t, current);
|
|
||||||
|
|
||||||
hrtimer_start_expires(&t.timer, mode);
|
|
||||||
|
|
||||||
if (likely(t.task))
|
if (likely(t.task))
|
||||||
schedule();
|
schedule();
|
||||||
|
|
|
@ -55,15 +55,10 @@ static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id,
|
||||||
val = it->expires;
|
val = it->expires;
|
||||||
interval = it->incr;
|
interval = it->incr;
|
||||||
if (val) {
|
if (val) {
|
||||||
struct task_cputime cputime;
|
u64 t, samples[CPUCLOCK_MAX];
|
||||||
u64 t;
|
|
||||||
|
|
||||||
thread_group_cputimer(tsk, &cputime);
|
thread_group_sample_cputime(tsk, samples);
|
||||||
if (clock_id == CPUCLOCK_PROF)
|
t = samples[clock_id];
|
||||||
t = cputime.utime + cputime.stime;
|
|
||||||
else
|
|
||||||
/* CPUCLOCK_VIRT */
|
|
||||||
t = cputime.utime;
|
|
||||||
|
|
||||||
if (val < t)
|
if (val < t)
|
||||||
/* about to fire */
|
/* about to fire */
|
||||||
|
@ -213,6 +208,7 @@ again:
|
||||||
/* We are sharing ->siglock with it_real_fn() */
|
/* We are sharing ->siglock with it_real_fn() */
|
||||||
if (hrtimer_try_to_cancel(timer) < 0) {
|
if (hrtimer_try_to_cancel(timer) < 0) {
|
||||||
spin_unlock_irq(&tsk->sighand->siglock);
|
spin_unlock_irq(&tsk->sighand->siglock);
|
||||||
|
hrtimer_cancel_wait_running(timer);
|
||||||
goto again;
|
goto again;
|
||||||
}
|
}
|
||||||
expires = timeval_to_ktime(value->it_value);
|
expires = timeval_to_ktime(value->it_value);
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -442,7 +442,7 @@ static struct k_itimer * alloc_posix_timer(void)
|
||||||
|
|
||||||
static void k_itimer_rcu_free(struct rcu_head *head)
|
static void k_itimer_rcu_free(struct rcu_head *head)
|
||||||
{
|
{
|
||||||
struct k_itimer *tmr = container_of(head, struct k_itimer, it.rcu);
|
struct k_itimer *tmr = container_of(head, struct k_itimer, rcu);
|
||||||
|
|
||||||
kmem_cache_free(posix_timers_cache, tmr);
|
kmem_cache_free(posix_timers_cache, tmr);
|
||||||
}
|
}
|
||||||
|
@ -459,7 +459,7 @@ static void release_posix_timer(struct k_itimer *tmr, int it_id_set)
|
||||||
}
|
}
|
||||||
put_pid(tmr->it_pid);
|
put_pid(tmr->it_pid);
|
||||||
sigqueue_free(tmr->sigq);
|
sigqueue_free(tmr->sigq);
|
||||||
call_rcu(&tmr->it.rcu, k_itimer_rcu_free);
|
call_rcu(&tmr->rcu, k_itimer_rcu_free);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int common_timer_create(struct k_itimer *new_timer)
|
static int common_timer_create(struct k_itimer *new_timer)
|
||||||
|
@ -805,6 +805,35 @@ static int common_hrtimer_try_to_cancel(struct k_itimer *timr)
|
||||||
return hrtimer_try_to_cancel(&timr->it.real.timer);
|
return hrtimer_try_to_cancel(&timr->it.real.timer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void common_timer_wait_running(struct k_itimer *timer)
|
||||||
|
{
|
||||||
|
hrtimer_cancel_wait_running(&timer->it.real.timer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* On PREEMPT_RT this prevent priority inversion against softirq kthread in
|
||||||
|
* case it gets preempted while executing a timer callback. See comments in
|
||||||
|
* hrtimer_cancel_wait_running. For PREEMPT_RT=n this just results in a
|
||||||
|
* cpu_relax().
|
||||||
|
*/
|
||||||
|
static struct k_itimer *timer_wait_running(struct k_itimer *timer,
|
||||||
|
unsigned long *flags)
|
||||||
|
{
|
||||||
|
const struct k_clock *kc = READ_ONCE(timer->kclock);
|
||||||
|
timer_t timer_id = READ_ONCE(timer->it_id);
|
||||||
|
|
||||||
|
/* Prevent kfree(timer) after dropping the lock */
|
||||||
|
rcu_read_lock();
|
||||||
|
unlock_timer(timer, *flags);
|
||||||
|
|
||||||
|
if (!WARN_ON_ONCE(!kc->timer_wait_running))
|
||||||
|
kc->timer_wait_running(timer);
|
||||||
|
|
||||||
|
rcu_read_unlock();
|
||||||
|
/* Relock the timer. It might be not longer hashed. */
|
||||||
|
return lock_timer(timer_id, flags);
|
||||||
|
}
|
||||||
|
|
||||||
/* Set a POSIX.1b interval timer. */
|
/* Set a POSIX.1b interval timer. */
|
||||||
int common_timer_set(struct k_itimer *timr, int flags,
|
int common_timer_set(struct k_itimer *timr, int flags,
|
||||||
struct itimerspec64 *new_setting,
|
struct itimerspec64 *new_setting,
|
||||||
|
@ -844,13 +873,13 @@ int common_timer_set(struct k_itimer *timr, int flags,
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int do_timer_settime(timer_t timer_id, int flags,
|
static int do_timer_settime(timer_t timer_id, int tmr_flags,
|
||||||
struct itimerspec64 *new_spec64,
|
struct itimerspec64 *new_spec64,
|
||||||
struct itimerspec64 *old_spec64)
|
struct itimerspec64 *old_spec64)
|
||||||
{
|
{
|
||||||
const struct k_clock *kc;
|
const struct k_clock *kc;
|
||||||
struct k_itimer *timr;
|
struct k_itimer *timr;
|
||||||
unsigned long flag;
|
unsigned long flags;
|
||||||
int error = 0;
|
int error = 0;
|
||||||
|
|
||||||
if (!timespec64_valid(&new_spec64->it_interval) ||
|
if (!timespec64_valid(&new_spec64->it_interval) ||
|
||||||
|
@ -859,8 +888,9 @@ static int do_timer_settime(timer_t timer_id, int flags,
|
||||||
|
|
||||||
if (old_spec64)
|
if (old_spec64)
|
||||||
memset(old_spec64, 0, sizeof(*old_spec64));
|
memset(old_spec64, 0, sizeof(*old_spec64));
|
||||||
|
|
||||||
|
timr = lock_timer(timer_id, &flags);
|
||||||
retry:
|
retry:
|
||||||
timr = lock_timer(timer_id, &flag);
|
|
||||||
if (!timr)
|
if (!timr)
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
||||||
|
@ -868,13 +898,16 @@ retry:
|
||||||
if (WARN_ON_ONCE(!kc || !kc->timer_set))
|
if (WARN_ON_ONCE(!kc || !kc->timer_set))
|
||||||
error = -EINVAL;
|
error = -EINVAL;
|
||||||
else
|
else
|
||||||
error = kc->timer_set(timr, flags, new_spec64, old_spec64);
|
error = kc->timer_set(timr, tmr_flags, new_spec64, old_spec64);
|
||||||
|
|
||||||
unlock_timer(timr, flag);
|
|
||||||
if (error == TIMER_RETRY) {
|
if (error == TIMER_RETRY) {
|
||||||
old_spec64 = NULL; // We already got the old time...
|
// We already got the old time...
|
||||||
|
old_spec64 = NULL;
|
||||||
|
/* Unlocks and relocks the timer if it still exists */
|
||||||
|
timr = timer_wait_running(timr, &flags);
|
||||||
goto retry;
|
goto retry;
|
||||||
}
|
}
|
||||||
|
unlock_timer(timr, flags);
|
||||||
|
|
||||||
return error;
|
return error;
|
||||||
}
|
}
|
||||||
|
@ -951,13 +984,15 @@ SYSCALL_DEFINE1(timer_delete, timer_t, timer_id)
|
||||||
struct k_itimer *timer;
|
struct k_itimer *timer;
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
|
|
||||||
retry_delete:
|
|
||||||
timer = lock_timer(timer_id, &flags);
|
timer = lock_timer(timer_id, &flags);
|
||||||
|
|
||||||
|
retry_delete:
|
||||||
if (!timer)
|
if (!timer)
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
||||||
if (timer_delete_hook(timer) == TIMER_RETRY) {
|
if (unlikely(timer_delete_hook(timer) == TIMER_RETRY)) {
|
||||||
unlock_timer(timer, flags);
|
/* Unlocks and relocks the timer if it still exists */
|
||||||
|
timer = timer_wait_running(timer, &flags);
|
||||||
goto retry_delete;
|
goto retry_delete;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1238,6 +1273,7 @@ static const struct k_clock clock_realtime = {
|
||||||
.timer_forward = common_hrtimer_forward,
|
.timer_forward = common_hrtimer_forward,
|
||||||
.timer_remaining = common_hrtimer_remaining,
|
.timer_remaining = common_hrtimer_remaining,
|
||||||
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
|
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
|
||||||
|
.timer_wait_running = common_timer_wait_running,
|
||||||
.timer_arm = common_hrtimer_arm,
|
.timer_arm = common_hrtimer_arm,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1253,6 +1289,7 @@ static const struct k_clock clock_monotonic = {
|
||||||
.timer_forward = common_hrtimer_forward,
|
.timer_forward = common_hrtimer_forward,
|
||||||
.timer_remaining = common_hrtimer_remaining,
|
.timer_remaining = common_hrtimer_remaining,
|
||||||
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
|
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
|
||||||
|
.timer_wait_running = common_timer_wait_running,
|
||||||
.timer_arm = common_hrtimer_arm,
|
.timer_arm = common_hrtimer_arm,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1283,6 +1320,7 @@ static const struct k_clock clock_tai = {
|
||||||
.timer_forward = common_hrtimer_forward,
|
.timer_forward = common_hrtimer_forward,
|
||||||
.timer_remaining = common_hrtimer_remaining,
|
.timer_remaining = common_hrtimer_remaining,
|
||||||
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
|
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
|
||||||
|
.timer_wait_running = common_timer_wait_running,
|
||||||
.timer_arm = common_hrtimer_arm,
|
.timer_arm = common_hrtimer_arm,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1298,6 +1336,7 @@ static const struct k_clock clock_boottime = {
|
||||||
.timer_forward = common_hrtimer_forward,
|
.timer_forward = common_hrtimer_forward,
|
||||||
.timer_remaining = common_hrtimer_remaining,
|
.timer_remaining = common_hrtimer_remaining,
|
||||||
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
|
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
|
||||||
|
.timer_wait_running = common_timer_wait_running,
|
||||||
.timer_arm = common_hrtimer_arm,
|
.timer_arm = common_hrtimer_arm,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,7 @@ struct k_clock {
|
||||||
int (*timer_try_to_cancel)(struct k_itimer *timr);
|
int (*timer_try_to_cancel)(struct k_itimer *timr);
|
||||||
void (*timer_arm)(struct k_itimer *timr, ktime_t expires,
|
void (*timer_arm)(struct k_itimer *timr, ktime_t expires,
|
||||||
bool absolute, bool sigev_none);
|
bool absolute, bool sigev_none);
|
||||||
|
void (*timer_wait_running)(struct k_itimer *timr);
|
||||||
};
|
};
|
||||||
|
|
||||||
extern const struct k_clock clock_posix_cpu;
|
extern const struct k_clock clock_posix_cpu;
|
||||||
|
|
|
@ -59,11 +59,16 @@ static int bc_set_next(ktime_t expires, struct clock_event_device *bc)
|
||||||
* hrtimer_{start/cancel} functions call into tracing,
|
* hrtimer_{start/cancel} functions call into tracing,
|
||||||
* calls to these functions must be bound within RCU_NONIDLE.
|
* calls to these functions must be bound within RCU_NONIDLE.
|
||||||
*/
|
*/
|
||||||
RCU_NONIDLE({
|
RCU_NONIDLE(
|
||||||
|
{
|
||||||
bc_moved = hrtimer_try_to_cancel(&bctimer) >= 0;
|
bc_moved = hrtimer_try_to_cancel(&bctimer) >= 0;
|
||||||
if (bc_moved)
|
if (bc_moved) {
|
||||||
hrtimer_start(&bctimer, expires,
|
hrtimer_start(&bctimer, expires,
|
||||||
HRTIMER_MODE_ABS_PINNED);});
|
HRTIMER_MODE_ABS_PINNED_HARD);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
if (bc_moved) {
|
if (bc_moved) {
|
||||||
/* Bind the "device" to the cpu */
|
/* Bind the "device" to the cpu */
|
||||||
bc->bound_on = smp_processor_id();
|
bc->bound_on = smp_processor_id();
|
||||||
|
@ -104,7 +109,7 @@ static enum hrtimer_restart bc_handler(struct hrtimer *t)
|
||||||
|
|
||||||
void tick_setup_hrtimer_broadcast(void)
|
void tick_setup_hrtimer_broadcast(void)
|
||||||
{
|
{
|
||||||
hrtimer_init(&bctimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
|
hrtimer_init(&bctimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
|
||||||
bctimer.function = bc_handler;
|
bctimer.function = bc_handler;
|
||||||
clockevents_register_device(&ce_broadcast_hrtimer);
|
clockevents_register_device(&ce_broadcast_hrtimer);
|
||||||
}
|
}
|
||||||
|
|
|
@ -634,10 +634,12 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
|
||||||
/* Forward the time to expire in the future */
|
/* Forward the time to expire in the future */
|
||||||
hrtimer_forward(&ts->sched_timer, now, tick_period);
|
hrtimer_forward(&ts->sched_timer, now, tick_period);
|
||||||
|
|
||||||
if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
|
if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
|
||||||
hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
|
hrtimer_start_expires(&ts->sched_timer,
|
||||||
else
|
HRTIMER_MODE_ABS_PINNED_HARD);
|
||||||
|
} else {
|
||||||
tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
|
tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Reset to make sure next tick stop doesn't get fooled by past
|
* Reset to make sure next tick stop doesn't get fooled by past
|
||||||
|
@ -802,7 +804,8 @@ static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
|
if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
|
||||||
hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED);
|
hrtimer_start(&ts->sched_timer, tick,
|
||||||
|
HRTIMER_MODE_ABS_PINNED_HARD);
|
||||||
} else {
|
} else {
|
||||||
hrtimer_set_expires(&ts->sched_timer, tick);
|
hrtimer_set_expires(&ts->sched_timer, tick);
|
||||||
tick_program_event(tick, 1);
|
tick_program_event(tick, 1);
|
||||||
|
@ -1230,7 +1233,7 @@ static void tick_nohz_switch_to_nohz(void)
|
||||||
* Recycle the hrtimer in ts, so we can share the
|
* Recycle the hrtimer in ts, so we can share the
|
||||||
* hrtimer_forward with the highres code.
|
* hrtimer_forward with the highres code.
|
||||||
*/
|
*/
|
||||||
hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
|
hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
|
||||||
/* Get the next period */
|
/* Get the next period */
|
||||||
next = tick_init_jiffy_update();
|
next = tick_init_jiffy_update();
|
||||||
|
|
||||||
|
@ -1327,7 +1330,7 @@ void tick_setup_sched_timer(void)
|
||||||
/*
|
/*
|
||||||
* Emulate tick processing via per-CPU hrtimers:
|
* Emulate tick processing via per-CPU hrtimers:
|
||||||
*/
|
*/
|
||||||
hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
|
hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
|
||||||
ts->sched_timer.function = tick_sched_timer;
|
ts->sched_timer.function = tick_sched_timer;
|
||||||
|
|
||||||
/* Get the next period (per-CPU) */
|
/* Get the next period (per-CPU) */
|
||||||
|
@ -1342,7 +1345,7 @@ void tick_setup_sched_timer(void)
|
||||||
}
|
}
|
||||||
|
|
||||||
hrtimer_forward(&ts->sched_timer, now, tick_period);
|
hrtimer_forward(&ts->sched_timer, now, tick_period);
|
||||||
hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
|
hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
|
||||||
tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
|
tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
|
||||||
}
|
}
|
||||||
#endif /* HIGH_RES_TIMERS */
|
#endif /* HIGH_RES_TIMERS */
|
||||||
|
|
|
@ -196,6 +196,10 @@ EXPORT_SYMBOL(jiffies_64);
|
||||||
struct timer_base {
|
struct timer_base {
|
||||||
raw_spinlock_t lock;
|
raw_spinlock_t lock;
|
||||||
struct timer_list *running_timer;
|
struct timer_list *running_timer;
|
||||||
|
#ifdef CONFIG_PREEMPT_RT
|
||||||
|
spinlock_t expiry_lock;
|
||||||
|
atomic_t timer_waiters;
|
||||||
|
#endif
|
||||||
unsigned long clk;
|
unsigned long clk;
|
||||||
unsigned long next_expiry;
|
unsigned long next_expiry;
|
||||||
unsigned int cpu;
|
unsigned int cpu;
|
||||||
|
@ -1227,7 +1231,78 @@ int try_to_del_timer_sync(struct timer_list *timer)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(try_to_del_timer_sync);
|
EXPORT_SYMBOL(try_to_del_timer_sync);
|
||||||
|
|
||||||
#ifdef CONFIG_SMP
|
#ifdef CONFIG_PREEMPT_RT
|
||||||
|
static __init void timer_base_init_expiry_lock(struct timer_base *base)
|
||||||
|
{
|
||||||
|
spin_lock_init(&base->expiry_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void timer_base_lock_expiry(struct timer_base *base)
|
||||||
|
{
|
||||||
|
spin_lock(&base->expiry_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void timer_base_unlock_expiry(struct timer_base *base)
|
||||||
|
{
|
||||||
|
spin_unlock(&base->expiry_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The counterpart to del_timer_wait_running().
|
||||||
|
*
|
||||||
|
* If there is a waiter for base->expiry_lock, then it was waiting for the
|
||||||
|
* timer callback to finish. Drop expiry_lock and reaquire it. That allows
|
||||||
|
* the waiter to acquire the lock and make progress.
|
||||||
|
*/
|
||||||
|
static void timer_sync_wait_running(struct timer_base *base)
|
||||||
|
{
|
||||||
|
if (atomic_read(&base->timer_waiters)) {
|
||||||
|
spin_unlock(&base->expiry_lock);
|
||||||
|
spin_lock(&base->expiry_lock);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This function is called on PREEMPT_RT kernels when the fast path
|
||||||
|
* deletion of a timer failed because the timer callback function was
|
||||||
|
* running.
|
||||||
|
*
|
||||||
|
* This prevents priority inversion, if the softirq thread on a remote CPU
|
||||||
|
* got preempted, and it prevents a life lock when the task which tries to
|
||||||
|
* delete a timer preempted the softirq thread running the timer callback
|
||||||
|
* function.
|
||||||
|
*/
|
||||||
|
static void del_timer_wait_running(struct timer_list *timer)
|
||||||
|
{
|
||||||
|
u32 tf;
|
||||||
|
|
||||||
|
tf = READ_ONCE(timer->flags);
|
||||||
|
if (!(tf & TIMER_MIGRATING)) {
|
||||||
|
struct timer_base *base = get_timer_base(tf);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Mark the base as contended and grab the expiry lock,
|
||||||
|
* which is held by the softirq across the timer
|
||||||
|
* callback. Drop the lock immediately so the softirq can
|
||||||
|
* expire the next timer. In theory the timer could already
|
||||||
|
* be running again, but that's more than unlikely and just
|
||||||
|
* causes another wait loop.
|
||||||
|
*/
|
||||||
|
atomic_inc(&base->timer_waiters);
|
||||||
|
spin_lock_bh(&base->expiry_lock);
|
||||||
|
atomic_dec(&base->timer_waiters);
|
||||||
|
spin_unlock_bh(&base->expiry_lock);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
static inline void timer_base_init_expiry_lock(struct timer_base *base) { }
|
||||||
|
static inline void timer_base_lock_expiry(struct timer_base *base) { }
|
||||||
|
static inline void timer_base_unlock_expiry(struct timer_base *base) { }
|
||||||
|
static inline void timer_sync_wait_running(struct timer_base *base) { }
|
||||||
|
static inline void del_timer_wait_running(struct timer_list *timer) { }
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
|
||||||
/**
|
/**
|
||||||
* del_timer_sync - deactivate a timer and wait for the handler to finish.
|
* del_timer_sync - deactivate a timer and wait for the handler to finish.
|
||||||
* @timer: the timer to be deactivated
|
* @timer: the timer to be deactivated
|
||||||
|
@ -1266,6 +1341,8 @@ EXPORT_SYMBOL(try_to_del_timer_sync);
|
||||||
*/
|
*/
|
||||||
int del_timer_sync(struct timer_list *timer)
|
int del_timer_sync(struct timer_list *timer)
|
||||||
{
|
{
|
||||||
|
int ret;
|
||||||
|
|
||||||
#ifdef CONFIG_LOCKDEP
|
#ifdef CONFIG_LOCKDEP
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
|
|
||||||
|
@ -1283,12 +1360,17 @@ int del_timer_sync(struct timer_list *timer)
|
||||||
* could lead to deadlock.
|
* could lead to deadlock.
|
||||||
*/
|
*/
|
||||||
WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));
|
WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));
|
||||||
for (;;) {
|
|
||||||
int ret = try_to_del_timer_sync(timer);
|
do {
|
||||||
if (ret >= 0)
|
ret = try_to_del_timer_sync(timer);
|
||||||
return ret;
|
|
||||||
cpu_relax();
|
if (unlikely(ret < 0)) {
|
||||||
}
|
del_timer_wait_running(timer);
|
||||||
|
cpu_relax();
|
||||||
|
}
|
||||||
|
} while (ret < 0);
|
||||||
|
|
||||||
|
return ret;
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(del_timer_sync);
|
EXPORT_SYMBOL(del_timer_sync);
|
||||||
#endif
|
#endif
|
||||||
|
@ -1360,10 +1442,13 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head)
|
||||||
if (timer->flags & TIMER_IRQSAFE) {
|
if (timer->flags & TIMER_IRQSAFE) {
|
||||||
raw_spin_unlock(&base->lock);
|
raw_spin_unlock(&base->lock);
|
||||||
call_timer_fn(timer, fn, baseclk);
|
call_timer_fn(timer, fn, baseclk);
|
||||||
|
base->running_timer = NULL;
|
||||||
raw_spin_lock(&base->lock);
|
raw_spin_lock(&base->lock);
|
||||||
} else {
|
} else {
|
||||||
raw_spin_unlock_irq(&base->lock);
|
raw_spin_unlock_irq(&base->lock);
|
||||||
call_timer_fn(timer, fn, baseclk);
|
call_timer_fn(timer, fn, baseclk);
|
||||||
|
base->running_timer = NULL;
|
||||||
|
timer_sync_wait_running(base);
|
||||||
raw_spin_lock_irq(&base->lock);
|
raw_spin_lock_irq(&base->lock);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1643,7 +1728,7 @@ void update_process_times(int user_tick)
|
||||||
#endif
|
#endif
|
||||||
scheduler_tick();
|
scheduler_tick();
|
||||||
if (IS_ENABLED(CONFIG_POSIX_TIMERS))
|
if (IS_ENABLED(CONFIG_POSIX_TIMERS))
|
||||||
run_posix_cpu_timers(p);
|
run_posix_cpu_timers();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1658,6 +1743,7 @@ static inline void __run_timers(struct timer_base *base)
|
||||||
if (!time_after_eq(jiffies, base->clk))
|
if (!time_after_eq(jiffies, base->clk))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
timer_base_lock_expiry(base);
|
||||||
raw_spin_lock_irq(&base->lock);
|
raw_spin_lock_irq(&base->lock);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -1684,8 +1770,8 @@ static inline void __run_timers(struct timer_base *base)
|
||||||
while (levels--)
|
while (levels--)
|
||||||
expire_timers(base, heads + levels);
|
expire_timers(base, heads + levels);
|
||||||
}
|
}
|
||||||
base->running_timer = NULL;
|
|
||||||
raw_spin_unlock_irq(&base->lock);
|
raw_spin_unlock_irq(&base->lock);
|
||||||
|
timer_base_unlock_expiry(base);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -1930,6 +2016,7 @@ static void __init init_timer_cpu(int cpu)
|
||||||
base->cpu = cpu;
|
base->cpu = cpu;
|
||||||
raw_spin_lock_init(&base->lock);
|
raw_spin_lock_init(&base->lock);
|
||||||
base->clk = jiffies;
|
base->clk = jiffies;
|
||||||
|
timer_base_init_expiry_lock(base);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -490,10 +490,10 @@ static void watchdog_enable(unsigned int cpu)
|
||||||
* Start the timer first to prevent the NMI watchdog triggering
|
* Start the timer first to prevent the NMI watchdog triggering
|
||||||
* before the timer has a chance to fire.
|
* before the timer has a chance to fire.
|
||||||
*/
|
*/
|
||||||
hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
|
||||||
hrtimer->function = watchdog_timer_fn;
|
hrtimer->function = watchdog_timer_fn;
|
||||||
hrtimer_start(hrtimer, ns_to_ktime(sample_period),
|
hrtimer_start(hrtimer, ns_to_ktime(sample_period),
|
||||||
HRTIMER_MODE_REL_PINNED);
|
HRTIMER_MODE_REL_PINNED_HARD);
|
||||||
|
|
||||||
/* Initialize timestamp */
|
/* Initialize timestamp */
|
||||||
__touch_watchdog();
|
__touch_watchdog();
|
||||||
|
|
|
@ -26,9 +26,10 @@
|
||||||
*/
|
*/
|
||||||
bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node)
|
bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node)
|
||||||
{
|
{
|
||||||
struct rb_node **p = &head->head.rb_node;
|
struct rb_node **p = &head->rb_root.rb_root.rb_node;
|
||||||
struct rb_node *parent = NULL;
|
struct rb_node *parent = NULL;
|
||||||
struct timerqueue_node *ptr;
|
struct timerqueue_node *ptr;
|
||||||
|
bool leftmost = true;
|
||||||
|
|
||||||
/* Make sure we don't add nodes that are already added */
|
/* Make sure we don't add nodes that are already added */
|
||||||
WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node));
|
WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node));
|
||||||
|
@ -36,19 +37,17 @@ bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node)
|
||||||
while (*p) {
|
while (*p) {
|
||||||
parent = *p;
|
parent = *p;
|
||||||
ptr = rb_entry(parent, struct timerqueue_node, node);
|
ptr = rb_entry(parent, struct timerqueue_node, node);
|
||||||
if (node->expires < ptr->expires)
|
if (node->expires < ptr->expires) {
|
||||||
p = &(*p)->rb_left;
|
p = &(*p)->rb_left;
|
||||||
else
|
} else {
|
||||||
p = &(*p)->rb_right;
|
p = &(*p)->rb_right;
|
||||||
|
leftmost = false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
rb_link_node(&node->node, parent, p);
|
rb_link_node(&node->node, parent, p);
|
||||||
rb_insert_color(&node->node, &head->head);
|
rb_insert_color_cached(&node->node, &head->rb_root, leftmost);
|
||||||
|
|
||||||
if (!head->next || node->expires < head->next->expires) {
|
return leftmost;
|
||||||
head->next = node;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(timerqueue_add);
|
EXPORT_SYMBOL_GPL(timerqueue_add);
|
||||||
|
|
||||||
|
@ -65,15 +64,10 @@ bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node)
|
||||||
{
|
{
|
||||||
WARN_ON_ONCE(RB_EMPTY_NODE(&node->node));
|
WARN_ON_ONCE(RB_EMPTY_NODE(&node->node));
|
||||||
|
|
||||||
/* update next pointer */
|
rb_erase_cached(&node->node, &head->rb_root);
|
||||||
if (head->next == node) {
|
|
||||||
struct rb_node *rbn = rb_next(&node->node);
|
|
||||||
|
|
||||||
head->next = rb_entry_safe(rbn, struct timerqueue_node, node);
|
|
||||||
}
|
|
||||||
rb_erase(&node->node, &head->head);
|
|
||||||
RB_CLEAR_NODE(&node->node);
|
RB_CLEAR_NODE(&node->node);
|
||||||
return head->next != NULL;
|
|
||||||
|
return !RB_EMPTY_ROOT(&head->rb_root.rb_root);
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(timerqueue_del);
|
EXPORT_SYMBOL_GPL(timerqueue_del);
|
||||||
|
|
||||||
|
|
|
@ -2156,7 +2156,7 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
|
||||||
s64 remaining;
|
s64 remaining;
|
||||||
struct hrtimer_sleeper t;
|
struct hrtimer_sleeper t;
|
||||||
|
|
||||||
hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
|
hrtimer_init_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
|
||||||
hrtimer_set_expires(&t.timer, spin_until);
|
hrtimer_set_expires(&t.timer, spin_until);
|
||||||
|
|
||||||
remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer));
|
remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer));
|
||||||
|
@ -2170,11 +2170,9 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
|
||||||
end_time = ktime_get();
|
end_time = ktime_get();
|
||||||
} while (ktime_compare(end_time, spin_until) < 0);
|
} while (ktime_compare(end_time, spin_until) < 0);
|
||||||
} else {
|
} else {
|
||||||
/* see do_nanosleep */
|
|
||||||
hrtimer_init_sleeper(&t, current);
|
|
||||||
do {
|
do {
|
||||||
set_current_state(TASK_INTERRUPTIBLE);
|
set_current_state(TASK_INTERRUPTIBLE);
|
||||||
hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS);
|
hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS);
|
||||||
|
|
||||||
if (likely(t.task))
|
if (likely(t.task))
|
||||||
schedule();
|
schedule();
|
||||||
|
|
Loading…
Reference in New Issue