!250 Backport pseudo NMI-based watchdog patch for OpenCloudOS-Kernel

Merge pull request !250 from lcy/devel-37

arch/arm64/kernel/Makefile

@@ -62,6 +62,7 @@ obj-$(CONFIG_CRASH_DUMP)		+= crash_dump.o
 obj-$(CONFIG_CRASH_CORE)		+= crash_core.o
 obj-$(CONFIG_ARM_SDE_INTERFACE)		+= sdei.o
 obj-$(CONFIG_ARM64_SSBD)		+= ssbd.o
+obj-$(CONFIG_SDEI_WATCHDOG)		+= watchdog_sdei.o
 obj-$(CONFIG_ARM64_PTR_AUTH)		+= pointer_auth.o
 
 obj-y					+= vdso/ probes/

arch/arm64/kernel/machine_kexec.c

@@ -10,6 +10,7 @@
 #include <linux/irq.h>
 #include <linux/kernel.h>
 #include <linux/kexec.h>
+#include <linux/nmi.h>
 #include <linux/page-flags.h>
 #include <linux/smp.h>
 
@@ -266,6 +267,15 @@ void machine_crash_shutdown(struct pt_regs *regs)
 	/* shutdown non-crashing cpus */
 	crash_smp_send_stop();
 
+	/*
+	 * when we panic in hardlockup detected by sdei_watchdog, the secure
+	 * timer interrupt remains activate here because firmware clear eoi
+	 * after dispatch is completed. This will cause arm_arch_timer
+	 * interrupt failed to trigger in the second kernel. So we clear eoi
+	 * of the secure timer before booting the second kernel.
+	 */
+	sdei_watchdog_clear_eoi();
+
 	/* for crashing cpu */
 	crash_save_cpu(regs, smp_processor_id());
 	machine_kexec_mask_interrupts();

arch/arm64/kernel/watchdog_sdei.c

@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Detect hard lockups on a system
+ *
+ * Note: Most of this code is borrowed heavily from the perf hardlockup
+ * detector, so thanks to Don for the initial implementation.
+ */
+
+#define pr_fmt(fmt) "SDEI NMI watchdog: " fmt
+
+#include <asm/irq_regs.h>
+#include <asm/kvm_hyp.h>
+#include <asm/smp_plat.h>
+#include <asm/sdei.h>
+#include <asm/virt.h>
+#include <linux/arm_sdei.h>
+#include <linux/kprobes.h>
+#include <linux/nmi.h>
+
+/* We use the secure physical timer as SDEI NMI watchdog timer */
+#define SDEI_NMI_WATCHDOG_HWIRQ		29
+
+static int sdei_watchdog_event_num;
+static bool disable_sdei_nmi_watchdog;
+static bool sdei_watchdog_registered;
+static DEFINE_PER_CPU(ktime_t, last_check_time);
+
+int watchdog_nmi_enable(unsigned int cpu)
+{
+	int ret;
+
+	if (!sdei_watchdog_registered)
+		return -EINVAL;
+
+#ifdef CONFIG_HARDLOCKUP_CHECK_TIMESTAMP
+	refresh_hld_last_timestamp();
+#endif
+
+	__this_cpu_write(last_check_time, ktime_get_mono_fast_ns());
+	sdei_api_set_secure_timer_period(watchdog_thresh);
+
+	ret = sdei_api_event_enable(sdei_watchdog_event_num);
+	if (ret) {
+		pr_err("Enable NMI Watchdog failed on cpu%d\n",
+				smp_processor_id());
+		return ret;
+	}
+
+	return 0;
+}
+
+void watchdog_nmi_disable(unsigned int cpu)
+{
+	int ret;
+
+	if (!sdei_watchdog_registered)
+		return;
+
+	ret = sdei_api_event_disable(sdei_watchdog_event_num);
+	if (ret)
+		pr_err("Disable NMI Watchdog failed on cpu%d\n",
+				smp_processor_id());
+}
+
+static int sdei_watchdog_callback(u32 event,
+		struct pt_regs *regs, void *arg)
+{
+	ktime_t delta, now = ktime_get_mono_fast_ns();
+
+	delta = now - __this_cpu_read(last_check_time);
+	__this_cpu_write(last_check_time, now);
+
+	/*
+	 * Set delta to 4/5 of the actual watchdog threshold period so the
+	 * hrtimer is guaranteed to fire at least once within the real
+	 * watchdog threshold.
+	 */
+	if (delta < watchdog_thresh * (u64)NSEC_PER_SEC * 4 / 5) {
+		pr_err(FW_BUG "SDEI Watchdog event triggered too soon, "
+			"time to last check:%lld ns\n", delta);
+		WARN_ON(1);
+		return 0;
+	}
+
+	watchdog_hardlockup_check(regs);
+
+	return 0;
+}
+NOKPROBE_SYMBOL(sdei_watchdog_callback);
+
+static void sdei_nmi_watchdog_bind(void *data)
+{
+	int ret;
+
+	ret = sdei_api_event_interrupt_bind(SDEI_NMI_WATCHDOG_HWIRQ);
+	if (ret < 0)
+		pr_err("SDEI bind failed on cpu%d, return %d\n",
+				smp_processor_id(), ret);
+}
+
+static int __init disable_sdei_nmi_watchdog_setup(char *str)
+{
+	disable_sdei_nmi_watchdog = true;
+	return 1;
+}
+__setup("disable_sdei_nmi_watchdog", disable_sdei_nmi_watchdog_setup);
+
+void sdei_watchdog_clear_eoi(void)
+{
+	if (sdei_watchdog_registered)
+		sdei_api_clear_eoi(SDEI_NMI_WATCHDOG_HWIRQ);
+}
+
+int __init watchdog_nmi_probe(void)
+{
+	int ret;
+
+	if (disable_sdei_nmi_watchdog)
+		return -EINVAL;
+
+	if (!is_hyp_mode_available()) {
+		pr_err("Disable SDEI NMI Watchdog in VM\n");
+		return -EINVAL;
+	}
+
+	sdei_watchdog_event_num = sdei_api_event_interrupt_bind(SDEI_NMI_WATCHDOG_HWIRQ);
+	if (sdei_watchdog_event_num < 0) {
+		pr_err("Bind interrupt failed. Firmware may not support SDEI !\n");
+		return sdei_watchdog_event_num;
+	}
+
+	/*
+	 * After we introduced 'sdei_api_set_secure_timer_period', we disselect
+	 * 'CONFIG_HARDLOCKUP_CHECK_TIMESTAMP'. So we need to make sure that
+	 * firmware can set the period of the secure timer and the timer
+	 * interrupt doesn't trigger too soon.
+	 */
+	if (sdei_api_set_secure_timer_period(watchdog_thresh)) {
+		pr_err("Firmware doesn't support setting the secure timer period, please update your BIOS !\n");
+		return -EINVAL;
+	}
+
+	on_each_cpu(sdei_nmi_watchdog_bind, NULL, true);
+
+	ret = sdei_event_register(sdei_watchdog_event_num,
+					sdei_watchdog_callback, NULL);
+	if (ret) {
+		pr_err("SDEI Watchdog register callback failed\n");
+		return ret;
+	}
+
+	sdei_watchdog_registered = true;
+	pr_info("SDEI Watchdog registered successfully\n");
+
+	return 0;
+}

drivers/firmware/arm_sdei.c

@@ -192,6 +192,28 @@ int sdei_api_event_context(u32 query, u64 *result)
 }
 NOKPROBE_SYMBOL(sdei_api_event_context);
 
+int sdei_api_event_interrupt_bind(int hwirq)
+{
+	u64 event_number;
+
+	invoke_sdei_fn(SDEI_1_0_FN_SDEI_INTERRUPT_BIND, hwirq, 0, 0, 0, 0,
+			&event_number);
+
+	return (int)event_number;
+}
+
+int sdei_api_clear_eoi(int hwirq)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_CLEAR_EOI, hwirq, 0, 0, 0, 0,
+			NULL);
+}
+
+int sdei_api_set_secure_timer_period(int sec)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SET_SECURE_TIMER_PERIOD, sec, 0, 0, 0,
+			0, NULL);
+}
+
 static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
 {
 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
@@ -381,7 +403,7 @@ static int sdei_platform_reset(void)
 	return err;
 }
 
-static int sdei_api_event_enable(u32 event_num)
+int sdei_api_event_enable(u32 event_num)
 {
 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
 			      0, NULL);
@@ -431,7 +453,7 @@ int sdei_event_enable(u32 event_num)
 }
 EXPORT_SYMBOL(sdei_event_enable);
 
-static int sdei_api_event_disable(u32 event_num)
+int sdei_api_event_disable(u32 event_num)
 {
 	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
 			      0, 0, NULL);

include/linux/arm_sdei.h

@@ -42,6 +42,11 @@ int sdei_event_unregister(u32 event_num);
 
 int sdei_event_enable(u32 event_num);
 int sdei_event_disable(u32 event_num);
+int sdei_api_event_interrupt_bind(int hwirq);
+int sdei_api_event_disable(u32 event_num);
+int sdei_api_event_enable(u32 event_num);
+int sdei_api_clear_eoi(int hwirq);
+int sdei_api_set_secure_timer_period(int sec);
 
 /* GHES register/unregister helpers */
 int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb,

include/linux/nmi.h

@@ -7,7 +7,7 @@
 
 #include <linux/sched.h>
 #include <asm/irq.h>
-#if defined(CONFIG_HAVE_NMI_WATCHDOG)
+#if defined(CONFIG_HAVE_NMI_WATCHDOG) && !defined(CONFIG_SDEI_WATCHDOG)
 #include <asm/nmi.h>
 #endif
 
@@ -83,6 +83,7 @@ static inline void reset_hung_task_detector(void) { }
 
 #if defined(CONFIG_HARDLOCKUP_DETECTOR)
 extern void hardlockup_detector_disable(void);
+extern void watchdog_hardlockup_check(struct pt_regs *regs);
 extern unsigned int hardlockup_panic;
 #else
 static inline void hardlockup_detector_disable(void) {}
@@ -94,8 +95,17 @@ static inline void hardlockup_detector_disable(void) {}
 # define NMI_WATCHDOG_SYSCTL_PERM	0444
 #endif
 
-#if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF)
+#if defined(CONFIG_HARDLOCKUP_DETECTOR)
+#ifndef CONFIG_PPC_WATCHDOG
 extern void arch_touch_nmi_watchdog(void);
+#endif
+#else
+# if !defined(CONFIG_HAVE_NMI_WATCHDOG)
+static inline void arch_touch_nmi_watchdog(void) {}
+# endif
+#endif
+
+#if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF)
 extern void hardlockup_detector_perf_stop(void);
 extern void hardlockup_detector_perf_restart(void);
 extern void hardlockup_detector_perf_disable(void);
@@ -110,7 +120,6 @@ static inline void hardlockup_detector_perf_enable(void) { }
 static inline void hardlockup_detector_perf_cleanup(void) { }
 # if !defined(CONFIG_HAVE_NMI_WATCHDOG)
 static inline int hardlockup_detector_perf_init(void) { return -ENODEV; }
-static inline void arch_touch_nmi_watchdog(void) {}
 # else
 static inline int hardlockup_detector_perf_init(void) { return 0; }
 # endif
@@ -199,6 +208,7 @@ u64 hw_nmi_get_sample_period(int watchdog_thresh);
 #if defined(CONFIG_HARDLOCKUP_CHECK_TIMESTAMP) && \
     defined(CONFIG_HARDLOCKUP_DETECTOR)
 void watchdog_update_hrtimer_threshold(u64 period);
+void refresh_hld_last_timestamp(void);
 #else
 static inline void watchdog_update_hrtimer_threshold(u64 period) { }
 #endif
@@ -219,4 +229,10 @@ extern int proc_watchdog_cpumask(struct ctl_table *, int,
 #include <asm/nmi.h>
 #endif
 
+#ifdef CONFIG_SDEI_WATCHDOG
+void sdei_watchdog_clear_eoi(void);
+#else
+static inline void sdei_watchdog_clear_eoi(void) { }
+#endif
+
 #endif

include/uapi/linux/arm_sdei.h

@@ -24,6 +24,8 @@
 #define SDEI_1_0_FN_SDEI_INTERRUPT_RELEASE		SDEI_1_0_FN(0x0E)
 #define SDEI_1_0_FN_SDEI_PRIVATE_RESET			SDEI_1_0_FN(0x11)
 #define SDEI_1_0_FN_SDEI_SHARED_RESET			SDEI_1_0_FN(0x12)
+#define SDEI_1_0_FN_SDEI_CLEAR_EOI			SDEI_1_0_FN(0x18)
+#define SDEI_1_0_FN_SET_SECURE_TIMER_PERIOD		SDEI_1_0_FN(0x19)
 
 #define SDEI_VERSION_MAJOR_SHIFT			48
 #define SDEI_VERSION_MAJOR_MASK				0x7fff

init/main.c

@@ -1196,7 +1196,6 @@ static noinline void __init kernel_init_freeable(void)
 	init_mm_internals();
 
 	do_pre_smp_initcalls();
-	lockup_detector_init();
 
 	smp_init();
 	sched_init_smp();
@@ -1207,6 +1206,8 @@ static noinline void __init kernel_init_freeable(void)
 
 	do_basic_setup();
 
+	lockup_detector_init();
+
 	/* Open the /dev/console on the rootfs, this should never fail */
 	if (ksys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0)
 		pr_err("Warning: unable to open an initial console.\n");

kernel/Makefile

@@ -89,7 +89,7 @@ obj-$(CONFIG_FAIL_FUNCTION) += fail_function.o
 obj-$(CONFIG_KGDB) += debug/
 obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_LOCKUP_DETECTOR) += watchdog.o
-obj-$(CONFIG_HARDLOCKUP_DETECTOR_PERF) += watchdog_hld.o
+obj-$(CONFIG_HARDLOCKUP_DETECTOR) += watchdog_hld.o
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RELAY) += relay.o
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o

kernel/watchdog.c

@@ -16,6 +16,7 @@
 #include <linux/cpu.h>
 #include <linux/nmi.h>
 #include <linux/init.h>
+#include <linux/kprobes.h>
 #include <linux/module.h>
 #include <linux/sysctl.h>
 #include <linux/tick.h>
@@ -333,6 +334,7 @@ bool is_hardlockup(void)
 	__this_cpu_write(hrtimer_interrupts_saved, hrint);
 	return false;
 }
+NOKPROBE_SYMBOL(is_hardlockup);
 
 static void watchdog_interrupt_count(void)
 {

kernel/watchdog_hld.c

@@ -14,6 +14,7 @@
 
 #include <linux/nmi.h>
 #include <linux/atomic.h>
+#include <linux/kprobes.h>
 #include <linux/module.h>
 #include <linux/sched/debug.h>
 
@@ -22,13 +23,10 @@
 
 static DEFINE_PER_CPU(bool, hard_watchdog_warn);
 static DEFINE_PER_CPU(bool, watchdog_nmi_touch);
-static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
-static DEFINE_PER_CPU(struct perf_event *, dead_event);
-static struct cpumask dead_events_mask;
 
 static unsigned long hardlockup_allcpu_dumped;
-static atomic_t watchdog_cpus = ATOMIC_INIT(0);
 
+#ifndef CONFIG_PPC_WATCHDOG
 notrace void arch_touch_nmi_watchdog(void)
 {
 	/*
@@ -41,6 +39,7 @@ notrace void arch_touch_nmi_watchdog(void)
 	raw_cpu_write(watchdog_nmi_touch, true);
 }
 EXPORT_SYMBOL(arch_touch_nmi_watchdog);
+#endif
 
 #ifdef CONFIG_HARDLOCKUP_CHECK_TIMESTAMP
 static DEFINE_PER_CPU(ktime_t, last_timestamp);
@@ -91,6 +90,15 @@ static bool watchdog_check_timestamp(void)
 	__this_cpu_write(last_timestamp, now);
 	return true;
 }
+
+void refresh_hld_last_timestamp(void)
+{
+	ktime_t now;
+
+	now = ktime_get_mono_fast_ns();
+	__this_cpu_write(last_timestamp, now);
+
+}
 #else
 static inline bool watchdog_check_timestamp(void)
 {
@@ -98,22 +106,8 @@ static inline bool watchdog_check_timestamp(void)
 }
 #endif
 
-static struct perf_event_attr wd_hw_attr = {
+void watchdog_hardlockup_check(struct pt_regs *regs)
-	.type		= PERF_TYPE_HARDWARE,
-	.config		= PERF_COUNT_HW_CPU_CYCLES,
-	.size		= sizeof(struct perf_event_attr),
-	.pinned		= 1,
-	.disabled	= 1,
-};
-
-/* Callback function for perf event subsystem */
-static void watchdog_overflow_callback(struct perf_event *event,
-				       struct perf_sample_data *data,
-				       struct pt_regs *regs)
 {
-	/* Ensure the watchdog never gets throttled */
-	event->hw.interrupts = 0;
-
 	if (__this_cpu_read(watchdog_nmi_touch) == true) {
 		__this_cpu_write(watchdog_nmi_touch, false);
 		return;
@@ -162,6 +156,32 @@ static void watchdog_overflow_callback(struct perf_event *event,
 	__this_cpu_write(hard_watchdog_warn, false);
 	return;
 }
+NOKPROBE_SYMBOL(watchdog_hardlockup_check);
+
+#ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF
+static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
+static DEFINE_PER_CPU(struct perf_event *, dead_event);
+static struct cpumask dead_events_mask;
+static atomic_t watchdog_cpus = ATOMIC_INIT(0);
+
+static struct perf_event_attr wd_hw_attr = {
+	.type		= PERF_TYPE_HARDWARE,
+	.config		= PERF_COUNT_HW_CPU_CYCLES,
+	.size		= sizeof(struct perf_event_attr),
+	.pinned		= 1,
+	.disabled	= 1,
+};
+
+/* Callback function for perf event subsystem */
+static void watchdog_overflow_callback(struct perf_event *event,
+				       struct perf_sample_data *data,
+				       struct pt_regs *regs)
+{
+	/* Ensure the watchdog never gets throttled */
+	event->hw.interrupts = 0;
+
+	watchdog_hardlockup_check(regs);
+}
 
 static int hardlockup_detector_event_create(void)
 {
@@ -294,3 +314,4 @@ int __init hardlockup_detector_perf_init(void)
 	}
 	return ret;
 }
+#endif /* CONFIG_HARDLOCKUP_DETECTOR_PERF */

lib/Kconfig.debug

@@ -853,6 +853,12 @@ config HARDLOCKUP_DETECTOR_PERF
 	bool
 	select SOFTLOCKUP_DETECTOR
 
+config SDEI_WATCHDOG
+	bool "SDEI NMI Watchdog support"
+	depends on ARM_SDE_INTERFACE && !HARDLOCKUP_CHECK_TIMESTAMP
+	select HAVE_HARDLOCKUP_DETECTOR_ARCH
+	select HARDLOCKUP_DETECTOR
+
 #
 # Enables a timestamp based low pass filter to compensate for perf based
 # hard lockup detection which runs too fast due to turbo modes.
@@ -868,7 +874,7 @@ config HARDLOCKUP_DETECTOR
 	bool "Detect Hard Lockups"
 	depends on DEBUG_KERNEL && !S390
 	depends on HAVE_HARDLOCKUP_DETECTOR_PERF || HAVE_HARDLOCKUP_DETECTOR_ARCH
-	select LOCKUP_DETECTOR
+	select SOFTLOCKUP_DETECTOR
 	select HARDLOCKUP_DETECTOR_PERF if HAVE_HARDLOCKUP_DETECTOR_PERF
 	help
 	  Say Y here to enable the kernel to act as a watchdog to detect