hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branches 'for-next/misc', 'for-next/kselftest', 'for-next/xntable', 'for-next/vdso', 'for-next/fiq', 'for-next/epan', 'for-next/kasan-vmalloc', 'for-next/fgt-boot-init', 'for-next/vhe-only' and 'for-next/neon-softirqs-disabled', remote-tracking branch 'arm64/for-next/perf' into for-next/core 

* for-next/misc:
  : Miscellaneous patches
  arm64/sve: Add compile time checks for SVE hooks in generic functions
  arm64/kernel/probes: Use BUG_ON instead of if condition followed by BUG.
  arm64/sve: Remove redundant system_supports_sve() tests
  arm64: mte: Remove unused mte_assign_mem_tag_range()
  arm64: Add __init section marker to some functions
  arm64/sve: Rework SVE access trap to convert state in registers
  docs: arm64: Fix a grammar error
  arm64: smp: Add missing prototype for some smp.c functions
  arm64: setup: name `tcr` register
  arm64: setup: name `mair` register
  arm64: stacktrace: Move start_backtrace() out of the header
  arm64: barrier: Remove spec_bar() macro
  arm64: entry: remove test_irqs_unmasked macro
  ARM64: enable GENERIC_FIND_FIRST_BIT
  arm64: defconfig: Use DEBUG_INFO_REDUCED

* for-next/kselftest:
  : Various kselftests for arm64
  kselftest: arm64: Add BTI tests
  kselftest/arm64: mte: Report filename on failing temp file creation
  kselftest/arm64: mte: Fix clang warning
  kselftest/arm64: mte: Makefile: Fix clang compilation
  kselftest/arm64: mte: Output warning about failing compiler
  kselftest/arm64: mte: Use cross-compiler if specified
  kselftest/arm64: mte: Fix MTE feature detection
  kselftest/arm64: mte: common: Fix write() warnings
  kselftest/arm64: mte: user_mem: Fix write() warning
  kselftest/arm64: mte: ksm_options: Fix fscanf warning
  kselftest/arm64: mte: Fix pthread linking
  kselftest/arm64: mte: Fix compilation with native compiler

* for-next/xntable:
  : Add hierarchical XN permissions for all page tables
  arm64: mm: use XN table mapping attributes for user/kernel mappings
  arm64: mm: use XN table mapping attributes for the linear region
  arm64: mm: add missing P4D definitions and use them consistently

* for-next/vdso:
  : Minor improvements to the compat vdso and sigpage
  arm64: compat: Poison the compat sigpage
  arm64: vdso: Avoid ISB after reading from cntvct_el0
  arm64: compat: Allow signal page to be remapped
  arm64: vdso: Remove redundant calls to flush_dcache_page()
  arm64: vdso: Use GFP_KERNEL for allocating compat vdso and signal pages

* for-next/fiq:
  : Support arm64 FIQ controller registration
  arm64: irq: allow FIQs to be handled
  arm64: Always keep DAIF.[IF] in sync
  arm64: entry: factor irq triage logic into macros
  arm64: irq: rework root IRQ handler registration
  arm64: don't use GENERIC_IRQ_MULTI_HANDLER
  genirq: Allow architectures to override set_handle_irq() fallback

* for-next/epan:
  : Support for Enhanced PAN (execute-only permissions)
  arm64: Support execute-only permissions with Enhanced PAN

* for-next/kasan-vmalloc:
  : Support CONFIG_KASAN_VMALLOC on arm64
  arm64: Kconfig: select KASAN_VMALLOC if KANSAN_GENERIC is enabled
  arm64: kaslr: support randomized module area with KASAN_VMALLOC
  arm64: Kconfig: support CONFIG_KASAN_VMALLOC
  arm64: kasan: abstract _text and _end to KERNEL_START/END
  arm64: kasan: don't populate vmalloc area for CONFIG_KASAN_VMALLOC

* for-next/fgt-boot-init:
  : Booting clarifications and fine grained traps setup
  arm64: Require that system registers at all visible ELs be initialized
  arm64: Disable fine grained traps on boot
  arm64: Document requirements for fine grained traps at boot

* for-next/vhe-only:
  : Dealing with VHE-only CPUs (a.k.a. M1)
  arm64: Get rid of CONFIG_ARM64_VHE
  arm64: Cope with CPUs stuck in VHE mode
  arm64: cpufeature: Allow early filtering of feature override

* arm64/for-next/perf:
  arm64: perf: Remove redundant initialization in perf_event.c
  perf/arm_pmu_platform: Clean up with dev_printk
  perf/arm_pmu_platform: Fix error handling
  perf/arm_pmu_platform: Use dev_err_probe() for IRQ errors
  docs: perf: Address some html build warnings
  docs: perf: Add new description on HiSilicon uncore PMU v2
  drivers/perf: hisi: Add support for HiSilicon PA PMU driver
  drivers/perf: hisi: Add support for HiSilicon SLLC PMU driver
  drivers/perf: hisi: Update DDRC PMU for programmable counter
  drivers/perf: hisi: Add new functions for HHA PMU
  drivers/perf: hisi: Add new functions for L3C PMU
  drivers/perf: hisi: Add PMU version for uncore PMU drivers.
  drivers/perf: hisi: Refactor code for more uncore PMUs
  drivers/perf: hisi: Remove unnecessary check of counter index
  drivers/perf: Simplify the SMMUv3 PMU event attributes
  drivers/perf: convert sysfs sprintf family to sysfs_emit
  drivers/perf: convert sysfs scnprintf family to sysfs_emit_at() and sysfs_emit()
  drivers/perf: convert sysfs snprintf family to sysfs_emit

* for-next/neon-softirqs-disabled:
  : Run kernel mode SIMD with softirqs disabled
  arm64: fpsimd: run kernel mode NEON with softirqs disabled
  arm64: assembler: introduce wxN aliases for wN registers
  arm64: assembler: remove conditional NEON yield macros
This commit is contained in:
Catalin Marinas 2021-04-15 14:00:38 +01:00
parent 087dfa5ca7 314bcbf09f 59511cfd08 6e554abd07 3889ba7010 18107f8a2d acc3042d62 230800cd31 2d726d0db6 2c2e21e78a 13150149aa
commit a1e1eddef2
86 changed files with 3146 additions and 732 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Documentation/admin-guide/kernel-parameters.txt
View File

@ -2279,8 +2279,7 @@
state is kept private from the host.
Not valid if the kernel is running in EL2.
Defaults to VHE/nVHE based on hardware support and
the value of CONFIG_ARM64_VHE.
Defaults to VHE/nVHE based on hardware support.
kvm-arm.vgic_v3_group0_trap=
[KVM,ARM] Trap guest accesses to GICv3 group-0

54
Documentation/admin-guide/perf/hisi-pmu.rst
View File

@ -53,6 +53,60 @@ Example usage of perf::
$# perf stat -a -e hisi_sccl3_l3c0/rd_hit_cpipe/ sleep 5
$# perf stat -a -e hisi_sccl3_l3c0/config=0x02/ sleep 5
For HiSilicon uncore PMU v2 whose identifier is 0x30, the topology is the same
as PMU v1, but some new functions are added to the hardware.
(a) L3C PMU supports filtering by core/thread within the cluster which can be
specified as a bitmap::
$# perf stat -a -e hisi_sccl3_l3c0/config=0x02,tt_core=0x3/ sleep 5
This will only count the operations from core/thread 0 and 1 in this cluster.
(b) Tracetag allow the user to chose to count only read, write or atomic
operations via the tt_req parameeter in perf. The default value counts all
operations. tt_req is 3bits, 3'b100 represents read operations, 3'b101
represents write operations, 3'b110 represents atomic store operations and
3'b111 represents atomic non-store operations, other values are reserved::
$# perf stat -a -e hisi_sccl3_l3c0/config=0x02,tt_req=0x4/ sleep 5
This will only count the read operations in this cluster.
(c) Datasrc allows the user to check where the data comes from. It is 5 bits.
Some important codes are as follows:
5'b00001: comes from L3C in this die;
5'b01000: comes from L3C in the cross-die;
5'b01001: comes from L3C which is in another socket;
5'b01110: comes from the local DDR;
5'b01111: comes from the cross-die DDR;
5'b10000: comes from cross-socket DDR;
etc, it is mainly helpful to find that the data source is nearest from the CPU
cores. If datasrc_cfg is used in the multi-chips, the datasrc_skt shall be
configured in perf command::
$# perf stat -a -e hisi_sccl3_l3c0/config=0xb9,datasrc_cfg=0xE/,
hisi_sccl3_l3c0/config=0xb9,datasrc_cfg=0xF/ sleep 5
(d)Some HiSilicon SoCs encapsulate multiple CPU and IO dies. Each CPU die
contains several Compute Clusters (CCLs). The I/O dies are called Super I/O
clusters (SICL) containing multiple I/O clusters (ICLs). Each CCL/ICL in the
SoC has a unique ID. Each ID is 11bits, include a 6-bit SCCL-ID and 5-bit
CCL/ICL-ID. For I/O die, the ICL-ID is followed by:
5'b00000: I/O_MGMT_ICL;
5'b00001: Network_ICL;
5'b00011: HAC_ICL;
5'b10000: PCIe_ICL;
Users could configure IDs to count data come from specific CCL/ICL, by setting
srcid_cmd & srcid_msk, and data desitined for specific CCL/ICL by setting
tgtid_cmd & tgtid_msk. A set bit in srcid_msk/tgtid_msk means the PMU will not
check the bit when matching against the srcid_cmd/tgtid_cmd.
If all of these options are disabled, it can works by the default value that
doesn't distinguish the filter condition and ID information and will return
the total counter values in the PMU counters.
The current driver does not support sampling. So "perf record" is unsupported.
Also attach to a task is unsupported as the events are all uncore.

13
Documentation/arm64/booting.rst
View File

@ -202,9 +202,10 @@ Before jumping into the kernel, the following conditions must be met:
- System registers
All writable architected system registers at the exception level where
the kernel image will be entered must be initialised by software at a
higher exception level to prevent execution in an UNKNOWN state.
All writable architected system registers at or below the exception
level where the kernel image will be entered must be initialised by
software at a higher exception level to prevent execution in an UNKNOWN
state.
- SCR_EL3.FIQ must have the same value across all CPUs the kernel is
executing on.
@ -270,6 +271,12 @@ Before jumping into the kernel, the following conditions must be met:
having 0b1 set for the corresponding bit for each of the auxiliary
counters present.
For CPUs with the Fine Grained Traps (FEAT_FGT) extension present:
- If EL3 is present and the kernel is entered at EL2:
- SCR_EL3.FGTEn (bit 27) must be initialised to 0b1.
The requirements described above for CPU mode, caches, MMUs, architected
timers, coherency and system registers apply to all CPUs. All CPUs must
enter the kernel in the same exception level.

40
arch/arm64/Kconfig
View File

@ -111,7 +111,6 @@ config ARM64
select GENERIC_FIND_FIRST_BIT
select GENERIC_IDLE_POLL_SETUP
select GENERIC_IRQ_IPI
select GENERIC_IRQ_MULTI_HANDLER
select GENERIC_IRQ_PROBE
select GENERIC_IRQ_SHOW
select GENERIC_IRQ_SHOW_LEVEL
@ -139,6 +138,7 @@ config ARM64
select HAVE_ARCH_JUMP_LABEL
select HAVE_ARCH_JUMP_LABEL_RELATIVE
select HAVE_ARCH_KASAN if !(ARM64_16K_PAGES && ARM64_VA_BITS_48)
select HAVE_ARCH_KASAN_VMALLOC if HAVE_ARCH_KASAN
select HAVE_ARCH_KASAN_SW_TAGS if HAVE_ARCH_KASAN
select HAVE_ARCH_KASAN_HW_TAGS if (HAVE_ARCH_KASAN && ARM64_MTE)
select HAVE_ARCH_KFENCE
@ -195,6 +195,7 @@ config ARM64
select IOMMU_DMA if IOMMU_SUPPORT
select IRQ_DOMAIN
select IRQ_FORCED_THREADING
select KASAN_VMALLOC if KASAN_GENERIC
select MODULES_USE_ELF_RELA
select NEED_DMA_MAP_STATE
select NEED_SG_DMA_LENGTH
@ -1059,6 +1060,9 @@ config SYS_SUPPORTS_HUGETLBFS
config ARCH_HAS_CACHE_LINE_SIZE
def_bool y
config ARCH_HAS_FILTER_PGPROT
def_bool y
config ARCH_ENABLE_SPLIT_PMD_PTLOCK
def_bool y if PGTABLE_LEVELS > 2
@ -1417,19 +1421,6 @@ config ARM64_USE_LSE_ATOMICS
built with binutils >= 2.25 in order for the new instructions
to be used.
config ARM64_VHE
bool "Enable support for Virtualization Host Extensions (VHE)"
default y
help
Virtualization Host Extensions (VHE) allow the kernel to run
directly at EL2 (instead of EL1) on processors that support
it. This leads to better performance for KVM, as they reduce
the cost of the world switch.
Selecting this option allows the VHE feature to be detected
at runtime, and does not affect processors that do not
implement this feature.
endmenu
menu "ARMv8.2 architectural features"
@ -1682,10 +1673,23 @@ config ARM64_MTE
endmenu
menu "ARMv8.7 architectural features"
config ARM64_EPAN
bool "Enable support for Enhanced Privileged Access Never (EPAN)"
default y
depends on ARM64_PAN
help
Enhanced Privileged Access Never (EPAN) allows Privileged
Access Never to be used with Execute-only mappings.
The feature is detected at runtime, and will remain disabled
if the cpu does not implement the feature.
endmenu
config ARM64_SVE
bool "ARM Scalable Vector Extension support"
default y
depends on !KVM || ARM64_VHE
help
The Scalable Vector Extension (SVE) is an extension to the AArch64
execution state which complements and extends the SIMD functionality
@ -1714,12 +1718,6 @@ config ARM64_SVE
booting the kernel. If unsure and you are not observing these
symptoms, you should assume that it is safe to say Y.
CPUs that support SVE are architecturally required to support the
Virtualization Host Extensions (VHE), so the kernel makes no
provision for supporting SVE alongside KVM without VHE enabled.
Thus, you will need to enable CONFIG_ARM64_VHE if you want to support
KVM in the same kernel image.
config ARM64_MODULE_PLTS
bool "Use PLTs to allow module memory to spill over into vmalloc area"
depends on MODULES

2
arch/arm64/crypto/aes-modes.S
View File

@ -700,7 +700,7 @@ AES_FUNC_START(aes_mac_update)
cbz w5, .Lmacout
encrypt_block v0, w2, x1, x7, w8
st1 {v0.16b}, [x4] /* return dg */
cond_yield .Lmacout, x7
cond_yield .Lmacout, x7, x8
b .Lmacloop4x
.Lmac1x:
add w3, w3, #4

2
arch/arm64/crypto/sha1-ce-core.S
View File

@ -121,7 +121,7 @@ CPU_LE( rev32 v11.16b, v11.16b )
add dgav.4s, dgav.4s, dg0v.4s
cbz w2, 2f
cond_yield 3f, x5
cond_yield 3f, x5, x6
b 0b
/*

2
arch/arm64/crypto/sha2-ce-core.S
View File

@ -129,7 +129,7 @@ CPU_LE( rev32 v19.16b, v19.16b )
/* handled all input blocks? */
cbz w2, 2f
cond_yield 3f, x5
cond_yield 3f, x5, x6
b 0b
/*

4
arch/arm64/crypto/sha3-ce-core.S
View File

@ -184,11 +184,11 @@ SYM_FUNC_START(sha3_ce_transform)
eor v0.16b, v0.16b, v31.16b
cbnz w8, 3b
cond_yield 3f, x8
cond_yield 4f, x8, x9
cbnz w2, 0b
/* save state */
3: st1 { v0.1d- v3.1d}, [x0], #32
4: st1 { v0.1d- v3.1d}, [x0], #32
st1 { v4.1d- v7.1d}, [x0], #32
st1 { v8.1d-v11.1d}, [x0], #32
st1 {v12.1d-v15.1d}, [x0], #32

2
arch/arm64/crypto/sha512-ce-core.S
View File

@ -195,7 +195,7 @@ CPU_LE( rev64 v19.16b, v19.16b )
add v10.2d, v10.2d, v2.2d
add v11.2d, v11.2d, v3.2d
cond_yield 3f, x4
cond_yield 3f, x4, x5
/* handled all input blocks? */
cbnz w2, 0b

2
arch/arm64/include/asm/arch_gicv3.h
View File

@ -173,7 +173,7 @@ static inline void gic_pmr_mask_irqs(void)
static inline void gic_arch_enable_irqs(void)
{
asm volatile ("msr daifclr, #2" : : : "memory");
asm volatile ("msr daifclr, #3" : : : "memory");
}
#endif /* __ASSEMBLY__ */

21
arch/arm64/include/asm/arch_timer.h
View File

@ -165,25 +165,6 @@ static inline void arch_timer_set_cntkctl(u32 cntkctl)
isb();
}
/*
* Ensure that reads of the counter are treated the same as memory reads
* for the purposes of ordering by subsequent memory barriers.
*
* This insanity brought to you by speculative system register reads,
* out-of-order memory accesses, sequence locks and Thomas Gleixner.
*
* http://lists.infradead.org/pipermail/linux-arm-kernel/2019-February/631195.html
*/
#define arch_counter_enforce_ordering(val) do { \
u64 tmp, _val = (val); \
\
asm volatile( \
" eor %0, %1, %1\n" \
" add %0, sp, %0\n" \
" ldr xzr, [%0]" \
: "=r" (tmp) : "r" (_val)); \
} while (0)
static __always_inline u64 __arch_counter_get_cntpct_stable(void)
{
u64 cnt;
@ -224,8 +205,6 @@ static __always_inline u64 __arch_counter_get_cntvct(void)
return cnt;
}
#undef arch_counter_enforce_ordering
static inline int arch_timer_arch_init(void)
{
return 0;

114
arch/arm64/include/asm/assembler.h
View File

@ -15,6 +15,7 @@
#include <asm-generic/export.h>
#include <asm/asm-offsets.h>
#include <asm/alternative.h>
#include <asm/cpufeature.h>
#include <asm/cputype.h>
#include <asm/debug-monitors.h>
@ -23,6 +24,14 @@
#include <asm/ptrace.h>
#include <asm/thread_info.h>
/*
* Provide a wxN alias for each wN register so what we can paste a xN
* reference after a 'w' to obtain the 32-bit version.
*/
.irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
wx\n .req w\n
.endr
.macro save_and_disable_daif, flags
mrs \flags, daif
msr daifset, #0xf
@ -40,9 +49,9 @@
msr daif, \flags
.endm
/* IRQ is the lowest priority flag, unconditionally unmask the rest. */
.macro enable_da_f
msr daifclr, #(8 | 4 | 1)
/* IRQ/FIQ are the lowest priority flags, unconditionally unmask the rest. */
.macro enable_da
msr daifclr, #(8 | 4)
.endm
/*
@ -50,7 +59,7 @@
*/
.macro save_and_disable_irq, flags
mrs \flags, daif
msr daifset, #2
msr daifset, #3
.endm
.macro restore_irq, flags
@ -692,90 +701,33 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU
isb
.endm
/*
* Check whether to yield to another runnable task from kernel mode NEON code
* (which runs with preemption disabled).
*
* if_will_cond_yield_neon
* // pre-yield patchup code
* do_cond_yield_neon
* // post-yield patchup code
* endif_yield_neon <label>
*
* where <label> is optional, and marks the point where execution will resume
* after a yield has been performed. If omitted, execution resumes right after
* the endif_yield_neon invocation. Note that the entire sequence, including
* the provided patchup code, will be omitted from the image if
* CONFIG_PREEMPTION is not defined.
*
* As a convenience, in the case where no patchup code is required, the above
* sequence may be abbreviated to
*
* cond_yield_neon <label>
*
* Note that the patchup code does not support assembler directives that change
* the output section, any use of such directives is undefined.
*
* The yield itself consists of the following:
* - Check whether the preempt count is exactly 1 and a reschedule is also
* needed. If so, calling of preempt_enable() in kernel_neon_end() will
* trigger a reschedule. If it is not the case, yielding is pointless.
* - Disable and re-enable kernel mode NEON, and branch to the yield fixup
* code.
*
* This macro sequence may clobber all CPU state that is not guaranteed by the
* AAPCS to be preserved across an ordinary function call.
*/
.macro cond_yield_neon, lbl
if_will_cond_yield_neon
do_cond_yield_neon
endif_yield_neon \lbl
.endm
.macro if_will_cond_yield_neon
#ifdef CONFIG_PREEMPTION
get_current_task x0
ldr x0, [x0, #TSK_TI_PREEMPT]
sub x0, x0, #PREEMPT_DISABLE_OFFSET
cbz x0, .Lyield_\@
/* fall through to endif_yield_neon */
.subsection 1
.Lyield_\@ :
#else
.section ".discard.cond_yield_neon", "ax"
#endif
.endm
.macro do_cond_yield_neon
bl kernel_neon_end
bl kernel_neon_begin
.endm
.macro endif_yield_neon, lbl
.ifnb \lbl
b \lbl
.else
b .Lyield_out_\@
.endif
.previous
.Lyield_out_\@ :
.endm
/*
* Check whether preempt-disabled code should yield as soon as it
* is able. This is the case if re-enabling preemption a single
* time results in a preempt count of zero, and the TIF_NEED_RESCHED
* flag is set. (Note that the latter is stored negated in the
* top word of the thread_info::preempt_count field)
* Check whether preempt/bh-disabled asm code should yield as soon as
* it is able. This is the case if we are currently running in task
* context, and either a softirq is pending, or the TIF_NEED_RESCHED
* flag is set and re-enabling preemption a single time would result in
* a preempt count of zero. (Note that the TIF_NEED_RESCHED flag is
* stored negated in the top word of the thread_info::preempt_count
* field)
*/
.macro cond_yield, lbl:req, tmp:req
#ifdef CONFIG_PREEMPTION
.macro cond_yield, lbl:req, tmp:req, tmp2:req
get_current_task \tmp
ldr \tmp, [\tmp, #TSK_TI_PREEMPT]
/*
* If we are serving a softirq, there is no point in yielding: the
* softirq will not be preempted no matter what we do, so we should
* run to completion as quickly as we can.
*/
tbnz \tmp, #SOFTIRQ_SHIFT, .Lnoyield_\@
#ifdef CONFIG_PREEMPTION
sub \tmp, \tmp, #PREEMPT_DISABLE_OFFSET
cbz \tmp, \lbl
#endif
adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING
this_cpu_offset \tmp2
ldr w\tmp, [\tmp, \tmp2]
cbnz w\tmp, \lbl // yield on pending softirq in task context
.Lnoyield_\@:
.endm
/*

19
arch/arm64/include/asm/barrier.h
View File

@ -66,6 +66,25 @@ static inline unsigned long array_index_mask_nospec(unsigned long idx,
return mask;
}
/*
* Ensure that reads of the counter are treated the same as memory reads
* for the purposes of ordering by subsequent memory barriers.
*
* This insanity brought to you by speculative system register reads,
* out-of-order memory accesses, sequence locks and Thomas Gleixner.
*
* http://lists.infradead.org/pipermail/linux-arm-kernel/2019-February/631195.html
*/
#define arch_counter_enforce_ordering(val) do { \
u64 tmp, _val = (val); \
\
asm volatile( \
" eor %0, %1, %1\n" \
" add %0, sp, %0\n" \
" ldr xzr, [%0]" \
: "=r" (tmp) : "r" (_val)); \
} while (0)
#define __smp_mb() dmb(ish)
#define __smp_rmb() dmb(ishld)
#define __smp_wmb() dmb(ishst)

3
arch/arm64/include/asm/cpucaps.h
View File

@ -66,7 +66,8 @@
#define ARM64_WORKAROUND_1508412 58
#define ARM64_HAS_LDAPR 59
#define ARM64_KVM_PROTECTED_MODE 60
#define ARM64_HAS_EPAN 61
#define ARM64_NCAPS 61
#define ARM64_NCAPS 62
#endif /* __ASM_CPUCAPS_H */

17
arch/arm64/include/asm/cpufeature.h
View File

@ -63,6 +63,23 @@ struct arm64_ftr_bits {
s64 safe_val; /* safe value for FTR_EXACT features */
};
/*
* Describe the early feature override to the core override code:
*
* @val Values that are to be merged into the final
* sanitised value of the register. Only the bitfields
* set to 1 in @mask are valid
* @mask Mask of the features that are overridden by @val
*
* A @mask field set to full-1 indicates that the corresponding field
* in @val is a valid override.
*
* A @mask field set to full-0 with the corresponding @val field set
* to full-0 denotes that this field has no override
*
* A @mask field set to full-0 with the corresponding @val field set
* to full-1 denotes thath this field has an invalid override.
*/
struct arm64_ftr_override {
u64 val;
u64 mask;

10
arch/arm64/include/asm/daifflags.h
View File

@ -13,8 +13,8 @@
#include <asm/ptrace.h>
#define DAIF_PROCCTX 0
#define DAIF_PROCCTX_NOIRQ PSR_I_BIT
#define DAIF_ERRCTX (PSR_I_BIT | PSR_A_BIT)
#define DAIF_PROCCTX_NOIRQ (PSR_I_BIT | PSR_F_BIT)
#define DAIF_ERRCTX (PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
#define DAIF_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
@ -47,7 +47,7 @@ static inline unsigned long local_daif_save_flags(void)
if (system_uses_irq_prio_masking()) {
/* If IRQs are masked with PMR, reflect it in the flags */
if (read_sysreg_s(SYS_ICC_PMR_EL1) != GIC_PRIO_IRQON)
flags |= PSR_I_BIT;
flags |= PSR_I_BIT | PSR_F_BIT;
}
return flags;
@ -69,7 +69,7 @@ static inline void local_daif_restore(unsigned long flags)
bool irq_disabled = flags & PSR_I_BIT;
WARN_ON(system_has_prio_mask_debugging() &&
!(read_sysreg(daif) & PSR_I_BIT));
(read_sysreg(daif) & (PSR_I_BIT | PSR_F_BIT)) != (PSR_I_BIT | PSR_F_BIT));
if (!irq_disabled) {
trace_hardirqs_on();
@ -86,7 +86,7 @@ static inline void local_daif_restore(unsigned long flags)
* If interrupts are disabled but we can take
* asynchronous errors, we can take NMIs
*/
flags &= ~PSR_I_BIT;
flags &= ~(PSR_I_BIT | PSR_F_BIT);
pmr = GIC_PRIO_IRQOFF;
} else {
pmr = GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET;

21
arch/arm64/include/asm/el2_setup.h
View File

@ -131,6 +131,26 @@
.Lskip_sve_\@:
.endm
/* Disable any fine grained traps */
.macro __init_el2_fgt
mrs x1, id_aa64mmfr0_el1
ubfx x1, x1, #ID_AA64MMFR0_FGT_SHIFT, #4
cbz x1, .Lskip_fgt_\@
msr_s SYS_HDFGRTR_EL2, xzr
msr_s SYS_HDFGWTR_EL2, xzr
msr_s SYS_HFGRTR_EL2, xzr
msr_s SYS_HFGWTR_EL2, xzr
msr_s SYS_HFGITR_EL2, xzr
mrs x1, id_aa64pfr0_el1 // AMU traps UNDEF without AMU
ubfx x1, x1, #ID_AA64PFR0_AMU_SHIFT, #4
cbz x1, .Lskip_fgt_\@
msr_s SYS_HAFGRTR_EL2, xzr
.Lskip_fgt_\@:
.endm
.macro __init_el2_nvhe_prepare_eret
mov x0, #INIT_PSTATE_EL1
msr spsr_el2, x0
@ -155,6 +175,7 @@
__init_el2_nvhe_idregs
__init_el2_nvhe_cptr
__init_el2_nvhe_sve
__init_el2_fgt
__init_el2_nvhe_prepare_eret
.endm

4
arch/arm64/include/asm/irq.h
View File

@ -8,6 +8,10 @@
struct pt_regs;
int set_handle_irq(void (*handle_irq)(struct pt_regs *));
#define set_handle_irq set_handle_irq
int set_handle_fiq(void (*handle_fiq)(struct pt_regs *));
static inline int nr_legacy_irqs(void)
{
return 0;

16
arch/arm64/include/asm/irqflags.h
View File

@ -12,15 +12,13 @@
/*
* Aarch64 has flags for masking: Debug, Asynchronous (serror), Interrupts and
* FIQ exceptions, in the 'daif' register. We mask and unmask them in 'dai'
* FIQ exceptions, in the 'daif' register. We mask and unmask them in 'daif'
* order:
* Masking debug exceptions causes all other exceptions to be masked too/
* Masking SError masks irq, but not debug exceptions. Masking irqs has no
* side effects for other flags. Keeping to this order makes it easier for
* entry.S to know which exceptions should be unmasked.
*
* FIQ is never expected, but we mask it when we disable debug exceptions, and
* unmask it at all other times.
* Masking SError masks IRQ/FIQ, but not debug exceptions. IRQ and FIQ are
* always masked and unmasked together, and have no side effects for other
* flags. Keeping to this order makes it easier for entry.S to know which
* exceptions should be unmasked.
*/
/*
@ -35,7 +33,7 @@ static inline void arch_local_irq_enable(void)
}
asm volatile(ALTERNATIVE(
"msr daifclr, #2 // arch_local_irq_enable",
"msr daifclr, #3 // arch_local_irq_enable",
__msr_s(SYS_ICC_PMR_EL1, "%0"),
ARM64_HAS_IRQ_PRIO_MASKING)
:
@ -54,7 +52,7 @@ static inline void arch_local_irq_disable(void)
}
asm volatile(ALTERNATIVE(
"msr daifset, #2 // arch_local_irq_disable",
"msr daifset, #3 // arch_local_irq_disable",
__msr_s(SYS_ICC_PMR_EL1, "%0"),
ARM64_HAS_IRQ_PRIO_MASKING)
:

19
arch/arm64/include/asm/pgalloc.h
View File

@ -27,7 +27,10 @@ static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)
{
__pud_populate(pudp, __pa(pmdp), PMD_TYPE_TABLE);
pudval_t pudval = PUD_TYPE_TABLE;
pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN;
__pud_populate(pudp, __pa(pmdp), pudval);
}
#else
static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
@ -45,7 +48,10 @@ static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp)
{
__p4d_populate(p4dp, __pa(pudp), PUD_TYPE_TABLE);
p4dval_t p4dval = P4D_TYPE_TABLE;
p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN;
__p4d_populate(p4dp, __pa(pudp), p4dval);
}
#else
static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
@ -70,16 +76,15 @@ static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep,
static inline void
pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
{
/*
* The pmd must be loaded with the physical address of the PTE table
*/
__pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE);
VM_BUG_ON(mm != &init_mm);
__pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE | PMD_TABLE_UXN);
}
static inline void
pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
{
__pmd_populate(pmdp, page_to_phys(ptep), PMD_TYPE_TABLE);
VM_BUG_ON(mm == &init_mm);
__pmd_populate(pmdp, page_to_phys(ptep), PMD_TYPE_TABLE | PMD_TABLE_PXN);
}
#define pmd_pgtable(pmd) pmd_page(pmd)

15
arch/arm64/include/asm/pgtable-hwdef.h
View File

@ -94,6 +94,17 @@
/*
* Hardware page table definitions.
*
* Level 0 descriptor (P4D).
*/
#define P4D_TYPE_TABLE (_AT(p4dval_t, 3) << 0)
#define P4D_TABLE_BIT (_AT(p4dval_t, 1) << 1)
#define P4D_TYPE_MASK (_AT(p4dval_t, 3) << 0)
#define P4D_TYPE_SECT (_AT(p4dval_t, 1) << 0)
#define P4D_SECT_RDONLY (_AT(p4dval_t, 1) << 7) /* AP[2] */
#define P4D_TABLE_PXN (_AT(p4dval_t, 1) << 59)
#define P4D_TABLE_UXN (_AT(p4dval_t, 1) << 60)
/*
* Level 1 descriptor (PUD).
*/
#define PUD_TYPE_TABLE (_AT(pudval_t, 3) << 0)
@ -101,6 +112,8 @@
#define PUD_TYPE_MASK (_AT(pudval_t, 3) << 0)
#define PUD_TYPE_SECT (_AT(pudval_t, 1) << 0)
#define PUD_SECT_RDONLY (_AT(pudval_t, 1) << 7) /* AP[2] */
#define PUD_TABLE_PXN (_AT(pudval_t, 1) << 59)
#define PUD_TABLE_UXN (_AT(pudval_t, 1) << 60)
/*
* Level 2 descriptor (PMD).
@ -122,6 +135,8 @@
#define PMD_SECT_CONT (_AT(pmdval_t, 1) << 52)
#define PMD_SECT_PXN (_AT(pmdval_t, 1) << 53)
#define PMD_SECT_UXN (_AT(pmdval_t, 1) << 54)
#define PMD_TABLE_PXN (_AT(pmdval_t, 1) << 59)
#define PMD_TABLE_UXN (_AT(pmdval_t, 1) << 60)
/*
* AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).

5
arch/arm64/include/asm/pgtable-prot.h
View File

@ -87,12 +87,13 @@ extern bool arm64_use_ng_mappings;
#define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_WRITE)
#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)
#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN)
#define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_RDONLY | PTE_NG | PTE_PXN)
#define __P000 PAGE_NONE
#define __P001 PAGE_READONLY
#define __P010 PAGE_READONLY
#define __P011 PAGE_READONLY
#define __P100 PAGE_READONLY_EXEC
#define __P100 PAGE_EXECONLY
#define __P101 PAGE_READONLY_EXEC
#define __P110 PAGE_READONLY_EXEC
#define __P111 PAGE_READONLY_EXEC
@ -101,7 +102,7 @@ extern bool arm64_use_ng_mappings;
#define __S001 PAGE_READONLY
#define __S010 PAGE_SHARED
#define __S011 PAGE_SHARED
#define __S100 PAGE_READONLY_EXEC
#define __S100 PAGE_EXECONLY
#define __S101 PAGE_READONLY_EXEC
#define __S110 PAGE_SHARED_EXEC
#define __S111 PAGE_SHARED_EXEC

31
arch/arm64/include/asm/pgtable.h
View File

@ -113,11 +113,12 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte))
#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID))
/*
* Execute-only user mappings do not have the PTE_USER bit set. All valid
* kernel mappings have the PTE_UXN bit set.
*/
#define pte_valid_not_user(pte) \
((pte_val(pte) & (PTE_VALID | PTE_USER)) == PTE_VALID)
#define pte_valid_user(pte) \
((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER))
((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))
/*
* Could the pte be present in the TLB? We must check mm_tlb_flush_pending
* so that we don't erroneously return false for pages that have been
@ -130,12 +131,14 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))
/*
* p??_access_permitted() is true for valid user mappings (subject to the
* write permission check). PROT_NONE mappings do not have the PTE_VALID bit
* set.
* p??_access_permitted() is true for valid user mappings (PTE_USER
* bit set, subject to the write permission check). For execute-only
* mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits
* not set) must return false. PROT_NONE mappings do not have the
* PTE_VALID bit set.
*/
#define pte_access_permitted(pte, write) \
(pte_valid_user(pte) && (!(write) || pte_write(pte)))
(((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte)))
#define pmd_access_permitted(pmd, write) \
(pte_access_permitted(pmd_pte(pmd), (write)))
#define pud_access_permitted(pud, write) \
@ -995,6 +998,18 @@ static inline bool arch_wants_old_prefaulted_pte(void)
}
#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte
static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
{
if (cpus_have_const_cap(ARM64_HAS_EPAN))
return prot;
if (pgprot_val(prot) != pgprot_val(PAGE_EXECONLY))
return prot;
return PAGE_READONLY_EXEC;
}
#endif /* !__ASSEMBLY__ */
#endif /* __ASM_PGTABLE_H */

9
arch/arm64/include/asm/sysreg.h
View File

@ -475,9 +475,15 @@
#define SYS_PMCCFILTR_EL0 sys_reg(3, 3, 14, 15, 7)
#define SYS_SCTLR_EL2 sys_reg(3, 4, 1, 0, 0)
#define SYS_HFGRTR_EL2 sys_reg(3, 4, 1, 1, 4)
#define SYS_HFGWTR_EL2 sys_reg(3, 4, 1, 1, 5)
#define SYS_HFGITR_EL2 sys_reg(3, 4, 1, 1, 6)
#define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0)
#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1)
#define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0)
#define SYS_HDFGRTR_EL2 sys_reg(3, 4, 3, 1, 4)
#define SYS_HDFGWTR_EL2 sys_reg(3, 4, 3, 1, 5)
#define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6)
#define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0)
#define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1)
#define SYS_IFSR32_EL2 sys_reg(3, 4, 5, 0, 1)
@ -597,6 +603,7 @@
(SCTLR_EL2_RES1 | ENDIAN_SET_EL2)
/* SCTLR_EL1 specific flags. */
#define SCTLR_EL1_EPAN (BIT(57))
#define SCTLR_EL1_ATA0 (BIT(42))
#define SCTLR_EL1_TCF0_SHIFT 38
@ -637,7 +644,7 @@
SCTLR_EL1_SED | SCTLR_ELx_I | SCTLR_EL1_DZE | SCTLR_EL1_UCT | \
SCTLR_EL1_NTWE | SCTLR_ELx_IESB | SCTLR_EL1_SPAN | SCTLR_ELx_ITFSB | \
SCTLR_ELx_ATA | SCTLR_EL1_ATA0 | ENDIAN_SET_EL1 | SCTLR_EL1_UCI | \
SCTLR_EL1_RES1)
SCTLR_EL1_EPAN | SCTLR_EL1_RES1)
/* MAIR_ELx memory attributes (used by Linux) */
#define MAIR_ATTR_DEVICE_nGnRnE UL(0x00)

6
arch/arm64/include/asm/vdso/gettimeofday.h
View File

@ -83,11 +83,7 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
*/
isb();
asm volatile("mrs %0, cntvct_el0" : "=r" (res) :: "memory");
/*
* This isb() is required to prevent that the seq lock is
* speculated.#
*/
isb();
arch_counter_enforce_ordering(res);
return res;
}

2
arch/arm64/kernel/asm-offsets.c
View File

@ -95,6 +95,8 @@ int main(void)
DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE);
BLANK();
DEFINE(PREEMPT_DISABLE_OFFSET, PREEMPT_DISABLE_OFFSET);
DEFINE(SOFTIRQ_SHIFT, SOFTIRQ_SHIFT);
DEFINE(IRQ_CPUSTAT_SOFTIRQ_PENDING, offsetof(irq_cpustat_t, __softirq_pending));
BLANK();
DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack));
DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task));

22
arch/arm64/kernel/cpufeature.c
View File

@ -809,6 +809,12 @@ static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
reg->name,
ftrp->shift + ftrp->width - 1,
ftrp->shift, str, tmp);
} else if ((ftr_mask & reg->override->val) == ftr_mask) {
reg->override->val &= ~ftr_mask;
pr_warn("%s[%d:%d]: impossible override, ignored\n",
reg->name,
ftrp->shift + ftrp->width - 1,
ftrp->shift);
}
val = arm64_ftr_set_value(ftrp, val, ftr_new);
@ -1617,7 +1623,6 @@ int get_cpu_with_amu_feat(void)
}
#endif
#ifdef CONFIG_ARM64_VHE
static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
{
return is_kernel_in_hyp_mode();
@ -1636,7 +1641,6 @@ static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
if (!alternative_is_applied(ARM64_HAS_VIRT_HOST_EXTN))
write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
}
#endif
static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused)
{
@ -1821,6 +1825,18 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.cpu_enable = cpu_enable_pan,
},
#endif /* CONFIG_ARM64_PAN */
#ifdef CONFIG_ARM64_EPAN
{
.desc = "Enhanced Privileged Access Never",
.capability = ARM64_HAS_EPAN,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
.sys_reg = SYS_ID_AA64MMFR1_EL1,
.field_pos = ID_AA64MMFR1_PAN_SHIFT,
.sign = FTR_UNSIGNED,
.min_field_value = 3,
},
#endif /* CONFIG_ARM64_EPAN */
#ifdef CONFIG_ARM64_LSE_ATOMICS
{
.desc = "LSE atomic instructions",
@ -1839,7 +1855,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
.matches = has_no_hw_prefetch,
},
#ifdef CONFIG_ARM64_VHE
{
.desc = "Virtualization Host Extensions",
.capability = ARM64_HAS_VIRT_HOST_EXTN,
@ -1847,7 +1862,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.matches = runs_at_el2,
.cpu_enable = cpu_copy_el2regs,
},
#endif /* CONFIG_ARM64_VHE */
{
.desc = "32-bit EL0 Support",
.capability = ARM64_HAS_32BIT_EL0,

114
arch/arm64/kernel/entry.S
View File

@ -491,8 +491,8 @@ tsk .req x28 // current thread_info
/*
* Interrupt handling.
*/
.macro irq_handler
ldr_l x1, handle_arch_irq
.macro irq_handler, handler:req
ldr_l x1, \handler
mov x0, sp
irq_stack_entry
blr x1
@ -517,6 +517,47 @@ tsk .req x28 // current thread_info
#endif
.endm
.macro el1_interrupt_handler, handler:req
gic_prio_irq_setup pmr=x20, tmp=x1
enable_da
mov x0, sp
bl enter_el1_irq_or_nmi
irq_handler \handler
#ifdef CONFIG_PREEMPTION
ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count
alternative_if ARM64_HAS_IRQ_PRIO_MASKING
/*
* DA were cleared at start of handling, and IF are cleared by
* the GIC irqchip driver using gic_arch_enable_irqs() for
* normal IRQs. If anything is set, it means we come back from
* an NMI instead of a normal IRQ, so skip preemption
*/
mrs x0, daif
orr x24, x24, x0
alternative_else_nop_endif
cbnz x24, 1f // preempt count != 0 || NMI return path
bl arm64_preempt_schedule_irq // irq en/disable is done inside
1:
#endif
mov x0, sp
bl exit_el1_irq_or_nmi
.endm
.macro el0_interrupt_handler, handler:req
gic_prio_irq_setup pmr=x20, tmp=x0
user_exit_irqoff
enable_da
tbz x22, #55, 1f
bl do_el0_irq_bp_hardening
1:
irq_handler \handler
.endm
.text
/*
@ -533,18 +574,18 @@ SYM_CODE_START(vectors)
kernel_ventry 1, sync // Synchronous EL1h
kernel_ventry 1, irq // IRQ EL1h
kernel_ventry 1, fiq_invalid // FIQ EL1h
kernel_ventry 1, fiq // FIQ EL1h
kernel_ventry 1, error // Error EL1h
kernel_ventry 0, sync // Synchronous 64-bit EL0
kernel_ventry 0, irq // IRQ 64-bit EL0
kernel_ventry 0, fiq_invalid // FIQ 64-bit EL0
kernel_ventry 0, fiq // FIQ 64-bit EL0
kernel_ventry 0, error // Error 64-bit EL0
#ifdef CONFIG_COMPAT
kernel_ventry 0, sync_compat, 32 // Synchronous 32-bit EL0
kernel_ventry 0, irq_compat, 32 // IRQ 32-bit EL0
kernel_ventry 0, fiq_invalid_compat, 32 // FIQ 32-bit EL0
kernel_ventry 0, fiq_compat, 32 // FIQ 32-bit EL0
kernel_ventry 0, error_compat, 32 // Error 32-bit EL0
#else
kernel_ventry 0, sync_invalid, 32 // Synchronous 32-bit EL0
@ -610,12 +651,6 @@ SYM_CODE_START_LOCAL(el0_error_invalid)
inv_entry 0, BAD_ERROR
SYM_CODE_END(el0_error_invalid)
#ifdef CONFIG_COMPAT
SYM_CODE_START_LOCAL(el0_fiq_invalid_compat)
inv_entry 0, BAD_FIQ, 32
SYM_CODE_END(el0_fiq_invalid_compat)
#endif
SYM_CODE_START_LOCAL(el1_sync_invalid)
inv_entry 1, BAD_SYNC
SYM_CODE_END(el1_sync_invalid)
@ -646,35 +681,16 @@ SYM_CODE_END(el1_sync)
.align 6
SYM_CODE_START_LOCAL_NOALIGN(el1_irq)
kernel_entry 1
gic_prio_irq_setup pmr=x20, tmp=x1
enable_da_f
mov x0, sp
bl enter_el1_irq_or_nmi
irq_handler
#ifdef CONFIG_PREEMPTION
ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count
alternative_if ARM64_HAS_IRQ_PRIO_MASKING
/*
* DA_F were cleared at start of handling. If anything is set in DAIF,
* we come back from an NMI, so skip preemption
*/
mrs x0, daif
orr x24, x24, x0
alternative_else_nop_endif
cbnz x24, 1f // preempt count != 0 || NMI return path
bl arm64_preempt_schedule_irq // irq en/disable is done inside
1:
#endif
mov x0, sp
bl exit_el1_irq_or_nmi
el1_interrupt_handler handle_arch_irq
kernel_exit 1
SYM_CODE_END(el1_irq)
SYM_CODE_START_LOCAL_NOALIGN(el1_fiq)
kernel_entry 1
el1_interrupt_handler handle_arch_fiq
kernel_exit 1
SYM_CODE_END(el1_fiq)
/*
* EL0 mode handlers.
*/
@ -701,6 +717,11 @@ SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat)
b el0_irq_naked
SYM_CODE_END(el0_irq_compat)
SYM_CODE_START_LOCAL_NOALIGN(el0_fiq_compat)
kernel_entry 0, 32
b el0_fiq_naked
SYM_CODE_END(el0_fiq_compat)
SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat)
kernel_entry 0, 32
b el0_error_naked
@ -711,18 +732,17 @@ SYM_CODE_END(el0_error_compat)
SYM_CODE_START_LOCAL_NOALIGN(el0_irq)
kernel_entry 0
el0_irq_naked:
gic_prio_irq_setup pmr=x20, tmp=x0
user_exit_irqoff
enable_da_f
tbz x22, #55, 1f
bl do_el0_irq_bp_hardening
1:
irq_handler
el0_interrupt_handler handle_arch_irq
b ret_to_user
SYM_CODE_END(el0_irq)
SYM_CODE_START_LOCAL_NOALIGN(el0_fiq)
kernel_entry 0
el0_fiq_naked:
el0_interrupt_handler handle_arch_fiq
b ret_to_user
SYM_CODE_END(el0_fiq)
SYM_CODE_START_LOCAL(el1_error)
kernel_entry 1
mrs x1, esr_el1
@ -743,7 +763,7 @@ el0_error_naked:
mov x0, sp
mov x1, x25
bl do_serror
enable_da_f
enable_da
b ret_to_user
SYM_CODE_END(el0_error)

4
arch/arm64/kernel/fpsimd.c
View File

@ -180,7 +180,7 @@ static void __get_cpu_fpsimd_context(void)
*/
static void get_cpu_fpsimd_context(void)
{
preempt_disable();
local_bh_disable();
__get_cpu_fpsimd_context();
}
@ -201,7 +201,7 @@ static void __put_cpu_fpsimd_context(void)
static void put_cpu_fpsimd_context(void)
{
__put_cpu_fpsimd_context();
preempt_enable();
local_bh_enable();
}
static bool have_cpu_fpsimd_context(void)

39
arch/arm64/kernel/head.S
View File

@ -477,14 +477,13 @@ EXPORT_SYMBOL(kimage_vaddr)
* booted in EL1 or EL2 respectively.
*/
SYM_FUNC_START(init_kernel_el)
mov_q x0, INIT_SCTLR_EL1_MMU_OFF
msr sctlr_el1, x0
mrs x0, CurrentEL
cmp x0, #CurrentEL_EL2
b.eq init_el2
SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
mov_q x0, INIT_SCTLR_EL1_MMU_OFF
msr sctlr_el1, x0
isb
mov_q x0, INIT_PSTATE_EL1
msr spsr_el1, x0
@ -504,9 +503,43 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
msr vbar_el2, x0
isb
/*
* Fruity CPUs seem to have HCR_EL2.E2H set to RES1,
* making it impossible to start in nVHE mode. Is that
* compliant with the architecture? Absolutely not!
*/
mrs x0, hcr_el2
and x0, x0, #HCR_E2H
cbz x0, 1f
/* Switching to VHE requires a sane SCTLR_EL1 as a start */
mov_q x0, INIT_SCTLR_EL1_MMU_OFF
msr_s SYS_SCTLR_EL12, x0
/*
* Force an eret into a helper "function", and let it return
* to our original caller... This makes sure that we have
* initialised the basic PSTATE state.
*/
mov x0, #INIT_PSTATE_EL2
msr spsr_el1, x0
adr x0, __cpu_stick_to_vhe
msr elr_el1, x0
eret
1:
mov_q x0, INIT_SCTLR_EL1_MMU_OFF
msr sctlr_el1, x0
msr elr_el2, lr
mov w0, #BOOT_CPU_MODE_EL2
eret
__cpu_stick_to_vhe:
mov x0, #HVC_VHE_RESTART
hvc #0
mov x0, #BOOT_CPU_MODE_EL2
ret
SYM_FUNC_END(init_kernel_el)
/*

10
arch/arm64/kernel/hyp-stub.S
View File

@ -27,12 +27,12 @@ SYM_CODE_START(__hyp_stub_vectors)
ventry el2_fiq_invalid // FIQ EL2t
ventry el2_error_invalid // Error EL2t
ventry el2_sync_invalid // Synchronous EL2h
ventry elx_sync // Synchronous EL2h
ventry el2_irq_invalid // IRQ EL2h
ventry el2_fiq_invalid // FIQ EL2h
ventry el2_error_invalid // Error EL2h
ventry el1_sync // Synchronous 64-bit EL1
ventry elx_sync // Synchronous 64-bit EL1
ventry el1_irq_invalid // IRQ 64-bit EL1
ventry el1_fiq_invalid // FIQ 64-bit EL1
ventry el1_error_invalid // Error 64-bit EL1
@ -45,7 +45,7 @@ SYM_CODE_END(__hyp_stub_vectors)
.align 11
SYM_CODE_START_LOCAL(el1_sync)
SYM_CODE_START_LOCAL(elx_sync)
cmp x0, #HVC_SET_VECTORS
b.ne 1f
msr vbar_el2, x1
@ -71,7 +71,7 @@ SYM_CODE_START_LOCAL(el1_sync)
9: mov x0, xzr
eret
SYM_CODE_END(el1_sync)
SYM_CODE_END(elx_sync)
// nVHE? No way! Give me the real thing!
SYM_CODE_START_LOCAL(mutate_to_vhe)
@ -224,7 +224,6 @@ SYM_FUNC_END(__hyp_reset_vectors)
* Entry point to switch to VHE if deemed capable
*/
SYM_FUNC_START(switch_to_vhe)
#ifdef CONFIG_ARM64_VHE
// Need to have booted at EL2
adr_l x1, __boot_cpu_mode
ldr w0, [x1]
@ -240,6 +239,5 @@ SYM_FUNC_START(switch_to_vhe)
mov x0, #HVC_VHE_RESTART
hvc #0
1:
#endif
ret
SYM_FUNC_END(switch_to_vhe)

26
arch/arm64/kernel/idreg-override.c
View File

@ -25,14 +25,26 @@ struct ftr_set_desc {
struct {
char name[FTR_DESC_FIELD_LEN];
u8 shift;
bool (*filter)(u64 val);
} fields[];
};
static bool __init mmfr1_vh_filter(u64 val)
{
/*
* If we ever reach this point while running VHE, we're
* guaranteed to be on one of these funky, VHE-stuck CPUs. If
* the user was trying to force nVHE on us, proceed with
* attitude adjustment.
*/
return !(is_kernel_in_hyp_mode() && val == 0);
}
static const struct ftr_set_desc mmfr1 __initconst = {
.name = "id_aa64mmfr1",
.override = &id_aa64mmfr1_override,
.fields = {
{ "vh", ID_AA64MMFR1_VHE_SHIFT },
{ "vh", ID_AA64MMFR1_VHE_SHIFT, mmfr1_vh_filter },
{}
},
};
@ -124,6 +136,18 @@ static void __init match_options(const char *cmdline)
if (find_field(cmdline, regs[i], f, &v))
continue;
/*
* If an override gets filtered out, advertise
* it by setting the value to 0xf, but
* clearing the mask... Yes, this is fragile.
*/
if (regs[i]->fields[f].filter &&
!regs[i]->fields[f].filter(v)) {
regs[i]->override->val |= mask;
regs[i]->override->mask &= ~mask;
continue;
}
regs[i]->override->val &= ~mask;
regs[i]->override->val |= (v << shift) & mask;
regs[i]->override->mask |= mask;

35
arch/arm64/kernel/irq.c
View File

@ -71,13 +71,44 @@ static void init_irq_stacks(void)
}
#endif
static void default_handle_irq(struct pt_regs *regs)
{
panic("IRQ taken without a root IRQ handler\n");
}
static void default_handle_fiq(struct pt_regs *regs)
{
panic("FIQ taken without a root FIQ handler\n");
}
void (*handle_arch_irq)(struct pt_regs *) __ro_after_init = default_handle_irq;
void (*handle_arch_fiq)(struct pt_regs *) __ro_after_init = default_handle_fiq;
int __init set_handle_irq(void (*handle_irq)(struct pt_regs *))
{
if (handle_arch_irq != default_handle_irq)
return -EBUSY;
handle_arch_irq = handle_irq;
pr_info("Root IRQ handler: %ps\n", handle_irq);
return 0;
}
int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *))
{
if (handle_arch_fiq != default_handle_fiq)
return -EBUSY;
handle_arch_fiq = handle_fiq;
pr_info("Root FIQ handler: %ps\n", handle_fiq);
return 0;
}
void __init init_IRQ(void)
{
init_irq_stacks();
init_irq_scs();
irqchip_init();
if (!handle_arch_irq)
panic("No interrupt controller found.");
if (system_uses_irq_prio_masking()) {
/*

18
arch/arm64/kernel/kaslr.c
View File

@ -128,15 +128,17 @@ u64 __init kaslr_early_init(void)
/* use the top 16 bits to randomize the linear region */
memstart_offset_seed = seed >> 48;
if (IS_ENABLED(CONFIG_KASAN_GENERIC) ||
IS_ENABLED(CONFIG_KASAN_SW_TAGS))
if (!IS_ENABLED(CONFIG_KASAN_VMALLOC) &&
(IS_ENABLED(CONFIG_KASAN_GENERIC) ||
IS_ENABLED(CONFIG_KASAN_SW_TAGS)))
/*
* KASAN does not expect the module region to intersect the
* vmalloc region, since shadow memory is allocated for each
* module at load time, whereas the vmalloc region is shadowed
* by KASAN zero pages. So keep modules out of the vmalloc
* region if KASAN is enabled, and put the kernel well within
* 4 GB of the module region.
* KASAN without KASAN_VMALLOC does not expect the module region
* to intersect the vmalloc region, since shadow memory is
* allocated for each module at load time, whereas the vmalloc
* region is shadowed by KASAN zero pages. So keep modules
* out of the vmalloc region if KASAN is enabled without
* KASAN_VMALLOC, and put the kernel well within 4 GB of the
* module region.
*/
return offset % SZ_2G;

16
arch/arm64/kernel/module.c
View File

@ -40,14 +40,16 @@ void *module_alloc(unsigned long size)
NUMA_NO_NODE, __builtin_return_address(0));
if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
!IS_ENABLED(CONFIG_KASAN_SW_TAGS))
(IS_ENABLED(CONFIG_KASAN_VMALLOC) ||
(!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
!IS_ENABLED(CONFIG_KASAN_SW_TAGS))))
/*
* KASAN can only deal with module allocations being served
* from the reserved module region, since the remainder of
* the vmalloc region is already backed by zero shadow pages,
* and punching holes into it is non-trivial. Since the module
* region is not randomized when KASAN is enabled, it is even
* KASAN without KASAN_VMALLOC can only deal with module
* allocations being served from the reserved module region,
* since the remainder of the vmalloc region is already
* backed by zero shadow pages, and punching holes into it
* is non-trivial. Since the module region is not randomized
* when KASAN is enabled without KASAN_VMALLOC, it is even
* less likely that the module region gets exhausted, so we
* can simply omit this fallback in that case.
*/

5
arch/arm64/kernel/perf_event.c
View File

@ -470,9 +470,8 @@ static inline u64 armv8pmu_read_evcntr(int idx)
static inline u64 armv8pmu_read_hw_counter(struct perf_event *event)
{
int idx = event->hw.idx;
u64 val = 0;
u64 val = armv8pmu_read_evcntr(idx);
val = armv8pmu_read_evcntr(idx);
if (armv8pmu_event_is_chained(event))
val = (val << 32) | armv8pmu_read_evcntr(idx - 1);
return val;
@ -520,7 +519,7 @@ static u64 armv8pmu_read_counter(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
int idx = hwc->idx;
u64 value = 0;
u64 value;
if (idx == ARMV8_IDX_CYCLE_COUNTER)
value = read_sysreg(pmccntr_el0);

2
arch/arm64/kernel/process.c
View File

@ -84,7 +84,7 @@ static void noinstr __cpu_do_idle_irqprio(void)
unsigned long daif_bits;
daif_bits = read_sysreg(daif);
write_sysreg(daif_bits | PSR_I_BIT, daif);
write_sysreg(daif_bits | PSR_I_BIT | PSR_F_BIT, daif);
/*
* Unmask PMR before going idle to make sure interrupts can

1
arch/arm64/kernel/smp.c
View File

@ -188,6 +188,7 @@ static void init_gic_priority_masking(void)
cpuflags = read_sysreg(daif);
WARN_ON(!(cpuflags & PSR_I_BIT));
WARN_ON(!(cpuflags & PSR_F_BIT));
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
}

22
arch/arm64/kernel/vdso.c
View File

@ -271,6 +271,14 @@ enum aarch32_map {
static struct page *aarch32_vectors_page __ro_after_init;
static struct page *aarch32_sig_page __ro_after_init;
static int aarch32_sigpage_mremap(const struct vm_special_mapping *sm,
struct vm_area_struct *new_vma)
{
current->mm->context.sigpage = (void *)new_vma->vm_start;
return 0;
}
static struct vm_special_mapping aarch32_vdso_maps[] = {
[AA32_MAP_VECTORS] = {
.name = "[vectors]", /* ABI */
@ -279,6 +287,7 @@ static struct vm_special_mapping aarch32_vdso_maps[] = {
[AA32_MAP_SIGPAGE] = {
.name = "[sigpage]", /* ABI */
.pages = &aarch32_sig_page,
.mremap = aarch32_sigpage_mremap,
},
[AA32_MAP_VVAR] = {
.name = "[vvar]",
@ -299,30 +308,31 @@ static int aarch32_alloc_kuser_vdso_page(void)
if (!IS_ENABLED(CONFIG_KUSER_HELPERS))
return 0;
vdso_page = get_zeroed_page(GFP_ATOMIC);
vdso_page = get_zeroed_page(GFP_KERNEL);
if (!vdso_page)
return -ENOMEM;
memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start,
kuser_sz);
aarch32_vectors_page = virt_to_page(vdso_page);
flush_dcache_page(aarch32_vectors_page);
return 0;
}
#define COMPAT_SIGPAGE_POISON_WORD 0xe7fddef1
static int aarch32_alloc_sigpage(void)
{
extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;
unsigned long sigpage;
__le32 poison = cpu_to_le32(COMPAT_SIGPAGE_POISON_WORD);
void *sigpage;
sigpage = get_zeroed_page(GFP_ATOMIC);
sigpage = (void *)__get_free_page(GFP_KERNEL);
if (!sigpage)
return -ENOMEM;
memcpy((void *)sigpage, __aarch32_sigret_code_start, sigret_sz);
memset32(sigpage, (__force u32)poison, PAGE_SIZE / sizeof(poison));
memcpy(sigpage, __aarch32_sigret_code_start, sigret_sz);
aarch32_sig_page = virt_to_page(sigpage);
flush_dcache_page(aarch32_sig_page);
return 0;
}

18
arch/arm64/mm/fault.c
View File

@ -527,7 +527,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
const struct fault_info *inf;
struct mm_struct *mm = current->mm;
vm_fault_t fault;
unsigned long vm_flags = VM_ACCESS_FLAGS;
unsigned long vm_flags;
unsigned int mm_flags = FAULT_FLAG_DEFAULT;
unsigned long addr = untagged_addr(far);
@ -544,12 +544,28 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
if (user_mode(regs))
mm_flags |= FAULT_FLAG_USER;
/*
* vm_flags tells us what bits we must have in vma->vm_flags
* for the fault to be benign, __do_page_fault() would check
* vma->vm_flags & vm_flags and returns an error if the
* intersection is empty
*/
if (is_el0_instruction_abort(esr)) {
/* It was exec fault */
vm_flags = VM_EXEC;
mm_flags |= FAULT_FLAG_INSTRUCTION;
} else if (is_write_abort(esr)) {
/* It was write fault */
vm_flags = VM_WRITE;
mm_flags |= FAULT_FLAG_WRITE;
} else {
/* It was read fault */
vm_flags = VM_READ;
/* Write implies read */
vm_flags |= VM_WRITE;
/* If EPAN is absent then exec implies read */
if (!cpus_have_const_cap(ARM64_HAS_EPAN))
vm_flags |= VM_EXEC;
}
if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {

29
arch/arm64/mm/kasan_init.c
View File

@ -79,7 +79,7 @@ static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,
phys_addr_t pmd_phys = early ?
__pa_symbol(kasan_early_shadow_pmd)
: kasan_alloc_zeroed_page(node);
__pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE);
__pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);
}
return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr);
@ -92,7 +92,7 @@ static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node,
phys_addr_t pud_phys = early ?
__pa_symbol(kasan_early_shadow_pud)
: kasan_alloc_zeroed_page(node);
__p4d_populate(p4dp, pud_phys, PMD_TYPE_TABLE);
__p4d_populate(p4dp, pud_phys, P4D_TYPE_TABLE);
}
return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr);
@ -214,15 +214,18 @@ static void __init kasan_init_shadow(void)
{
u64 kimg_shadow_start, kimg_shadow_end;
u64 mod_shadow_start, mod_shadow_end;
u64 vmalloc_shadow_end;
phys_addr_t pa_start, pa_end;
u64 i;
kimg_shadow_start = (u64)kasan_mem_to_shadow(_text) & PAGE_MASK;
kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(_end));
kimg_shadow_start = (u64)kasan_mem_to_shadow(KERNEL_START) & PAGE_MASK;
kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(KERNEL_END));
mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR);
mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END);
vmalloc_shadow_end = (u64)kasan_mem_to_shadow((void *)VMALLOC_END);
/*
* We are going to perform proper setup of shadow memory.
* At first we should unmap early shadow (clear_pgds() call below).
@ -237,16 +240,22 @@ static void __init kasan_init_shadow(void)
clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
kasan_map_populate(kimg_shadow_start, kimg_shadow_end,
early_pfn_to_nid(virt_to_pfn(lm_alias(_text))));
early_pfn_to_nid(virt_to_pfn(lm_alias(KERNEL_START))));
kasan_populate_early_shadow(kasan_mem_to_shadow((void *)PAGE_END),
(void *)mod_shadow_start);
kasan_populate_early_shadow((void *)kimg_shadow_end,
(void *)KASAN_SHADOW_END);
if (kimg_shadow_start > mod_shadow_end)
kasan_populate_early_shadow((void *)mod_shadow_end,
(void *)kimg_shadow_start);
if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
BUILD_BUG_ON(VMALLOC_START != MODULES_END);
kasan_populate_early_shadow((void *)vmalloc_shadow_end,
(void *)KASAN_SHADOW_END);
} else {
kasan_populate_early_shadow((void *)kimg_shadow_end,
(void *)KASAN_SHADOW_END);
if (kimg_shadow_start > mod_shadow_end)
kasan_populate_early_shadow((void *)mod_shadow_end,
(void *)kimg_shadow_start);
}
for_each_mem_range(i, &pa_start, &pa_end) {
void *start = (void *)__phys_to_virt(pa_start);

41
arch/arm64/mm/mmu.c
View File

@ -39,6 +39,7 @@
#define NO_BLOCK_MAPPINGS BIT(0)
#define NO_CONT_MAPPINGS BIT(1)
#define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */
u64 idmap_t0sz = TCR_T0SZ(VA_BITS_MIN);
u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
@ -185,10 +186,14 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
BUG_ON(pmd_sect(pmd));
if (pmd_none(pmd)) {
pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN;
phys_addr_t pte_phys;
if (flags & NO_EXEC_MAPPINGS)
pmdval |= PMD_TABLE_PXN;
BUG_ON(!pgtable_alloc);
pte_phys = pgtable_alloc(PAGE_SHIFT);
__pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);
__pmd_populate(pmdp, pte_phys, pmdval);
pmd = READ_ONCE(*pmdp);
}
BUG_ON(pmd_bad(pmd));
@ -259,10 +264,14 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
*/
BUG_ON(pud_sect(pud));
if (pud_none(pud)) {
pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN;
phys_addr_t pmd_phys;
if (flags & NO_EXEC_MAPPINGS)
pudval |= PUD_TABLE_PXN;
BUG_ON(!pgtable_alloc);
pmd_phys = pgtable_alloc(PMD_SHIFT);
__pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);
__pud_populate(pudp, pmd_phys, pudval);
pud = READ_ONCE(*pudp);
}
BUG_ON(pud_bad(pud));
@ -306,10 +315,14 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
p4d_t p4d = READ_ONCE(*p4dp);
if (p4d_none(p4d)) {
p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN;
phys_addr_t pud_phys;
if (flags & NO_EXEC_MAPPINGS)
p4dval |= P4D_TABLE_PXN;
BUG_ON(!pgtable_alloc);
pud_phys = pgtable_alloc(PUD_SHIFT);
__p4d_populate(p4dp, pud_phys, PUD_TYPE_TABLE);
__p4d_populate(p4dp, pud_phys, p4dval);
p4d = READ_ONCE(*p4dp);
}
BUG_ON(p4d_bad(p4d));
@ -486,14 +499,24 @@ early_param("crashkernel", enable_crash_mem_map);
static void __init map_mem(pgd_t *pgdp)
{
static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN);
phys_addr_t kernel_start = __pa_symbol(_stext);
phys_addr_t kernel_end = __pa_symbol(__init_begin);
phys_addr_t start, end;
int flags = 0;
int flags = NO_EXEC_MAPPINGS;
u64 i;
/*
* Setting hierarchical PXNTable attributes on table entries covering
* the linear region is only possible if it is guaranteed that no table
* entries at any level are being shared between the linear region and
* the vmalloc region. Check whether this is true for the PGD level, in
* which case it is guaranteed to be true for all other levels as well.
*/
BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end));
if (rodata_full || crash_mem_map || debug_pagealloc_enabled())
flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
/*
* Take care not to create a writable alias for the
@ -1210,11 +1233,11 @@ void __init early_fixmap_init(void)
pudp = pud_offset_kimg(p4dp, addr);
} else {
if (p4d_none(p4d))
__p4d_populate(p4dp, __pa_symbol(bm_pud), PUD_TYPE_TABLE);
__p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE);
pudp = fixmap_pud(addr);
}
if (pud_none(READ_ONCE(*pudp)))
__pud_populate(pudp, __pa_symbol(bm_pmd), PMD_TYPE_TABLE);
__pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE);
pmdp = fixmap_pmd(addr);
__pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE);
@ -1463,7 +1486,7 @@ struct range arch_get_mappable_range(void)
int arch_add_memory(int nid, u64 start, u64 size,
struct mhp_params *params)
{
int ret, flags = 0;
int ret, flags = NO_EXEC_MAPPINGS;
VM_BUG_ON(!mhp_range_allowed(start, size, true));
@ -1473,7 +1496,7 @@ int arch_add_memory(int nid, u64 start, u64 size,
*/
if (rodata_full || debug_pagealloc_enabled() ||
IS_ENABLED(CONFIG_KFENCE))
flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
__create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),
size, params->pgprot, __pgd_pgtable_alloc,

12
drivers/perf/arm-cci.c
View File

@ -306,7 +306,7 @@ static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
{
struct dev_ext_attribute *eattr = container_of(attr,
struct dev_ext_attribute, attr);
return snprintf(buf, PAGE_SIZE, "config=0x%lx\n", (unsigned long)eattr->var);
return sysfs_emit(buf, "config=0x%lx\n", (unsigned long)eattr->var);
}
static int cci400_get_event_idx(struct cci_pmu *cci_pmu,
@ -525,8 +525,8 @@ static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
struct dev_ext_attribute *eattr = container_of(attr,
struct dev_ext_attribute, attr);
/* Global events have single fixed source code */
return snprintf(buf, PAGE_SIZE, "event=0x%lx,source=0x%x\n",
(unsigned long)eattr->var, CCI5xx_PORT_GLOBAL);
return sysfs_emit(buf, "event=0x%lx,source=0x%x\n",
(unsigned long)eattr->var, CCI5xx_PORT_GLOBAL);
}
/*
@ -696,7 +696,7 @@ static ssize_t cci_pmu_format_show(struct device *dev,
{
struct dev_ext_attribute *eattr = container_of(attr,
struct dev_ext_attribute, attr);
return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var);
return sysfs_emit(buf, "%s\n", (char *)eattr->var);
}
static ssize_t cci_pmu_event_show(struct device *dev,
@ -705,8 +705,8 @@ static ssize_t cci_pmu_event_show(struct device *dev,
struct dev_ext_attribute *eattr = container_of(attr,
struct dev_ext_attribute, attr);
/* source parameter is mandatory for normal PMU events */
return snprintf(buf, PAGE_SIZE, "source=?,event=0x%lx\n",
(unsigned long)eattr->var);
return sysfs_emit(buf, "source=?,event=0x%lx\n",
(unsigned long)eattr->var);
}
static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx)

31
drivers/perf/arm-ccn.c
View File

@ -221,7 +221,7 @@ static ssize_t arm_ccn_pmu_format_show(struct device *dev,
struct dev_ext_attribute *ea = container_of(attr,
struct dev_ext_attribute, attr);
return snprintf(buf, PAGE_SIZE, "%s\n", (char *)ea->var);
return sysfs_emit(buf, "%s\n", (char *)ea->var);
}
#define CCN_FORMAT_ATTR(_name, _config) \
@ -326,43 +326,38 @@ static ssize_t arm_ccn_pmu_event_show(struct device *dev,
struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
struct arm_ccn_pmu_event *event = container_of(attr,
struct arm_ccn_pmu_event, attr);
ssize_t res;
int res;
res = scnprintf(buf, PAGE_SIZE, "type=0x%x", event->type);
res = sysfs_emit(buf, "type=0x%x", event->type);
if (event->event)
res += scnprintf(buf + res, PAGE_SIZE - res, ",event=0x%x",
event->event);
res += sysfs_emit_at(buf, res, ",event=0x%x", event->event);
if (event->def)
res += scnprintf(buf + res, PAGE_SIZE - res, ",%s",
event->def);
res += sysfs_emit_at(buf, res, ",%s", event->def);
if (event->mask)
res += scnprintf(buf + res, PAGE_SIZE - res, ",mask=0x%x",
event->mask);
res += sysfs_emit_at(buf, res, ",mask=0x%x", event->mask);
/* Arguments required by an event */
switch (event->type) {
case CCN_TYPE_CYCLES:
break;
case CCN_TYPE_XP:
res += scnprintf(buf + res, PAGE_SIZE - res,
",xp=?,vc=?");
res += sysfs_emit_at(buf, res, ",xp=?,vc=?");
if (event->event == CCN_EVENT_WATCHPOINT)
res += scnprintf(buf + res, PAGE_SIZE - res,
res += sysfs_emit_at(buf, res,
",port=?,dir=?,cmp_l=?,cmp_h=?,mask=?");
else
res += scnprintf(buf + res, PAGE_SIZE - res,
",bus=?");
res += sysfs_emit_at(buf, res, ",bus=?");
break;
case CCN_TYPE_MN:
res += scnprintf(buf + res, PAGE_SIZE - res, ",node=%d", ccn->mn_id);
res += sysfs_emit_at(buf, res, ",node=%d", ccn->mn_id);
break;
default:
res += scnprintf(buf + res, PAGE_SIZE - res, ",node=?");
res += sysfs_emit_at(buf, res, ",node=?");
break;
}
res += scnprintf(buf + res, PAGE_SIZE - res, "\n");
res += sysfs_emit_at(buf, res, "\n");
return res;
}
@ -476,7 +471,7 @@ static ssize_t arm_ccn_pmu_cmp_mask_show(struct device *dev,
struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name);
return mask ? snprintf(buf, PAGE_SIZE, "0x%016llx\n", *mask) : -EINVAL;
return mask ? sysfs_emit(buf, "0x%016llx\n", *mask) : -EINVAL;
}
static ssize_t arm_ccn_pmu_cmp_mask_store(struct device *dev,

22
drivers/perf/arm-cmn.c
View File

@ -348,19 +348,19 @@ static ssize_t arm_cmn_event_show(struct device *dev,
eattr = container_of(attr, typeof(*eattr), attr);
if (eattr->type == CMN_TYPE_DTC)
return snprintf(buf, PAGE_SIZE, "type=0x%x\n", eattr->type);
return sysfs_emit(buf, "type=0x%x\n", eattr->type);
if (eattr->type == CMN_TYPE_WP)
return snprintf(buf, PAGE_SIZE,
"type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n",
eattr->type, eattr->eventid);
return sysfs_emit(buf,
"type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n",
eattr->type, eattr->eventid);
if (arm_cmn_is_occup_event(eattr->type, eattr->eventid))
return snprintf(buf, PAGE_SIZE, "type=0x%x,eventid=0x%x,occupid=0x%x\n",
eattr->type, eattr->eventid, eattr->occupid);
return sysfs_emit(buf, "type=0x%x,eventid=0x%x,occupid=0x%x\n",
eattr->type, eattr->eventid, eattr->occupid);
return snprintf(buf, PAGE_SIZE, "type=0x%x,eventid=0x%x\n",
eattr->type, eattr->eventid);
return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type,
eattr->eventid);
}
static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj,
@ -560,12 +560,12 @@ static ssize_t arm_cmn_format_show(struct device *dev,
int lo = __ffs(fmt->field), hi = __fls(fmt->field);
if (lo == hi)
return snprintf(buf, PAGE_SIZE, "config:%d\n", lo);
return sysfs_emit(buf, "config:%d\n", lo);
if (!fmt->config)
return snprintf(buf, PAGE_SIZE, "config:%d-%d\n", lo, hi);
return sysfs_emit(buf, "config:%d-%d\n", lo, hi);
return snprintf(buf, PAGE_SIZE, "config%d:%d-%d\n", fmt->config, lo, hi);
return sysfs_emit(buf, "config%d:%d-%d\n", fmt->config, lo, hi);
}
#define _CMN_FORMAT_ATTR(_name, _cfg, _fld) \

2
drivers/perf/arm_dmc620_pmu.c
View File

@ -113,7 +113,7 @@ dmc620_pmu_event_show(struct device *dev,
eattr = container_of(attr, typeof(*eattr), attr);
return sprintf(page, "event=0x%x,clkdiv2=0x%x\n", eattr->eventid, eattr->clkdiv2);
return sysfs_emit(page, "event=0x%x,clkdiv2=0x%x\n", eattr->eventid, eattr->clkdiv2);
}
#define DMC620_PMU_EVENT_ATTR(_name, _eventid, _clkdiv2) \

5
drivers/perf/arm_dsu_pmu.c
View File

@ -136,8 +136,7 @@ static ssize_t dsu_pmu_sysfs_event_show(struct device *dev,
{
struct dev_ext_attribute *eattr = container_of(attr,
struct dev_ext_attribute, attr);
return snprintf(buf, PAGE_SIZE, "event=0x%lx\n",
(unsigned long)eattr->var);
return sysfs_emit(buf, "event=0x%lx\n", (unsigned long)eattr->var);
}
static ssize_t dsu_pmu_sysfs_format_show(struct device *dev,
@ -146,7 +145,7 @@ static ssize_t dsu_pmu_sysfs_format_show(struct device *dev,
{
struct dev_ext_attribute *eattr = container_of(attr,
struct dev_ext_attribute, attr);
return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var);
return sysfs_emit(buf, "%s\n", (char *)eattr->var);
}
static ssize_t dsu_pmu_cpumask_show(struct device *dev,

54
drivers/perf/arm_pmu_platform.c
View File

@ -6,6 +6,7 @@
* Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
*/
#define pr_fmt(fmt) "hw perfevents: " fmt
#define dev_fmt pr_fmt
#include <linux/bug.h>
#include <linux/cpumask.h>
@ -62,7 +63,7 @@ static bool pmu_has_irq_affinity(struct device_node *node)
return !!of_find_property(node, "interrupt-affinity", NULL);
}
static int pmu_parse_irq_affinity(struct device_node *node, int i)
static int pmu_parse_irq_affinity(struct device *dev, int i)
{
struct device_node *dn;
int cpu;
@ -72,19 +73,18 @@ static int pmu_parse_irq_affinity(struct device_node *node, int i)
* affinity matches our logical CPU order, as we used to assume.
* This is fragile, so we'll warn in pmu_parse_irqs().
*/
if (!pmu_has_irq_affinity(node))
if (!pmu_has_irq_affinity(dev->of_node))
return i;
dn = of_parse_phandle(node, "interrupt-affinity", i);
dn = of_parse_phandle(dev->of_node, "interrupt-affinity", i);
if (!dn) {
pr_warn("failed to parse interrupt-affinity[%d] for %pOFn\n",
i, node);
dev_warn(dev, "failed to parse interrupt-affinity[%d]\n", i);
return -EINVAL;
}
cpu = of_cpu_node_to_id(dn);
if (cpu < 0) {
pr_warn("failed to find logical CPU for %pOFn\n", dn);
dev_warn(dev, "failed to find logical CPU for %pOFn\n", dn);
cpu = nr_cpu_ids;
}
@ -98,19 +98,18 @@ static int pmu_parse_irqs(struct arm_pmu *pmu)
int i = 0, num_irqs;
struct platform_device *pdev = pmu->plat_device;
struct pmu_hw_events __percpu *hw_events = pmu->hw_events;
struct device *dev = &pdev->dev;
num_irqs = platform_irq_count(pdev);
if (num_irqs < 0) {
pr_err("unable to count PMU IRQs\n");
return num_irqs;
}
if (num_irqs < 0)
return dev_err_probe(dev, num_irqs, "unable to count PMU IRQs\n");
/*
* In this case we have no idea which CPUs are covered by the PMU.
* To match our prior behaviour, we assume all CPUs in this case.
*/
if (num_irqs == 0) {
pr_warn("no irqs for PMU, sampling events not supported\n");
dev_warn(dev, "no irqs for PMU, sampling events not supported\n");
pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
cpumask_setall(&pmu->supported_cpus);
return 0;
@ -122,10 +121,8 @@ static int pmu_parse_irqs(struct arm_pmu *pmu)
return pmu_parse_percpu_irq(pmu, irq);
}
if (nr_cpu_ids != 1 && !pmu_has_irq_affinity(pdev->dev.of_node)) {
pr_warn("no interrupt-affinity property for %pOF, guessing.\n",
pdev->dev.of_node);
}
if (nr_cpu_ids != 1 && !pmu_has_irq_affinity(dev->of_node))
dev_warn(dev, "no interrupt-affinity property, guessing.\n");
for (i = 0; i < num_irqs; i++) {
int cpu, irq;
@ -135,18 +132,18 @@ static int pmu_parse_irqs(struct arm_pmu *pmu)
continue;
if (irq_is_percpu_devid(irq)) {
pr_warn("multiple PPIs or mismatched SPI/PPI detected\n");
dev_warn(dev, "multiple PPIs or mismatched SPI/PPI detected\n");
return -EINVAL;
}
cpu = pmu_parse_irq_affinity(pdev->dev.of_node, i);
cpu = pmu_parse_irq_affinity(dev, i);
if (cpu < 0)
return cpu;
if (cpu >= nr_cpu_ids)
continue;
if (per_cpu(hw_events->irq, cpu)) {
pr_warn("multiple PMU IRQs for the same CPU detected\n");
dev_warn(dev, "multiple PMU IRQs for the same CPU detected\n");
return -EINVAL;
}
@ -191,9 +188,8 @@ int arm_pmu_device_probe(struct platform_device *pdev,
const struct of_device_id *of_table,
const struct pmu_probe_info *probe_table)
{
const struct of_device_id *of_id;
armpmu_init_fn init_fn;
struct device_node *node = pdev->dev.of_node;
struct device *dev = &pdev->dev;
struct arm_pmu *pmu;
int ret = -ENODEV;
@ -207,15 +203,14 @@ int arm_pmu_device_probe(struct platform_device *pdev,
if (ret)
goto out_free;
if (node && (of_id = of_match_node(of_table, pdev->dev.of_node))) {
init_fn = of_id->data;
pmu->secure_access = of_property_read_bool(pdev->dev.of_node,
init_fn = of_device_get_match_data(dev);
if (init_fn) {
pmu->secure_access = of_property_read_bool(dev->of_node,
"secure-reg-access");
/* arm64 systems boot only as non-secure */
if (IS_ENABLED(CONFIG_ARM64) && pmu->secure_access) {
pr_warn("ignoring \"secure-reg-access\" property for arm64\n");
dev_warn(dev, "ignoring \"secure-reg-access\" property for arm64\n");
pmu->secure_access = false;
}
@ -226,7 +221,7 @@ int arm_pmu_device_probe(struct platform_device *pdev,
}
if (ret) {
pr_info("%pOF: failed to probe PMU!\n", node);
dev_err(dev, "failed to probe PMU!\n");
goto out_free;
}
@ -235,15 +230,16 @@ int arm_pmu_device_probe(struct platform_device *pdev,
goto out_free_irqs;
ret = armpmu_register(pmu);
if (ret)
goto out_free;
if (ret) {
dev_err(dev, "failed to register PMU devices!\n");
goto out_free_irqs;
}
return 0;
out_free_irqs:
armpmu_free_irqs(pmu);
out_free:
pr_info("%pOF: failed to register PMU devices!\n", node);
armpmu_free(pmu);
return ret;
}

36
drivers/perf/arm_smmuv3_pmu.c
View File

@ -506,30 +506,24 @@ static ssize_t smmu_pmu_event_show(struct device *dev,
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
}
#define SMMU_EVENT_ATTR(name, config) \
PMU_EVENT_ATTR(name, smmu_event_attr_##name, \
config, smmu_pmu_event_show)
SMMU_EVENT_ATTR(cycles, 0);
SMMU_EVENT_ATTR(transaction, 1);
SMMU_EVENT_ATTR(tlb_miss, 2);
SMMU_EVENT_ATTR(config_cache_miss, 3);
SMMU_EVENT_ATTR(trans_table_walk_access, 4);
SMMU_EVENT_ATTR(config_struct_access, 5);
SMMU_EVENT_ATTR(pcie_ats_trans_rq, 6);
SMMU_EVENT_ATTR(pcie_ats_trans_passed, 7);
#define SMMU_EVENT_ATTR(name, config) \
(&((struct perf_pmu_events_attr) { \
.attr = __ATTR(name, 0444, smmu_pmu_event_show, NULL), \
.id = config, \
}).attr.attr)
static struct attribute *smmu_pmu_events[] = {
&smmu_event_attr_cycles.attr.attr,
&smmu_event_attr_transaction.attr.attr,
&smmu_event_attr_tlb_miss.attr.attr,
&smmu_event_attr_config_cache_miss.attr.attr,
&smmu_event_attr_trans_table_walk_access.attr.attr,
&smmu_event_attr_config_struct_access.attr.attr,
&smmu_event_attr_pcie_ats_trans_rq.attr.attr,
&smmu_event_attr_pcie_ats_trans_passed.attr.attr,
SMMU_EVENT_ATTR(cycles, 0),
SMMU_EVENT_ATTR(transaction, 1),
SMMU_EVENT_ATTR(tlb_miss, 2),
SMMU_EVENT_ATTR(config_cache_miss, 3),
SMMU_EVENT_ATTR(trans_table_walk_access, 4),
SMMU_EVENT_ATTR(config_struct_access, 5),
SMMU_EVENT_ATTR(pcie_ats_trans_rq, 6),
SMMU_EVENT_ATTR(pcie_ats_trans_passed, 7),
NULL
};
@ -560,7 +554,7 @@ static ssize_t smmu_pmu_identifier_attr_show(struct device *dev,
{
struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev));
return snprintf(page, PAGE_SIZE, "0x%08x\n", smmu_pmu->iidr);
return sysfs_emit(page, "0x%08x\n", smmu_pmu->iidr);
}
static umode_t smmu_pmu_identifier_attr_visible(struct kobject *kobj,

3
drivers/perf/arm_spe_pmu.c
View File

@ -126,8 +126,7 @@ static ssize_t arm_spe_pmu_cap_show(struct device *dev,
container_of(attr, struct dev_ext_attribute, attr);
int cap = (long)ea->var;
return snprintf(buf, PAGE_SIZE, "%u\n",
arm_spe_pmu_cap_get(spe_pmu, cap));
return sysfs_emit(buf, "%u\n", arm_spe_pmu_cap_get(spe_pmu, cap));
}
#define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \

7
drivers/perf/fsl_imx8_ddr_perf.c
View File

@ -110,7 +110,7 @@ static ssize_t ddr_perf_identifier_show(struct device *dev,
{
struct ddr_pmu *pmu = dev_get_drvdata(dev);
return sprintf(page, "%s\n", pmu->devtype_data->identifier);
return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);
}
static umode_t ddr_perf_identifier_attr_visible(struct kobject *kobj,
@ -170,8 +170,7 @@ static ssize_t ddr_perf_filter_cap_show(struct device *dev,
container_of(attr, struct dev_ext_attribute, attr);
int cap = (long)ea->var;
return snprintf(buf, PAGE_SIZE, "%u\n",
ddr_perf_filter_cap_get(pmu, cap));
return sysfs_emit(buf, "%u\n", ddr_perf_filter_cap_get(pmu, cap));
}
#define PERF_EXT_ATTR_ENTRY(_name, _func, _var) \
@ -220,7 +219,7 @@ ddr_pmu_event_show(struct device *dev, struct device_attribute *attr,
struct perf_pmu_events_attr *pmu_attr;
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
}
#define IMX8_DDR_PMU_EVENT_ATTR(_name, _id) \

3
drivers/perf/hisilicon/Makefile
View File

@ -1,3 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_HISI_PMU) += hisi_uncore_pmu.o hisi_uncore_l3c_pmu.o \
hisi_uncore_hha_pmu.o hisi_uncore_ddrc_pmu.o
hisi_uncore_hha_pmu.o hisi_uncore_ddrc_pmu.o hisi_uncore_sllc_pmu.o \
hisi_uncore_pa_pmu.o

354
drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c
View File

@ -14,12 +14,11 @@
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/list.h>
#include <linux/platform_device.h>
#include <linux/smp.h>
#include "hisi_uncore_pmu.h"
/* DDRC register definition */
/* DDRC register definition in v1 */
#define DDRC_PERF_CTRL 0x010
#define DDRC_FLUX_WR 0x380
#define DDRC_FLUX_RD 0x384
@ -35,12 +34,24 @@
#define DDRC_INT_CLEAR 0x6d0
#define DDRC_VERSION 0x710
/* DDRC register definition in v2 */
#define DDRC_V2_INT_MASK 0x528
#define DDRC_V2_INT_STATUS 0x52c
#define DDRC_V2_INT_CLEAR 0x530
#define DDRC_V2_EVENT_CNT 0xe00
#define DDRC_V2_EVENT_CTRL 0xe70
#define DDRC_V2_EVENT_TYPE 0xe74
#define DDRC_V2_PERF_CTRL 0xeA0
/* DDRC has 8-counters */
#define DDRC_NR_COUNTERS 0x8
#define DDRC_PERF_CTRL_EN 0x2
#define DDRC_V1_PERF_CTRL_EN 0x2
#define DDRC_V2_PERF_CTRL_EN 0x1
#define DDRC_V1_NR_EVENTS 0x7
#define DDRC_V2_NR_EVENTS 0x90
/*
* For DDRC PMU, there are eight-events and every event has been mapped
* For PMU v1, there are eight-events and every event has been mapped
* to fixed-purpose counters which register offset is not consistent.
* Therefore there is no write event type and we assume that event
* code (0 to 7) is equal to counter index in PMU driver.
@ -54,73 +65,85 @@ static const u32 ddrc_reg_off[] = {
/*
* Select the counter register offset using the counter index.
* In DDRC there are no programmable counter, the count
* is readed form the statistics counter register itself.
* In PMU v1, there are no programmable counter, the count
* is read form the statistics counter register itself.
*/
static u32 hisi_ddrc_pmu_get_counter_offset(int cntr_idx)
static u32 hisi_ddrc_pmu_v1_get_counter_offset(int cntr_idx)
{
return ddrc_reg_off[cntr_idx];
}
static u64 hisi_ddrc_pmu_read_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
static u32 hisi_ddrc_pmu_v2_get_counter_offset(int cntr_idx)
{
/* Use event code as counter index */
u32 idx = GET_DDRC_EVENTID(hwc);
if (!hisi_uncore_pmu_counter_valid(ddrc_pmu, idx)) {
dev_err(ddrc_pmu->dev, "Unsupported event index:%d!\n", idx);
return 0;
}
return readl(ddrc_pmu->base + hisi_ddrc_pmu_get_counter_offset(idx));
return DDRC_V2_EVENT_CNT + cntr_idx * 8;
}
static void hisi_ddrc_pmu_write_counter(struct hisi_pmu *ddrc_pmu,
static u64 hisi_ddrc_pmu_v1_read_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
return readl(ddrc_pmu->base +
hisi_ddrc_pmu_v1_get_counter_offset(hwc->idx));
}
static void hisi_ddrc_pmu_v1_write_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc, u64 val)
{
u32 idx = GET_DDRC_EVENTID(hwc);
if (!hisi_uncore_pmu_counter_valid(ddrc_pmu, idx)) {
dev_err(ddrc_pmu->dev, "Unsupported event index:%d!\n", idx);
return;
}
writel((u32)val,
ddrc_pmu->base + hisi_ddrc_pmu_get_counter_offset(idx));
ddrc_pmu->base + hisi_ddrc_pmu_v1_get_counter_offset(hwc->idx));
}
static u64 hisi_ddrc_pmu_v2_read_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
return readq(ddrc_pmu->base +
hisi_ddrc_pmu_v2_get_counter_offset(hwc->idx));
}
static void hisi_ddrc_pmu_v2_write_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc, u64 val)
{
writeq(val,
ddrc_pmu->base + hisi_ddrc_pmu_v2_get_counter_offset(hwc->idx));
}
/*
* For DDRC PMU, event has been mapped to fixed-purpose counter by hardware,
* so there is no need to write event type.
* For DDRC PMU v1, event has been mapped to fixed-purpose counter by hardware,
* so there is no need to write event type, while it is programmable counter in
* PMU v2.
*/
static void hisi_ddrc_pmu_write_evtype(struct hisi_pmu *hha_pmu, int idx,
u32 type)
{
u32 offset;
if (hha_pmu->identifier >= HISI_PMU_V2) {
offset = DDRC_V2_EVENT_TYPE + 4 * idx;
writel(type, hha_pmu->base + offset);
}
}
static void hisi_ddrc_pmu_start_counters(struct hisi_pmu *ddrc_pmu)
static void hisi_ddrc_pmu_v1_start_counters(struct hisi_pmu *ddrc_pmu)
{
u32 val;
/* Set perf_enable in DDRC_PERF_CTRL to start event counting */
val = readl(ddrc_pmu->base + DDRC_PERF_CTRL);
val |= DDRC_PERF_CTRL_EN;
val |= DDRC_V1_PERF_CTRL_EN;
writel(val, ddrc_pmu->base + DDRC_PERF_CTRL);
}
static void hisi_ddrc_pmu_stop_counters(struct hisi_pmu *ddrc_pmu)
static void hisi_ddrc_pmu_v1_stop_counters(struct hisi_pmu *ddrc_pmu)
{
u32 val;
/* Clear perf_enable in DDRC_PERF_CTRL to stop event counting */
val = readl(ddrc_pmu->base + DDRC_PERF_CTRL);
val &= ~DDRC_PERF_CTRL_EN;
val &= ~DDRC_V1_PERF_CTRL_EN;
writel(val, ddrc_pmu->base + DDRC_PERF_CTRL);
}
static void hisi_ddrc_pmu_enable_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
static void hisi_ddrc_pmu_v1_enable_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
@ -130,8 +153,8 @@ static void hisi_ddrc_pmu_enable_counter(struct hisi_pmu *ddrc_pmu,
writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL);
}
static void hisi_ddrc_pmu_disable_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
static void hisi_ddrc_pmu_v1_disable_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
@ -141,7 +164,7 @@ static void hisi_ddrc_pmu_disable_counter(struct hisi_pmu *ddrc_pmu,
writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL);
}
static int hisi_ddrc_pmu_get_event_idx(struct perf_event *event)
static int hisi_ddrc_pmu_v1_get_event_idx(struct perf_event *event)
{
struct hisi_pmu *ddrc_pmu = to_hisi_pmu(event->pmu);
unsigned long *used_mask = ddrc_pmu->pmu_events.used_mask;
@ -157,87 +180,117 @@ static int hisi_ddrc_pmu_get_event_idx(struct perf_event *event)
return idx;
}
static void hisi_ddrc_pmu_enable_counter_int(struct hisi_pmu *ddrc_pmu,
static int hisi_ddrc_pmu_v2_get_event_idx(struct perf_event *event)
{
return hisi_uncore_pmu_get_event_idx(event);
}
static void hisi_ddrc_pmu_v2_start_counters(struct hisi_pmu *ddrc_pmu)
{
u32 val;
val = readl(ddrc_pmu->base + DDRC_V2_PERF_CTRL);
val |= DDRC_V2_PERF_CTRL_EN;
writel(val, ddrc_pmu->base + DDRC_V2_PERF_CTRL);
}
static void hisi_ddrc_pmu_v2_stop_counters(struct hisi_pmu *ddrc_pmu)
{
u32 val;
val = readl(ddrc_pmu->base + DDRC_V2_PERF_CTRL);
val &= ~DDRC_V2_PERF_CTRL_EN;
writel(val, ddrc_pmu->base + DDRC_V2_PERF_CTRL);
}
static void hisi_ddrc_pmu_v2_enable_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
val = readl(ddrc_pmu->base + DDRC_V2_EVENT_CTRL);
val |= 1 << hwc->idx;
writel(val, ddrc_pmu->base + DDRC_V2_EVENT_CTRL);
}
static void hisi_ddrc_pmu_v2_disable_counter(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
val = readl(ddrc_pmu->base + DDRC_V2_EVENT_CTRL);
val &= ~(1 << hwc->idx);
writel(val, ddrc_pmu->base + DDRC_V2_EVENT_CTRL);
}
static void hisi_ddrc_pmu_v1_enable_counter_int(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
/* Write 0 to enable interrupt */
val = readl(ddrc_pmu->base + DDRC_INT_MASK);
val &= ~(1 << GET_DDRC_EVENTID(hwc));
val &= ~(1 << hwc->idx);
writel(val, ddrc_pmu->base + DDRC_INT_MASK);
}
static void hisi_ddrc_pmu_disable_counter_int(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
static void hisi_ddrc_pmu_v1_disable_counter_int(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
/* Write 1 to mask interrupt */
val = readl(ddrc_pmu->base + DDRC_INT_MASK);
val |= (1 << GET_DDRC_EVENTID(hwc));
val |= 1 << hwc->idx;
writel(val, ddrc_pmu->base + DDRC_INT_MASK);
}
static irqreturn_t hisi_ddrc_pmu_isr(int irq, void *dev_id)
static void hisi_ddrc_pmu_v2_enable_counter_int(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
struct hisi_pmu *ddrc_pmu = dev_id;
struct perf_event *event;
unsigned long overflown;
int idx;
u32 val;
/* Read the DDRC_INT_STATUS register */
overflown = readl(ddrc_pmu->base + DDRC_INT_STATUS);
if (!overflown)
return IRQ_NONE;
/*
* Find the counter index which overflowed if the bit was set
* and handle it
*/
for_each_set_bit(idx, &overflown, DDRC_NR_COUNTERS) {
/* Write 1 to clear the IRQ status flag */
writel((1 << idx), ddrc_pmu->base + DDRC_INT_CLEAR);
/* Get the corresponding event struct */
event = ddrc_pmu->pmu_events.hw_events[idx];
if (!event)
continue;
hisi_uncore_pmu_event_update(event);
hisi_uncore_pmu_set_event_period(event);
}
return IRQ_HANDLED;
val = readl(ddrc_pmu->base + DDRC_V2_INT_MASK);
val &= ~(1 << hwc->idx);
writel(val, ddrc_pmu->base + DDRC_V2_INT_MASK);
}
static int hisi_ddrc_pmu_init_irq(struct hisi_pmu *ddrc_pmu,
struct platform_device *pdev)
static void hisi_ddrc_pmu_v2_disable_counter_int(struct hisi_pmu *ddrc_pmu,
struct hw_perf_event *hwc)
{
int irq, ret;
u32 val;
/* Read and init IRQ */
irq = platform_get_irq(pdev, 0);
if (irq < 0)
return irq;
val = readl(ddrc_pmu->base + DDRC_V2_INT_MASK);
val |= 1 << hwc->idx;
writel(val, ddrc_pmu->base + DDRC_V2_INT_MASK);
}
ret = devm_request_irq(&pdev->dev, irq, hisi_ddrc_pmu_isr,
IRQF_NOBALANCING | IRQF_NO_THREAD,
dev_name(&pdev->dev), ddrc_pmu);
if (ret < 0) {
dev_err(&pdev->dev,
"Fail to request IRQ:%d ret:%d\n", irq, ret);
return ret;
}
static u32 hisi_ddrc_pmu_v1_get_int_status(struct hisi_pmu *ddrc_pmu)
{
return readl(ddrc_pmu->base + DDRC_INT_STATUS);
}
ddrc_pmu->irq = irq;
static void hisi_ddrc_pmu_v1_clear_int_status(struct hisi_pmu *ddrc_pmu,
int idx)
{
writel(1 << idx, ddrc_pmu->base + DDRC_INT_CLEAR);
}
return 0;
static u32 hisi_ddrc_pmu_v2_get_int_status(struct hisi_pmu *ddrc_pmu)
{
return readl(ddrc_pmu->base + DDRC_V2_INT_STATUS);
}
static void hisi_ddrc_pmu_v2_clear_int_status(struct hisi_pmu *ddrc_pmu,
int idx)
{
writel(1 << idx, ddrc_pmu->base + DDRC_V2_INT_CLEAR);
}
static const struct acpi_device_id hisi_ddrc_pmu_acpi_match[] = {
{ "HISI0233", },
{},
{ "HISI0234", },
{}
};
MODULE_DEVICE_TABLE(acpi, hisi_ddrc_pmu_acpi_match);
@ -269,21 +322,38 @@ static int hisi_ddrc_pmu_init_data(struct platform_device *pdev,
}
ddrc_pmu->identifier = readl(ddrc_pmu->base + DDRC_VERSION);
if (ddrc_pmu->identifier >= HISI_PMU_V2) {
if (device_property_read_u32(&pdev->dev, "hisilicon,sub-id",
&ddrc_pmu->sub_id)) {
dev_err(&pdev->dev, "Can not read sub-id!\n");
return -EINVAL;
}
}
return 0;
}
static struct attribute *hisi_ddrc_pmu_format_attr[] = {
static struct attribute *hisi_ddrc_pmu_v1_format_attr[] = {
HISI_PMU_FORMAT_ATTR(event, "config:0-4"),
NULL,
};
static const struct attribute_group hisi_ddrc_pmu_format_group = {
static const struct attribute_group hisi_ddrc_pmu_v1_format_group = {
.name = "format",
.attrs = hisi_ddrc_pmu_format_attr,
.attrs = hisi_ddrc_pmu_v1_format_attr,
};
static struct attribute *hisi_ddrc_pmu_events_attr[] = {
static struct attribute *hisi_ddrc_pmu_v2_format_attr[] = {
HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
NULL
};
static const struct attribute_group hisi_ddrc_pmu_v2_format_group = {
.name = "format",
.attrs = hisi_ddrc_pmu_v2_format_attr,
};
static struct attribute *hisi_ddrc_pmu_v1_events_attr[] = {
HISI_PMU_EVENT_ATTR(flux_wr, 0x00),
HISI_PMU_EVENT_ATTR(flux_rd, 0x01),
HISI_PMU_EVENT_ATTR(flux_wcmd, 0x02),
@ -295,9 +365,21 @@ static struct attribute *hisi_ddrc_pmu_events_attr[] = {
NULL,
};
static const struct attribute_group hisi_ddrc_pmu_events_group = {
static const struct attribute_group hisi_ddrc_pmu_v1_events_group = {
.name = "events",
.attrs = hisi_ddrc_pmu_events_attr,
.attrs = hisi_ddrc_pmu_v1_events_attr,
};
static struct attribute *hisi_ddrc_pmu_v2_events_attr[] = {
HISI_PMU_EVENT_ATTR(cycles, 0x00),
HISI_PMU_EVENT_ATTR(flux_wr, 0x83),
HISI_PMU_EVENT_ATTR(flux_rd, 0x84),
NULL
};
static const struct attribute_group hisi_ddrc_pmu_v2_events_group = {
.name = "events",
.attrs = hisi_ddrc_pmu_v2_events_attr,
};
static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
@ -323,25 +405,50 @@ static const struct attribute_group hisi_ddrc_pmu_identifier_group = {
.attrs = hisi_ddrc_pmu_identifier_attrs,
};
static const struct attribute_group *hisi_ddrc_pmu_attr_groups[] = {
&hisi_ddrc_pmu_format_group,
&hisi_ddrc_pmu_events_group,
static const struct attribute_group *hisi_ddrc_pmu_v1_attr_groups[] = {
&hisi_ddrc_pmu_v1_format_group,
&hisi_ddrc_pmu_v1_events_group,
&hisi_ddrc_pmu_cpumask_attr_group,
&hisi_ddrc_pmu_identifier_group,
NULL,
};
static const struct hisi_uncore_ops hisi_uncore_ddrc_ops = {
static const struct attribute_group *hisi_ddrc_pmu_v2_attr_groups[] = {
&hisi_ddrc_pmu_v2_format_group,
&hisi_ddrc_pmu_v2_events_group,
&hisi_ddrc_pmu_cpumask_attr_group,
&hisi_ddrc_pmu_identifier_group,
NULL
};
static const struct hisi_uncore_ops hisi_uncore_ddrc_v1_ops = {
.write_evtype = hisi_ddrc_pmu_write_evtype,
.get_event_idx = hisi_ddrc_pmu_get_event_idx,
.start_counters = hisi_ddrc_pmu_start_counters,
.stop_counters = hisi_ddrc_pmu_stop_counters,
.enable_counter = hisi_ddrc_pmu_enable_counter,
.disable_counter = hisi_ddrc_pmu_disable_counter,
.enable_counter_int = hisi_ddrc_pmu_enable_counter_int,
.disable_counter_int = hisi_ddrc_pmu_disable_counter_int,
.write_counter = hisi_ddrc_pmu_write_counter,
.read_counter = hisi_ddrc_pmu_read_counter,
.get_event_idx = hisi_ddrc_pmu_v1_get_event_idx,
.start_counters = hisi_ddrc_pmu_v1_start_counters,
.stop_counters = hisi_ddrc_pmu_v1_stop_counters,
.enable_counter = hisi_ddrc_pmu_v1_enable_counter,
.disable_counter = hisi_ddrc_pmu_v1_disable_counter,
.enable_counter_int = hisi_ddrc_pmu_v1_enable_counter_int,
.disable_counter_int = hisi_ddrc_pmu_v1_disable_counter_int,
.write_counter = hisi_ddrc_pmu_v1_write_counter,
.read_counter = hisi_ddrc_pmu_v1_read_counter,
.get_int_status = hisi_ddrc_pmu_v1_get_int_status,
.clear_int_status = hisi_ddrc_pmu_v1_clear_int_status,
};
static const struct hisi_uncore_ops hisi_uncore_ddrc_v2_ops = {
.write_evtype = hisi_ddrc_pmu_write_evtype,
.get_event_idx = hisi_ddrc_pmu_v2_get_event_idx,
.start_counters = hisi_ddrc_pmu_v2_start_counters,
.stop_counters = hisi_ddrc_pmu_v2_stop_counters,
.enable_counter = hisi_ddrc_pmu_v2_enable_counter,
.disable_counter = hisi_ddrc_pmu_v2_disable_counter,
.enable_counter_int = hisi_ddrc_pmu_v2_enable_counter_int,
.disable_counter_int = hisi_ddrc_pmu_v2_disable_counter_int,
.write_counter = hisi_ddrc_pmu_v2_write_counter,
.read_counter = hisi_ddrc_pmu_v2_read_counter,
.get_int_status = hisi_ddrc_pmu_v2_get_int_status,
.clear_int_status = hisi_ddrc_pmu_v2_clear_int_status,
};
static int hisi_ddrc_pmu_dev_probe(struct platform_device *pdev,
@ -353,16 +460,25 @@ static int hisi_ddrc_pmu_dev_probe(struct platform_device *pdev,
if (ret)
return ret;
ret = hisi_ddrc_pmu_init_irq(ddrc_pmu, pdev);
ret = hisi_uncore_pmu_init_irq(ddrc_pmu, pdev);
if (ret)
return ret;
if (ddrc_pmu->identifier >= HISI_PMU_V2) {
ddrc_pmu->counter_bits = 48;
ddrc_pmu->check_event = DDRC_V2_NR_EVENTS;
ddrc_pmu->pmu_events.attr_groups = hisi_ddrc_pmu_v2_attr_groups;
ddrc_pmu->ops = &hisi_uncore_ddrc_v2_ops;
} else {
ddrc_pmu->counter_bits = 32;
ddrc_pmu->check_event = DDRC_V1_NR_EVENTS;
ddrc_pmu->pmu_events.attr_groups = hisi_ddrc_pmu_v1_attr_groups;
ddrc_pmu->ops = &hisi_uncore_ddrc_v1_ops;
}
ddrc_pmu->num_counters = DDRC_NR_COUNTERS;
ddrc_pmu->counter_bits = 32;
ddrc_pmu->ops = &hisi_uncore_ddrc_ops;
ddrc_pmu->dev = &pdev->dev;
ddrc_pmu->on_cpu = -1;
ddrc_pmu->check_event = 7;
return 0;
}
@ -390,8 +506,16 @@ static int hisi_ddrc_pmu_probe(struct platform_device *pdev)
return ret;
}
name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_ddrc%u",
ddrc_pmu->sccl_id, ddrc_pmu->index_id);
if (ddrc_pmu->identifier >= HISI_PMU_V2)
name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
"hisi_sccl%u_ddrc%u_%u",
ddrc_pmu->sccl_id, ddrc_pmu->index_id,
ddrc_pmu->sub_id);
else
name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
"hisi_sccl%u_ddrc%u", ddrc_pmu->sccl_id,
ddrc_pmu->index_id);
ddrc_pmu->pmu = (struct pmu) {
.name = name,
.module = THIS_MODULE,
@ -404,7 +528,7 @@ static int hisi_ddrc_pmu_probe(struct platform_device *pdev)
.start = hisi_uncore_pmu_start,
.stop = hisi_uncore_pmu_stop,
.read = hisi_uncore_pmu_read,
.attr_groups = hisi_ddrc_pmu_attr_groups,
.attr_groups = ddrc_pmu->pmu_events.attr_groups,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};

301
drivers/perf/hisilicon/hisi_uncore_hha_pmu.c
View File

@ -14,7 +14,6 @@
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/list.h>
#include <linux/platform_device.h>
#include <linux/smp.h>
#include "hisi_uncore_pmu.h"
@ -26,18 +25,136 @@
#define HHA_VERSION 0x1cf0
#define HHA_PERF_CTRL 0x1E00
#define HHA_EVENT_CTRL 0x1E04
#define HHA_SRCID_CTRL 0x1E08
#define HHA_DATSRC_CTRL 0x1BF0
#define HHA_EVENT_TYPE0 0x1E80
/*
* Each counter is 48-bits and [48:63] are reserved
* which are Read-As-Zero and Writes-Ignored.
* If the HW version only supports a 48-bit counter, then
* bits [63:48] are reserved, which are Read-As-Zero and
* Writes-Ignored.
*/
#define HHA_CNT0_LOWER 0x1F00
/* HHA has 16-counters */
#define HHA_NR_COUNTERS 0x10
/* HHA PMU v1 has 16 counters and v2 only has 8 counters */
#define HHA_V1_NR_COUNTERS 0x10
#define HHA_V2_NR_COUNTERS 0x8
#define HHA_PERF_CTRL_EN 0x1
#define HHA_TRACETAG_EN BIT(31)
#define HHA_SRCID_EN BIT(2)
#define HHA_SRCID_CMD_SHIFT 6
#define HHA_SRCID_MSK_SHIFT 20
#define HHA_SRCID_CMD GENMASK(16, 6)
#define HHA_SRCID_MSK GENMASK(30, 20)
#define HHA_DATSRC_SKT_EN BIT(23)
#define HHA_EVTYPE_NONE 0xff
#define HHA_V1_NR_EVENT 0x65
#define HHA_V2_NR_EVENT 0xCE
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_cmd, config1, 10, 0);
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_msk, config1, 21, 11);
HISI_PMU_EVENT_ATTR_EXTRACTOR(tracetag_en, config1, 22, 22);
HISI_PMU_EVENT_ATTR_EXTRACTOR(datasrc_skt, config1, 23, 23);
static void hisi_hha_pmu_enable_tracetag(struct perf_event *event)
{
struct hisi_pmu *hha_pmu = to_hisi_pmu(event->pmu);
u32 tt_en = hisi_get_tracetag_en(event);
if (tt_en) {
u32 val;
val = readl(hha_pmu->base + HHA_SRCID_CTRL);
val |= HHA_TRACETAG_EN;
writel(val, hha_pmu->base + HHA_SRCID_CTRL);
}
}
static void hisi_hha_pmu_clear_tracetag(struct perf_event *event)
{
struct hisi_pmu *hha_pmu = to_hisi_pmu(event->pmu);
u32 val;
val = readl(hha_pmu->base + HHA_SRCID_CTRL);
val &= ~HHA_TRACETAG_EN;
writel(val, hha_pmu->base + HHA_SRCID_CTRL);
}
static void hisi_hha_pmu_config_ds(struct perf_event *event)
{
struct hisi_pmu *hha_pmu = to_hisi_pmu(event->pmu);
u32 ds_skt = hisi_get_datasrc_skt(event);
if (ds_skt) {
u32 val;
val = readl(hha_pmu->base + HHA_DATSRC_CTRL);
val |= HHA_DATSRC_SKT_EN;
writel(ds_skt, hha_pmu->base + HHA_DATSRC_CTRL);
}
}
static void hisi_hha_pmu_clear_ds(struct perf_event *event)
{
struct hisi_pmu *hha_pmu = to_hisi_pmu(event->pmu);
u32 ds_skt = hisi_get_datasrc_skt(event);
if (ds_skt) {
u32 val;
val = readl(hha_pmu->base + HHA_DATSRC_CTRL);
val &= ~HHA_DATSRC_SKT_EN;
writel(ds_skt, hha_pmu->base + HHA_DATSRC_CTRL);
}
}
static void hisi_hha_pmu_config_srcid(struct perf_event *event)
{
struct hisi_pmu *hha_pmu = to_hisi_pmu(event->pmu);
u32 cmd = hisi_get_srcid_cmd(event);
if (cmd) {
u32 val, msk;
msk = hisi_get_srcid_msk(event);
val = readl(hha_pmu->base + HHA_SRCID_CTRL);
val |= HHA_SRCID_EN | (cmd << HHA_SRCID_CMD_SHIFT) |
(msk << HHA_SRCID_MSK_SHIFT);
writel(val, hha_pmu->base + HHA_SRCID_CTRL);
}
}
static void hisi_hha_pmu_disable_srcid(struct perf_event *event)
{
struct hisi_pmu *hha_pmu = to_hisi_pmu(event->pmu);
u32 cmd = hisi_get_srcid_cmd(event);
if (cmd) {
u32 val;
val = readl(hha_pmu->base + HHA_SRCID_CTRL);
val &= ~(HHA_SRCID_EN | HHA_SRCID_MSK | HHA_SRCID_CMD);
writel(val, hha_pmu->base + HHA_SRCID_CTRL);
}
}
static void hisi_hha_pmu_enable_filter(struct perf_event *event)
{
if (event->attr.config1 != 0x0) {
hisi_hha_pmu_enable_tracetag(event);
hisi_hha_pmu_config_ds(event);
hisi_hha_pmu_config_srcid(event);
}
}
static void hisi_hha_pmu_disable_filter(struct perf_event *event)
{
if (event->attr.config1 != 0x0) {
hisi_hha_pmu_disable_srcid(event);
hisi_hha_pmu_clear_ds(event);
hisi_hha_pmu_clear_tracetag(event);
}
}
/*
* Select the counter register offset using the counter index
@ -51,29 +168,15 @@ static u32 hisi_hha_pmu_get_counter_offset(int cntr_idx)
static u64 hisi_hha_pmu_read_counter(struct hisi_pmu *hha_pmu,
struct hw_perf_event *hwc)
{
u32 idx = hwc->idx;
if (!hisi_uncore_pmu_counter_valid(hha_pmu, idx)) {
dev_err(hha_pmu->dev, "Unsupported event index:%d!\n", idx);
return 0;
}
/* Read 64 bits and like L3C, top 16 bits are RAZ */
return readq(hha_pmu->base + hisi_hha_pmu_get_counter_offset(idx));
return readq(hha_pmu->base + hisi_hha_pmu_get_counter_offset(hwc->idx));
}
static void hisi_hha_pmu_write_counter(struct hisi_pmu *hha_pmu,
struct hw_perf_event *hwc, u64 val)
{
u32 idx = hwc->idx;
if (!hisi_uncore_pmu_counter_valid(hha_pmu, idx)) {
dev_err(hha_pmu->dev, "Unsupported event index:%d!\n", idx);
return;
}
/* Write 64 bits and like L3C, top 16 bits are WI */
writeq(val, hha_pmu->base + hisi_hha_pmu_get_counter_offset(idx));
writeq(val, hha_pmu->base + hisi_hha_pmu_get_counter_offset(hwc->idx));
}
static void hisi_hha_pmu_write_evtype(struct hisi_pmu *hha_pmu, int idx,
@ -169,65 +272,20 @@ static void hisi_hha_pmu_disable_counter_int(struct hisi_pmu *hha_pmu,
writel(val, hha_pmu->base + HHA_INT_MASK);
}
static irqreturn_t hisi_hha_pmu_isr(int irq, void *dev_id)
static u32 hisi_hha_pmu_get_int_status(struct hisi_pmu *hha_pmu)
{
struct hisi_pmu *hha_pmu = dev_id;
struct perf_event *event;
unsigned long overflown;
int idx;
/* Read HHA_INT_STATUS register */
overflown = readl(hha_pmu->base + HHA_INT_STATUS);
if (!overflown)
return IRQ_NONE;
/*
* Find the counter index which overflowed if the bit was set
* and handle it
*/
for_each_set_bit(idx, &overflown, HHA_NR_COUNTERS) {
/* Write 1 to clear the IRQ status flag */
writel((1 << idx), hha_pmu->base + HHA_INT_CLEAR);
/* Get the corresponding event struct */
event = hha_pmu->pmu_events.hw_events[idx];
if (!event)
continue;
hisi_uncore_pmu_event_update(event);
hisi_uncore_pmu_set_event_period(event);
}
return IRQ_HANDLED;
return readl(hha_pmu->base + HHA_INT_STATUS);
}
static int hisi_hha_pmu_init_irq(struct hisi_pmu *hha_pmu,
struct platform_device *pdev)
static void hisi_hha_pmu_clear_int_status(struct hisi_pmu *hha_pmu, int idx)
{
int irq, ret;
/* Read and init IRQ */
irq = platform_get_irq(pdev, 0);
if (irq < 0)
return irq;
ret = devm_request_irq(&pdev->dev, irq, hisi_hha_pmu_isr,
IRQF_NOBALANCING | IRQF_NO_THREAD,
dev_name(&pdev->dev), hha_pmu);
if (ret < 0) {
dev_err(&pdev->dev,
"Fail to request IRQ:%d ret:%d\n", irq, ret);
return ret;
}
hha_pmu->irq = irq;
return 0;
writel(1 << idx, hha_pmu->base + HHA_INT_CLEAR);
}
static const struct acpi_device_id hisi_hha_pmu_acpi_match[] = {
{ "HISI0243", },
{},
{ "HISI0244", },
{}
};
MODULE_DEVICE_TABLE(acpi, hisi_hha_pmu_acpi_match);
@ -237,13 +295,6 @@ static int hisi_hha_pmu_init_data(struct platform_device *pdev,
unsigned long long id;
acpi_status status;
status = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
"_UID", NULL, &id);
if (ACPI_FAILURE(status))
return -EINVAL;
hha_pmu->index_id = id;
/*
* Use SCCL_ID and UID to identify the HHA PMU, while
* SCCL_ID is in MPIDR[aff2].
@ -253,6 +304,22 @@ static int hisi_hha_pmu_init_data(struct platform_device *pdev,
dev_err(&pdev->dev, "Can not read hha sccl-id!\n");
return -EINVAL;
}
/*
* Early versions of BIOS support _UID by mistake, so we support
* both "hisilicon, idx-id" as preference, if available.
*/
if (device_property_read_u32(&pdev->dev, "hisilicon,idx-id",
&hha_pmu->index_id)) {
status = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
"_UID", NULL, &id);
if (ACPI_FAILURE(status)) {
dev_err(&pdev->dev, "Cannot read idx-id!\n");
return -EINVAL;
}
hha_pmu->index_id = id;
}
/* HHA PMUs only share the same SCCL */
hha_pmu->ccl_id = -1;
@ -267,17 +334,31 @@ static int hisi_hha_pmu_init_data(struct platform_device *pdev,
return 0;
}
static struct attribute *hisi_hha_pmu_format_attr[] = {
static struct attribute *hisi_hha_pmu_v1_format_attr[] = {
HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
NULL,
};
static const struct attribute_group hisi_hha_pmu_format_group = {
static const struct attribute_group hisi_hha_pmu_v1_format_group = {
.name = "format",
.attrs = hisi_hha_pmu_format_attr,
.attrs = hisi_hha_pmu_v1_format_attr,
};
static struct attribute *hisi_hha_pmu_events_attr[] = {
static struct attribute *hisi_hha_pmu_v2_format_attr[] = {
HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
HISI_PMU_FORMAT_ATTR(srcid_cmd, "config1:0-10"),
HISI_PMU_FORMAT_ATTR(srcid_msk, "config1:11-21"),
HISI_PMU_FORMAT_ATTR(tracetag_en, "config1:22"),
HISI_PMU_FORMAT_ATTR(datasrc_skt, "config1:23"),
NULL
};
static const struct attribute_group hisi_hha_pmu_v2_format_group = {
.name = "format",
.attrs = hisi_hha_pmu_v2_format_attr,
};
static struct attribute *hisi_hha_pmu_v1_events_attr[] = {
HISI_PMU_EVENT_ATTR(rx_ops_num, 0x00),
HISI_PMU_EVENT_ATTR(rx_outer, 0x01),
HISI_PMU_EVENT_ATTR(rx_sccl, 0x02),
@ -307,9 +388,23 @@ static struct attribute *hisi_hha_pmu_events_attr[] = {
NULL,
};
static const struct attribute_group hisi_hha_pmu_events_group = {
static const struct attribute_group hisi_hha_pmu_v1_events_group = {
.name = "events",
.attrs = hisi_hha_pmu_events_attr,
.attrs = hisi_hha_pmu_v1_events_attr,
};
static struct attribute *hisi_hha_pmu_v2_events_attr[] = {
HISI_PMU_EVENT_ATTR(rx_ops_num, 0x00),
HISI_PMU_EVENT_ATTR(rx_outer, 0x01),
HISI_PMU_EVENT_ATTR(rx_sccl, 0x02),
HISI_PMU_EVENT_ATTR(hha_retry, 0x2e),
HISI_PMU_EVENT_ATTR(cycles, 0x55),
NULL
};
static const struct attribute_group hisi_hha_pmu_v2_events_group = {
.name = "events",
.attrs = hisi_hha_pmu_v2_events_attr,
};
static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
@ -335,14 +430,22 @@ static const struct attribute_group hisi_hha_pmu_identifier_group = {
.attrs = hisi_hha_pmu_identifier_attrs,
};
static const struct attribute_group *hisi_hha_pmu_attr_groups[] = {
&hisi_hha_pmu_format_group,
&hisi_hha_pmu_events_group,
static const struct attribute_group *hisi_hha_pmu_v1_attr_groups[] = {
&hisi_hha_pmu_v1_format_group,
&hisi_hha_pmu_v1_events_group,
&hisi_hha_pmu_cpumask_attr_group,
&hisi_hha_pmu_identifier_group,
NULL,
};
static const struct attribute_group *hisi_hha_pmu_v2_attr_groups[] = {
&hisi_hha_pmu_v2_format_group,
&hisi_hha_pmu_v2_events_group,
&hisi_hha_pmu_cpumask_attr_group,
&hisi_hha_pmu_identifier_group,
NULL
};
static const struct hisi_uncore_ops hisi_uncore_hha_ops = {
.write_evtype = hisi_hha_pmu_write_evtype,
.get_event_idx = hisi_uncore_pmu_get_event_idx,
@ -354,6 +457,10 @@ static const struct hisi_uncore_ops hisi_uncore_hha_ops = {
.disable_counter_int = hisi_hha_pmu_disable_counter_int,
.write_counter = hisi_hha_pmu_write_counter,
.read_counter = hisi_hha_pmu_read_counter,
.get_int_status = hisi_hha_pmu_get_int_status,
.clear_int_status = hisi_hha_pmu_clear_int_status,
.enable_filter = hisi_hha_pmu_enable_filter,
.disable_filter = hisi_hha_pmu_disable_filter,
};
static int hisi_hha_pmu_dev_probe(struct platform_device *pdev,
@ -365,16 +472,24 @@ static int hisi_hha_pmu_dev_probe(struct platform_device *pdev,
if (ret)
return ret;
ret = hisi_hha_pmu_init_irq(hha_pmu, pdev);
ret = hisi_uncore_pmu_init_irq(hha_pmu, pdev);
if (ret)
return ret;
hha_pmu->num_counters = HHA_NR_COUNTERS;
hha_pmu->counter_bits = 48;
if (hha_pmu->identifier >= HISI_PMU_V2) {
hha_pmu->counter_bits = 64;
hha_pmu->check_event = HHA_V2_NR_EVENT;
hha_pmu->pmu_events.attr_groups = hisi_hha_pmu_v2_attr_groups;
hha_pmu->num_counters = HHA_V2_NR_COUNTERS;
} else {
hha_pmu->counter_bits = 48;
hha_pmu->check_event = HHA_V1_NR_EVENT;
hha_pmu->pmu_events.attr_groups = hisi_hha_pmu_v1_attr_groups;
hha_pmu->num_counters = HHA_V1_NR_COUNTERS;
}
hha_pmu->ops = &hisi_uncore_hha_ops;
hha_pmu->dev = &pdev->dev;
hha_pmu->on_cpu = -1;
hha_pmu->check_event = 0x65;
return 0;
}
@ -416,7 +531,7 @@ static int hisi_hha_pmu_probe(struct platform_device *pdev)
.start = hisi_uncore_pmu_start,
.stop = hisi_uncore_pmu_stop,
.read = hisi_uncore_pmu_read,
.attr_groups = hisi_hha_pmu_attr_groups,
.attr_groups = hha_pmu->pmu_events.attr_groups,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};

355
drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c
View File

@ -14,7 +14,6 @@
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/list.h>
#include <linux/platform_device.h>
#include <linux/smp.h>
#include "hisi_uncore_pmu.h"
@ -24,12 +23,17 @@
#define L3C_INT_MASK 0x0800
#define L3C_INT_STATUS 0x0808
#define L3C_INT_CLEAR 0x080c
#define L3C_CORE_CTRL 0x1b04
#define L3C_TRACETAG_CTRL 0x1b20
#define L3C_DATSRC_TYPE 0x1b48
#define L3C_DATSRC_CTRL 0x1bf0
#define L3C_EVENT_CTRL 0x1c00
#define L3C_VERSION 0x1cf0
#define L3C_EVENT_TYPE0 0x1d00
/*
* Each counter is 48-bits and [48:63] are reserved
* which are Read-As-Zero and Writes-Ignored.
* If the HW version only supports a 48-bit counter, then
* bits [63:48] are reserved, which are Read-As-Zero and
* Writes-Ignored.
*/
#define L3C_CNTR0_LOWER 0x1e00
@ -37,7 +41,186 @@
#define L3C_NR_COUNTERS 0x8
#define L3C_PERF_CTRL_EN 0x10000
#define L3C_TRACETAG_EN BIT(31)
#define L3C_TRACETAG_REQ_SHIFT 7
#define L3C_TRACETAG_MARK_EN BIT(0)
#define L3C_TRACETAG_REQ_EN (L3C_TRACETAG_MARK_EN | BIT(2))
#define L3C_TRACETAG_CORE_EN (L3C_TRACETAG_MARK_EN | BIT(3))
#define L3C_CORE_EN BIT(20)
#define L3C_COER_NONE 0x0
#define L3C_DATSRC_MASK 0xFF
#define L3C_DATSRC_SKT_EN BIT(23)
#define L3C_DATSRC_NONE 0x0
#define L3C_EVTYPE_NONE 0xff
#define L3C_V1_NR_EVENTS 0x59
#define L3C_V2_NR_EVENTS 0xFF
HISI_PMU_EVENT_ATTR_EXTRACTOR(tt_core, config1, 7, 0);
HISI_PMU_EVENT_ATTR_EXTRACTOR(tt_req, config1, 10, 8);
HISI_PMU_EVENT_ATTR_EXTRACTOR(datasrc_cfg, config1, 15, 11);
HISI_PMU_EVENT_ATTR_EXTRACTOR(datasrc_skt, config1, 16, 16);
static void hisi_l3c_pmu_config_req_tracetag(struct perf_event *event)
{
struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
u32 tt_req = hisi_get_tt_req(event);
if (tt_req) {
u32 val;
/* Set request-type for tracetag */
val = readl(l3c_pmu->base + L3C_TRACETAG_CTRL);
val |= tt_req << L3C_TRACETAG_REQ_SHIFT;
val |= L3C_TRACETAG_REQ_EN;
writel(val, l3c_pmu->base + L3C_TRACETAG_CTRL);
/* Enable request-tracetag statistics */
val = readl(l3c_pmu->base + L3C_PERF_CTRL);
val |= L3C_TRACETAG_EN;
writel(val, l3c_pmu->base + L3C_PERF_CTRL);
}
}
static void hisi_l3c_pmu_clear_req_tracetag(struct perf_event *event)
{
struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
u32 tt_req = hisi_get_tt_req(event);
if (tt_req) {
u32 val;
/* Clear request-type */
val = readl(l3c_pmu->base + L3C_TRACETAG_CTRL);
val &= ~(tt_req << L3C_TRACETAG_REQ_SHIFT);
val &= ~L3C_TRACETAG_REQ_EN;
writel(val, l3c_pmu->base + L3C_TRACETAG_CTRL);
/* Disable request-tracetag statistics */
val = readl(l3c_pmu->base + L3C_PERF_CTRL);
val &= ~L3C_TRACETAG_EN;
writel(val, l3c_pmu->base + L3C_PERF_CTRL);
}
}
static void hisi_l3c_pmu_write_ds(struct perf_event *event, u32 ds_cfg)
{
struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
struct hw_perf_event *hwc = &event->hw;
u32 reg, reg_idx, shift, val;
int idx = hwc->idx;
/*
* Select the appropriate datasource register(L3C_DATSRC_TYPE0/1).
* There are 2 datasource ctrl register for the 8 hardware counters.
* Datasrc is 8-bits and for the former 4 hardware counters,
* L3C_DATSRC_TYPE0 is chosen. For the latter 4 hardware counters,
* L3C_DATSRC_TYPE1 is chosen.
*/
reg = L3C_DATSRC_TYPE + (idx / 4) * 4;
reg_idx = idx % 4;
shift = 8 * reg_idx;
val = readl(l3c_pmu->base + reg);
val &= ~(L3C_DATSRC_MASK << shift);
val |= ds_cfg << shift;
writel(val, l3c_pmu->base + reg);
}
static void hisi_l3c_pmu_config_ds(struct perf_event *event)
{
struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
u32 ds_cfg = hisi_get_datasrc_cfg(event);
u32 ds_skt = hisi_get_datasrc_skt(event);
if (ds_cfg)
hisi_l3c_pmu_write_ds(event, ds_cfg);
if (ds_skt) {
u32 val;
val = readl(l3c_pmu->base + L3C_DATSRC_CTRL);
val |= L3C_DATSRC_SKT_EN;
writel(val, l3c_pmu->base + L3C_DATSRC_CTRL);
}
}
static void hisi_l3c_pmu_clear_ds(struct perf_event *event)
{
struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
u32 ds_cfg = hisi_get_datasrc_cfg(event);
u32 ds_skt = hisi_get_datasrc_skt(event);
if (ds_cfg)
hisi_l3c_pmu_write_ds(event, L3C_DATSRC_NONE);
if (ds_skt) {
u32 val;
val = readl(l3c_pmu->base + L3C_DATSRC_CTRL);
val &= ~L3C_DATSRC_SKT_EN;
writel(val, l3c_pmu->base + L3C_DATSRC_CTRL);
}
}
static void hisi_l3c_pmu_config_core_tracetag(struct perf_event *event)
{
struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
u32 core = hisi_get_tt_core(event);
if (core) {
u32 val;
/* Config and enable core information */
writel(core, l3c_pmu->base + L3C_CORE_CTRL);
val = readl(l3c_pmu->base + L3C_PERF_CTRL);
val |= L3C_CORE_EN;
writel(val, l3c_pmu->base + L3C_PERF_CTRL);
/* Enable core-tracetag statistics */
val = readl(l3c_pmu->base + L3C_TRACETAG_CTRL);
val |= L3C_TRACETAG_CORE_EN;
writel(val, l3c_pmu->base + L3C_TRACETAG_CTRL);
}
}
static void hisi_l3c_pmu_clear_core_tracetag(struct perf_event *event)
{
struct hisi_pmu *l3c_pmu = to_hisi_pmu(event->pmu);
u32 core = hisi_get_tt_core(event);
if (core) {
u32 val;
/* Clear core information */
writel(L3C_COER_NONE, l3c_pmu->base + L3C_CORE_CTRL);
val = readl(l3c_pmu->base + L3C_PERF_CTRL);
val &= ~L3C_CORE_EN;
writel(val, l3c_pmu->base + L3C_PERF_CTRL);
/* Disable core-tracetag statistics */
val = readl(l3c_pmu->base + L3C_TRACETAG_CTRL);
val &= ~L3C_TRACETAG_CORE_EN;
writel(val, l3c_pmu->base + L3C_TRACETAG_CTRL);
}
}
static void hisi_l3c_pmu_enable_filter(struct perf_event *event)
{
if (event->attr.config1 != 0x0) {
hisi_l3c_pmu_config_req_tracetag(event);
hisi_l3c_pmu_config_core_tracetag(event);
hisi_l3c_pmu_config_ds(event);
}
}
static void hisi_l3c_pmu_disable_filter(struct perf_event *event)
{
if (event->attr.config1 != 0x0) {
hisi_l3c_pmu_clear_ds(event);
hisi_l3c_pmu_clear_core_tracetag(event);
hisi_l3c_pmu_clear_req_tracetag(event);
}
}
/*
* Select the counter register offset using the counter index
@ -50,29 +233,13 @@ static u32 hisi_l3c_pmu_get_counter_offset(int cntr_idx)
static u64 hisi_l3c_pmu_read_counter(struct hisi_pmu *l3c_pmu,
struct hw_perf_event *hwc)
{
u32 idx = hwc->idx;
if (!hisi_uncore_pmu_counter_valid(l3c_pmu, idx)) {
dev_err(l3c_pmu->dev, "Unsupported event index:%d!\n", idx);
return 0;
}
/* Read 64-bits and the upper 16 bits are RAZ */
return readq(l3c_pmu->base + hisi_l3c_pmu_get_counter_offset(idx));
return readq(l3c_pmu->base + hisi_l3c_pmu_get_counter_offset(hwc->idx));
}
static void hisi_l3c_pmu_write_counter(struct hisi_pmu *l3c_pmu,
struct hw_perf_event *hwc, u64 val)
{
u32 idx = hwc->idx;
if (!hisi_uncore_pmu_counter_valid(l3c_pmu, idx)) {
dev_err(l3c_pmu->dev, "Unsupported event index:%d!\n", idx);
return;
}
/* Write 64-bits and the upper 16 bits are WI */
writeq(val, l3c_pmu->base + hisi_l3c_pmu_get_counter_offset(idx));
writeq(val, l3c_pmu->base + hisi_l3c_pmu_get_counter_offset(hwc->idx));
}
static void hisi_l3c_pmu_write_evtype(struct hisi_pmu *l3c_pmu, int idx,
@ -168,81 +335,26 @@ static void hisi_l3c_pmu_disable_counter_int(struct hisi_pmu *l3c_pmu,
writel(val, l3c_pmu->base + L3C_INT_MASK);
}
static irqreturn_t hisi_l3c_pmu_isr(int irq, void *dev_id)
static u32 hisi_l3c_pmu_get_int_status(struct hisi_pmu *l3c_pmu)
{
struct hisi_pmu *l3c_pmu = dev_id;
struct perf_event *event;
unsigned long overflown;
int idx;
/* Read L3C_INT_STATUS register */
overflown = readl(l3c_pmu->base + L3C_INT_STATUS);
if (!overflown)
return IRQ_NONE;
/*
* Find the counter index which overflowed if the bit was set
* and handle it.
*/
for_each_set_bit(idx, &overflown, L3C_NR_COUNTERS) {
/* Write 1 to clear the IRQ status flag */
writel((1 << idx), l3c_pmu->base + L3C_INT_CLEAR);
/* Get the corresponding event struct */
event = l3c_pmu->pmu_events.hw_events[idx];
if (!event)
continue;
hisi_uncore_pmu_event_update(event);
hisi_uncore_pmu_set_event_period(event);
}
return IRQ_HANDLED;
return readl(l3c_pmu->base + L3C_INT_STATUS);
}
static int hisi_l3c_pmu_init_irq(struct hisi_pmu *l3c_pmu,
struct platform_device *pdev)
static void hisi_l3c_pmu_clear_int_status(struct hisi_pmu *l3c_pmu, int idx)
{
int irq, ret;
/* Read and init IRQ */
irq = platform_get_irq(pdev, 0);
if (irq < 0)
return irq;
ret = devm_request_irq(&pdev->dev, irq, hisi_l3c_pmu_isr,
IRQF_NOBALANCING | IRQF_NO_THREAD,
dev_name(&pdev->dev), l3c_pmu);
if (ret < 0) {
dev_err(&pdev->dev,
"Fail to request IRQ:%d ret:%d\n", irq, ret);
return ret;
}
l3c_pmu->irq = irq;
return 0;
writel(1 << idx, l3c_pmu->base + L3C_INT_CLEAR);
}
static const struct acpi_device_id hisi_l3c_pmu_acpi_match[] = {
{ "HISI0213", },
{},
{ "HISI0214", },
{}
};
MODULE_DEVICE_TABLE(acpi, hisi_l3c_pmu_acpi_match);
static int hisi_l3c_pmu_init_data(struct platform_device *pdev,
struct hisi_pmu *l3c_pmu)
{
unsigned long long id;
acpi_status status;
status = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
"_UID", NULL, &id);
if (ACPI_FAILURE(status))
return -EINVAL;
l3c_pmu->index_id = id;
/*
* Use the SCCL_ID and CCL_ID to identify the L3C PMU, while
* SCCL_ID is in MPIDR[aff2] and CCL_ID is in MPIDR[aff1].
@ -270,17 +382,31 @@ static int hisi_l3c_pmu_init_data(struct platform_device *pdev,
return 0;
}
static struct attribute *hisi_l3c_pmu_format_attr[] = {
static struct attribute *hisi_l3c_pmu_v1_format_attr[] = {
HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
NULL,
};
static const struct attribute_group hisi_l3c_pmu_format_group = {
static const struct attribute_group hisi_l3c_pmu_v1_format_group = {
.name = "format",
.attrs = hisi_l3c_pmu_format_attr,
.attrs = hisi_l3c_pmu_v1_format_attr,
};
static struct attribute *hisi_l3c_pmu_events_attr[] = {
static struct attribute *hisi_l3c_pmu_v2_format_attr[] = {
HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
HISI_PMU_FORMAT_ATTR(tt_core, "config1:0-7"),
HISI_PMU_FORMAT_ATTR(tt_req, "config1:8-10"),
HISI_PMU_FORMAT_ATTR(datasrc_cfg, "config1:11-15"),
HISI_PMU_FORMAT_ATTR(datasrc_skt, "config1:16"),
NULL
};
static const struct attribute_group hisi_l3c_pmu_v2_format_group = {
.name = "format",
.attrs = hisi_l3c_pmu_v2_format_attr,
};
static struct attribute *hisi_l3c_pmu_v1_events_attr[] = {
HISI_PMU_EVENT_ATTR(rd_cpipe, 0x00),
HISI_PMU_EVENT_ATTR(wr_cpipe, 0x01),
HISI_PMU_EVENT_ATTR(rd_hit_cpipe, 0x02),
@ -297,9 +423,22 @@ static struct attribute *hisi_l3c_pmu_events_attr[] = {
NULL,
};
static const struct attribute_group hisi_l3c_pmu_events_group = {
static const struct attribute_group hisi_l3c_pmu_v1_events_group = {
.name = "events",
.attrs = hisi_l3c_pmu_events_attr,
.attrs = hisi_l3c_pmu_v1_events_attr,
};
static struct attribute *hisi_l3c_pmu_v2_events_attr[] = {
HISI_PMU_EVENT_ATTR(l3c_hit, 0x48),
HISI_PMU_EVENT_ATTR(cycles, 0x7f),
HISI_PMU_EVENT_ATTR(l3c_ref, 0xb8),
HISI_PMU_EVENT_ATTR(dat_access, 0xb9),
NULL
};
static const struct attribute_group hisi_l3c_pmu_v2_events_group = {
.name = "events",
.attrs = hisi_l3c_pmu_v2_events_attr,
};
static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
@ -325,14 +464,22 @@ static const struct attribute_group hisi_l3c_pmu_identifier_group = {
.attrs = hisi_l3c_pmu_identifier_attrs,
};
static const struct attribute_group *hisi_l3c_pmu_attr_groups[] = {
&hisi_l3c_pmu_format_group,
&hisi_l3c_pmu_events_group,
static const struct attribute_group *hisi_l3c_pmu_v1_attr_groups[] = {
&hisi_l3c_pmu_v1_format_group,
&hisi_l3c_pmu_v1_events_group,
&hisi_l3c_pmu_cpumask_attr_group,
&hisi_l3c_pmu_identifier_group,
NULL,
};
static const struct attribute_group *hisi_l3c_pmu_v2_attr_groups[] = {
&hisi_l3c_pmu_v2_format_group,
&hisi_l3c_pmu_v2_events_group,
&hisi_l3c_pmu_cpumask_attr_group,
&hisi_l3c_pmu_identifier_group,
NULL
};
static const struct hisi_uncore_ops hisi_uncore_l3c_ops = {
.write_evtype = hisi_l3c_pmu_write_evtype,
.get_event_idx = hisi_uncore_pmu_get_event_idx,
@ -344,6 +491,10 @@ static const struct hisi_uncore_ops hisi_uncore_l3c_ops = {
.disable_counter_int = hisi_l3c_pmu_disable_counter_int,
.write_counter = hisi_l3c_pmu_write_counter,
.read_counter = hisi_l3c_pmu_read_counter,
.get_int_status = hisi_l3c_pmu_get_int_status,
.clear_int_status = hisi_l3c_pmu_clear_int_status,
.enable_filter = hisi_l3c_pmu_enable_filter,
.disable_filter = hisi_l3c_pmu_disable_filter,
};
static int hisi_l3c_pmu_dev_probe(struct platform_device *pdev,
@ -355,16 +506,24 @@ static int hisi_l3c_pmu_dev_probe(struct platform_device *pdev,
if (ret)
return ret;
ret = hisi_l3c_pmu_init_irq(l3c_pmu, pdev);
ret = hisi_uncore_pmu_init_irq(l3c_pmu, pdev);
if (ret)
return ret;
if (l3c_pmu->identifier >= HISI_PMU_V2) {
l3c_pmu->counter_bits = 64;
l3c_pmu->check_event = L3C_V2_NR_EVENTS;
l3c_pmu->pmu_events.attr_groups = hisi_l3c_pmu_v2_attr_groups;
} else {
l3c_pmu->counter_bits = 48;
l3c_pmu->check_event = L3C_V1_NR_EVENTS;
l3c_pmu->pmu_events.attr_groups = hisi_l3c_pmu_v1_attr_groups;
}
l3c_pmu->num_counters = L3C_NR_COUNTERS;
l3c_pmu->counter_bits = 48;
l3c_pmu->ops = &hisi_uncore_l3c_ops;
l3c_pmu->dev = &pdev->dev;
l3c_pmu->on_cpu = -1;
l3c_pmu->check_event = 0x59;
return 0;
}
@ -392,8 +551,12 @@ static int hisi_l3c_pmu_probe(struct platform_device *pdev)
return ret;
}
/*
* CCL_ID is used to identify the L3C in the same SCCL which was
* used _UID by mistake.
*/
name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_l3c%u",
l3c_pmu->sccl_id, l3c_pmu->index_id);
l3c_pmu->sccl_id, l3c_pmu->ccl_id);
l3c_pmu->pmu = (struct pmu) {
.name = name,
.module = THIS_MODULE,
@ -406,7 +569,7 @@ static int hisi_l3c_pmu_probe(struct platform_device *pdev)
.start = hisi_uncore_pmu_start,
.stop = hisi_uncore_pmu_stop,
.read = hisi_uncore_pmu_read,
.attr_groups = hisi_l3c_pmu_attr_groups,
.attr_groups = l3c_pmu->pmu_events.attr_groups,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};

500
drivers/perf/hisilicon/hisi_uncore_pa_pmu.c Normal file
View File

@ -0,0 +1,500 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* HiSilicon PA uncore Hardware event counters support
*
* Copyright (C) 2020 HiSilicon Limited
* Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
*
* This code is based on the uncore PMUs like arm-cci and arm-ccn.
*/
#include <linux/acpi.h>
#include <linux/cpuhotplug.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/list.h>
#include <linux/smp.h>
#include "hisi_uncore_pmu.h"
/* PA register definition */
#define PA_PERF_CTRL 0x1c00
#define PA_EVENT_CTRL 0x1c04
#define PA_TT_CTRL 0x1c08
#define PA_TGTID_CTRL 0x1c14
#define PA_SRCID_CTRL 0x1c18
#define PA_INT_MASK 0x1c70
#define PA_INT_STATUS 0x1c78
#define PA_INT_CLEAR 0x1c7c
#define PA_EVENT_TYPE0 0x1c80
#define PA_PMU_VERSION 0x1cf0
#define PA_EVENT_CNT0_L 0x1f00
#define PA_EVTYPE_MASK 0xff
#define PA_NR_COUNTERS 0x8
#define PA_PERF_CTRL_EN BIT(0)
#define PA_TRACETAG_EN BIT(4)
#define PA_TGTID_EN BIT(11)
#define PA_SRCID_EN BIT(11)
#define PA_TGTID_NONE 0
#define PA_SRCID_NONE 0
#define PA_TGTID_MSK_SHIFT 12
#define PA_SRCID_MSK_SHIFT 12
HISI_PMU_EVENT_ATTR_EXTRACTOR(tgtid_cmd, config1, 10, 0);
HISI_PMU_EVENT_ATTR_EXTRACTOR(tgtid_msk, config1, 21, 11);
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_cmd, config1, 32, 22);
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_msk, config1, 43, 33);
HISI_PMU_EVENT_ATTR_EXTRACTOR(tracetag_en, config1, 44, 44);
static void hisi_pa_pmu_enable_tracetag(struct perf_event *event)
{
struct hisi_pmu *pa_pmu = to_hisi_pmu(event->pmu);
u32 tt_en = hisi_get_tracetag_en(event);
if (tt_en) {
u32 val;
val = readl(pa_pmu->base + PA_TT_CTRL);
val |= PA_TRACETAG_EN;
writel(val, pa_pmu->base + PA_TT_CTRL);
}
}
static void hisi_pa_pmu_clear_tracetag(struct perf_event *event)
{
struct hisi_pmu *pa_pmu = to_hisi_pmu(event->pmu);
u32 tt_en = hisi_get_tracetag_en(event);
if (tt_en) {
u32 val;
val = readl(pa_pmu->base + PA_TT_CTRL);
val &= ~PA_TRACETAG_EN;
writel(val, pa_pmu->base + PA_TT_CTRL);
}
}
static void hisi_pa_pmu_config_tgtid(struct perf_event *event)
{
struct hisi_pmu *pa_pmu = to_hisi_pmu(event->pmu);
u32 cmd = hisi_get_tgtid_cmd(event);
if (cmd) {
u32 msk = hisi_get_tgtid_msk(event);
u32 val = cmd | PA_TGTID_EN | (msk << PA_TGTID_MSK_SHIFT);
writel(val, pa_pmu->base + PA_TGTID_CTRL);
}
}
static void hisi_pa_pmu_clear_tgtid(struct perf_event *event)
{
struct hisi_pmu *pa_pmu = to_hisi_pmu(event->pmu);
u32 cmd = hisi_get_tgtid_cmd(event);
if (cmd)
writel(PA_TGTID_NONE, pa_pmu->base + PA_TGTID_CTRL);
}
static void hisi_pa_pmu_config_srcid(struct perf_event *event)
{
struct hisi_pmu *pa_pmu = to_hisi_pmu(event->pmu);
u32 cmd = hisi_get_srcid_cmd(event);
if (cmd) {
u32 msk = hisi_get_srcid_msk(event);
u32 val = cmd | PA_SRCID_EN | (msk << PA_SRCID_MSK_SHIFT);
writel(val, pa_pmu->base + PA_SRCID_CTRL);
}
}
static void hisi_pa_pmu_clear_srcid(struct perf_event *event)
{
struct hisi_pmu *pa_pmu = to_hisi_pmu(event->pmu);
u32 cmd = hisi_get_srcid_cmd(event);
if (cmd)
writel(PA_SRCID_NONE, pa_pmu->base + PA_SRCID_CTRL);
}
static void hisi_pa_pmu_enable_filter(struct perf_event *event)
{
if (event->attr.config1 != 0x0) {
hisi_pa_pmu_enable_tracetag(event);
hisi_pa_pmu_config_srcid(event);
hisi_pa_pmu_config_tgtid(event);
}
}
static void hisi_pa_pmu_disable_filter(struct perf_event *event)
{
if (event->attr.config1 != 0x0) {
hisi_pa_pmu_clear_tgtid(event);
hisi_pa_pmu_clear_srcid(event);
hisi_pa_pmu_clear_tracetag(event);
}
}
static u32 hisi_pa_pmu_get_counter_offset(int idx)
{
return (PA_EVENT_CNT0_L + idx * 8);
}
static u64 hisi_pa_pmu_read_counter(struct hisi_pmu *pa_pmu,
struct hw_perf_event *hwc)
{
return readq(pa_pmu->base + hisi_pa_pmu_get_counter_offset(hwc->idx));
}
static void hisi_pa_pmu_write_counter(struct hisi_pmu *pa_pmu,
struct hw_perf_event *hwc, u64 val)
{
writeq(val, pa_pmu->base + hisi_pa_pmu_get_counter_offset(hwc->idx));
}
static void hisi_pa_pmu_write_evtype(struct hisi_pmu *pa_pmu, int idx,
u32 type)
{
u32 reg, reg_idx, shift, val;
/*
* Select the appropriate event select register(PA_EVENT_TYPE0/1).
* There are 2 event select registers for the 8 hardware counters.
* Event code is 8-bits and for the former 4 hardware counters,
* PA_EVENT_TYPE0 is chosen. For the latter 4 hardware counters,
* PA_EVENT_TYPE1 is chosen.
*/
reg = PA_EVENT_TYPE0 + (idx / 4) * 4;
reg_idx = idx % 4;
shift = 8 * reg_idx;
/* Write event code to pa_EVENT_TYPEx Register */
val = readl(pa_pmu->base + reg);
val &= ~(PA_EVTYPE_MASK << shift);
val |= (type << shift);
writel(val, pa_pmu->base + reg);
}
static void hisi_pa_pmu_start_counters(struct hisi_pmu *pa_pmu)
{
u32 val;
val = readl(pa_pmu->base + PA_PERF_CTRL);
val |= PA_PERF_CTRL_EN;
writel(val, pa_pmu->base + PA_PERF_CTRL);
}
static void hisi_pa_pmu_stop_counters(struct hisi_pmu *pa_pmu)
{
u32 val;
val = readl(pa_pmu->base + PA_PERF_CTRL);
val &= ~(PA_PERF_CTRL_EN);
writel(val, pa_pmu->base + PA_PERF_CTRL);
}
static void hisi_pa_pmu_enable_counter(struct hisi_pmu *pa_pmu,
struct hw_perf_event *hwc)
{
u32 val;
/* Enable counter index in PA_EVENT_CTRL register */
val = readl(pa_pmu->base + PA_EVENT_CTRL);
val |= 1 << hwc->idx;
writel(val, pa_pmu->base + PA_EVENT_CTRL);
}
static void hisi_pa_pmu_disable_counter(struct hisi_pmu *pa_pmu,
struct hw_perf_event *hwc)
{
u32 val;
/* Clear counter index in PA_EVENT_CTRL register */
val = readl(pa_pmu->base + PA_EVENT_CTRL);
val &= ~(1 << hwc->idx);
writel(val, pa_pmu->base + PA_EVENT_CTRL);
}
static void hisi_pa_pmu_enable_counter_int(struct hisi_pmu *pa_pmu,
struct hw_perf_event *hwc)
{
u32 val;
/* Write 0 to enable interrupt */
val = readl(pa_pmu->base + PA_INT_MASK);
val &= ~(1 << hwc->idx);
writel(val, pa_pmu->base + PA_INT_MASK);
}
static void hisi_pa_pmu_disable_counter_int(struct hisi_pmu *pa_pmu,
struct hw_perf_event *hwc)
{
u32 val;
/* Write 1 to mask interrupt */
val = readl(pa_pmu->base + PA_INT_MASK);
val |= 1 << hwc->idx;
writel(val, pa_pmu->base + PA_INT_MASK);
}
static u32 hisi_pa_pmu_get_int_status(struct hisi_pmu *pa_pmu)
{
return readl(pa_pmu->base + PA_INT_STATUS);
}
static void hisi_pa_pmu_clear_int_status(struct hisi_pmu *pa_pmu, int idx)
{
writel(1 << idx, pa_pmu->base + PA_INT_CLEAR);
}
static const struct acpi_device_id hisi_pa_pmu_acpi_match[] = {
{ "HISI0273", },
{}
};
MODULE_DEVICE_TABLE(acpi, hisi_pa_pmu_acpi_match);
static int hisi_pa_pmu_init_data(struct platform_device *pdev,
struct hisi_pmu *pa_pmu)
{
/*
* Use the SCCL_ID and the index ID to identify the PA PMU,
* while SCCL_ID is the nearst SCCL_ID from this SICL and
* CPU core is chosen from this SCCL to manage this PMU.
*/
if (device_property_read_u32(&pdev->dev, "hisilicon,scl-id",
&pa_pmu->sccl_id)) {
dev_err(&pdev->dev, "Cannot read sccl-id!\n");
return -EINVAL;
}
if (device_property_read_u32(&pdev->dev, "hisilicon,idx-id",
&pa_pmu->index_id)) {
dev_err(&pdev->dev, "Cannot read idx-id!\n");
return -EINVAL;
}
pa_pmu->ccl_id = -1;
pa_pmu->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(pa_pmu->base)) {
dev_err(&pdev->dev, "ioremap failed for pa_pmu resource.\n");
return PTR_ERR(pa_pmu->base);
}
pa_pmu->identifier = readl(pa_pmu->base + PA_PMU_VERSION);
return 0;
}
static struct attribute *hisi_pa_pmu_v2_format_attr[] = {
HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
HISI_PMU_FORMAT_ATTR(tgtid_cmd, "config1:0-10"),
HISI_PMU_FORMAT_ATTR(tgtid_msk, "config1:11-21"),
HISI_PMU_FORMAT_ATTR(srcid_cmd, "config1:22-32"),
HISI_PMU_FORMAT_ATTR(srcid_msk, "config1:33-43"),
HISI_PMU_FORMAT_ATTR(tracetag_en, "config1:44"),
NULL,
};
static const struct attribute_group hisi_pa_pmu_v2_format_group = {
.name = "format",
.attrs = hisi_pa_pmu_v2_format_attr,
};
static struct attribute *hisi_pa_pmu_v2_events_attr[] = {
HISI_PMU_EVENT_ATTR(rx_req, 0x40),
HISI_PMU_EVENT_ATTR(tx_req, 0x5c),
HISI_PMU_EVENT_ATTR(cycle, 0x78),
NULL
};
static const struct attribute_group hisi_pa_pmu_v2_events_group = {
.name = "events",
.attrs = hisi_pa_pmu_v2_events_attr,
};
static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
static struct attribute *hisi_pa_pmu_cpumask_attrs[] = {
&dev_attr_cpumask.attr,
NULL
};
static const struct attribute_group hisi_pa_pmu_cpumask_attr_group = {
.attrs = hisi_pa_pmu_cpumask_attrs,
};
static struct device_attribute hisi_pa_pmu_identifier_attr =
__ATTR(identifier, 0444, hisi_uncore_pmu_identifier_attr_show, NULL);
static struct attribute *hisi_pa_pmu_identifier_attrs[] = {
&hisi_pa_pmu_identifier_attr.attr,
NULL
};
static struct attribute_group hisi_pa_pmu_identifier_group = {
.attrs = hisi_pa_pmu_identifier_attrs,
};
static const struct attribute_group *hisi_pa_pmu_v2_attr_groups[] = {
&hisi_pa_pmu_v2_format_group,
&hisi_pa_pmu_v2_events_group,
&hisi_pa_pmu_cpumask_attr_group,
&hisi_pa_pmu_identifier_group,
NULL
};
static const struct hisi_uncore_ops hisi_uncore_pa_ops = {
.write_evtype = hisi_pa_pmu_write_evtype,
.get_event_idx = hisi_uncore_pmu_get_event_idx,
.start_counters = hisi_pa_pmu_start_counters,
.stop_counters = hisi_pa_pmu_stop_counters,
.enable_counter = hisi_pa_pmu_enable_counter,
.disable_counter = hisi_pa_pmu_disable_counter,
.enable_counter_int = hisi_pa_pmu_enable_counter_int,
.disable_counter_int = hisi_pa_pmu_disable_counter_int,
.write_counter = hisi_pa_pmu_write_counter,
.read_counter = hisi_pa_pmu_read_counter,
.get_int_status = hisi_pa_pmu_get_int_status,
.clear_int_status = hisi_pa_pmu_clear_int_status,
.enable_filter = hisi_pa_pmu_enable_filter,
.disable_filter = hisi_pa_pmu_disable_filter,
};
static int hisi_pa_pmu_dev_probe(struct platform_device *pdev,
struct hisi_pmu *pa_pmu)
{
int ret;
ret = hisi_pa_pmu_init_data(pdev, pa_pmu);
if (ret)
return ret;
ret = hisi_uncore_pmu_init_irq(pa_pmu, pdev);
if (ret)
return ret;
pa_pmu->pmu_events.attr_groups = hisi_pa_pmu_v2_attr_groups;
pa_pmu->num_counters = PA_NR_COUNTERS;
pa_pmu->ops = &hisi_uncore_pa_ops;
pa_pmu->check_event = 0xB0;
pa_pmu->counter_bits = 64;
pa_pmu->dev = &pdev->dev;
pa_pmu->on_cpu = -1;
return 0;
}
static int hisi_pa_pmu_probe(struct platform_device *pdev)
{
struct hisi_pmu *pa_pmu;
char *name;
int ret;
pa_pmu = devm_kzalloc(&pdev->dev, sizeof(*pa_pmu), GFP_KERNEL);
if (!pa_pmu)
return -ENOMEM;
ret = hisi_pa_pmu_dev_probe(pdev, pa_pmu);
if (ret)
return ret;
/*
* PA is attached in SICL and the CPU core is chosen to manage this
* PMU which is the nearest SCCL, while its SCCL_ID is greater than
* one with the SICL_ID.
*/
name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sicl%u_pa%u",
pa_pmu->sccl_id - 1, pa_pmu->index_id);
if (!name)
return -ENOMEM;
ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,
&pa_pmu->node);
if (ret) {
dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
return ret;
}
pa_pmu->pmu = (struct pmu) {
.module = THIS_MODULE,
.task_ctx_nr = perf_invalid_context,
.event_init = hisi_uncore_pmu_event_init,
.pmu_enable = hisi_uncore_pmu_enable,
.pmu_disable = hisi_uncore_pmu_disable,
.add = hisi_uncore_pmu_add,
.del = hisi_uncore_pmu_del,
.start = hisi_uncore_pmu_start,
.stop = hisi_uncore_pmu_stop,
.read = hisi_uncore_pmu_read,
.attr_groups = pa_pmu->pmu_events.attr_groups,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};
ret = perf_pmu_register(&pa_pmu->pmu, name, -1);
if (ret) {
dev_err(pa_pmu->dev, "PMU register failed, ret = %d\n", ret);
cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,
&pa_pmu->node);
irq_set_affinity_hint(pa_pmu->irq, NULL);
return ret;
}
platform_set_drvdata(pdev, pa_pmu);
return ret;
}
static int hisi_pa_pmu_remove(struct platform_device *pdev)
{
struct hisi_pmu *pa_pmu = platform_get_drvdata(pdev);
perf_pmu_unregister(&pa_pmu->pmu);
cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,
&pa_pmu->node);
irq_set_affinity_hint(pa_pmu->irq, NULL);
return 0;
}
static struct platform_driver hisi_pa_pmu_driver = {
.driver = {
.name = "hisi_pa_pmu",
.acpi_match_table = hisi_pa_pmu_acpi_match,
.suppress_bind_attrs = true,
},
.probe = hisi_pa_pmu_probe,
.remove = hisi_pa_pmu_remove,
};
static int __init hisi_pa_pmu_module_init(void)
{
int ret;
ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,
"AP_PERF_ARM_HISI_PA_ONLINE",
hisi_uncore_pmu_online_cpu,
hisi_uncore_pmu_offline_cpu);
if (ret) {
pr_err("PA PMU: cpuhp state setup failed, ret = %d\n", ret);
return ret;
}
ret = platform_driver_register(&hisi_pa_pmu_driver);
if (ret)
cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE);
return ret;
}
module_init(hisi_pa_pmu_module_init);
static void __exit hisi_pa_pmu_module_exit(void)
{
platform_driver_unregister(&hisi_pa_pmu_driver);
cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE);
}
module_exit(hisi_pa_pmu_module_exit);
MODULE_DESCRIPTION("HiSilicon Protocol Adapter uncore PMU driver");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");
MODULE_AUTHOR("Qi Liu <liuqi115@huawei.com>");

81
drivers/perf/hisilicon/hisi_uncore_pmu.c
View File

@ -21,7 +21,7 @@
#include "hisi_uncore_pmu.h"
#define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff)
#define HISI_MAX_PERIOD(nr) (BIT_ULL(nr) - 1)
#define HISI_MAX_PERIOD(nr) (GENMASK_ULL((nr) - 1, 0))
/*
* PMU format attributes
@ -33,7 +33,7 @@ ssize_t hisi_format_sysfs_show(struct device *dev,
eattr = container_of(attr, struct dev_ext_attribute, attr);
return sprintf(buf, "%s\n", (char *)eattr->var);
return sysfs_emit(buf, "%s\n", (char *)eattr->var);
}
EXPORT_SYMBOL_GPL(hisi_format_sysfs_show);
@ -47,7 +47,7 @@ ssize_t hisi_event_sysfs_show(struct device *dev,
eattr = container_of(attr, struct dev_ext_attribute, attr);
return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
return sysfs_emit(page, "config=0x%lx\n", (unsigned long)eattr->var);
}
EXPORT_SYMBOL_GPL(hisi_event_sysfs_show);
@ -59,7 +59,7 @@ ssize_t hisi_cpumask_sysfs_show(struct device *dev,
{
struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
return sprintf(buf, "%d\n", hisi_pmu->on_cpu);
return sysfs_emit(buf, "%d\n", hisi_pmu->on_cpu);
}
EXPORT_SYMBOL_GPL(hisi_cpumask_sysfs_show);
@ -96,12 +96,6 @@ static bool hisi_validate_event_group(struct perf_event *event)
return counters <= hisi_pmu->num_counters;
}
int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx)
{
return idx >= 0 && idx < hisi_pmu->num_counters;
}
EXPORT_SYMBOL_GPL(hisi_uncore_pmu_counter_valid);
int hisi_uncore_pmu_get_event_idx(struct perf_event *event)
{
struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
@ -125,20 +119,69 @@ ssize_t hisi_uncore_pmu_identifier_attr_show(struct device *dev,
{
struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
return snprintf(page, PAGE_SIZE, "0x%08x\n", hisi_pmu->identifier);
return sysfs_emit(page, "0x%08x\n", hisi_pmu->identifier);
}
EXPORT_SYMBOL_GPL(hisi_uncore_pmu_identifier_attr_show);
static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)
{
if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) {
dev_err(hisi_pmu->dev, "Unsupported event index:%d!\n", idx);
return;
}
clear_bit(idx, hisi_pmu->pmu_events.used_mask);
}
static irqreturn_t hisi_uncore_pmu_isr(int irq, void *data)
{
struct hisi_pmu *hisi_pmu = data;
struct perf_event *event;
unsigned long overflown;
int idx;
overflown = hisi_pmu->ops->get_int_status(hisi_pmu);
if (!overflown)
return IRQ_NONE;
/*
* Find the counter index which overflowed if the bit was set
* and handle it.
*/
for_each_set_bit(idx, &overflown, hisi_pmu->num_counters) {
/* Write 1 to clear the IRQ status flag */
hisi_pmu->ops->clear_int_status(hisi_pmu, idx);
/* Get the corresponding event struct */
event = hisi_pmu->pmu_events.hw_events[idx];
if (!event)
continue;
hisi_uncore_pmu_event_update(event);
hisi_uncore_pmu_set_event_period(event);
}
return IRQ_HANDLED;
}
int hisi_uncore_pmu_init_irq(struct hisi_pmu *hisi_pmu,
struct platform_device *pdev)
{
int irq, ret;
irq = platform_get_irq(pdev, 0);
if (irq < 0)
return irq;
ret = devm_request_irq(&pdev->dev, irq, hisi_uncore_pmu_isr,
IRQF_NOBALANCING | IRQF_NO_THREAD,
dev_name(&pdev->dev), hisi_pmu);
if (ret < 0) {
dev_err(&pdev->dev,
"Fail to request IRQ: %d ret: %d.\n", irq, ret);
return ret;
}
hisi_pmu->irq = irq;
return 0;
}
EXPORT_SYMBOL_GPL(hisi_uncore_pmu_init_irq);
int hisi_uncore_pmu_event_init(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
@ -202,6 +245,9 @@ static void hisi_uncore_pmu_enable_event(struct perf_event *event)
hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx,
HISI_GET_EVENTID(event));
if (hisi_pmu->ops->enable_filter)
hisi_pmu->ops->enable_filter(event);
hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc);
hisi_pmu->ops->enable_counter(hisi_pmu, hwc);
}
@ -216,6 +262,9 @@ static void hisi_uncore_pmu_disable_event(struct perf_event *event)
hisi_pmu->ops->disable_counter(hisi_pmu, hwc);
hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc);
if (hisi_pmu->ops->disable_filter)
hisi_pmu->ops->disable_filter(event);
}
void hisi_uncore_pmu_set_event_period(struct perf_event *event)

20
drivers/perf/hisilicon/hisi_uncore_pmu.h
View File

@ -11,16 +11,19 @@
#ifndef __HISI_UNCORE_PMU_H__
#define __HISI_UNCORE_PMU_H__
#include <linux/bitfield.h>
#include <linux/cpumask.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/perf_event.h>
#include <linux/platform_device.h>
#include <linux/types.h>
#undef pr_fmt
#define pr_fmt(fmt) "hisi_pmu: " fmt
#define HISI_PMU_V2 0x30
#define HISI_MAX_COUNTERS 0x10
#define to_hisi_pmu(p) (container_of(p, struct hisi_pmu, pmu))
@ -34,6 +37,12 @@
#define HISI_PMU_EVENT_ATTR(_name, _config) \
HISI_PMU_ATTR(_name, hisi_event_sysfs_show, (unsigned long)_config)
#define HISI_PMU_EVENT_ATTR_EXTRACTOR(name, config, hi, lo) \
static inline u32 hisi_get_##name(struct perf_event *event) \
{ \
return FIELD_GET(GENMASK_ULL(hi, lo), event->attr.config); \
}
struct hisi_pmu;
struct hisi_uncore_ops {
@ -47,11 +56,16 @@ struct hisi_uncore_ops {
void (*disable_counter_int)(struct hisi_pmu *, struct hw_perf_event *);
void (*start_counters)(struct hisi_pmu *);
void (*stop_counters)(struct hisi_pmu *);
u32 (*get_int_status)(struct hisi_pmu *hisi_pmu);
void (*clear_int_status)(struct hisi_pmu *hisi_pmu, int idx);
void (*enable_filter)(struct perf_event *event);
void (*disable_filter)(struct perf_event *event);
};
struct hisi_pmu_hwevents {
struct perf_event *hw_events[HISI_MAX_COUNTERS];
DECLARE_BITMAP(used_mask, HISI_MAX_COUNTERS);
const struct attribute_group **attr_groups;
};
/* Generic pmu struct for different pmu types */
@ -71,6 +85,8 @@ struct hisi_pmu {
void __iomem *base;
/* the ID of the PMU modules */
u32 index_id;
/* For DDRC PMU v2: each DDRC has more than one DMC */
u32 sub_id;
int num_counters;
int counter_bits;
/* check event code range */
@ -78,7 +94,6 @@ struct hisi_pmu {
u32 identifier;
};
int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx);
int hisi_uncore_pmu_get_event_idx(struct perf_event *event);
void hisi_uncore_pmu_read(struct perf_event *event);
int hisi_uncore_pmu_add(struct perf_event *event, int flags);
@ -102,6 +117,7 @@ int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node);
ssize_t hisi_uncore_pmu_identifier_attr_show(struct device *dev,
struct device_attribute *attr,
char *page);
int hisi_uncore_pmu_init_irq(struct hisi_pmu *hisi_pmu,
struct platform_device *pdev);
#endif /* __HISI_UNCORE_PMU_H__ */

530
drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c Normal file
View File

@ -0,0 +1,530 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* HiSilicon SLLC uncore Hardware event counters support
*
* Copyright (C) 2020 Hisilicon Limited
* Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
*
* This code is based on the uncore PMUs like arm-cci and arm-ccn.
*/
#include <linux/acpi.h>
#include <linux/cpuhotplug.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/list.h>
#include <linux/smp.h>
#include "hisi_uncore_pmu.h"
/* SLLC register definition */
#define SLLC_INT_MASK 0x0814
#define SLLC_INT_STATUS 0x0818
#define SLLC_INT_CLEAR 0x081c
#define SLLC_PERF_CTRL 0x1c00
#define SLLC_SRCID_CTRL 0x1c04
#define SLLC_TGTID_CTRL 0x1c08
#define SLLC_EVENT_CTRL 0x1c14
#define SLLC_EVENT_TYPE0 0x1c18
#define SLLC_VERSION 0x1cf0
#define SLLC_EVENT_CNT0_L 0x1d00
#define SLLC_EVTYPE_MASK 0xff
#define SLLC_PERF_CTRL_EN BIT(0)
#define SLLC_FILT_EN BIT(1)
#define SLLC_TRACETAG_EN BIT(2)
#define SLLC_SRCID_EN BIT(4)
#define SLLC_SRCID_NONE 0x0
#define SLLC_TGTID_EN BIT(5)
#define SLLC_TGTID_NONE 0x0
#define SLLC_TGTID_MIN_SHIFT 1
#define SLLC_TGTID_MAX_SHIFT 12
#define SLLC_SRCID_CMD_SHIFT 1
#define SLLC_SRCID_MSK_SHIFT 12
#define SLLC_NR_EVENTS 0x80
HISI_PMU_EVENT_ATTR_EXTRACTOR(tgtid_min, config1, 10, 0);
HISI_PMU_EVENT_ATTR_EXTRACTOR(tgtid_max, config1, 21, 11);
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_cmd, config1, 32, 22);
HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_msk, config1, 43, 33);
HISI_PMU_EVENT_ATTR_EXTRACTOR(tracetag_en, config1, 44, 44);
static bool tgtid_is_valid(u32 max, u32 min)
{
return max > 0 && max >= min;
}
static void hisi_sllc_pmu_enable_tracetag(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
u32 tt_en = hisi_get_tracetag_en(event);
if (tt_en) {
u32 val;
val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
val |= SLLC_TRACETAG_EN | SLLC_FILT_EN;
writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
}
}
static void hisi_sllc_pmu_disable_tracetag(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
u32 tt_en = hisi_get_tracetag_en(event);
if (tt_en) {
u32 val;
val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
val &= ~(SLLC_TRACETAG_EN | SLLC_FILT_EN);
writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
}
}
static void hisi_sllc_pmu_config_tgtid(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
u32 min = hisi_get_tgtid_min(event);
u32 max = hisi_get_tgtid_max(event);
if (tgtid_is_valid(max, min)) {
u32 val = (max << SLLC_TGTID_MAX_SHIFT) | (min << SLLC_TGTID_MIN_SHIFT);
writel(val, sllc_pmu->base + SLLC_TGTID_CTRL);
/* Enable the tgtid */
val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
val |= SLLC_TGTID_EN | SLLC_FILT_EN;
writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
}
}
static void hisi_sllc_pmu_clear_tgtid(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
u32 min = hisi_get_tgtid_min(event);
u32 max = hisi_get_tgtid_max(event);
if (tgtid_is_valid(max, min)) {
u32 val;
writel(SLLC_TGTID_NONE, sllc_pmu->base + SLLC_TGTID_CTRL);
/* Disable the tgtid */
val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
val &= ~(SLLC_TGTID_EN | SLLC_FILT_EN);
writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
}
}
static void hisi_sllc_pmu_config_srcid(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
u32 cmd = hisi_get_srcid_cmd(event);
if (cmd) {
u32 val, msk;
msk = hisi_get_srcid_msk(event);
val = (cmd << SLLC_SRCID_CMD_SHIFT) | (msk << SLLC_SRCID_MSK_SHIFT);
writel(val, sllc_pmu->base + SLLC_SRCID_CTRL);
/* Enable the srcid */
val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
val |= SLLC_SRCID_EN | SLLC_FILT_EN;
writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
}
}
static void hisi_sllc_pmu_clear_srcid(struct perf_event *event)
{
struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu);
u32 cmd = hisi_get_srcid_cmd(event);
if (cmd) {
u32 val;
writel(SLLC_SRCID_NONE, sllc_pmu->base + SLLC_SRCID_CTRL);
/* Disable the srcid */
val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
val &= ~(SLLC_SRCID_EN | SLLC_FILT_EN);
writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
}
}
static void hisi_sllc_pmu_enable_filter(struct perf_event *event)
{
if (event->attr.config1 != 0x0) {
hisi_sllc_pmu_enable_tracetag(event);
hisi_sllc_pmu_config_srcid(event);
hisi_sllc_pmu_config_tgtid(event);
}
}
static void hisi_sllc_pmu_clear_filter(struct perf_event *event)
{
if (event->attr.config1 != 0x0) {
hisi_sllc_pmu_disable_tracetag(event);
hisi_sllc_pmu_clear_srcid(event);
hisi_sllc_pmu_clear_tgtid(event);
}
}
static u32 hisi_sllc_pmu_get_counter_offset(int idx)
{
return (SLLC_EVENT_CNT0_L + idx * 8);
}
static u64 hisi_sllc_pmu_read_counter(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
return readq(sllc_pmu->base +
hisi_sllc_pmu_get_counter_offset(hwc->idx));
}
static void hisi_sllc_pmu_write_counter(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc, u64 val)
{
writeq(val, sllc_pmu->base +
hisi_sllc_pmu_get_counter_offset(hwc->idx));
}
static void hisi_sllc_pmu_write_evtype(struct hisi_pmu *sllc_pmu, int idx,
u32 type)
{
u32 reg, reg_idx, shift, val;
/*
* Select the appropriate event select register(SLLC_EVENT_TYPE0/1).
* There are 2 event select registers for the 8 hardware counters.
* Event code is 8-bits and for the former 4 hardware counters,
* SLLC_EVENT_TYPE0 is chosen. For the latter 4 hardware counters,
* SLLC_EVENT_TYPE1 is chosen.
*/
reg = SLLC_EVENT_TYPE0 + (idx / 4) * 4;
reg_idx = idx % 4;
shift = 8 * reg_idx;
/* Write event code to SLLC_EVENT_TYPEx Register */
val = readl(sllc_pmu->base + reg);
val &= ~(SLLC_EVTYPE_MASK << shift);
val |= (type << shift);
writel(val, sllc_pmu->base + reg);
}
static void hisi_sllc_pmu_start_counters(struct hisi_pmu *sllc_pmu)
{
u32 val;
val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
val |= SLLC_PERF_CTRL_EN;
writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
}
static void hisi_sllc_pmu_stop_counters(struct hisi_pmu *sllc_pmu)
{
u32 val;
val = readl(sllc_pmu->base + SLLC_PERF_CTRL);
val &= ~(SLLC_PERF_CTRL_EN);
writel(val, sllc_pmu->base + SLLC_PERF_CTRL);
}
static void hisi_sllc_pmu_enable_counter(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
val = readl(sllc_pmu->base + SLLC_EVENT_CTRL);
val |= 1 << hwc->idx;
writel(val, sllc_pmu->base + SLLC_EVENT_CTRL);
}
static void hisi_sllc_pmu_disable_counter(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
val = readl(sllc_pmu->base + SLLC_EVENT_CTRL);
val &= ~(1 << hwc->idx);
writel(val, sllc_pmu->base + SLLC_EVENT_CTRL);
}
static void hisi_sllc_pmu_enable_counter_int(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
val = readl(sllc_pmu->base + SLLC_INT_MASK);
/* Write 0 to enable interrupt */
val &= ~(1 << hwc->idx);
writel(val, sllc_pmu->base + SLLC_INT_MASK);
}
static void hisi_sllc_pmu_disable_counter_int(struct hisi_pmu *sllc_pmu,
struct hw_perf_event *hwc)
{
u32 val;
val = readl(sllc_pmu->base + SLLC_INT_MASK);
/* Write 1 to mask interrupt */
val |= 1 << hwc->idx;
writel(val, sllc_pmu->base + SLLC_INT_MASK);
}
static u32 hisi_sllc_pmu_get_int_status(struct hisi_pmu *sllc_pmu)
{
return readl(sllc_pmu->base + SLLC_INT_STATUS);
}
static void hisi_sllc_pmu_clear_int_status(struct hisi_pmu *sllc_pmu, int idx)
{
writel(1 << idx, sllc_pmu->base + SLLC_INT_CLEAR);
}
static const struct acpi_device_id hisi_sllc_pmu_acpi_match[] = {
{ "HISI0263", },
{}
};
MODULE_DEVICE_TABLE(acpi, hisi_sllc_pmu_acpi_match);
static int hisi_sllc_pmu_init_data(struct platform_device *pdev,
struct hisi_pmu *sllc_pmu)
{
/*
* Use the SCCL_ID and the index ID to identify the SLLC PMU,
* while SCCL_ID is from MPIDR_EL1 by CPU.
*/
if (device_property_read_u32(&pdev->dev, "hisilicon,scl-id",
&sllc_pmu->sccl_id)) {
dev_err(&pdev->dev, "Cannot read sccl-id!\n");
return -EINVAL;
}
if (device_property_read_u32(&pdev->dev, "hisilicon,idx-id",
&sllc_pmu->index_id)) {
dev_err(&pdev->dev, "Cannot read idx-id!\n");
return -EINVAL;
}
/* SLLC PMUs only share the same SCCL */
sllc_pmu->ccl_id = -1;
sllc_pmu->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(sllc_pmu->base)) {
dev_err(&pdev->dev, "ioremap failed for sllc_pmu resource.\n");
return PTR_ERR(sllc_pmu->base);
}
sllc_pmu->identifier = readl(sllc_pmu->base + SLLC_VERSION);
return 0;
}
static struct attribute *hisi_sllc_pmu_v2_format_attr[] = {
HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
HISI_PMU_FORMAT_ATTR(tgtid_min, "config1:0-10"),
HISI_PMU_FORMAT_ATTR(tgtid_max, "config1:11-21"),
HISI_PMU_FORMAT_ATTR(srcid_cmd, "config1:22-32"),
HISI_PMU_FORMAT_ATTR(srcid_msk, "config1:33-43"),
HISI_PMU_FORMAT_ATTR(tracetag_en, "config1:44"),
NULL
};
static const struct attribute_group hisi_sllc_pmu_v2_format_group = {
.name = "format",
.attrs = hisi_sllc_pmu_v2_format_attr,
};
static struct attribute *hisi_sllc_pmu_v2_events_attr[] = {
HISI_PMU_EVENT_ATTR(rx_req, 0x30),
HISI_PMU_EVENT_ATTR(rx_data, 0x31),
HISI_PMU_EVENT_ATTR(tx_req, 0x34),
HISI_PMU_EVENT_ATTR(tx_data, 0x35),
HISI_PMU_EVENT_ATTR(cycles, 0x09),
NULL
};
static const struct attribute_group hisi_sllc_pmu_v2_events_group = {
.name = "events",
.attrs = hisi_sllc_pmu_v2_events_attr,
};
static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
static struct attribute *hisi_sllc_pmu_cpumask_attrs[] = {
&dev_attr_cpumask.attr,
NULL
};
static const struct attribute_group hisi_sllc_pmu_cpumask_attr_group = {
.attrs = hisi_sllc_pmu_cpumask_attrs,
};
static struct device_attribute hisi_sllc_pmu_identifier_attr =
__ATTR(identifier, 0444, hisi_uncore_pmu_identifier_attr_show, NULL);
static struct attribute *hisi_sllc_pmu_identifier_attrs[] = {
&hisi_sllc_pmu_identifier_attr.attr,
NULL
};
static struct attribute_group hisi_sllc_pmu_identifier_group = {
.attrs = hisi_sllc_pmu_identifier_attrs,
};
static const struct attribute_group *hisi_sllc_pmu_v2_attr_groups[] = {
&hisi_sllc_pmu_v2_format_group,
&hisi_sllc_pmu_v2_events_group,
&hisi_sllc_pmu_cpumask_attr_group,
&hisi_sllc_pmu_identifier_group,
NULL
};
static const struct hisi_uncore_ops hisi_uncore_sllc_ops = {
.write_evtype = hisi_sllc_pmu_write_evtype,
.get_event_idx = hisi_uncore_pmu_get_event_idx,
.start_counters = hisi_sllc_pmu_start_counters,
.stop_counters = hisi_sllc_pmu_stop_counters,
.enable_counter = hisi_sllc_pmu_enable_counter,
.disable_counter = hisi_sllc_pmu_disable_counter,
.enable_counter_int = hisi_sllc_pmu_enable_counter_int,
.disable_counter_int = hisi_sllc_pmu_disable_counter_int,
.write_counter = hisi_sllc_pmu_write_counter,
.read_counter = hisi_sllc_pmu_read_counter,
.get_int_status = hisi_sllc_pmu_get_int_status,
.clear_int_status = hisi_sllc_pmu_clear_int_status,
.enable_filter = hisi_sllc_pmu_enable_filter,
.disable_filter = hisi_sllc_pmu_clear_filter,
};
static int hisi_sllc_pmu_dev_probe(struct platform_device *pdev,
struct hisi_pmu *sllc_pmu)
{
int ret;
ret = hisi_sllc_pmu_init_data(pdev, sllc_pmu);
if (ret)
return ret;
ret = hisi_uncore_pmu_init_irq(sllc_pmu, pdev);
if (ret)
return ret;
sllc_pmu->pmu_events.attr_groups = hisi_sllc_pmu_v2_attr_groups;
sllc_pmu->ops = &hisi_uncore_sllc_ops;
sllc_pmu->check_event = SLLC_NR_EVENTS;
sllc_pmu->counter_bits = 64;
sllc_pmu->num_counters = 8;
sllc_pmu->dev = &pdev->dev;
sllc_pmu->on_cpu = -1;
return 0;
}
static int hisi_sllc_pmu_probe(struct platform_device *pdev)
{
struct hisi_pmu *sllc_pmu;
char *name;
int ret;
sllc_pmu = devm_kzalloc(&pdev->dev, sizeof(*sllc_pmu), GFP_KERNEL);
if (!sllc_pmu)
return -ENOMEM;
ret = hisi_sllc_pmu_dev_probe(pdev, sllc_pmu);
if (ret)
return ret;
name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_sllc%u",
sllc_pmu->sccl_id, sllc_pmu->index_id);
if (!name)
return -ENOMEM;
ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,
&sllc_pmu->node);
if (ret) {
dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
return ret;
}
sllc_pmu->pmu = (struct pmu) {
.module = THIS_MODULE,
.task_ctx_nr = perf_invalid_context,
.event_init = hisi_uncore_pmu_event_init,
.pmu_enable = hisi_uncore_pmu_enable,
.pmu_disable = hisi_uncore_pmu_disable,
.add = hisi_uncore_pmu_add,
.del = hisi_uncore_pmu_del,
.start = hisi_uncore_pmu_start,
.stop = hisi_uncore_pmu_stop,
.read = hisi_uncore_pmu_read,
.attr_groups = sllc_pmu->pmu_events.attr_groups,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};
ret = perf_pmu_register(&sllc_pmu->pmu, name, -1);
if (ret) {
dev_err(sllc_pmu->dev, "PMU register failed, ret = %d\n", ret);
cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,
&sllc_pmu->node);
irq_set_affinity_hint(sllc_pmu->irq, NULL);
return ret;
}
platform_set_drvdata(pdev, sllc_pmu);
return ret;
}
static int hisi_sllc_pmu_remove(struct platform_device *pdev)
{
struct hisi_pmu *sllc_pmu = platform_get_drvdata(pdev);
perf_pmu_unregister(&sllc_pmu->pmu);
cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,
&sllc_pmu->node);
irq_set_affinity_hint(sllc_pmu->irq, NULL);
return 0;
}
static struct platform_driver hisi_sllc_pmu_driver = {
.driver = {
.name = "hisi_sllc_pmu",
.acpi_match_table = hisi_sllc_pmu_acpi_match,
.suppress_bind_attrs = true,
},
.probe = hisi_sllc_pmu_probe,
.remove = hisi_sllc_pmu_remove,
};
static int __init hisi_sllc_pmu_module_init(void)
{
int ret;
ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,
"AP_PERF_ARM_HISI_SLLC_ONLINE",
hisi_uncore_pmu_online_cpu,
hisi_uncore_pmu_offline_cpu);
if (ret) {
pr_err("SLLC PMU: cpuhp state setup failed, ret = %d\n", ret);
return ret;
}
ret = platform_driver_register(&hisi_sllc_pmu_driver);
if (ret)
cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE);
return ret;
}
module_init(hisi_sllc_pmu_module_init);
static void __exit hisi_sllc_pmu_module_exit(void)
{
platform_driver_unregister(&hisi_sllc_pmu_driver);
cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE);
}
module_exit(hisi_sllc_pmu_module_exit);
MODULE_DESCRIPTION("HiSilicon SLLC uncore PMU driver");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");
MODULE_AUTHOR("Qi Liu <liuqi115@huawei.com>");

2
drivers/perf/qcom_l2_pmu.c
View File

@ -676,7 +676,7 @@ static ssize_t l2cache_pmu_event_show(struct device *dev,
struct perf_pmu_events_attr *pmu_attr;
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
}
#define L2CACHE_EVENT_ATTR(_name, _id) \

4
drivers/perf/qcom_l3_pmu.c
View File

@ -615,7 +615,7 @@ static ssize_t l3cache_pmu_format_show(struct device *dev,
struct dev_ext_attribute *eattr;
eattr = container_of(attr, struct dev_ext_attribute, attr);
return sprintf(buf, "%s\n", (char *) eattr->var);
return sysfs_emit(buf, "%s\n", (char *) eattr->var);
}
#define L3CACHE_PMU_FORMAT_ATTR(_name, _config) \
@ -643,7 +643,7 @@ static ssize_t l3cache_pmu_event_show(struct device *dev,
struct perf_pmu_events_attr *pmu_attr;
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
}
#define L3CACHE_EVENT_ATTR(_name, _id) \

4
drivers/perf/thunderx2_pmu.c
View File

@ -128,7 +128,7 @@ __tx2_pmu_##_var##_show(struct device *dev, \
char *page) \
{ \
BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
return sprintf(page, _format "\n"); \
return sysfs_emit(page, _format "\n"); \
} \
\
static struct device_attribute format_attr_##_var = \
@ -176,7 +176,7 @@ static ssize_t tx2_pmu_event_show(struct device *dev,
struct dev_ext_attribute *eattr;
eattr = container_of(attr, struct dev_ext_attribute, attr);
return sprintf(buf, "event=0x%lx\n", (unsigned long) eattr->var);
return sysfs_emit(buf, "event=0x%lx\n", (unsigned long) eattr->var);
}
#define TX2_EVENT_ATTR(name, config) \

4
drivers/perf/xgene_pmu.c
View File

@ -170,7 +170,7 @@ static ssize_t xgene_pmu_format_show(struct device *dev,
struct dev_ext_attribute *eattr;
eattr = container_of(attr, struct dev_ext_attribute, attr);
return sprintf(buf, "%s\n", (char *) eattr->var);
return sysfs_emit(buf, "%s\n", (char *) eattr->var);
}
#define XGENE_PMU_FORMAT_ATTR(_name, _config) \
@ -281,7 +281,7 @@ static ssize_t xgene_pmu_event_show(struct device *dev,
struct dev_ext_attribute *eattr;
eattr = container_of(attr, struct dev_ext_attribute, attr);
return sprintf(buf, "config=0x%lx\n", (unsigned long) eattr->var);
return sysfs_emit(buf, "config=0x%lx\n", (unsigned long) eattr->var);
}
#define XGENE_PMU_EVENT_ATTR(_name, _config) \

2
include/linux/cpuhotplug.h
View File

@ -175,6 +175,8 @@ enum cpuhp_state {
CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,
CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,
CPUHP_AP_PERF_ARM_L2X0_ONLINE,
CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE,

2
include/linux/irq.h
View File

@ -1258,11 +1258,13 @@ int __init set_handle_irq(void (*handle_irq)(struct pt_regs *));
*/
extern void (*handle_arch_irq)(struct pt_regs *) __ro_after_init;
#else
#ifndef set_handle_irq
#define set_handle_irq(handle_irq) \
do { \
(void)handle_irq; \
WARN_ON(1); \
} while (0)
#endif
#endif
#endif /* _LINUX_IRQ_H */

6
mm/mmap.c
View File

@ -93,6 +93,12 @@ static void unmap_region(struct mm_struct *mm,
* MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
* w: (no) no w: (no) no w: (copy) copy w: (no) no
* x: (no) no x: (no) yes x: (no) yes x: (yes) yes
*
* On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and
* MAP_PRIVATE (with Enhanced PAN supported):
* r: (no) no
* w: (no) no
* x: (yes) yes
*/
pgprot_t protection_map[16] __ro_after_init = {
__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,

2
tools/testing/selftests/arm64/Makefile
View File

@ -4,7 +4,7 @@
ARCH ?= $(shell uname -m 2>/dev/null || echo not)
ifneq (,$(filter $(ARCH),aarch64 arm64))
ARM64_SUBTARGETS ?= tags signal pauth fp mte
ARM64_SUBTARGETS ?= tags signal pauth fp mte bti
else
ARM64_SUBTARGETS :=
endif

2
tools/testing/selftests/arm64/bti/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
btitest
nobtitest

61
tools/testing/selftests/arm64/bti/Makefile Normal file
View File

@ -0,0 +1,61 @@
# SPDX-License-Identifier: GPL-2.0
TEST_GEN_PROGS := btitest nobtitest
PROGS := $(patsubst %,gen/%,$(TEST_GEN_PROGS))
# These tests are built as freestanding binaries since otherwise BTI
# support in ld.so is required which is not currently widespread; when
# it is available it will still be useful to test this separately as the
# cases for statically linked and dynamically lined binaries are
# slightly different.
CFLAGS_NOBTI = -DBTI=0
CFLAGS_BTI = -mbranch-protection=standard -DBTI=1
CFLAGS_COMMON = -ffreestanding -Wall -Wextra $(CFLAGS)
BTI_CC_COMMAND = $(CC) $(CFLAGS_BTI) $(CFLAGS_COMMON) -c -o $@ $<
NOBTI_CC_COMMAND = $(CC) $(CFLAGS_NOBTI) $(CFLAGS_COMMON) -c -o $@ $<
%-bti.o: %.c
$(BTI_CC_COMMAND)
%-bti.o: %.S
$(BTI_CC_COMMAND)
%-nobti.o: %.c
$(NOBTI_CC_COMMAND)
%-nobti.o: %.S
$(NOBTI_CC_COMMAND)
BTI_OBJS = \
test-bti.o \
signal-bti.o \
start-bti.o \
syscall-bti.o \
system-bti.o \
teststubs-bti.o \
trampoline-bti.o
gen/btitest: $(BTI_OBJS)
$(CC) $(CFLAGS_BTI) $(CFLAGS_COMMON) -nostdlib -o $@ $^
NOBTI_OBJS = \
test-nobti.o \
signal-nobti.o \
start-nobti.o \
syscall-nobti.o \
system-nobti.o \
teststubs-nobti.o \
trampoline-nobti.o
gen/nobtitest: $(NOBTI_OBJS)
$(CC) $(CFLAGS_BTI) $(CFLAGS_COMMON) -nostdlib -o $@ $^
# Including KSFT lib.mk here will also mangle the TEST_GEN_PROGS list
# to account for any OUTPUT target-dirs optionally provided by
# the toplevel makefile
include ../../lib.mk
$(TEST_GEN_PROGS): $(PROGS)
cp $(PROGS) $(OUTPUT)/

80
tools/testing/selftests/arm64/bti/assembler.h Normal file
View File

@ -0,0 +1,80 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#ifndef ASSEMBLER_H
#define ASSEMBLER_H
#define NT_GNU_PROPERTY_TYPE_0 5
#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
/* Bits for GNU_PROPERTY_AARCH64_FEATURE_1_BTI */
#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1)
.macro startfn name:req
.globl \name
\name:
.macro endfn
.size \name, . - \name
.type \name, @function
.purgem endfn
.endm
.endm
.macro emit_aarch64_feature_1_and
.pushsection .note.gnu.property, "a"
.align 3
.long 2f - 1f
.long 6f - 3f
.long NT_GNU_PROPERTY_TYPE_0
1: .string "GNU"
2:
.align 3
3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND
.long 5f - 4f
4:
#if BTI
.long GNU_PROPERTY_AARCH64_FEATURE_1_PAC | \
GNU_PROPERTY_AARCH64_FEATURE_1_BTI
#else
.long 0
#endif
5:
.align 3
6:
.popsection
.endm
.macro paciasp
hint 0x19
.endm
.macro autiasp
hint 0x1d
.endm
.macro __bti_
hint 0x20
.endm
.macro __bti_c
hint 0x22
.endm
.macro __bti_j
hint 0x24
.endm
.macro __bti_jc
hint 0x26
.endm
.macro bti what=
__bti_\what
.endm
#endif /* ! ASSEMBLER_H */

23
tools/testing/selftests/arm64/bti/btitest.h Normal file
View File

@ -0,0 +1,23 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#ifndef BTITEST_H
#define BTITEST_H
/* Trampolines for calling the test stubs: */
void call_using_br_x0(void (*)(void));
void call_using_br_x16(void (*)(void));
void call_using_blr(void (*)(void));
/* Test stubs: */
void nohint_func(void);
void bti_none_func(void);
void bti_c_func(void);
void bti_j_func(void);
void bti_jc_func(void);
void paciasp_func(void);
#endif /* !BTITEST_H */

21
tools/testing/selftests/arm64/bti/compiler.h Normal file
View File

@ -0,0 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#ifndef COMPILER_H
#define COMPILER_H
#define __always_unused __attribute__((__unused__))
#define __noreturn __attribute__((__noreturn__))
#define __unreachable() __builtin_unreachable()
/* curse(e) has value e, but the compiler cannot assume so */
#define curse(e) ({ \
__typeof__(e) __curse_e = (e); \
asm ("" : "+r" (__curse_e)); \
__curse_e; \
})
#endif /* ! COMPILER_H */

2
tools/testing/selftests/arm64/bti/gen/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
btitest
nobtitest

37
tools/testing/selftests/arm64/bti/signal.c Normal file
View File

@ -0,0 +1,37 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#include "system.h"
#include "signal.h"
int sigemptyset(sigset_t *s)
{
unsigned int i;
for (i = 0; i < _NSIG_WORDS; ++i)
s->sig[i] = 0;
return 0;
}
int sigaddset(sigset_t *s, int n)
{
if (n < 1 || n > _NSIG)
return -EINVAL;
s->sig[(n - 1) / _NSIG_BPW] |= 1UL << (n - 1) % _NSIG_BPW;
return 0;
}
int sigaction(int n, struct sigaction *sa, const struct sigaction *old)
{
return syscall(__NR_rt_sigaction, n, sa, old, sizeof(sa->sa_mask));
}
int sigprocmask(int how, const sigset_t *mask, sigset_t *old)
{
return syscall(__NR_rt_sigprocmask, how, mask, old, sizeof(*mask));
}

21
tools/testing/selftests/arm64/bti/signal.h Normal file
View File

@ -0,0 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#ifndef SIGNAL_H
#define SIGNAL_H
#include <linux/signal.h>
#include "system.h"
typedef __sighandler_t sighandler_t;
int sigemptyset(sigset_t *s);
int sigaddset(sigset_t *s, int n);
int sigaction(int n, struct sigaction *sa, const struct sigaction *old);
int sigprocmask(int how, const sigset_t *mask, sigset_t *old);
#endif /* ! SIGNAL_H */

14
tools/testing/selftests/arm64/bti/start.S Normal file
View File

@ -0,0 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#include "assembler.h"
startfn _start
mov x0, sp
b start
endfn
emit_aarch64_feature_1_and

23
tools/testing/selftests/arm64/bti/syscall.S Normal file
View File

@ -0,0 +1,23 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#include "assembler.h"
startfn syscall
bti c
mov w8, w0
mov x0, x1
mov x1, x2
mov x2, x3
mov x3, x4
mov x4, x5
mov x5, x6
mov x6, x7
svc #0
ret
endfn
emit_aarch64_feature_1_and

22
tools/testing/selftests/arm64/bti/system.c Normal file
View File

@ -0,0 +1,22 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#include "system.h"
#include <asm/unistd.h>
#include "compiler.h"
void __noreturn exit(int n)
{
syscall(__NR_exit, n);
__unreachable();
}
ssize_t write(int fd, const void *buf, size_t size)
{
return syscall(__NR_write, fd, buf, size);
}

28
tools/testing/selftests/arm64/bti/system.h Normal file
View File

@ -0,0 +1,28 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#ifndef SYSTEM_H
#define SYSTEM_H
#include <linux/types.h>
#include <linux/stddef.h>
typedef __kernel_size_t size_t;
typedef __kernel_ssize_t ssize_t;
#include <linux/errno.h>
#include <asm/hwcap.h>
#include <asm/ptrace.h>
#include <asm/unistd.h>
#include "compiler.h"
long syscall(int nr, ...);
void __noreturn exit(int n);
ssize_t write(int fd, const void *buf, size_t size);
#endif /* ! SYSTEM_H */

234
tools/testing/selftests/arm64/bti/test.c Normal file
View File

@ -0,0 +1,234 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019,2021 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#include "system.h"
#include <linux/errno.h>
#include <linux/auxvec.h>
#include <linux/signal.h>
#include <asm/sigcontext.h>
#include <asm/ucontext.h>
typedef struct ucontext ucontext_t;
#include "btitest.h"
#include "compiler.h"
#include "signal.h"
#define EXPECTED_TESTS 18
static volatile unsigned int test_num = 1;
static unsigned int test_passed;
static unsigned int test_failed;
static unsigned int test_skipped;
static void fdputs(int fd, const char *str)
{
size_t len = 0;
const char *p = str;
while (*p++)
++len;
write(fd, str, len);
}
static void putstr(const char *str)
{
fdputs(1, str);
}
static void putnum(unsigned int num)
{
char c;
if (num / 10)
putnum(num / 10);
c = '0' + (num % 10);
write(1, &c, 1);
}
#define puttestname(test_name, trampoline_name) do { \
putstr(test_name); \
putstr("/"); \
putstr(trampoline_name); \
} while (0)
void print_summary(void)
{
putstr("# Totals: pass:");
putnum(test_passed);
putstr(" fail:");
putnum(test_failed);
putstr(" xfail:0 xpass:0 skip:");
putnum(test_skipped);
putstr(" error:0\n");
}
static const char *volatile current_test_name;
static const char *volatile current_trampoline_name;
static volatile int sigill_expected, sigill_received;
static void handler(int n, siginfo_t *si __always_unused,
void *uc_ __always_unused)
{
ucontext_t *uc = uc_;
putstr("# \t[SIGILL in ");
puttestname(current_test_name, current_trampoline_name);
putstr(", BTYPE=");
write(1, &"00011011"[((uc->uc_mcontext.pstate & PSR_BTYPE_MASK)
>> PSR_BTYPE_SHIFT) * 2], 2);
if (!sigill_expected) {
putstr("]\n");
putstr("not ok ");
putnum(test_num);
putstr(" ");
puttestname(current_test_name, current_trampoline_name);
putstr("(unexpected SIGILL)\n");
print_summary();
exit(128 + n);
}
putstr(" (expected)]\n");
sigill_received = 1;
/* zap BTYPE so that resuming the faulting code will work */
uc->uc_mcontext.pstate &= ~PSR_BTYPE_MASK;
}
static int skip_all;
static void __do_test(void (*trampoline)(void (*)(void)),
void (*fn)(void),
const char *trampoline_name,
const char *name,
int expect_sigill)
{
if (skip_all) {
test_skipped++;
putstr("ok ");
putnum(test_num);
putstr(" ");
puttestname(name, trampoline_name);
putstr(" # SKIP\n");
return;
}
/* Branch Target exceptions should only happen in BTI binaries: */
if (!BTI)
expect_sigill = 0;
sigill_expected = expect_sigill;
sigill_received = 0;
current_test_name = name;
current_trampoline_name = trampoline_name;
trampoline(fn);
if (expect_sigill && !sigill_received) {
putstr("not ok ");
test_failed++;
} else {
putstr("ok ");
test_passed++;
}
putnum(test_num++);
putstr(" ");
puttestname(name, trampoline_name);
putstr("\n");
}
#define do_test(expect_sigill_br_x0, \
expect_sigill_br_x16, \
expect_sigill_blr, \
name) \
do { \
__do_test(call_using_br_x0, name, "call_using_br_x0", #name, \
expect_sigill_br_x0); \
__do_test(call_using_br_x16, name, "call_using_br_x16", #name, \
expect_sigill_br_x16); \
__do_test(call_using_blr, name, "call_using_blr", #name, \
expect_sigill_blr); \
} while (0)
void start(int *argcp)
{
struct sigaction sa;
void *const *p;
const struct auxv_entry {
unsigned long type;
unsigned long val;
} *auxv;
unsigned long hwcap = 0, hwcap2 = 0;
putstr("TAP version 13\n");
putstr("1..");
putnum(EXPECTED_TESTS);
putstr("\n");
/* Gross hack for finding AT_HWCAP2 from the initial process stack: */
p = (void *const *)argcp + 1 + *argcp + 1; /* start of environment */
/* step over environment */
while (*p++)
;
for (auxv = (const struct auxv_entry *)p; auxv->type != AT_NULL; ++auxv) {
switch (auxv->type) {
case AT_HWCAP:
hwcap = auxv->val;
break;
case AT_HWCAP2:
hwcap2 = auxv->val;
break;
default:
break;
}
}
if (hwcap & HWCAP_PACA)
putstr("# HWCAP_PACA present\n");
else
putstr("# HWCAP_PACA not present\n");
if (hwcap2 & HWCAP2_BTI) {
putstr("# HWCAP2_BTI present\n");
if (!(hwcap & HWCAP_PACA))
putstr("# Bad hardware? Expect problems.\n");
} else {
putstr("# HWCAP2_BTI not present\n");
skip_all = 1;
}
putstr("# Test binary");
if (!BTI)
putstr(" not");
putstr(" built for BTI\n");
sa.sa_handler = (sighandler_t)(void *)handler;
sa.sa_flags = SA_SIGINFO;
sigemptyset(&sa.sa_mask);
sigaction(SIGILL, &sa, NULL);
sigaddset(&sa.sa_mask, SIGILL);
sigprocmask(SIG_UNBLOCK, &sa.sa_mask, NULL);
do_test(1, 1, 1, nohint_func);
do_test(1, 1, 1, bti_none_func);
do_test(1, 0, 0, bti_c_func);
do_test(0, 0, 1, bti_j_func);
do_test(0, 0, 0, bti_jc_func);
do_test(1, 0, 0, paciasp_func);
print_summary();
if (test_num - 1 != EXPECTED_TESTS)
putstr("# WARNING - EXPECTED TEST COUNT WRONG\n");
if (test_failed)
exit(1);
else
exit(0);
}

39
tools/testing/selftests/arm64/bti/teststubs.S Normal file
View File

@ -0,0 +1,39 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#include "assembler.h"
startfn bti_none_func
bti
ret
endfn
startfn bti_c_func
bti c
ret
endfn
startfn bti_j_func
bti j
ret
endfn
startfn bti_jc_func
bti jc
ret
endfn
startfn paciasp_func
paciasp
autiasp
ret
endfn
startfn nohint_func
ret
endfn
emit_aarch64_feature_1_and

29
tools/testing/selftests/arm64/bti/trampoline.S Normal file
View File

@ -0,0 +1,29 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 Arm Limited
* Original author: Dave Martin <Dave.Martin@arm.com>
*/
#include "assembler.h"
startfn call_using_br_x0
bti c
br x0
endfn
startfn call_using_br_x16
bti c
mov x16, x0
br x16
endfn
startfn call_using_blr
paciasp
stp x29, x30, [sp, #-16]!
blr x0
ldp x29, x30, [sp], #16
autiasp
ret
endfn
emit_aarch64_feature_1_and

15
tools/testing/selftests/arm64/mte/Makefile
View File

@ -1,14 +1,18 @@
# SPDX-License-Identifier: GPL-2.0
# Copyright (C) 2020 ARM Limited
CFLAGS += -std=gnu99 -I. -lpthread
# preserve CC value from top level Makefile
ifeq ($(CC),cc)
CC := $(CROSS_COMPILE)gcc
endif
CFLAGS += -std=gnu99 -I. -pthread
LDFLAGS += -pthread
SRCS := $(filter-out mte_common_util.c,$(wildcard *.c))
PROGS := $(patsubst %.c,%,$(SRCS))
#Add mte compiler option
ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep gcc),)
CFLAGS += -march=armv8.5-a+memtag
endif
#check if the compiler works well
mte_cc_support := $(shell if ($(CC) $(CFLAGS) -E -x c /dev/null -o /dev/null 2>&1) then echo "1"; fi)
@ -19,11 +23,14 @@ TEST_GEN_PROGS := $(PROGS)
# Get Kernel headers installed and use them.
KSFT_KHDR_INSTALL := 1
else
$(warning compiler "$(CC)" does not support the ARMv8.5 MTE extension.)
$(warning test program "mte" will not be created.)
endif
# Include KSFT lib.mk.
include ../../lib.mk
ifeq ($(mte_cc_support),1)
$(TEST_GEN_PROGS): mte_common_util.c mte_common_util.h mte_helper.S
$(TEST_GEN_PROGS): mte_common_util.c mte_helper.S
endif

5
tools/testing/selftests/arm64/mte/check_ksm_options.c
View File

@ -33,7 +33,10 @@ static unsigned long read_sysfs(char *str)
ksft_print_msg("ERR: missing %s\n", str);
return 0;
}
fscanf(f, "%lu", &val);
if (fscanf(f, "%lu", &val) != 1) {
ksft_print_msg("ERR: parsing %s\n", str);
val = 0;
}
fclose(f);
return val;
}

3
tools/testing/selftests/arm64/mte/check_user_mem.c
View File

@ -33,7 +33,8 @@ static int check_usermem_access_fault(int mem_type, int mode, int mapping)
if (fd == -1)
return KSFT_FAIL;
for (i = 0; i < len; i++)
write(fd, &val, sizeof(val));
if (write(fd, &val, sizeof(val)) != sizeof(val))
return KSFT_FAIL;
lseek(fd, 0, 0);
ptr = mte_allocate_memory(len, mem_type, mapping, true);
if (check_allocated_memory(ptr, len, mem_type, true) != KSFT_PASS) {

39
tools/testing/selftests/arm64/mte/mte_common_util.c
View File

@ -181,10 +181,17 @@ void *mte_allocate_file_memory(size_t size, int mem_type, int mapping, bool tags
}
/* Initialize the file for mappable size */
lseek(fd, 0, SEEK_SET);
for (index = INIT_BUFFER_SIZE; index < size; index += INIT_BUFFER_SIZE)
write(fd, buffer, INIT_BUFFER_SIZE);
for (index = INIT_BUFFER_SIZE; index < size; index += INIT_BUFFER_SIZE) {
if (write(fd, buffer, INIT_BUFFER_SIZE) != INIT_BUFFER_SIZE) {
perror("initialising buffer");
return NULL;
}
}
index -= INIT_BUFFER_SIZE;
write(fd, buffer, size - index);
if (write(fd, buffer, size - index) != size - index) {
perror("initialising buffer");
return NULL;
}
return __mte_allocate_memory_range(size, mem_type, mapping, 0, 0, tags, fd);
}
@ -202,9 +209,15 @@ void *mte_allocate_file_memory_tag_range(size_t size, int mem_type, int mapping,
/* Initialize the file for mappable size */
lseek(fd, 0, SEEK_SET);
for (index = INIT_BUFFER_SIZE; index < map_size; index += INIT_BUFFER_SIZE)
write(fd, buffer, INIT_BUFFER_SIZE);
if (write(fd, buffer, INIT_BUFFER_SIZE) != INIT_BUFFER_SIZE) {
perror("initialising buffer");
return NULL;
}
index -= INIT_BUFFER_SIZE;
write(fd, buffer, map_size - index);
if (write(fd, buffer, map_size - index) != map_size - index) {
perror("initialising buffer");
return NULL;
}
return __mte_allocate_memory_range(size, mem_type, mapping, range_before,
range_after, true, fd);
}
@ -271,29 +284,20 @@ int mte_switch_mode(int mte_option, unsigned long incl_mask)
en |= (incl_mask << PR_MTE_TAG_SHIFT);
/* Enable address tagging ABI, mte error reporting mode and tag inclusion mask. */
if (!prctl(PR_SET_TAGGED_ADDR_CTRL, en, 0, 0, 0) == 0) {
if (prctl(PR_SET_TAGGED_ADDR_CTRL, en, 0, 0, 0) != 0) {
ksft_print_msg("FAIL:prctl PR_SET_TAGGED_ADDR_CTRL for mte mode\n");
return -EINVAL;
}
return 0;
}
#define ID_AA64PFR1_MTE_SHIFT 8
#define ID_AA64PFR1_MTE 2
int mte_default_setup(void)
{
unsigned long hwcaps = getauxval(AT_HWCAP);
unsigned long hwcaps2 = getauxval(AT_HWCAP2);
unsigned long en = 0;
int ret;
if (!(hwcaps & HWCAP_CPUID)) {
ksft_print_msg("FAIL: CPUID registers unavailable\n");
return KSFT_FAIL;
}
/* Read ID_AA64PFR1_EL1 register */
asm volatile("mrs %0, id_aa64pfr1_el1" : "=r"(hwcaps) : : "memory");
if (((hwcaps >> ID_AA64PFR1_MTE_SHIFT) & MT_TAG_MASK) != ID_AA64PFR1_MTE) {
if (!(hwcaps2 & HWCAP2_MTE)) {
ksft_print_msg("FAIL: MTE features unavailable\n");
return KSFT_SKIP;
}
@ -333,6 +337,7 @@ int create_temp_file(void)
/* Create a file in the tmpfs filesystem */
fd = mkstemp(&filename[0]);
if (fd == -1) {
perror(filename);
ksft_print_msg("FAIL: Unable to open temporary file\n");
return 0;
}