diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c index 7b60268d93c0..240f5006e278 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c @@ -21,6 +21,7 @@ */ #include "amdgpu.h" #include "amdgpu_amdkfd.h" +#include "amdgpu_amdkfd_gfx_v10.h" #include "gc/gc_10_1_0_offset.h" #include "gc/gc_10_1_0_sh_mask.h" #include "athub/athub_2_0_0_offset.h" @@ -709,6 +710,99 @@ static void set_vm_context_page_table_base(struct amdgpu_device *adev, adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base); } +/* + * GFX10 helper for wave launch stall requirements on debug trap setting. + * + * vmid: + * Target VMID to stall/unstall. + * + * stall: + * 0-unstall wave launch (enable), 1-stall wave launch (disable). + * After wavefront launch has been stalled, allocated waves must drain from + * SPI in order for debug trap settings to take effect on those waves. + * This is roughly a ~3500 clock cycle wait on SPI where a read on + * SPI_GDBG_WAVE_CNTL translates to ~32 clock cycles. + * KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY indicates the number of reads required. + * + * NOTE: We can afford to clear the entire STALL_VMID field on unstall + * because current GFX10 chips cannot support multi-process debugging due to + * trap configuration and masking being limited to global scope. Always + * assume single process conditions. + * + */ + +#define KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY 110 +static void kgd_gfx_v10_set_wave_launch_stall(struct amdgpu_device *adev, uint32_t vmid, bool stall) +{ + uint32_t data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL)); + int i; + + data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL, STALL_VMID, + stall ? 1 << vmid : 0); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), data); + + if (!stall) + return; + + for (i = 0; i < KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY; i++) + RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL)); +} + +uint32_t kgd_gfx_v10_enable_debug_trap(struct amdgpu_device *adev, + bool restore_dbg_registers, + uint32_t vmid) +{ + + mutex_lock(&adev->grbm_idx_mutex); + + kgd_gfx_v10_set_wave_launch_stall(adev, vmid, true); + + /* assume gfx off is disabled for the debug session if rlc restore not supported. */ + if (restore_dbg_registers) { + uint32_t data = 0; + + data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, + VMID_SEL, 1 << vmid); + data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, + TRAP_EN, 1); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0); + WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0); + + kgd_gfx_v10_set_wave_launch_stall(adev, vmid, false); + + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; + } + + WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); + + kgd_gfx_v10_set_wave_launch_stall(adev, vmid, false); + + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; +} + +uint32_t kgd_gfx_v10_disable_debug_trap(struct amdgpu_device *adev, + bool keep_trap_enabled, + uint32_t vmid) +{ + mutex_lock(&adev->grbm_idx_mutex); + + kgd_gfx_v10_set_wave_launch_stall(adev, vmid, true); + + WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); + + kgd_gfx_v10_set_wave_launch_stall(adev, vmid, false); + + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; +} + static void program_trap_handler_settings(struct amdgpu_device *adev, uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr, uint32_t inst) @@ -752,5 +846,7 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = { .get_atc_vmid_pasid_mapping_info = get_atc_vmid_pasid_mapping_info, .set_vm_context_page_table_base = set_vm_context_page_table_base, + .enable_debug_trap = kgd_gfx_v10_enable_debug_trap, + .disable_debug_trap = kgd_gfx_v10_disable_debug_trap, .program_trap_handler_settings = program_trap_handler_settings, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h new file mode 100644 index 000000000000..251d61fbde07 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h @@ -0,0 +1,28 @@ +/* + * Copyright 2023 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +uint32_t kgd_gfx_v10_enable_debug_trap(struct amdgpu_device *adev, + bool restore_dbg_registers, + uint32_t vmid); +uint32_t kgd_gfx_v10_disable_debug_trap(struct amdgpu_device *adev, + bool keep_trap_enabled, + uint32_t vmid); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c index 52d0d35fb58d..8b293f3dcbd2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c @@ -22,6 +22,7 @@ #include #include "amdgpu.h" #include "amdgpu_amdkfd.h" +#include "amdgpu_amdkfd_gfx_v10.h" #include "gc/gc_10_3_0_offset.h" #include "gc/gc_10_3_0_sh_mask.h" #include "oss/osssys_5_0_0_offset.h" @@ -654,143 +655,6 @@ static void program_trap_handler_settings_v10_3(struct amdgpu_device *adev, unlock_srbm(adev); } -#if 0 -uint32_t enable_debug_trap_v10_3(struct amdgpu_device *adev, - uint32_t trap_debug_wave_launch_mode, - uint32_t vmid) -{ - uint32_t data = 0; - uint32_t orig_wave_cntl_value; - uint32_t orig_stall_vmid; - - mutex_lock(&adev->grbm_idx_mutex); - - orig_wave_cntl_value = RREG32(SOC15_REG_OFFSET(GC, - 0, - mmSPI_GDBG_WAVE_CNTL)); - orig_stall_vmid = REG_GET_FIELD(orig_wave_cntl_value, - SPI_GDBG_WAVE_CNTL, - STALL_VMID); - - data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL, STALL_RA, 1); - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), data); - - data = 0; - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), data); - - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), orig_stall_vmid); - - mutex_unlock(&adev->grbm_idx_mutex); - - return 0; -} - -uint32_t disable_debug_trap_v10_3(struct amdgpu_device *adev) -{ - mutex_lock(&adev->grbm_idx_mutex); - - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); - - mutex_unlock(&adev->grbm_idx_mutex); - - return 0; -} - -uint32_t set_wave_launch_trap_override_v10_3(struct amdgpu_device *adev, - uint32_t trap_override, - uint32_t trap_mask) -{ - uint32_t data = 0; - - mutex_lock(&adev->grbm_idx_mutex); - - data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL)); - data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL, STALL_RA, 1); - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), data); - - data = 0; - data = REG_SET_FIELD(data, SPI_GDBG_TRAP_MASK, - EXCP_EN, trap_mask); - data = REG_SET_FIELD(data, SPI_GDBG_TRAP_MASK, - REPLACE, trap_override); - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), data); - - data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL)); - data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL, STALL_RA, 0); - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), data); - - mutex_unlock(&adev->grbm_idx_mutex); - - return 0; -} - -uint32_t set_wave_launch_mode_v10_3(struct amdgpu_device *adev, - uint8_t wave_launch_mode, - uint32_t vmid) -{ - uint32_t data = 0; - bool is_stall_mode; - bool is_mode_set; - - is_stall_mode = (wave_launch_mode == 4); - is_mode_set = (wave_launch_mode != 0 && wave_launch_mode != 4); - - mutex_lock(&adev->grbm_idx_mutex); - - data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL2, - VMID_MASK, is_mode_set ? 1 << vmid : 0); - data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL2, - MODE, is_mode_set ? wave_launch_mode : 0); - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL2), data); - - data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL)); - data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL, - STALL_VMID, is_stall_mode ? 1 << vmid : 0); - data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL, - STALL_RA, is_stall_mode ? 1 : 0); - WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), data); - - mutex_unlock(&adev->grbm_idx_mutex); - - return 0; -} - -/* kgd_get_iq_wait_times: Returns the mmCP_IQ_WAIT_TIME1/2 values - * The values read are: - * ib_offload_wait_time -- Wait Count for Indirect Buffer Offloads. - * atomic_offload_wait_time -- Wait Count for L2 and GDS Atomics Offloads. - * wrm_offload_wait_time -- Wait Count for WAIT_REG_MEM Offloads. - * gws_wait_time -- Wait Count for Global Wave Syncs. - * que_sleep_wait_time -- Wait Count for Dequeue Retry. - * sch_wave_wait_time -- Wait Count for Scheduling Wave Message. - * sem_rearm_wait_time -- Wait Count for Semaphore re-arm. - * deq_retry_wait_time -- Wait Count for Global Wave Syncs. - */ -void get_iq_wait_times_v10_3(struct amdgpu_device *adev, - uint32_t *wait_times, uint32_t inst) - -{ - *wait_times = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2)); -} - -void build_grace_period_packet_info_v10_3(struct amdgpu_device *adev, - uint32_t wait_times, - uint32_t grace_period, - uint32_t *reg_offset, - uint32_t *reg_data, - uint32_t inst) -{ - *reg_data = wait_times; - - *reg_data = REG_SET_FIELD(*reg_data, - CP_IQ_WAIT_TIME2, - SCH_WAVE, - grace_period); - - *reg_offset = mmCP_IQ_WAIT_TIME2; -} -#endif - const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v10_3, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v10_3, @@ -808,12 +672,6 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = { .get_atc_vmid_pasid_mapping_info = get_atc_vmid_pasid_mapping_info_v10_3, .set_vm_context_page_table_base = set_vm_context_page_table_base_v10_3, .program_trap_handler_settings = program_trap_handler_settings_v10_3, -#if 0 - .enable_debug_trap = enable_debug_trap_v10_3, - .disable_debug_trap = disable_debug_trap_v10_3, - .set_wave_launch_trap_override = set_wave_launch_trap_override_v10_3, - .set_wave_launch_mode = set_wave_launch_mode_v10_3, - .get_iq_wait_times = get_iq_wait_times_v10_3, - .build_grace_period_packet_info = build_grace_period_packet_info_v10_3, -#endif + .enable_debug_trap = kgd_gfx_v10_enable_debug_trap, + .disable_debug_trap = kgd_gfx_v10_disable_debug_trap };