375 lines
9.3 KiB
C
375 lines
9.3 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/*
|
|
* Copyright 2016-2019 HabanaLabs, Ltd.
|
|
* All Rights Reserved.
|
|
*/
|
|
|
|
#include <uapi/misc/habanalabs.h>
|
|
#include "habanalabs.h"
|
|
|
|
#include <linux/fs.h>
|
|
#include <linux/uaccess.h>
|
|
#include <linux/slab.h>
|
|
|
|
static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
|
|
[HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
|
|
[HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
|
|
[HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
|
|
[HL_DEBUG_OP_FUNNEL] = 0,
|
|
[HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
|
|
[HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
|
|
[HL_DEBUG_OP_TIMESTAMP] = 0
|
|
|
|
};
|
|
|
|
static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
|
|
{
|
|
struct hl_info_device_status dev_stat = {0};
|
|
u32 size = args->return_size;
|
|
void __user *out = (void __user *) (uintptr_t) args->return_pointer;
|
|
|
|
if ((!size) || (!out))
|
|
return -EINVAL;
|
|
|
|
dev_stat.status = hl_device_status(hdev);
|
|
|
|
return copy_to_user(out, &dev_stat,
|
|
min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
|
|
}
|
|
|
|
static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
|
|
{
|
|
struct hl_info_hw_ip_info hw_ip = {0};
|
|
u32 size = args->return_size;
|
|
void __user *out = (void __user *) (uintptr_t) args->return_pointer;
|
|
struct asic_fixed_properties *prop = &hdev->asic_prop;
|
|
u64 sram_kmd_size, dram_kmd_size;
|
|
|
|
if ((!size) || (!out))
|
|
return -EINVAL;
|
|
|
|
sram_kmd_size = (prop->sram_user_base_address -
|
|
prop->sram_base_address);
|
|
dram_kmd_size = (prop->dram_user_base_address -
|
|
prop->dram_base_address);
|
|
|
|
hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev);
|
|
hw_ip.sram_base_address = prop->sram_user_base_address;
|
|
hw_ip.dram_base_address = prop->dram_user_base_address;
|
|
hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask;
|
|
hw_ip.sram_size = prop->sram_size - sram_kmd_size;
|
|
hw_ip.dram_size = prop->dram_size - dram_kmd_size;
|
|
if (hw_ip.dram_size > 0)
|
|
hw_ip.dram_enabled = 1;
|
|
hw_ip.num_of_events = prop->num_of_events;
|
|
memcpy(hw_ip.armcp_version,
|
|
prop->armcp_info.armcp_version, VERSION_MAX_LEN);
|
|
hw_ip.armcp_cpld_version = __le32_to_cpu(prop->armcp_info.cpld_version);
|
|
hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
|
|
hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
|
|
hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
|
|
hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor;
|
|
|
|
return copy_to_user(out, &hw_ip,
|
|
min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0;
|
|
}
|
|
|
|
static int hw_events_info(struct hl_device *hdev, struct hl_info_args *args)
|
|
{
|
|
u32 size, max_size = args->return_size;
|
|
void __user *out = (void __user *) (uintptr_t) args->return_pointer;
|
|
void *arr;
|
|
|
|
if ((!max_size) || (!out))
|
|
return -EINVAL;
|
|
|
|
arr = hdev->asic_funcs->get_events_stat(hdev, &size);
|
|
|
|
return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0;
|
|
}
|
|
|
|
static int dram_usage_info(struct hl_device *hdev, struct hl_info_args *args)
|
|
{
|
|
struct hl_info_dram_usage dram_usage = {0};
|
|
u32 max_size = args->return_size;
|
|
void __user *out = (void __user *) (uintptr_t) args->return_pointer;
|
|
struct asic_fixed_properties *prop = &hdev->asic_prop;
|
|
u64 dram_kmd_size;
|
|
|
|
if ((!max_size) || (!out))
|
|
return -EINVAL;
|
|
|
|
dram_kmd_size = (prop->dram_user_base_address -
|
|
prop->dram_base_address);
|
|
dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) -
|
|
atomic64_read(&hdev->dram_used_mem);
|
|
dram_usage.ctx_dram_mem = atomic64_read(&hdev->user_ctx->dram_phys_mem);
|
|
|
|
return copy_to_user(out, &dram_usage,
|
|
min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0;
|
|
}
|
|
|
|
static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
|
|
{
|
|
struct hl_info_hw_idle hw_idle = {0};
|
|
u32 max_size = args->return_size;
|
|
void __user *out = (void __user *) (uintptr_t) args->return_pointer;
|
|
|
|
if ((!max_size) || (!out))
|
|
return -EINVAL;
|
|
|
|
hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
|
|
&hw_idle.busy_engines_mask, NULL);
|
|
|
|
return copy_to_user(out, &hw_idle,
|
|
min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
|
|
}
|
|
|
|
static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
|
|
{
|
|
struct hl_debug_params *params;
|
|
void *input = NULL, *output = NULL;
|
|
int rc;
|
|
|
|
params = kzalloc(sizeof(*params), GFP_KERNEL);
|
|
if (!params)
|
|
return -ENOMEM;
|
|
|
|
params->reg_idx = args->reg_idx;
|
|
params->enable = args->enable;
|
|
params->op = args->op;
|
|
|
|
if (args->input_ptr && args->input_size) {
|
|
input = memdup_user(u64_to_user_ptr(args->input_ptr),
|
|
args->input_size);
|
|
if (IS_ERR(input)) {
|
|
rc = PTR_ERR(input);
|
|
input = NULL;
|
|
dev_err(hdev->dev,
|
|
"error %d when copying input debug data\n", rc);
|
|
goto out;
|
|
}
|
|
|
|
params->input = input;
|
|
}
|
|
|
|
if (args->output_ptr && args->output_size) {
|
|
output = kzalloc(args->output_size, GFP_KERNEL);
|
|
if (!output) {
|
|
rc = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
params->output = output;
|
|
params->output_size = args->output_size;
|
|
}
|
|
|
|
rc = hdev->asic_funcs->debug_coresight(hdev, params);
|
|
if (rc) {
|
|
dev_err(hdev->dev,
|
|
"debug coresight operation failed %d\n", rc);
|
|
goto out;
|
|
}
|
|
|
|
if (output) {
|
|
if (copy_to_user((void __user *) (uintptr_t) args->output_ptr,
|
|
output,
|
|
args->output_size)) {
|
|
dev_err(hdev->dev,
|
|
"copy to user failed in debug ioctl\n");
|
|
rc = -EFAULT;
|
|
goto out;
|
|
}
|
|
}
|
|
|
|
out:
|
|
kfree(params);
|
|
kfree(output);
|
|
kfree(input);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
|
|
{
|
|
struct hl_info_args *args = data;
|
|
struct hl_device *hdev = hpriv->hdev;
|
|
int rc;
|
|
|
|
/* We want to return device status even if it disabled or in reset */
|
|
if (args->op == HL_INFO_DEVICE_STATUS)
|
|
return device_status_info(hdev, args);
|
|
|
|
if (hl_device_disabled_or_in_reset(hdev)) {
|
|
dev_warn_ratelimited(hdev->dev,
|
|
"Device is %s. Can't execute INFO IOCTL\n",
|
|
atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
|
|
return -EBUSY;
|
|
}
|
|
|
|
switch (args->op) {
|
|
case HL_INFO_HW_IP_INFO:
|
|
rc = hw_ip_info(hdev, args);
|
|
break;
|
|
|
|
case HL_INFO_HW_EVENTS:
|
|
rc = hw_events_info(hdev, args);
|
|
break;
|
|
|
|
case HL_INFO_DRAM_USAGE:
|
|
rc = dram_usage_info(hdev, args);
|
|
break;
|
|
|
|
case HL_INFO_HW_IDLE:
|
|
rc = hw_idle(hdev, args);
|
|
break;
|
|
|
|
default:
|
|
dev_err(hdev->dev, "Invalid request %d\n", args->op);
|
|
rc = -ENOTTY;
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
|
|
{
|
|
struct hl_debug_args *args = data;
|
|
struct hl_device *hdev = hpriv->hdev;
|
|
int rc = 0;
|
|
|
|
if (hl_device_disabled_or_in_reset(hdev)) {
|
|
dev_warn_ratelimited(hdev->dev,
|
|
"Device is %s. Can't execute DEBUG IOCTL\n",
|
|
atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
|
|
return -EBUSY;
|
|
}
|
|
|
|
switch (args->op) {
|
|
case HL_DEBUG_OP_ETR:
|
|
case HL_DEBUG_OP_ETF:
|
|
case HL_DEBUG_OP_STM:
|
|
case HL_DEBUG_OP_FUNNEL:
|
|
case HL_DEBUG_OP_BMON:
|
|
case HL_DEBUG_OP_SPMU:
|
|
case HL_DEBUG_OP_TIMESTAMP:
|
|
if (!hdev->in_debug) {
|
|
dev_err_ratelimited(hdev->dev,
|
|
"Rejecting debug configuration request because device not in debug mode\n");
|
|
return -EFAULT;
|
|
}
|
|
args->input_size =
|
|
min(args->input_size, hl_debug_struct_size[args->op]);
|
|
rc = debug_coresight(hdev, args);
|
|
break;
|
|
case HL_DEBUG_OP_SET_MODE:
|
|
rc = hl_device_set_debug_mode(hdev, (bool) args->enable);
|
|
break;
|
|
default:
|
|
dev_err(hdev->dev, "Invalid request %d\n", args->op);
|
|
rc = -ENOTTY;
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
#define HL_IOCTL_DEF(ioctl, _func) \
|
|
[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
|
|
|
|
static const struct hl_ioctl_desc hl_ioctls[] = {
|
|
HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
|
|
HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
|
|
HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
|
|
HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl),
|
|
HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
|
|
HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
|
|
};
|
|
|
|
#define HL_CORE_IOCTL_COUNT ARRAY_SIZE(hl_ioctls)
|
|
|
|
long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
|
|
{
|
|
struct hl_fpriv *hpriv = filep->private_data;
|
|
struct hl_device *hdev = hpriv->hdev;
|
|
hl_ioctl_t *func;
|
|
const struct hl_ioctl_desc *ioctl = NULL;
|
|
unsigned int nr = _IOC_NR(cmd);
|
|
char stack_kdata[128] = {0};
|
|
char *kdata = NULL;
|
|
unsigned int usize, asize;
|
|
int retcode;
|
|
|
|
if (hdev->hard_reset_pending) {
|
|
dev_crit_ratelimited(hdev->dev,
|
|
"Device HARD reset pending! Please close FD\n");
|
|
return -ENODEV;
|
|
}
|
|
|
|
if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
|
|
u32 hl_size;
|
|
|
|
ioctl = &hl_ioctls[nr];
|
|
|
|
hl_size = _IOC_SIZE(ioctl->cmd);
|
|
usize = asize = _IOC_SIZE(cmd);
|
|
if (hl_size > asize)
|
|
asize = hl_size;
|
|
|
|
cmd = ioctl->cmd;
|
|
} else {
|
|
dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
|
|
task_pid_nr(current), nr);
|
|
return -ENOTTY;
|
|
}
|
|
|
|
/* Do not trust userspace, use our own definition */
|
|
func = ioctl->func;
|
|
|
|
if (unlikely(!func)) {
|
|
dev_dbg(hdev->dev, "no function\n");
|
|
retcode = -ENOTTY;
|
|
goto out_err;
|
|
}
|
|
|
|
if (cmd & (IOC_IN | IOC_OUT)) {
|
|
if (asize <= sizeof(stack_kdata)) {
|
|
kdata = stack_kdata;
|
|
} else {
|
|
kdata = kzalloc(asize, GFP_KERNEL);
|
|
if (!kdata) {
|
|
retcode = -ENOMEM;
|
|
goto out_err;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (cmd & IOC_IN) {
|
|
if (copy_from_user(kdata, (void __user *)arg, usize)) {
|
|
retcode = -EFAULT;
|
|
goto out_err;
|
|
}
|
|
} else if (cmd & IOC_OUT) {
|
|
memset(kdata, 0, usize);
|
|
}
|
|
|
|
retcode = func(hpriv, kdata);
|
|
|
|
if (cmd & IOC_OUT)
|
|
if (copy_to_user((void __user *)arg, kdata, usize))
|
|
retcode = -EFAULT;
|
|
|
|
out_err:
|
|
if (retcode)
|
|
dev_dbg(hdev->dev,
|
|
"error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
|
|
task_pid_nr(current), cmd, nr);
|
|
|
|
if (kdata != stack_kdata)
|
|
kfree(kdata);
|
|
|
|
return retcode;
|
|
}
|