From 12b003b2e4e47f9f395ffd29719370023b4f0ba8 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Sun, 27 May 2018 09:56:13 +0300 Subject: [PATCH 1/3] mlxsw: reg: Add Management Reset and Shutdown Register Signed-off-by: Jiri Pirko Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/reg.h | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index 3f4d7e22cece..1877d9f8a11a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -7034,6 +7034,30 @@ static inline void mlxsw_reg_mpar_pack(char *payload, u8 local_port, mlxsw_reg_mpar_pa_id_set(payload, pa_id); } +/* MRSR - Management Reset and Shutdown Register + * --------------------------------------------- + * MRSR register is used to reset or shutdown the switch or + * the entire system (when applicable). + */ +#define MLXSW_REG_MRSR_ID 0x9023 +#define MLXSW_REG_MRSR_LEN 0x08 + +MLXSW_REG_DEFINE(mrsr, MLXSW_REG_MRSR_ID, MLXSW_REG_MRSR_LEN); + +/* reg_mrsr_command + * Reset/shutdown command + * 0 - do nothing + * 1 - software reset + * Access: WO + */ +MLXSW_ITEM32(reg, mrsr, command, 0x00, 0, 4); + +static inline void mlxsw_reg_mrsr_pack(char *payload) +{ + MLXSW_REG_ZERO(mrsr, payload); + mlxsw_reg_mrsr_command_set(payload, 1); +} + /* MLCR - Management LED Control Register * -------------------------------------- * Controls the system LEDs. @@ -7898,6 +7922,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(mcia), MLXSW_REG(mpat), MLXSW_REG(mpar), + MLXSW_REG(mrsr), MLXSW_REG(mlcr), MLXSW_REG(mpsc), MLXSW_REG(mcqi), From 2a360bf0f6608f61ef149220cfb9c369f5d086b7 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Sun, 27 May 2018 09:56:14 +0300 Subject: [PATCH 2/3] mlxsw: cmd: Handle error after reset gracefully There is an exception in command interface processing in case the MRSR register is written to. The register triggers FW reset and during the reset FW returns an error. So handle this by ignoring this error while writing to MRSR register. Signed-off-by: Jiri Pirko Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/cmd.h | 16 ++++++++----- drivers/net/ethernet/mellanox/mlxsw/core.c | 26 ++++++++++++++++------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/cmd.h b/drivers/net/ethernet/mellanox/mlxsw/cmd.h index 8da91b023b13..2bc48054b685 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/cmd.h +++ b/drivers/net/ethernet/mellanox/mlxsw/cmd.h @@ -58,7 +58,7 @@ static inline void mlxsw_cmd_mbox_zero(char *mbox) struct mlxsw_core; int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod, - u32 in_mod, bool out_mbox_direct, + u32 in_mod, bool out_mbox_direct, bool reset_ok, char *in_mbox, size_t in_mbox_size, char *out_mbox, size_t out_mbox_size); @@ -67,7 +67,7 @@ static inline int mlxsw_cmd_exec_in(struct mlxsw_core *mlxsw_core, u16 opcode, size_t in_mbox_size) { return mlxsw_cmd_exec(mlxsw_core, opcode, opcode_mod, in_mod, false, - in_mbox, in_mbox_size, NULL, 0); + false, in_mbox, in_mbox_size, NULL, 0); } static inline int mlxsw_cmd_exec_out(struct mlxsw_core *mlxsw_core, u16 opcode, @@ -76,7 +76,7 @@ static inline int mlxsw_cmd_exec_out(struct mlxsw_core *mlxsw_core, u16 opcode, char *out_mbox, size_t out_mbox_size) { return mlxsw_cmd_exec(mlxsw_core, opcode, opcode_mod, in_mod, - out_mbox_direct, NULL, 0, + out_mbox_direct, false, NULL, 0, out_mbox, out_mbox_size); } @@ -84,7 +84,7 @@ static inline int mlxsw_cmd_exec_none(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod, u32 in_mod) { return mlxsw_cmd_exec(mlxsw_core, opcode, opcode_mod, in_mod, false, - NULL, 0, NULL, 0); + false, NULL, 0, NULL, 0); } enum mlxsw_cmd_opcode { @@ -179,6 +179,8 @@ enum mlxsw_cmd_status { MLXSW_CMD_STATUS_BAD_INDEX = 0x0A, /* NVMEM checksum/CRC failed. */ MLXSW_CMD_STATUS_BAD_NVMEM = 0x0B, + /* Device is currently running reset */ + MLXSW_CMD_STATUS_RUNNING_RESET = 0x26, /* Bad management packet (silently discarded). */ MLXSW_CMD_STATUS_BAD_PKT = 0x30, }; @@ -208,6 +210,8 @@ static inline const char *mlxsw_cmd_status_str(u8 status) return "BAD_INDEX"; case MLXSW_CMD_STATUS_BAD_NVMEM: return "BAD_NVMEM"; + case MLXSW_CMD_STATUS_RUNNING_RESET: + return "RUNNING_RESET"; case MLXSW_CMD_STATUS_BAD_PKT: return "BAD_PKT"; default: @@ -869,10 +873,12 @@ MLXSW_ITEM32(cmd_mbox, config_profile, cqe_version, 0xB0, 0, 8); */ static inline int mlxsw_cmd_access_reg(struct mlxsw_core *mlxsw_core, + bool reset_ok, char *in_mbox, char *out_mbox) { return mlxsw_cmd_exec(mlxsw_core, MLXSW_CMD_OPCODE_ACCESS_REG, - 0, 0, false, in_mbox, MLXSW_CMD_MBOX_SIZE, + 0, 0, false, reset_ok, + in_mbox, MLXSW_CMD_MBOX_SIZE, out_mbox, MLXSW_CMD_MBOX_SIZE); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index a38faec45b30..1d9ecf89854e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -1480,6 +1480,7 @@ static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core, { enum mlxsw_emad_op_tlv_status status; int err, n_retry; + bool reset_ok; char *in_mbox, *out_mbox, *tmp; dev_dbg(mlxsw_core->bus_info->dev, "Reg cmd access (reg_id=%x(%s),type=%s)\n", @@ -1501,9 +1502,16 @@ static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core, tmp = in_mbox + MLXSW_EMAD_OP_TLV_LEN * sizeof(u32); mlxsw_emad_pack_reg_tlv(tmp, reg, payload); + /* There is a special treatment needed for MRSR (reset) register. + * The command interface will return error after the command + * is executed, so tell the lower layer to expect it + * and cope accordingly. + */ + reset_ok = reg->id == MLXSW_REG_MRSR_ID; + n_retry = 0; retry: - err = mlxsw_cmd_access_reg(mlxsw_core, in_mbox, out_mbox); + err = mlxsw_cmd_access_reg(mlxsw_core, reset_ok, in_mbox, out_mbox); if (!err) { err = mlxsw_emad_process_status(out_mbox, &status); if (err) { @@ -1793,7 +1801,7 @@ static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core, } int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod, - u32 in_mod, bool out_mbox_direct, + u32 in_mod, bool out_mbox_direct, bool reset_ok, char *in_mbox, size_t in_mbox_size, char *out_mbox, size_t out_mbox_size) { @@ -1816,7 +1824,15 @@ int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod, in_mbox, in_mbox_size, out_mbox, out_mbox_size, &status); - if (err == -EIO && status != MLXSW_CMD_STATUS_OK) { + if (!err && out_mbox) { + dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n"); + mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size); + } + + if (reset_ok && err == -EIO && + status == MLXSW_CMD_STATUS_RUNNING_RESET) { + err = 0; + } else if (err == -EIO && status != MLXSW_CMD_STATUS_OK) { dev_err(mlxsw_core->bus_info->dev, "Cmd exec failed (opcode=%x(%s),opcode_mod=%x,in_mod=%x,status=%x(%s))\n", opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod, in_mod, status, mlxsw_cmd_status_str(status)); @@ -1826,10 +1842,6 @@ int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod, in_mod); } - if (!err && out_mbox) { - dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n"); - mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size); - } return err; } EXPORT_SYMBOL(mlxsw_cmd_exec); From f3a52c6162f835727326a0a497e289494caf1fa5 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Sun, 27 May 2018 09:56:15 +0300 Subject: [PATCH 3/3] mlxsw: pci: Utilize MRSR register to perform FW reset So far, the PCI BAR0 register is used for triggering FW reset. However, that is a legacy attitude and it is recommended to use MRSR to perform reset instead. So do that. Move the reset into init() function as the cmd interface needs to be used. With that, IRQ initialization needs to be moved as well. As a side effect, the reset move simplifies the devlink reload flow. Signed-off-by: Jiri Pirko Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/core.c | 4 +- drivers/net/ethernet/mellanox/mlxsw/core.h | 2 +- drivers/net/ethernet/mellanox/mlxsw/pci.c | 130 ++++++++++----------- 3 files changed, 62 insertions(+), 74 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index 1d9ecf89854e..8a766fe28fa0 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -966,14 +966,12 @@ mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port, static int mlxsw_devlink_core_bus_device_reload(struct devlink *devlink) { struct mlxsw_core *mlxsw_core = devlink_priv(devlink); - const struct mlxsw_bus *mlxsw_bus = mlxsw_core->bus; int err; - if (!mlxsw_bus->reset) + if (!(mlxsw_core->bus->features & MLXSW_BUS_F_RESET)) return -EOPNOTSUPP; mlxsw_core_bus_device_unregister(mlxsw_core, true); - mlxsw_bus->reset(mlxsw_core->bus_priv); err = mlxsw_core_bus_device_register(mlxsw_core->bus_info, mlxsw_core->bus, mlxsw_core->bus_priv, true, diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 4eac7fbd07d5..4a8d4c7f89d9 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -337,6 +337,7 @@ u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core, mlxsw_core_res_get(mlxsw_core, MLXSW_RES_ID_##short_res_id) #define MLXSW_BUS_F_TXRX BIT(0) +#define MLXSW_BUS_F_RESET BIT(1) struct mlxsw_bus { const char *kind; @@ -344,7 +345,6 @@ struct mlxsw_bus { const struct mlxsw_config_profile *profile, struct mlxsw_res *res); void (*fini)(void *bus_priv); - void (*reset)(void *bus_priv); bool (*skb_transmit_busy)(void *bus_priv, const struct mlxsw_tx_info *tx_info); int (*skb_transmit)(void *bus_priv, struct sk_buff *skb, diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index db794a1a3a7e..fc4557245ff4 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -1371,6 +1371,51 @@ static void mlxsw_pci_mbox_free(struct mlxsw_pci *mlxsw_pci, mbox->mapaddr); } +static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci, + const struct pci_device_id *id) +{ + unsigned long end; + char mrsr_pl[MLXSW_REG_MRSR_LEN]; + int err; + + mlxsw_reg_mrsr_pack(mrsr_pl); + err = mlxsw_reg_write(mlxsw_pci->core, MLXSW_REG(mrsr), mrsr_pl); + if (err) + return err; + if (id->device == PCI_DEVICE_ID_MELLANOX_SWITCHX2) { + msleep(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS); + return 0; + } + + /* We must wait for the HW to become responsive once again. */ + msleep(MLXSW_PCI_SW_RESET_WAIT_MSECS); + + end = jiffies + msecs_to_jiffies(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS); + do { + u32 val = mlxsw_pci_read32(mlxsw_pci, FW_READY); + + if ((val & MLXSW_PCI_FW_READY_MASK) == MLXSW_PCI_FW_READY_MAGIC) + break; + cond_resched(); + } while (time_before(jiffies, end)); + return 0; +} + +static int mlxsw_pci_alloc_irq_vectors(struct mlxsw_pci *mlxsw_pci) +{ + int err; + + err = pci_alloc_irq_vectors(mlxsw_pci->pdev, 1, 1, PCI_IRQ_MSIX); + if (err < 0) + dev_err(&mlxsw_pci->pdev->dev, "MSI-X init failed\n"); + return err; +} + +static void mlxsw_pci_free_irq_vectors(struct mlxsw_pci *mlxsw_pci) +{ + pci_free_irq_vectors(mlxsw_pci->pdev); +} + static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core, const struct mlxsw_config_profile *profile, struct mlxsw_res *res) @@ -1398,6 +1443,16 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core, if (err) goto err_out_mbox_alloc; + err = mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id); + if (err) + goto err_sw_reset; + + err = mlxsw_pci_alloc_irq_vectors(mlxsw_pci); + if (err < 0) { + dev_err(&pdev->dev, "MSI-X init failed\n"); + goto err_alloc_irq; + } + err = mlxsw_cmd_query_fw(mlxsw_core, mbox); if (err) goto err_query_fw; @@ -1481,6 +1536,9 @@ err_fw_area_init: err_doorbell_page_bar: err_iface_rev: err_query_fw: + mlxsw_pci_free_irq_vectors(mlxsw_pci); +err_alloc_irq: +err_sw_reset: mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox); err_out_mbox_alloc: mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox); @@ -1496,6 +1554,7 @@ static void mlxsw_pci_fini(void *bus_priv) free_irq(pci_irq_vector(mlxsw_pci->pdev, 0), mlxsw_pci); mlxsw_pci_aqs_fini(mlxsw_pci); mlxsw_pci_fw_area_fini(mlxsw_pci); + mlxsw_pci_free_irq_vectors(mlxsw_pci); mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox); mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox); } @@ -1677,58 +1736,6 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, return err; } -static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci, - const struct pci_device_id *id) -{ - unsigned long end; - - mlxsw_pci_write32(mlxsw_pci, SW_RESET, MLXSW_PCI_SW_RESET_RST_BIT); - if (id->device == PCI_DEVICE_ID_MELLANOX_SWITCHX2) { - msleep(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS); - return 0; - } - - /* Reset needs to be written before we read control register, and - * we must wait for the HW to become responsive once again - */ - wmb(); - msleep(MLXSW_PCI_SW_RESET_WAIT_MSECS); - - end = jiffies + msecs_to_jiffies(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS); - do { - u32 val = mlxsw_pci_read32(mlxsw_pci, FW_READY); - - if ((val & MLXSW_PCI_FW_READY_MASK) == MLXSW_PCI_FW_READY_MAGIC) - break; - cond_resched(); - } while (time_before(jiffies, end)); - return 0; -} - -static void mlxsw_pci_free_irq_vectors(struct mlxsw_pci *mlxsw_pci) -{ - pci_free_irq_vectors(mlxsw_pci->pdev); -} - -static int mlxsw_pci_alloc_irq_vectors(struct mlxsw_pci *mlxsw_pci) -{ - int err; - - err = pci_alloc_irq_vectors(mlxsw_pci->pdev, 1, 1, PCI_IRQ_MSIX); - if (err < 0) - dev_err(&mlxsw_pci->pdev->dev, "MSI-X init failed\n"); - return err; -} - -static void mlxsw_pci_reset(void *bus_priv) -{ - struct mlxsw_pci *mlxsw_pci = bus_priv; - - mlxsw_pci_free_irq_vectors(mlxsw_pci); - mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id); - mlxsw_pci_alloc_irq_vectors(mlxsw_pci); -} - static const struct mlxsw_bus mlxsw_pci_bus = { .kind = "pci", .init = mlxsw_pci_init, @@ -1736,8 +1743,7 @@ static const struct mlxsw_bus mlxsw_pci_bus = { .skb_transmit_busy = mlxsw_pci_skb_transmit_busy, .skb_transmit = mlxsw_pci_skb_transmit, .cmd_exec = mlxsw_pci_cmd_exec, - .features = MLXSW_BUS_F_TXRX, - .reset = mlxsw_pci_reset, + .features = MLXSW_BUS_F_TXRX | MLXSW_BUS_F_RESET, }; static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -1795,18 +1801,6 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) mlxsw_pci->pdev = pdev; pci_set_drvdata(pdev, mlxsw_pci); - err = mlxsw_pci_sw_reset(mlxsw_pci, id); - if (err) { - dev_err(&pdev->dev, "Software reset failed\n"); - goto err_sw_reset; - } - - err = mlxsw_pci_alloc_irq_vectors(mlxsw_pci); - if (err < 0) { - dev_err(&pdev->dev, "MSI-X init failed\n"); - goto err_msix_init; - } - mlxsw_pci->bus_info.device_kind = driver_name; mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev); mlxsw_pci->bus_info.dev = &pdev->dev; @@ -1823,9 +1817,6 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; err_bus_device_register: - mlxsw_pci_free_irq_vectors(mlxsw_pci); -err_msix_init: -err_sw_reset: iounmap(mlxsw_pci->hw_addr); err_ioremap: err_pci_resource_len_check: @@ -1843,7 +1834,6 @@ static void mlxsw_pci_remove(struct pci_dev *pdev) struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev); mlxsw_core_bus_device_unregister(mlxsw_pci->core, false); - mlxsw_pci_free_irq_vectors(mlxsw_pci); iounmap(mlxsw_pci->hw_addr); pci_release_regions(mlxsw_pci->pdev); pci_disable_device(mlxsw_pci->pdev);