From e328d410826d52e9ee348aff9064c4a207f2adb1 Mon Sep 17 00:00:00 2001 From: Roland Dreier Date: Fri, 6 May 2011 08:32:53 +0000 Subject: [PATCH] vmxnet3: Consistently disable irqs when taking adapter->cmd_lock Using the vmxnet3 driver produces a lockdep warning because vmxnet3_set_mc(), which is called with mc->mca_lock held, takes adapter->cmd_lock. However, there are a couple of places where adapter->cmd_lock is taken with softirqs enabled, lockdep warns that a softirq that tries to take mc->mca_lock could happen while adapter->cmd_lock is held, leading to an AB-BA deadlock. I'm not sure if this is a real potential deadlock or not, but the simplest and best fix seems to be simply to make sure we take cmd_lock with spin_lock_irqsave() everywhere -- the places with plain spin_lock just look like oversights. The full enormous lockdep warning is: ========================================================= [ INFO: possible irq lock inversion dependency detected ] 2.6.39-rc6+ #1 --------------------------------------------------------- ifconfig/567 just changed the state of lock: (&(&mc->mca_lock)->rlock){+.-...}, at: [] mld_ifc_timer_expire+0xff/0x280 but this lock took another, SOFTIRQ-unsafe lock in the past: (&(&adapter->cmd_lock)->rlock){+.+...} and interrupts could create inverse lock ordering between them. other info that might help us debug this: 4 locks held by ifconfig/567: #0: (rtnl_mutex){+.+.+.}, at: [] rtnl_lock+0x17/0x20 #1: ((inetaddr_chain).rwsem){.+.+.+}, at: [] __blocking_notifier_call_chain+0x5f/0xb0 #2: (&idev->mc_ifc_timer){+.-...}, at: [] run_timer_softirq+0xeb/0x3f0 #3: (&ndev->lock){++.-..}, at: [] mld_ifc_timer_expire+0x32/0x280 the shortest dependencies between 2nd lock and 1st lock: -> (&(&adapter->cmd_lock)->rlock){+.+...} ops: 11 { HARDIRQ-ON-W at: [] __lock_acquire+0x7f6/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock+0x36/0x70 [] vmxnet3_alloc_intr_resources+0x22/0x230 [vmxnet3] [] vmxnet3_probe_device+0x5f6/0x15c5 [vmxnet3] [] local_pci_probe+0x5f/0xd0 [] pci_device_probe+0x119/0x120 [] driver_probe_device+0x96/0x1c0 [] __driver_attach+0xab/0xb0 [] bus_for_each_dev+0x5e/0x90 [] driver_attach+0x1e/0x20 [] bus_add_driver+0xc8/0x290 [] driver_register+0x76/0x140 [] __pci_register_driver+0x66/0xe0 [] serio_raw_poll+0x3a/0x60 [serio_raw] [] do_one_initcall+0x45/0x190 [] sys_init_module+0xfb/0x250 [] system_call_fastpath+0x16/0x1b SOFTIRQ-ON-W at: [] __lock_acquire+0x827/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock+0x36/0x70 [] vmxnet3_alloc_intr_resources+0x22/0x230 [vmxnet3] [] vmxnet3_probe_device+0x5f6/0x15c5 [vmxnet3] [] local_pci_probe+0x5f/0xd0 [] pci_device_probe+0x119/0x120 [] driver_probe_device+0x96/0x1c0 [] __driver_attach+0xab/0xb0 [] bus_for_each_dev+0x5e/0x90 [] driver_attach+0x1e/0x20 [] bus_add_driver+0xc8/0x290 [] driver_register+0x76/0x140 [] __pci_register_driver+0x66/0xe0 [] serio_raw_poll+0x3a/0x60 [serio_raw] [] do_one_initcall+0x45/0x190 [] sys_init_module+0xfb/0x250 [] system_call_fastpath+0x16/0x1b INITIAL USE at: [] __lock_acquire+0x459/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock+0x36/0x70 [] vmxnet3_alloc_intr_resources+0x22/0x230 [vmxnet3] [] vmxnet3_probe_device+0x5f6/0x15c5 [vmxnet3] [] local_pci_probe+0x5f/0xd0 [] pci_device_probe+0x119/0x120 [] driver_probe_device+0x96/0x1c0 [] __driver_attach+0xab/0xb0 [] bus_for_each_dev+0x5e/0x90 [] driver_attach+0x1e/0x20 [] bus_add_driver+0xc8/0x290 [] driver_register+0x76/0x140 [] __pci_register_driver+0x66/0xe0 [] serio_raw_poll+0x3a/0x60 [serio_raw] [] do_one_initcall+0x45/0x190 [] sys_init_module+0xfb/0x250 [] system_call_fastpath+0x16/0x1b } ... key at: [] __key.42516+0x0/0xffffffffffffda70 [vmxnet3] ... acquired at: [] lock_acquire+0x9d/0x130 [] _raw_spin_lock_irqsave+0x55/0xa0 [] vmxnet3_set_mc+0x97/0x1a0 [vmxnet3] [] __dev_set_rx_mode+0x40/0xb0 [] dev_set_rx_mode+0x30/0x50 [] __dev_open+0xc7/0x100 [] __dev_change_flags+0xa1/0x180 [] dev_change_flags+0x28/0x70 [] devinet_ioctl+0x730/0x800 [] inet_ioctl+0x88/0xa0 [] sock_do_ioctl+0x30/0x70 [] sock_ioctl+0x79/0x2f0 [] do_vfs_ioctl+0x98/0x570 [] sys_ioctl+0x91/0xa0 [] system_call_fastpath+0x16/0x1b -> (_xmit_ETHER){+.....} ops: 6 { HARDIRQ-ON-W at: [] __lock_acquire+0x7f6/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock_bh+0x3b/0x70 [] __dev_mc_add+0x38/0x90 [] dev_mc_add+0x10/0x20 [] igmp6_group_added+0x10e/0x1b0 [] ipv6_dev_mc_inc+0x2cd/0x430 [] ipv6_add_dev+0x357/0x450 [] addrconf_notify+0x2f7/0xb10 [] notifier_call_chain+0x8c/0xc0 [] raw_notifier_call_chain+0x16/0x20 [] call_netdevice_notifiers+0x37/0x70 [] register_netdevice+0x244/0x2d0 [] register_netdev+0x3f/0x60 [] vmxnet3_probe_device+0x760/0x15c5 [vmxnet3] [] local_pci_probe+0x5f/0xd0 [] pci_device_probe+0x119/0x120 [] driver_probe_device+0x96/0x1c0 [] __driver_attach+0xab/0xb0 [] bus_for_each_dev+0x5e/0x90 [] driver_attach+0x1e/0x20 [] bus_add_driver+0xc8/0x290 [] driver_register+0x76/0x140 [] __pci_register_driver+0x66/0xe0 [] serio_raw_poll+0x3a/0x60 [serio_raw] [] do_one_initcall+0x45/0x190 [] sys_init_module+0xfb/0x250 [] system_call_fastpath+0x16/0x1b INITIAL USE at: [] __lock_acquire+0x459/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock_bh+0x3b/0x70 [] __dev_mc_add+0x38/0x90 [] dev_mc_add+0x10/0x20 [] igmp6_group_added+0x10e/0x1b0 [] ipv6_dev_mc_inc+0x2cd/0x430 [] ipv6_add_dev+0x357/0x450 [] addrconf_notify+0x2f7/0xb10 [] notifier_call_chain+0x8c/0xc0 [] raw_notifier_call_chain+0x16/0x20 [] call_netdevice_notifiers+0x37/0x70 [] register_netdevice+0x244/0x2d0 [] register_netdev+0x3f/0x60 [] vmxnet3_probe_device+0x760/0x15c5 [vmxnet3] [] local_pci_probe+0x5f/0xd0 [] pci_device_probe+0x119/0x120 [] driver_probe_device+0x96/0x1c0 [] __driver_attach+0xab/0xb0 [] bus_for_each_dev+0x5e/0x90 [] driver_attach+0x1e/0x20 [] bus_add_driver+0xc8/0x290 [] driver_register+0x76/0x140 [] __pci_register_driver+0x66/0xe0 [] serio_raw_poll+0x3a/0x60 [serio_raw] [] do_one_initcall+0x45/0x190 [] sys_init_module+0xfb/0x250 [] system_call_fastpath+0x16/0x1b } ... key at: [] netdev_addr_lock_key+0x8/0x1e0 ... acquired at: [] lock_acquire+0x9d/0x130 [] _raw_spin_lock_bh+0x3b/0x70 [] __dev_mc_add+0x38/0x90 [] dev_mc_add+0x10/0x20 [] igmp6_group_added+0x10e/0x1b0 [] ipv6_dev_mc_inc+0x2cd/0x430 [] ipv6_add_dev+0x357/0x450 [] addrconf_notify+0x2f7/0xb10 [] notifier_call_chain+0x8c/0xc0 [] raw_notifier_call_chain+0x16/0x20 [] call_netdevice_notifiers+0x37/0x70 [] register_netdevice+0x244/0x2d0 [] register_netdev+0x3f/0x60 [] vmxnet3_probe_device+0x760/0x15c5 [vmxnet3] [] local_pci_probe+0x5f/0xd0 [] pci_device_probe+0x119/0x120 [] driver_probe_device+0x96/0x1c0 [] __driver_attach+0xab/0xb0 [] bus_for_each_dev+0x5e/0x90 [] driver_attach+0x1e/0x20 [] bus_add_driver+0xc8/0x290 [] driver_register+0x76/0x140 [] __pci_register_driver+0x66/0xe0 [] serio_raw_poll+0x3a/0x60 [serio_raw] [] do_one_initcall+0x45/0x190 [] sys_init_module+0xfb/0x250 [] system_call_fastpath+0x16/0x1b -> (&(&mc->mca_lock)->rlock){+.-...} ops: 6 { HARDIRQ-ON-W at: [] __lock_acquire+0x7f6/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock_bh+0x3b/0x70 [] igmp6_group_added+0x45/0x1b0 [] ipv6_dev_mc_inc+0x2cd/0x430 [] ipv6_add_dev+0x357/0x450 [] addrconf_init+0x4e/0x183 [] inet6_init+0x191/0x2a6 [] do_one_initcall+0x45/0x190 [] kernel_init+0xe3/0x168 [] kernel_thread_helper+0x4/0x10 IN-SOFTIRQ-W at: [] __lock_acquire+0x7ce/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock_bh+0x3b/0x70 [] mld_ifc_timer_expire+0xff/0x280 [] run_timer_softirq+0x179/0x3f0 [] __do_softirq+0xc0/0x210 [] call_softirq+0x1c/0x30 [] do_softirq+0xad/0xe0 [] irq_exit+0x9e/0xb0 [] smp_apic_timer_interrupt+0x70/0x9b [] apic_timer_interrupt+0x13/0x20 [] rt_do_flush+0x87/0x2a0 [] rt_cache_flush+0x46/0x60 [] fib_disable_ip+0x40/0x60 [] fib_inetaddr_event+0xd7/0xe0 [] notifier_call_chain+0x8c/0xc0 [] __blocking_notifier_call_chain+0x78/0xb0 [] blocking_notifier_call_chain+0x16/0x20 [] __inet_del_ifa+0xf1/0x2e0 [] inet_del_ifa+0x13/0x20 [] devinet_ioctl+0x501/0x800 [] inet_ioctl+0x88/0xa0 [] sock_do_ioctl+0x30/0x70 [] sock_ioctl+0x79/0x2f0 [] do_vfs_ioctl+0x98/0x570 [] sys_ioctl+0x91/0xa0 [] system_call_fastpath+0x16/0x1b INITIAL USE at: [] __lock_acquire+0x459/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock_bh+0x3b/0x70 [] igmp6_group_added+0x45/0x1b0 [] ipv6_dev_mc_inc+0x2cd/0x430 [] ipv6_add_dev+0x357/0x450 [] addrconf_init+0x4e/0x183 [] inet6_init+0x191/0x2a6 [] do_one_initcall+0x45/0x190 [] kernel_init+0xe3/0x168 [] kernel_thread_helper+0x4/0x10 } ... key at: [] __key.40877+0x0/0x8 ... acquired at: [] check_usage_forwards+0x9c/0x110 [] mark_lock+0x19c/0x400 [] __lock_acquire+0x7ce/0x1e10 [] lock_acquire+0x9d/0x130 [] _raw_spin_lock_bh+0x3b/0x70 [] mld_ifc_timer_expire+0xff/0x280 [] run_timer_softirq+0x179/0x3f0 [] __do_softirq+0xc0/0x210 [] call_softirq+0x1c/0x30 [] do_softirq+0xad/0xe0 [] irq_exit+0x9e/0xb0 [] smp_apic_timer_interrupt+0x70/0x9b [] apic_timer_interrupt+0x13/0x20 [] rt_do_flush+0x87/0x2a0 [] rt_cache_flush+0x46/0x60 [] fib_disable_ip+0x40/0x60 [] fib_inetaddr_event+0xd7/0xe0 [] notifier_call_chain+0x8c/0xc0 [] __blocking_notifier_call_chain+0x78/0xb0 [] blocking_notifier_call_chain+0x16/0x20 [] __inet_del_ifa+0xf1/0x2e0 [] inet_del_ifa+0x13/0x20 [] devinet_ioctl+0x501/0x800 [] inet_ioctl+0x88/0xa0 [] sock_do_ioctl+0x30/0x70 [] sock_ioctl+0x79/0x2f0 [] do_vfs_ioctl+0x98/0x570 [] sys_ioctl+0x91/0xa0 [] system_call_fastpath+0x16/0x1b stack backtrace: Pid: 567, comm: ifconfig Not tainted 2.6.39-rc6+ #1 Call Trace: [] print_irq_inversion_bug+0x146/0x170 [] ? print_irq_inversion_bug+0x170/0x170 [] check_usage_forwards+0x9c/0x110 [] mark_lock+0x19c/0x400 [] __lock_acquire+0x7ce/0x1e10 [] ? mark_lock+0x1f3/0x400 [] ? __lock_acquire+0xf07/0x1e10 [] ? native_sched_clock+0x15/0x70 [] lock_acquire+0x9d/0x130 [] ? mld_ifc_timer_expire+0xff/0x280 [] ? lock_release_holdtime+0x3d/0x1a0 [] _raw_spin_lock_bh+0x3b/0x70 [] ? mld_ifc_timer_expire+0xff/0x280 [] ? _raw_spin_unlock+0x2b/0x40 [] mld_ifc_timer_expire+0xff/0x280 [] run_timer_softirq+0x179/0x3f0 [] ? run_timer_softirq+0xeb/0x3f0 [] ? sched_clock+0x9/0x10 [] ? mld_gq_timer_expire+0x30/0x30 [] __do_softirq+0xc0/0x210 [] ? tick_program_event+0x1f/0x30 [] call_softirq+0x1c/0x30 [] do_softirq+0xad/0xe0 [] irq_exit+0x9e/0xb0 [] smp_apic_timer_interrupt+0x70/0x9b [] apic_timer_interrupt+0x13/0x20 [] ? retint_restore_args+0x13/0x13 [] ? lock_is_held+0x17/0xd0 [] rt_do_flush+0x87/0x2a0 [] rt_cache_flush+0x46/0x60 [] fib_disable_ip+0x40/0x60 [] fib_inetaddr_event+0xd7/0xe0 [] notifier_call_chain+0x8c/0xc0 [] __blocking_notifier_call_chain+0x78/0xb0 [] blocking_notifier_call_chain+0x16/0x20 [] __inet_del_ifa+0xf1/0x2e0 [] inet_del_ifa+0x13/0x20 [] devinet_ioctl+0x501/0x800 [] ? local_clock+0x6f/0x80 [] ? do_page_fault+0x268/0x560 [] inet_ioctl+0x88/0xa0 [] sock_do_ioctl+0x30/0x70 [] sock_ioctl+0x79/0x2f0 [] ? __call_rcu+0xa7/0x190 [] do_vfs_ioctl+0x98/0x570 [] ? fget_light+0x33e/0x430 [] ? retint_swapgs+0x13/0x1b [] sys_ioctl+0x91/0xa0 [] system_call_fastpath+0x16/0x1b Signed-off-by: Roland Dreier Signed-off-by: Shreyas N Bhatewara Signed-off-by: Scott J. Goldman Signed-off-by: David S. Miller --- drivers/net/vmxnet3/vmxnet3_drv.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c index 0d47c3a05307..c16ed961153a 100644 --- a/drivers/net/vmxnet3/vmxnet3_drv.c +++ b/drivers/net/vmxnet3/vmxnet3_drv.c @@ -178,6 +178,7 @@ static void vmxnet3_process_events(struct vmxnet3_adapter *adapter) { int i; + unsigned long flags; u32 events = le32_to_cpu(adapter->shared->ecr); if (!events) return; @@ -190,10 +191,10 @@ vmxnet3_process_events(struct vmxnet3_adapter *adapter) /* Check if there is an error on xmit/recv queues */ if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { - spin_lock(&adapter->cmd_lock); + spin_lock_irqsave(&adapter->cmd_lock, flags); VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_QUEUE_STATUS); - spin_unlock(&adapter->cmd_lock); + spin_unlock_irqrestore(&adapter->cmd_lock, flags); for (i = 0; i < adapter->num_tx_queues; i++) if (adapter->tqd_start[i].status.stopped) @@ -2733,13 +2734,14 @@ static void vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) { u32 cfg; + unsigned long flags; /* intr settings */ - spin_lock(&adapter->cmd_lock); + spin_lock_irqsave(&adapter->cmd_lock, flags); VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_CONF_INTR); cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); - spin_unlock(&adapter->cmd_lock); + spin_unlock_irqrestore(&adapter->cmd_lock, flags); adapter->intr.type = cfg & 0x3; adapter->intr.mask_mode = (cfg >> 2) & 0x3;