diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index d053110207eb..a1826f4c2e23 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -181,7 +181,7 @@ int ib_get_cached_subnet_prefix(struct ib_device *device,
 				u64              *sn_pfx);
 
 #ifdef CONFIG_SECURITY_INFINIBAND
-void ib_security_destroy_port_pkey_list(struct ib_device *device);
+void ib_security_release_port_pkey_list(struct ib_device *device);
 
 void ib_security_cache_change(struct ib_device *device,
 			      u8 port_num,
@@ -204,7 +204,7 @@ void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent);
 int ib_mad_enforce_security(struct ib_mad_agent_private *map, u16 pkey_index);
 void ib_mad_agent_security_change(void);
 #else
-static inline void ib_security_destroy_port_pkey_list(struct ib_device *device)
+static inline void ib_security_release_port_pkey_list(struct ib_device *device)
 {
 }
 
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 60083bde3e39..b997feac2c63 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -253,6 +253,8 @@ static void ib_device_release(struct device *device)
 		ib_cache_release_one(dev);
 		kfree(dev->port_immutable);
 	}
+	ib_security_release_port_pkey_list(dev);
+	kfree(dev->port_pkey_list);
 	kfree(dev);
 }
 
@@ -522,7 +524,6 @@ static void cleanup_device(struct ib_device *device)
 {
 	ib_cache_cleanup_one(device);
 	ib_cache_release_one(device);
-	kfree(device->port_pkey_list);
 	kfree(device->port_immutable);
 }
 
@@ -560,12 +561,10 @@ static int setup_device(struct ib_device *device)
 	if (ret) {
 		dev_warn(&device->dev,
 			 "Couldn't set up InfiniBand P_Key/GID cache\n");
-		goto pkey_cleanup;
+		return ret;
 	}
 	return 0;
 
-pkey_cleanup:
-	kfree(device->port_pkey_list);
 port_cleanup:
 	kfree(device->port_immutable);
 	return ret;
@@ -682,9 +681,6 @@ void ib_unregister_device(struct ib_device *device)
 
 	ib_cache_cleanup_one(device);
 
-	ib_security_destroy_port_pkey_list(device);
-	kfree(device->port_pkey_list);
-
 	down_write(&lists_rwsem);
 	write_lock_irqsave(&device->client_data_lock, flags);
 	list_for_each_entry_safe(context, tmp, &device->client_data_list,
diff --git a/drivers/infiniband/core/security.c b/drivers/infiniband/core/security.c
index a70d2ba312ed..dad6a94a43f3 100644
--- a/drivers/infiniband/core/security.c
+++ b/drivers/infiniband/core/security.c
@@ -558,13 +558,12 @@ void ib_security_cache_change(struct ib_device *device,
 	}
 }
 
-void ib_security_destroy_port_pkey_list(struct ib_device *device)
+void ib_security_release_port_pkey_list(struct ib_device *device)
 {
 	struct pkey_index_qp_list *pkey, *tmp_pkey;
 	int i;
 
 	for (i = rdma_start_port(device); i <= rdma_end_port(device); i++) {
-		spin_lock(&device->port_pkey_list[i].list_lock);
 		list_for_each_entry_safe(pkey,
 					 tmp_pkey,
 					 &device->port_pkey_list[i].pkey_list,
@@ -572,7 +571,6 @@ void ib_security_destroy_port_pkey_list(struct ib_device *device)
 			list_del(&pkey->pkey_index_list);
 			kfree(pkey);
 		}
-		spin_unlock(&device->port_pkey_list[i].list_lock);
 	}
 }