diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c
index ecff30ab22a0..2e2961a3cf6f 100644
--- a/net/bluetooth/hci_core.c
+++ b/net/bluetooth/hci_core.c
@@ -5468,6 +5468,12 @@ static u8 update_white_list(struct hci_request *req)
 			return 0x00;
 		}
 
+		if (hci_find_irk_by_addr(hdev, &params->addr,
+					 params->addr_type)) {
+			/* White list can not be used with RPAs */
+			return 0x00;
+		}
+
 		white_list_entries++;
 		add_to_white_list(req, params);
 	}
@@ -5486,6 +5492,12 @@ static u8 update_white_list(struct hci_request *req)
 			return 0x00;
 		}
 
+		if (hci_find_irk_by_addr(hdev, &params->addr,
+					 params->addr_type)) {
+			/* White list can not be used with RPAs */
+			return 0x00;
+		}
+
 		white_list_entries++;
 		add_to_white_list(req, params);
 	}
diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c
index e49c83d8b957..a7c344b4acbc 100644
--- a/net/bluetooth/smp.c
+++ b/net/bluetooth/smp.c
@@ -1291,6 +1291,22 @@ static void smp_notify_keys(struct l2cap_conn *conn)
 		bacpy(&hcon->dst, &smp->remote_irk->bdaddr);
 		hcon->dst_type = smp->remote_irk->addr_type;
 		l2cap_conn_update_id_addr(hcon);
+
+		/* When receiving an indentity resolving key for
+		 * a remote device that does not use a resolvable
+		 * private address, just remove the key so that
+		 * it is possible to use the controller white
+		 * list for scanning.
+		 *
+		 * Userspace will have been told to not store
+		 * this key at this point. So it is safe to
+		 * just remove it.
+		 */
+		if (!bacmp(&smp->remote_irk->rpa, BDADDR_ANY)) {
+			list_del(&smp->remote_irk->list);
+			kfree(smp->remote_irk);
+			smp->remote_irk = NULL;
+		}
 	}
 
 	/* The LTKs and CSRKs should be persistent only if both sides