diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index cd62f6ffde2a..4e1eb05689ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -496,7 +496,6 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 	struct amdgpu_bo_list_entry vm_pd;
 	struct ww_acquire_ctx ticket;
 	struct list_head list, duplicates;
-	unsigned domain;
 	int r;
 
 	INIT_LIST_HEAD(&list);
@@ -514,12 +513,18 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 		goto error_print;
 
 	list_for_each_entry(entry, &list, head) {
-		domain = amdgpu_mem_type_to_domain(entry->bo->mem.mem_type);
+		struct amdgpu_bo *bo =
+			container_of(entry->bo, struct amdgpu_bo, tbo);
+
 		/* if anything is swapped out don't swap it in here,
 		   just abort and wait for the next CS */
-		if (domain == AMDGPU_GEM_DOMAIN_CPU)
+		if (!amdgpu_bo_gpu_accessible(bo))
+			goto error_unreserve;
+
+		if (bo->shadow && !amdgpu_bo_gpu_accessible(bo->shadow))
 			goto error_unreserve;
 	}
+
 	r = amdgpu_vm_validate_pt_bos(adev, bo_va->vm, amdgpu_gem_va_check,
 				      NULL);
 	if (r)