diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index 65bb40c240fb..b914e373a478 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -1328,7 +1328,7 @@ static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev,
 
 	gpd_data->base.dev = dev;
 	gpd_data->td.constraint_changed = true;
-	gpd_data->td.effective_constraint_ns = -1;
+	gpd_data->td.effective_constraint_ns = 0;
 	gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier;
 
 	spin_lock_irq(&dev->power.lock);
diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c
index 281f949c5ffe..e4cca8adab32 100644
--- a/drivers/base/power/domain_governor.c
+++ b/drivers/base/power/domain_governor.c
@@ -14,22 +14,33 @@
 static int dev_update_qos_constraint(struct device *dev, void *data)
 {
 	s64 *constraint_ns_p = data;
-	s32 constraint_ns = -1;
+	s64 constraint_ns;
 
-	if (dev->power.subsys_data && dev->power.subsys_data->domain_data)
+	if (dev->power.subsys_data && dev->power.subsys_data->domain_data) {
+		/*
+		 * Only take suspend-time QoS constraints of devices into
+		 * account, because constraints updated after the device has
+		 * been suspended are not guaranteed to be taken into account
+		 * anyway.  In order for them to take effect, the device has to
+		 * be resumed and suspended again.
+		 */
 		constraint_ns = dev_gpd_data(dev)->td.effective_constraint_ns;
-
-	if (constraint_ns < 0) {
+	} else {
+		/*
+		 * The child is not in a domain and there's no info on its
+		 * suspend/resume latencies, so assume them to be negligible and
+		 * take its current PM QoS constraint (that's the only thing
+		 * known at this point anyway).
+		 */
 		constraint_ns = dev_pm_qos_read_value(dev);
-		constraint_ns *= NSEC_PER_USEC;
+		if (constraint_ns > 0)
+			constraint_ns *= NSEC_PER_USEC;
 	}
+
+	/* 0 means "no constraint" */
 	if (constraint_ns == 0)
 		return 0;
 
-	/*
-	 * constraint_ns cannot be negative here, because the device has been
-	 * suspended.
-	 */
 	if (constraint_ns < *constraint_ns_p || *constraint_ns_p == 0)
 		*constraint_ns_p = constraint_ns;
 
@@ -76,14 +87,32 @@ static bool default_suspend_ok(struct device *dev)
 		device_for_each_child(dev, &constraint_ns,
 				      dev_update_qos_constraint);
 
-	if (constraint_ns > 0) {
+	if (constraint_ns == 0) {
+		/* "No restriction", so the device is allowed to suspend. */
+		td->effective_constraint_ns = 0;
+		td->cached_suspend_ok = true;
+	} else if (constraint_ns < 0) {
+		/*
+		 * This triggers if one of the children that don't belong to a
+		 * domain has a negative PM QoS constraint and it's better not
+		 * to suspend then.  effective_constraint_ns is negative already
+		 * and cached_suspend_ok is false, so bail out.
+		 */
+		return false;
+	} else {
 		constraint_ns -= td->suspend_latency_ns +
 				td->resume_latency_ns;
-		if (constraint_ns == 0)
+		/*
+		 * effective_constraint_ns is negative already and
+		 * cached_suspend_ok is false, so if the computed value is not
+		 * positive, return right away.
+		 */
+		if (constraint_ns <= 0)
 			return false;
+
+		td->effective_constraint_ns = constraint_ns;
+		td->cached_suspend_ok = true;
 	}
-	td->effective_constraint_ns = constraint_ns;
-	td->cached_suspend_ok = constraint_ns >= 0;
 
 	/*
 	 * The children have been suspended already, so we don't need to take
@@ -144,18 +173,16 @@ static bool __default_power_down_ok(struct dev_pm_domain *pd,
 		 */
 		td = &to_gpd_data(pdd)->td;
 		constraint_ns = td->effective_constraint_ns;
-		/* default_suspend_ok() need not be called before us. */
-		if (constraint_ns < 0) {
-			constraint_ns = dev_pm_qos_read_value(pdd->dev);
-			constraint_ns *= NSEC_PER_USEC;
-		}
+		/*
+		 * Negative values mean "no suspend at all" and this runs only
+		 * when all devices in the domain are suspended, so it must be
+		 * 0 at least.
+		 *
+		 * 0 means "no constraint"
+		 */
 		if (constraint_ns == 0)
 			continue;
 
-		/*
-		 * constraint_ns cannot be negative here, because the device has
-		 * been suspended.
-		 */
 		if (constraint_ns <= off_on_time_ns)
 			return false;