diff --git a/sound/soc/codecs/arizona.c b/sound/soc/codecs/arizona.c
index ac948a671ea6..c14e7551a332 100644
--- a/sound/soc/codecs/arizona.c
+++ b/sound/soc/codecs/arizona.c
@@ -1079,7 +1079,7 @@ int arizona_set_fll(struct arizona_fll *fll, int source,
 {
 	struct arizona *arizona = fll->arizona;
 	struct arizona_fll_cfg cfg, sync;
-	unsigned int reg, val;
+	unsigned int reg;
 	int syncsrc;
 	bool ena;
 	int ret;
@@ -1096,16 +1096,7 @@ int arizona_set_fll(struct arizona_fll *fll, int source,
 	ena = reg & ARIZONA_FLL1_ENA;
 
 	if (Fout) {
-		/* Do we have a 32kHz reference? */
-		regmap_read(arizona->regmap, ARIZONA_CLOCK_32K_1, &val);
-		switch (val & ARIZONA_CLK_32K_SRC_MASK) {
-		case ARIZONA_CLK_SRC_MCLK1:
-		case ARIZONA_CLK_SRC_MCLK2:
-			syncsrc = val & ARIZONA_CLK_32K_SRC_MASK;
-			break;
-		default:
-			syncsrc = -1;
-		}
+		syncsrc = fll->ref_src;
 
 		if (source == syncsrc)
 			syncsrc = -1;
@@ -1115,7 +1106,7 @@ int arizona_set_fll(struct arizona_fll *fll, int source,
 			if (ret != 0)
 				return ret;
 
-			ret = arizona_calc_fll(fll, &cfg, 32768, Fout);
+			ret = arizona_calc_fll(fll, &cfg, fll->ref_freq, Fout);
 			if (ret != 0)
 				return ret;
 		} else {
@@ -1178,6 +1169,7 @@ int arizona_init_fll(struct arizona *arizona, int id, int base, int lock_irq,
 		     int ok_irq, struct arizona_fll *fll)
 {
 	int ret;
+	unsigned int val;
 
 	init_completion(&fll->ok);
 
@@ -1185,6 +1177,18 @@ int arizona_init_fll(struct arizona *arizona, int id, int base, int lock_irq,
 	fll->base = base;
 	fll->arizona = arizona;
 
+	/* Configure default refclk to 32kHz if we have one */
+	regmap_read(arizona->regmap, ARIZONA_CLOCK_32K_1, &val);
+	switch (val & ARIZONA_CLK_32K_SRC_MASK) {
+	case ARIZONA_CLK_SRC_MCLK1:
+	case ARIZONA_CLK_SRC_MCLK2:
+		fll->ref_src = val & ARIZONA_CLK_32K_SRC_MASK;
+		break;
+	default:
+		fll->ref_src = -1;
+	}
+	fll->ref_freq = 32768;
+
 	snprintf(fll->lock_name, sizeof(fll->lock_name), "FLL%d lock", id);
 	snprintf(fll->clock_ok_name, sizeof(fll->clock_ok_name),
 		 "FLL%d clock OK", id);
diff --git a/sound/soc/codecs/arizona.h b/sound/soc/codecs/arizona.h
index 116372c91f5d..124f9f0ef1ac 100644
--- a/sound/soc/codecs/arizona.h
+++ b/sound/soc/codecs/arizona.h
@@ -201,6 +201,9 @@ struct arizona_fll {
 	unsigned int fref;
 	unsigned int fout;
 
+	int ref_src;
+	unsigned int ref_freq;
+
 	char lock_name[ARIZONA_FLL_NAME_LEN];
 	char clock_ok_name[ARIZONA_FLL_NAME_LEN];
 };