only populate full local DynamicSettings cache when expired

rather than on every fetch call

closes #LA-900

Change-Id: If603066732e7e96aa6ad6fb4614f756c0433de42
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/233042
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Cody Cutrer <cody@instructure.com>
Product-Review: Cody Cutrer <cody@instructure.com>
QA-Review: Clint Furse <cfurse@instructure.com>

lib/canvas/dynamic_settings/prefix_proxy.rb

@@ -78,13 +78,21 @@ module Canvas
           ["global", tree, service, prefix, key].compact.join("/"),
         ].uniq - keys
 
-        # pre-cache an entire tree
-        tree_key = [tree, service, environment].compact.join("/")
-        subtree = LocalCache.fetch(CACHE_KEY_PREFIX + tree_key + '/', expires_in: ttl) do
-          result = @kv_client.get(tree_key, :recurse, :stale)
-          result.values if result&.status == 200
+        # try to get the local cache first right away
+        keys.each do |full_key|
+          result = LocalCache.fetch(CACHE_KEY_PREFIX + full_key)
+          return result if result
+        end
+
+        # okay now pre-cache an entire tree
+        tree_key = [tree, service, environment].compact.join("/")
+        LocalCache.fetch(CACHE_KEY_PREFIX + tree_key + '/', expires_in: ttl) do
+          result = @kv_client.get(tree_key, :recurse, :stale)
+          if result&.status == 200
+            populate_cache(tree_key, result.values, ttl) # only populate recursively when we missed
+            result.values
+          end
         end
-        populate_cache(tree_key, subtree, ttl)
 
         keys.each do |full_key|
           # these keys will have been populated (or not!) above; don't

spec/lib/canvas/dynamic_settings/prefix_proxy_spec.rb

@@ -57,6 +57,7 @@ module Canvas
         end
 
         it 'must use the dynamic settings cache for previously fetched values' do
+          expect(LocalCache).to receive(:fetch).with(DynamicSettings::CACHE_KEY_PREFIX + 'foo/bar/baz').ordered
           expect(LocalCache).to receive(:fetch).with(DynamicSettings::CACHE_KEY_PREFIX + '/', expires_in: 3.minutes).ordered
           expect(LocalCache).to receive(:fetch).with(DynamicSettings::CACHE_KEY_PREFIX + 'foo/bar/baz').ordered
           expect(LocalCache).to receive(:fetch).with(DynamicSettings::CACHE_KEY_PREFIX + 'global/foo/bar/baz', expires_in: 3.minutes).ordered