add visibility to unexpected consul consumption

refs FOO-965

Change-Id: I007877bfdd414244adc9a1aa618b66ab29313095
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/247900
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Simon Williams <simon@instructure.com>
QA-Review: Ethan Vizitei <evizitei@instructure.com>
Product-Review: Ethan Vizitei <evizitei@instructure.com>

lib/canvas/cache/fallback_expiration_cache.rb

@@ -16,6 +16,7 @@
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 module Canvas
   module Cache
+    class NilCacheEntry < StandardError; end
     # all local cache implementations share a goal of being able to expire
     # cache entries before they're actually gone (think credentials expiring soon, etc),
     # but to hold onto the stale version just in case.
@@ -40,6 +41,10 @@ module Canvas
       private
 
       def write_entry(key, entry, options)
+        if entry.value.nil?
+          Rails.logger.warn("[LOCAL_CACHE] Writing nil value for key #{key}")
+          Canvas::Errors.capture_exception(:local_cache, NilCacheEntry.new("No Config For Key: #{key}"))
+        end
         super(key, entry, options)
         forever_entry = entry.dup
         forever_entry.remove_instance_variable(:@expires_in)

lib/canvas/dynamic_settings/prefix_proxy.rb

@@ -15,6 +15,7 @@
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 module Canvas
   class DynamicSettings
+    class AbsentConfigError < StandardError; end
     # A class for reading values from Consul
     #
     # @attr prefix [String] The prefix to be prepended to keys for querying.
@@ -108,6 +109,8 @@ module Canvas
           end
           return result if result
         end
+        Rails.logger.warn("[DYNAMIC_SETTINGS] config requested which was found no-where (#{key})")
+        Canvas::Errors.capture_exception(:dynamic_settings, AbsentConfigError.new("No Config For Key: #{key}"))
         nil
       rescue Imperium::TimeoutError => exception
         LocalCache.fetch_without_expiration(CACHE_KEY_PREFIX + keys.first).tap do |val|

spec/lib/canvas/dynamic_settings/prefix_proxy_spec.rb

@@ -29,11 +29,21 @@ module Canvas
       end
 
       describe '.fetch(key, ttl: @default_ttl)' do
+        before(:each) do
+          # use in-memory cache to vaoid redis errors for old expirys.
+          # Using redis for local cache results in `ERR invalid expire time in set`
+          # when we try to write an already expired key
+          allow(ConfigFile).to receive(:load).with("local_cache").and_return({ store: "memory" })
+          LocalCache.reset
+        end
+
         it 'must return nil when no value was found' do
           allow(client).to receive(:get)
             .and_return(
               Imperium::Testing.kv_not_found_response(options: [:stale])
             )
+          allow(Canvas::Errors).to receive(:capture_exception).with(any_args).and_call_original
+          expect(Canvas::Errors).to receive(:capture_exception).with(:dynamic_settings, anything)
           expect(proxy.fetch('baz')).to be_nil
         end
 

spec/lib/local_cache_spec.rb

@@ -86,5 +86,16 @@ describe LocalCache do
         expect(LocalCache.fetch_without_expiration("test_key")).to eq("test_value")
       end
     end
+
+    it "allows writing nil, but will tell on you" do
+      expect(Canvas::Errors).to receive(:capture_exception) do |message, e|
+        expect(message).to eq(:local_cache)
+      end
+      LocalCache.fetch("never_seen_key") do
+        # nil on purpose
+        nil
+      end
+      expect(LocalCache.read("never_seen_key")).to be_nil
+    end
   end
 end