RCE resiliant to consul failure

closes CNVS-27666

this stops us from getting a page error in cases
where consul is suddenly gone and we don't have a
cached value for a CDN endpoint.

Users will get an error in their js console,
and the textarea will display with no editor chrome.

TEST PLAN:
 1) point your consul.yml to a fake host
 2) with the feature flag for RCE on, go to eportfolios
    and try to edit one
 3) you should still be able to play with the form, even though
    you won't have an editor

Change-Id: Ia5a1e4b9e243e2c10cdc12726a26f6a0eb2e0d1f
Reviewed-on: https://gerrit.instructure.com/73545
Tested-by: Jenkins
Reviewed-by: Simon Williams <simon@instructure.com>
QA-Review: Jeremy Putnam <jeremyp@instructure.com>
Product-Review: Ethan Vizitei <evizitei@instructure.com>

lib/services/rich_content.rb

@@ -24,6 +24,14 @@ module Services
           RICH_CONTENT_APP_HOST: settings["app-host"],
           RICH_CONTENT_CDN_HOST: settings["cdn-host"]
         }
+      rescue Faraday::ConnectionFailed,
+             Faraday::ClientError,
+             Canvas::DynamicSettings::ConsulError => e
+        Canvas::Errors.capture_exception(:rce_flag, e)
+        {
+          RICH_CONTENT_APP_HOST: "error",
+          RICH_CONTENT_CDN_HOST: "error"
+        }
       end
 
       def fine_grained_flags(root_account)

spec/lib/services/rich_content_spec.rb

@@ -23,6 +23,28 @@ module Services
         expect(env[:RICH_CONTENT_CDN_HOST]).to eq("rce-cdn")
       end
 
+      it "populates hosts with an error signal when consul is down" do
+
+        Canvas::DynamicSettings.stubs(:find).with("rich-content-service").
+          raises(Faraday::ConnectionFailed, "can't talk to consul")
+        root_account = stub("root_account", feature_enabled?: true)
+        env = described_class.env_for(root_account)
+        expect(env[:RICH_CONTENT_SERVICE_ENABLED]).to be_truthy
+        expect(env[:RICH_CONTENT_APP_HOST]).to eq("error")
+        expect(env[:RICH_CONTENT_CDN_HOST]).to eq("error")
+      end
+
+      it "logs errors for later consideration" do
+        Canvas::DynamicSettings.stubs(:find).with("rich-content-service").
+          raises(Canvas::DynamicSettings::ConsulError, "can't talk to consul")
+        root_account = stub("root_account", feature_enabled?: true)
+        Canvas::Errors.expects(:capture_exception).with do |type, e|
+          expect(type).to eq(:rce_flag)
+          expect(e.is_a?(Canvas::DynamicSettings::ConsulError)).to be_truthy
+        end
+        described_class.env_for(root_account)
+      end
+
       it "can enable only the non-sidebar use cases" do
         root_account = stub("root_account")
         root_account.stubs(:feature_enabled?).with(:rich_content_service).returns(true)

spec/spec_helper.rb

@@ -435,6 +435,7 @@ RSpec.configure do |config|
     Delayed::Job.redis.flushdb if Delayed::Job == Delayed::Backend::Redis::Job
     Rails::logger.try(:info, "Running #{self.class.description} #{@method_name}")
     Attachment.domain_namespace = nil
+    Canvas::DynamicSettings.reset_cache!
     $spec_api_tokens = {}
   end