Outcomes Service content migration service

closes OUT-3655

flag=outcome_alignments_course_migration

test plan:
  - start up canvas, quiz_api and outcomes
  - for an account, provision outcomes service
    and sync some course outcomes (see g/235665)
  - enable the outcome_alignments_course_migration
    and outcome_alignment_non_scoring_content
    feature options in the root account of a course
  - in a course, create a canvas page and align
    an outcome
  - export a copy of the course
  - create a new course
  - import the course content into the new course
  - re-sync course outcomes and contexts
  - re-import the course content into the new course
  - confirm that the new course contains the canvas page
    with the outcome aligned to it

Change-Id: Ie5cce9e1f8829736c03fd5f6bc2c7a8c000c8333
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/238973
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Pat Renner <prenner@instructure.com>
Reviewed-by: Michael Brewer-Davis <mbd@instructure.com>
QA-Review: Michael Brewer-Davis <mbd@instructure.com>
Product-Review: Michael Brewer-Davis <mbd@instructure.com>

app/models/outcomes_service/migration_service.rb

@@ -0,0 +1,151 @@
+#
+# Copyright (C) 2020 - present Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+module OutcomesService
+  class MigrationService
+    class << self
+      def applies_to_course?(course)
+        course.root_account.feature_enabled?(:outcome_alignments_course_migration) &&
+          OutcomesService::Service.enabled_in_context?(course)
+      end
+
+      def begin_export(course, opts)
+        artifacts = export_artifacts(course, opts)
+        return nil if artifacts.empty?
+        data = {
+          context_type: 'course',
+          context_id: course.id.to_s,
+          export_settings: {
+            format: 'canvas',
+            artifacts: artifacts
+          }
+        }
+        content_exports_url = "#{OutcomesService::Service.url(course)}/api/content_exports"
+        response = CanvasHttp.post(
+          content_exports_url,
+          headers_for(course, 'content_migration.export', context_type: 'course', context_id: course.id.to_s),
+          body: data.to_json,
+          content_type: 'application/json'
+        )
+        if response.code =~ /^2/
+          json = JSON.parse(response.body)
+          { export_id: json['id'], course: course }
+        else
+          raise "Error queueing export for Outcomes Service: #{response.body}"
+        end
+      end
+
+      def export_completed?(export_data)
+        content_export_url = "#{OutcomesService::Service.url(export_data[:course])}/api/content_exports/#{export_data[:export_id]}"
+        response = CanvasHttp.get(
+          content_export_url,
+          headers_for(export_data[:course], 'content_migration.export', id: export_data[:export_id])
+        )
+        if response.code =~ /^2/
+          json = JSON.parse(response.body)
+          case json['state']
+          when 'completed'
+            true
+          when 'failed'
+            raise "Content Export for Outcomes Service failed"
+          else
+            false
+          end
+        else
+          raise "Error retrieving export state for Outcomes Service: #{response.body}"
+        end
+      end
+
+      def retrieve_export(export_data)
+        content_export_url = "#{OutcomesService::Service.url(export_data[:course])}/api/content_exports/#{export_data[:export_id]}"
+        response = CanvasHttp.get(
+          content_export_url,
+          headers_for(export_data[:course], 'content_migration.export', id: export_data[:export_id])
+        )
+        if response.code =~ /^2/
+          json = JSON.parse(response.body)
+          json['data']
+        else
+          raise "Error retrieving export for Outcomes Service: #{response.body}"
+        end
+      end
+
+      def send_imported_content(course, content_migration, imported_content)
+        content_imports_url = "#{OutcomesService::Service.url(course)}/api/content_imports"
+        data = imported_content.merge(
+          context_type: 'course',
+          context_id: course.id.to_s,
+          external_migration_id: content_migration.id
+        )
+        response = CanvasHttp.post(
+          content_imports_url,
+          headers_for(course, 'content_migration.import', context_type: 'course', context_id: course.id.to_s),
+          body: data.to_json,
+          content_type: 'application/json'
+        )
+        if response.code =~ /^2/
+          json = JSON.parse(response.body)
+          { import_id: json['id'], course: course }
+        else
+          raise "Error sending import for Outcomes Service: #{response.body}"
+        end
+      end
+
+      def import_completed?(import_data)
+        content_import_url = "#{OutcomesService::Service.url(import_data[:course])}/api/content_imports/#{import_data[:import_id]}"
+        response = CanvasHttp.get(
+          content_import_url,
+          headers_for(import_data[:course], 'content_migration.import', id: import_data[:import_id])
+        )
+        if response.code =~ /^2/
+          json = JSON.parse(response.body)
+          case json['state']
+          when 'completed'
+            true
+          when 'failed'
+            raise 'Content Import for Outcomes Service failed'
+          else
+            false
+          end
+        else
+          raise "Error retrieving import state for Outcomes Service: #{response.body}"
+        end
+      end
+
+      private
+
+      def headers_for(course, scope, overrides = {})
+        {
+          'Authorization' => OutcomesService::Service.jwt(course, scope, overrides: overrides)
+        }
+      end
+
+      def export_artifacts(course, opts)
+        page_ids = if opts[:selective]
+          opts[:exported_assets].map{|asset| (match = asset.match(/wiki_page_(\d+)/)) && match[1]}.compact
+        else
+          course.wiki_pages.pluck(:id)
+        end
+        return [] unless page_ids.any?
+        [{
+          external_type: 'canvas.page',
+          external_id: page_ids
+        }]
+      end
+    end
+  end
+end

app/models/outcomes_service/service.rb

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2020 - present Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+module OutcomesService
+  class Service
+    class << self
+      def url(context)
+        settings = settings(context)
+        protocol = ENV.fetch('OUTCOMES_SERVICE_PROTOCOL', Rails.env.production? ? 'https' : 'http')
+        domain = settings[:domain]
+        "#{protocol}://#{domain}" if domain.present?
+      end
+
+      def enabled_in_context?(context)
+        settings = settings(context)
+        settings[:consumer_key].present? && settings[:jwt_secret].present? && settings[:domain].present?
+      end
+
+      def jwt(context, scope, expiration = 1.day.from_now.to_i, overrides: {})
+        settings = settings(context)
+        if settings.key?(:consumer_key) && settings.key?(:jwt_secret) && settings.key?(:domain)
+          consumer_key = settings[:consumer_key]
+          jwt_secret = settings[:jwt_secret]
+          domain = settings[:domain]
+          payload = {
+            host: domain,
+            consumer_key: consumer_key,
+            scope: scope,
+            exp: expiration
+          }.merge(overrides)
+          JWT.encode(payload, jwt_secret, 'HS512')
+        end
+      end
+
+      private
+
+      def settings(context)
+        context.root_account.settings.dig(:provision, 'outcomes') || {}
+      end
+    end
+  end
+end
+

config/feature_flags/outcomes_feature_flags.yml

@@ -41,3 +41,9 @@ limit_section_visibility_in_lmgb:
   display_name: Restricted view for teachers in LMGB
   description: Hides certain students and sections in the LMGB if the teacher only
     has access to a particular section
+outcome_alignments_course_migration:
+  state: hidden
+  applies_to: RootAccount
+  display_name: Outcomes Service Alignment Migration
+  description: Includes Outcomes Service alignments when exporting and importing
+    course content

config/initializers/external_migrations.rb

@@ -16,4 +16,5 @@
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 
 Canvas::Migration::ExternalContent::Migrator.register_service('conditional_release', ConditionalRelease::MigrationService)
+Canvas::Migration::ExternalContent::Migrator.register_service('outcomes_service', OutcomesService::MigrationService)
 Canvas::Migration::ExternalContent::Migrator.register_service('quizzes_next_export', QuizzesNext::ExportService)

spec/models/outcomes_service/migration_service_spec.rb

@@ -0,0 +1,277 @@
+#
+# Copyright (C) 2020 - present Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require_relative '../../spec_helper'
+require_relative '../../sharding_spec_helper'
+require 'webmock/rspec'
+
+describe OutcomesService::MigrationService do
+  around(:example) do |example|
+    WebMock.disable_net_connect!(allow_localhost: true)
+    example.run
+    WebMock.enable_net_connect!
+  end
+
+  let(:root_account) { account_model }
+  let(:course) { course_model(root_account: root_account) }
+
+  context 'without settings' do
+    describe '.applies_to_course?' do
+      it 'returns false' do
+        expect(described_class.applies_to_course?(course)).to eq false
+      end
+    end
+  end
+
+  context 'with settings' do
+    before do
+      root_account.settings[:provision] = { 'outcomes' => {
+        domain: 'canvas.test',
+        consumer_key: 'blah',
+        jwt_secret: 'woo'
+      }}
+      root_account.save!
+    end
+
+    context 'with feature flag disabled' do
+      before do
+        root_account.disable_feature!(:outcome_alignments_course_migration)
+      end
+
+      describe '.applies_to_course?' do
+        it 'returns false' do
+          expect(described_class.applies_to_course?(course)).to eq false
+        end
+      end
+    end
+
+    context 'with feature flag enabled' do
+      before do
+        root_account.enable_feature!(:outcome_alignments_course_migration)
+      end
+
+      describe '.applies_to_course?' do
+        it 'returns true' do
+          expect(described_class.applies_to_course?(course)).to eq true
+        end
+      end
+
+      describe '.begin_export' do
+        before do
+          course.wiki_pages.create(
+            title: 'Some random wiki page',
+            body: 'wiki page content'
+          )
+        end
+
+        def stub_post_content_export(external_ids = [course.wiki_pages.first.id])
+          stub_request(:post, 'http://canvas.test/api/content_exports').with({
+            body: {
+              context_type: 'course',
+              context_id: course.id.to_s,
+              export_settings: {
+                format: 'canvas',
+                artifacts: [{
+                  external_type: 'canvas.page',
+                  external_id: external_ids
+                }]
+              }
+            },
+            headers: {
+              Authorization: /.+/
+            }
+          })
+        end
+
+        it 'should request course wiki page' do
+          stub_post_content_export.to_return(status: 200, body: '{"id":123}')
+          expect(described_class.begin_export(course, {})).to eq({
+            export_id: 123,
+            course: course
+          })
+        end
+
+        it 'should raise error on non 2xx response' do
+          stub_post_content_export.to_return(status: 401, body: '{"valid_jwt":false}')
+          expect { described_class.begin_export(course, {}) }.to raise_error(/Error queueing export for Outcomes Service/)
+        end
+
+        it 'should request selective course wiki page' do
+          stub_post_content_export(['2']).to_return(status: 200, body: '{"id":123}')
+          expect(described_class.begin_export(course, {
+            selective: true,
+            exported_assets: ['wiki_page_2']
+          })).to eq({
+            export_id: 123,
+            course: course
+          })
+        end
+
+        it 'should return no export if no artifacts requested' do
+          expect(described_class.begin_export(course, {
+            selective: true,
+            exported_assets: []
+          })).to eq nil
+        end
+      end
+
+      describe '.export_completed?' do
+        let(:export_data) do
+          {
+            course: course,
+            export_id: 1
+          }
+        end
+
+        def stub_get_content_export
+          stub_request(:get, 'http://canvas.test/api/content_exports/1').with({
+            headers: {
+              Authorization: /\+*/
+            }
+          })
+        end
+
+        it 'returns true on completed' do
+          stub_get_content_export.to_return(status: 200, body: '{"state":"completed"}')
+          expect(described_class.export_completed?(export_data)).to eq true
+        end
+
+        it 'returns false on pending' do
+          stub_get_content_export.to_return(status: 200, body: '{"state":"in_progress"}')
+          expect(described_class.export_completed?(export_data)).to eq false
+        end
+
+        it 'raises error on failed' do
+          stub_get_content_export.to_return(status: 200, body: '{"state":"failed"}')
+          expect { described_class.export_completed?(export_data) }.to raise_error('Content Export for Outcomes Service failed')
+        end
+
+        it 'raises error on non 2xx response' do
+          stub_request(:get, 'http://canvas.test/api/content_exports/1').to_return(status: 401, body: '{"valid_jwt":false}')
+          export_data = {
+            course: course,
+            export_id: 1
+          }
+          expect { described_class.export_completed?(export_data) }.to raise_error('Error retrieving export state for Outcomes Service: {"valid_jwt":false}')
+        end
+      end
+
+      describe '.retrieve_export' do
+        let(:export_data) do
+          {
+            course: course,
+            export_id: 1
+          }
+        end
+
+        def stub_get_content_export
+          stub_request(:get, 'http://canvas.test/api/content_exports/1').with({
+            headers: {
+              Authorization: /\+*/
+            }
+          })
+        end
+
+        it 'returns export data when complete' do
+          stub_get_content_export.to_return(status: 200, body: '{"data":"stuff"}')
+          expect(described_class.retrieve_export(export_data)).to eq 'stuff'
+        end
+
+        it 'raises error on non 2xx response' do
+          stub_get_content_export.to_return(status: 401, body: '{"valid_jwt":false}')
+          expect { described_class.retrieve_export(export_data) }.to raise_error('Error retrieving export for Outcomes Service: {"valid_jwt":false}')
+        end
+      end
+
+      describe '.send_imported_content' do
+        let(:content_migration) { ContentMigration.create!(context: course) }
+        let(:imported_content) do
+          {
+            data: 'stuff'
+          }
+        end
+
+        def stub_post_content_import
+          stub_request(:post, 'http://canvas.test/api/content_imports').with(
+            body: {
+              context_type: 'course',
+              context_id: course.id.to_s,
+              external_migration_id: content_migration.id,
+              data: 'stuff'
+            },
+            headers: {
+              Authorization: /\+*/
+            }
+          )
+        end
+
+        it 'returns import id on import creation' do
+          stub_post_content_import.to_return(status: 200, body: '{"id":123}')
+          expect(described_class.send_imported_content(course, content_migration, imported_content)). to eq({
+            import_id: 123,
+            course: course
+          })
+        end
+
+        it 'raises error on non 2xx response' do
+          stub_post_content_import.to_return(status: 401, body: '{"valid_jwt":false}')
+          expect { described_class.send_imported_content(course, content_migration, imported_content) }.to raise_error(
+            'Error sending import for Outcomes Service: {"valid_jwt":false}'
+          )
+        end
+      end
+
+      describe '.import_completed?' do
+        let(:import_data) do
+          {
+            course: course,
+            import_id: 1
+          }
+        end
+
+        def stub_get_content_import
+          stub_request(:get, 'http://canvas.test/api/content_imports/1').with({
+            headers: {
+              Authorization: /\+*/
+            }
+          })
+        end
+
+        it 'returns true on completed' do
+          stub_get_content_import.to_return(status: 200, body: '{"state":"completed"}')
+          expect(described_class.import_completed?(import_data)).to eq true
+        end
+
+        it 'returns false on pending' do
+          stub_get_content_import.to_return(status: 200, body: '{"state":"in_progress"}')
+          expect(described_class.import_completed?(import_data)).to eq false
+        end
+
+        it 'raises error on failed' do
+          stub_get_content_import.to_return(status: 200, body: '{"state":"failed"}')
+          expect { described_class.import_completed?(import_data) }.to raise_error('Content Import for Outcomes Service failed')
+        end
+
+        it 'raises error on non 2xx response' do
+          stub_get_content_import.to_return(status: 401, body: '{"valid_jwt":false}')
+          expect { described_class.import_completed?(import_data) }.to raise_error('Error retrieving import state for Outcomes Service: {"valid_jwt":false}')
+        end
+      end
+    end
+  end
+end

spec/models/outcomes_service/service_spec.rb

@@ -0,0 +1,87 @@
+#
+# Copyright (C) 2020 - present Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require_relative '../../spec_helper'
+require_relative '../../sharding_spec_helper'
+
+describe OutcomesService::Service do
+  Service = OutcomesService::Service
+
+  let(:root_account) { account_model }
+  let(:course) { course_model(root_account: root_account) }
+
+  context 'without settings' do
+    describe '.url' do
+      it 'returns nil url' do
+        expect(Service.url(course)).to be_nil
+      end
+    end
+
+    describe '.enabled_in_context?' do
+      it 'returns not enabled' do
+        expect(Service.enabled_in_context?(course)).to eq false
+      end
+    end
+
+    describe '.jwt' do
+      it 'returns nil jwt' do
+        expect(Service.jwt(course, 'outcomes.show')).to be_nil
+      end
+    end
+  end
+
+  context 'with settings' do
+    before do
+      root_account.settings[:provision] = { 'outcomes' => {
+        domain: 'canvas.test',
+        consumer_key: 'blah',
+        jwt_secret: 'woo'
+      }}
+      root_account.save!
+    end
+
+    describe '.url' do
+      it 'returns url' do
+        expect(Service.url(course)).to eq 'http://canvas.test'
+      end
+    end
+
+    describe '.enabled_in_context?' do
+      it 'returns enabled' do
+        expect(Service.enabled_in_context?(course)).to eq true
+      end
+    end
+
+    describe '.jwt' do
+      it 'returns valid jwt' do
+        expect(Service.jwt(course, 'outcomes.show')).not_to be_nil
+      end
+
+      it 'includes overrides' do
+        token = Service.jwt(course, 'outcomes.list', overrides: { context_uuid: 'xyz' })
+        decoded = JWT.decode(token, 'woo', true, algorithm: 'HS512')
+        expect(decoded[0]).to include(
+          'host' => 'canvas.test',
+          'consumer_key' => 'blah',
+          'scope' => 'outcomes.list',
+          'context_uuid' => 'xyz'
+        )
+      end
+    end
+  end
+end