diff --git a/app/controllers/content_migrations_controller.rb b/app/controllers/content_migrations_controller.rb index f335b3e92db..405f873c135 100644 --- a/app/controllers/content_migrations_controller.rb +++ b/app/controllers/content_migrations_controller.rb @@ -249,6 +249,58 @@ class ContentMigrationsController < ApplicationController render :json => json end + # @undocumented Leaving undocumented for now because format is expected to change + # Get list of items in the migration for selective import of content + # + # If no type is sent you will get a list of the top-level sections in the content + # It will look something like this: + # [ + # { + # "type": "course_settings", + # "property": "copy[all_course_settings]", + # "title": "Course Settings" + # }, + # { + # "type": "syllabus_body", + # "property": "copy[all_syllabus_body]", + # "title": "Syllabus Body" + # }, + # { + # "type": "context_modules", + # "property": "copy[all_context_modules]", + # "title": "Modules", + # "count": 1 + # }, + # { + # "type": "discussion_topics", + # "property": "copy[all_discussion_topics]", + # "title": "Discussion Topics", + # "count": 1 + # }, + # { + # "type": "wiki_pages", + # "property": "copy[all_wiki_pages]", + # "title": "Wiki Pages", + # "count": 1 + # }, + # { + # "type": "attachments", + # "property": "copy[all_attachments]", + # "title": "Files", + # "count": 1 + # } + # ] + # + # If there is no count for an item that means there are no sub-items and you + # shouldn't try to fetch them + # + # @argument type [string] (optional) Return list of specified type + # + # @returns list of content items + def content_list + @content_migration = @context.content_migrations.find(params[:id]) + render :json => @content_migration.get_content_list(params[:type]) + end protected diff --git a/app/models/content_migration.rb b/app/models/content_migration.rb index 7bfefda7878..ffe14cd00da 100644 --- a/app/models/content_migration.rb +++ b/app/models/content_migration.rb @@ -320,7 +320,7 @@ class ContentMigration < ActiveRecord::Base p end - def export_content + def queue_migration reset_job_progress check_quiz_id_prepender plugin = Canvas::Plugin.find(migration_type) @@ -350,7 +350,7 @@ class ContentMigration < ActiveRecord::Base self.save end end - alias_method :queue_migration, :export_content + alias_method :export_content, :queue_migration def check_quiz_id_prepender if !migration_settings[:id_prepender] && (!migration_settings[:overwrite_questions] || !migration_settings[:overwrite_quizzes]) @@ -562,4 +562,11 @@ class ContentMigration < ActiveRecord::Base end end end + + # returns a list of content for selective content migrations + # If no section is specified the top-level areas with content are returned + def get_content_list(type=nil) + Canvas::Migration::Helpers::SelectiveContentFormatter.new(self).get_content_list(type) + end + end diff --git a/config/routes.rb b/config/routes.rb index cdfd5511e3d..2c3eba7bc77 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -768,6 +768,7 @@ ActionController::Routing::Routes.draw do |map| cm.get 'courses/:course_id/content_migrations', :action => :index, :path_name => 'course_content_migration_list' cm.post 'courses/:course_id/content_migrations', :action => :create, :path_name => 'course_content_migration_create' cm.put 'courses/:course_id/content_migrations/:id', :action => :update, :path_name => 'course_content_migration_update' + cm.get 'courses/:course_id/content_migrations/:id/selective_data', :action => :content_list, :path_name => 'course_content_migration_selective_data' end api.with_options(:controller => :migration_issues) do |mi| diff --git a/lib/canvas/migration/helpers/selective_content_formatter.rb b/lib/canvas/migration/helpers/selective_content_formatter.rb new file mode 100644 index 00000000000..f1a7e086539 --- /dev/null +++ b/lib/canvas/migration/helpers/selective_content_formatter.rb @@ -0,0 +1,229 @@ +module Canvas::Migration::Helpers + class SelectiveContentFormatter + SELECTIVE_CONTENT_TYPES = [ + ['course_settings', -> { I18n.t('lib.canvas.migration.course_settings', 'Course Settings') }], + ['syllabus_body', -> { I18n.t('lib.canvas.migration.syllabus_body', 'Syllabus Body') }], + ['context_modules', -> { I18n.t('lib.canvas.migration.context_modules', 'Modules') }], + ['assignments', -> { I18n.t('lib.canvas.migration.assignments', 'Assignments') }], + ['quizzes', -> { I18n.t('lib.canvas.migration.quizzes', 'Quizzes') }], + ['assessment_question_banks', -> { I18n.t('lib.canvas.migration.assessment_question_banks', 'Question Banks') }], + ['discussion_topics', -> { I18n.t('lib.canvas.migration.discussion_topics', 'Discussion Topics') }], + ['wiki_pages', -> { I18n.t('lib.canvas.migration.wikis', 'Wiki Pages') }], + ['context_external_tools', -> { I18n.t('lib.canvas.migration.external_tools', 'External Tools') }], + ['announcements', -> { I18n.t('lib.canvas.migration.announcements', 'Announcements') }], + ['calendar_events', -> { I18n.t('lib.canvas.migration.calendar_events', 'Calendar Events') }], + ['rubrics', -> { I18n.t('lib.canvas.migration.rubrics', 'Rubrics') }], + ['groups', -> { I18n.t('lib.canvas.migration.groups', 'Student Groups') }], + ['learning_outcomes', -> { I18n.t('lib.canvas.migration.learning_outcomes', 'Learning Outcomes') }], + ['attachments', -> { I18n.t('lib.canvas.migration.attachments', 'Files') }], + ] + + def initialize(migration) + @migration = migration + end + + def get_content_list(type=nil) + raise "unsupported migration type" if type && !SELECTIVE_CONTENT_TYPES.any?{|t|t[0] == type} + + if @migration.migration_type == 'course_copy_importer' + get_content_from_course(type) + elsif @migration.overview_attachment + get_content_from_overview(type) + else + raise "course hasn't been converted" + end + end + + private + + # pulls the available items from the overview attachment on the content migration + def get_content_from_overview(type=nil) + course_data = Rails.cache.fetch(['migration_selective_cache', @migration.shard, @migration].cache_key, :expires_in => 5.minutes) do + att = @migration.overview_attachment.open + data = JSON.parse(att.read) + data['attachments'] ||= data['file_map'] ? data['file_map'].values : nil + data['quizzes'] ||= data['assessments'] + data['context_modules'] ||= data['modules'] + data['wiki_pages'] ||= data['wikis'] + data["context_external_tools"] ||= data["external_tools"] + data["learning_outcomes"] ||= data["outcomes"] + att.close + data + end + + content_list = [] + if type + if course_data[type] + case type + when 'assignments' + assignment_data(content_list, course_data) + when 'attachments' + attachment_data(content_list, course_data) + else + course_data[type].each do |item| + content_list << item_hash(type, item) + end + end + end + else + SELECTIVE_CONTENT_TYPES.each do |type, title| + if course_data[type] && course_data[type].count > 0 + content_list << {type: type, property: "copy[all_#{type}]", title: title.call, count: course_data[type].count} + end + end + end + + content_list + end + + # Returns all the assignments in their assignment groups + def assignment_data(content_list, course_data) + added_asmnts = [] + if course_data['assignment_groups'] + course_data['assignment_groups'].each do |group| + item = item_hash('assignment_groups', group) + sub_items = [] + course_data['assignments'].select { |a| a['assignment_group_migration_id'] == group['migration_id'] }.each do |asmnt| + sub_items << item_hash('assignments', asmnt) + added_asmnts << asmnt['migration_id'] + end + if sub_items.any? + item['sub_items'] = sub_items + end + content_list << item + end + end + course_data['assignments'].each do |asmnt| + next if added_asmnts.member? asmnt['migration_id'] + content_list << item_hash('assignments', asmnt) + end + end + + def attachment_data(content_list, course_data) + return [] unless course_data['attachments'] && course_data['attachments'].length > 0 + remove_name_regex = %r{/[^/]*\z} + course_data['attachments'].each{|a| next unless a['path_name']; a['path_name'].gsub!(remove_name_regex, '') } + folder_groups = course_data['attachments'].group_by{|a|a['path_name']} + sorted = folder_groups.sort_by{|i|i.first} + sorted.each do |folder_name, atts| + if atts.length == 1 && atts[0]['file_name'] == folder_name + content_list << item_hash('attachments', atts[0]) + else + mig_id = Digest::MD5.hexdigest(folder_name) + folder = {type: 'folders', property: "copy[folders][#{mig_id}]", title: folder_name, migration_id: mig_id, sub_items: []} + content_list << folder + atts.each {|att| folder[:sub_items] << item_hash('attachments', att)} + end + end + end + + def item_hash(type, item) + hash = { + type: type, + property: "copy[#{type}][#{item['migration_id']}]", + title: item['title'], + migration_id: item['migration_id'] + } + if type == 'attachments' + hash[:path] = item['path_name'] + hash[:title] = item['file_name'] + end + + hash + end + + + # returns lists of available content from a source course + def get_content_from_course(type=nil) + content_list = [] + if source = @migration.source_course || Course.find(@migration.migration_settings[:source_course_id]) + if type + case type + when 'assignments' + course_assignment_data(content_list, source) + when 'attachments' + course_attachments_data(content_list, source) + when 'wiki_pages' + source.wiki.wiki_pages.select("id, title").each do |item| + content_list << course_item_hash(type, item) + end + when 'discussion_topics' + source.discussion_topics.select("id, title, user_id").except(:user).each do |item| + content_list << course_item_hash(type, item) + end + else + if source.respond_to?(type) + scope = source.send(type).select(:id) + # We only need the id and name, so don't fetch everything from DB + if type == 'learning_outcomes' + scope = scope.select(:short_description) + elsif type == 'context_modules' || type == 'external_tools' + scope = scope.select(:name) + else + scope = scope.select(:title) + end + + scope.each do |item| + content_list << course_item_hash(type, item) + end + end + end + else + SELECTIVE_CONTENT_TYPES.each do |type, title| + if type == 'course_settings' || type == 'syllabus_body' + content_list << {type: type, property: "copy[all_#{type}]", title: title.call} + elsif type == 'wiki_pages' + count = source.wiki.wiki_pages.count + content_list << {type: type, property: "copy[all_#{type}]", title: title.call, count: count} if count > 0 + elsif source.respond_to?(type) && source.send(type).respond_to?(:count) + count = source.send(type).count + next if count == 0 + content_list << {type: type, property: "copy[all_#{type}]", title: title.call, count: count} + end + end + end + end + + content_list + end + + def course_item_hash(type, item) + mig_id = CC::CCHelper.create_key(item) + title = nil + title ||= item.title if item.respond_to?(:title) + title ||= item.full_name if item.respond_to?(:full_name) + title ||= item.display_name if item.respond_to?(:display_name) + title ||= item.name if item.respond_to?(:name) + title ||= item.short_description if item.respond_to?(:short_description) + title ||= '' + + {type: type, property: "copy[#{type}][#{mig_id}]", title: title, migration_id: mig_id} + end + + def course_assignment_data(content_list, source_course) + source_course.assignment_groups.includes(:assignments).select("id, name").each do |group| + item = course_item_hash('assignment_groups', group) + content_list << item + group.assignments.select(:id).select(:title).each do |asmnt| + item[:sub_items] ||= [] + item[:sub_items] << course_item_hash('assignments', asmnt) + end + end + end + + def course_attachments_data(content_list, source_course) + source_course.folders.active.select('id, full_name, name').includes(:active_file_attachments).sort_by{|f| f.full_name}.each do |folder| + next if folder.active_file_attachments.length == 0 + + item = course_item_hash('folders', folder) + item[:sub_items] = [] + content_list << item + folder.active_file_attachments.each do |att| + item[:sub_items] << course_item_hash('attachments', att) + end + end + end + + + end +end \ No newline at end of file diff --git a/spec/apis/v1/content_migrations_api_spec.rb b/spec/apis/v1/content_migrations_api_spec.rb index d2ea9ba11bb..fc868cf0365 100644 --- a/spec/apis/v1/content_migrations_api_spec.rb +++ b/spec/apis/v1/content_migrations_api_spec.rb @@ -260,6 +260,38 @@ describe ContentMigrationsController, :type => :integration do end end + describe 'content selection' do + before do + @migration_url = "/api/v1/courses/#{@course.id}/content_migrations/#{@migration.id}/selective_data" + @params = {:controller => 'content_migrations', :format => 'json', :course_id => @course.id.to_param, :action => 'content_list', :id => @migration.id.to_param} + + course + @dt1 = @course.discussion_topics.create!(:message => "hi", :title => "discussion title") + @cm = @course.context_modules.create!(:name => "some module") + @att = Attachment.create!(:filename => 'first.txt', :uploaded_data => StringIO.new('ohai'), :folder => Folder.unfiled_folder(@course), :context => @course) + @wiki = @course.wiki.wiki_pages.create!(:title => "wiki", :body => "ohai") + @migration.migration_type = 'course_copy_importer' + @migration.migration_settings[:source_course_id] = @course.id + @migration.save! + end + + it "should return the top-level list" do + json = api_call(:get, @migration_url, @params) + json.should == [{"type"=>"course_settings", "property"=>"copy[all_course_settings]", "title"=>"Course Settings"}, + {"type"=>"syllabus_body", "property"=>"copy[all_syllabus_body]", "title"=>"Syllabus Body"}, + {"type"=>"context_modules", "property"=>"copy[all_context_modules]", "title"=>"Modules", "count"=>1}, + {"type"=>"discussion_topics", "property"=>"copy[all_discussion_topics]", "title"=>"Discussion Topics", "count"=>1}, + {"type"=>"wiki_pages", "property"=>"copy[all_wiki_pages]", "title"=>"Wiki Pages", "count"=>1}, + {"type"=>"attachments", "property"=>"copy[all_attachments]", "title"=>"Files", "count"=>1}] + end + + it "should return individual types" do + json = api_call(:get, @migration_url + '?type=context_modules', @params.merge({type: 'context_modules'})) + json.length.should == 1 + json.first["type"].should == 'context_modules' + json.first["title"].should == @cm.name + end + end end diff --git a/spec/factories/attachment_factory.rb b/spec/factories/attachment_factory.rb index 84828e91eb0..5790ccaff77 100644 --- a/spec/factories/attachment_factory.rb +++ b/spec/factories/attachment_factory.rb @@ -28,14 +28,19 @@ end def valid_attachment_attributes(opts={}) @context = opts[:context] || @context @context ||= Course.first || course_model(:reusable => true) - if @context.respond_to?(:folders) - @folder = Folder.root_folders(@context).find{|f| f.name == 'unfiled'} || Folder.root_folders(@context).first + if opts[:folder] + folder = opts[:folder] + else + if @context.respond_to?(:folders) + @folder = Folder.root_folders(@context).find{|f| f.name == 'unfiled'} || Folder.root_folders(@context).first + end + @folder ||= folder_model + folder = @folder end - @folder ||= folder_model @attributes_res = { :context => @context, :size => 100, - :folder => @folder, + :folder => folder, :content_type => 'application/loser', :filename => 'unknown.loser' } diff --git a/spec/lib/canvas/migration/selective_content_formatter_spec.rb b/spec/lib/canvas/migration/selective_content_formatter_spec.rb new file mode 100644 index 00000000000..35c8a1d6839 --- /dev/null +++ b/spec/lib/canvas/migration/selective_content_formatter_spec.rb @@ -0,0 +1,182 @@ +# +# Copyright (C) 2011 Instructure, Inc. +# +# This file is part of Canvas. +# +# Canvas is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the Free +# Software Foundation, version 3 of the License. +# +# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along +# with this program. If not, see . +# + +require File.expand_path(File.dirname(__FILE__) + '/../../../spec_helper.rb') + +describe Canvas::Migration::Helpers::SelectiveContentFormatter do + context "overview json data" do + before do + @migration = mock() + @migration.stubs(:migration_type).returns('common_cartridge_importer') + @migration.stubs(:overview_attachment).returns(@migration) + @migration.stubs(:open).returns(@migration) + @migration.stubs(:shard).returns('1') + @migration.stubs(:cache_key).returns('1') + @migration.stubs(:close) + @migration.stubs(:read).returns({ + 'assessments' => [{'title' => 'a1', 'migration_id' => 'a1'}], + 'modules' => [{'title' => 'a1', 'migration_id' => 'a1'}], + 'wikis' => [{'title' => 'a1', 'migration_id' => 'a1'}], + 'external_tools' => [{'title' => 'a1', 'migration_id' => 'a1'}], + 'outcomes' => [{'title' => 'a1', 'migration_id' => 'a1'}], + 'file_map' => {'oi' => {'title' => 'a1', 'migration_id' => 'a1'}}, + 'assignments' => [{'title' => 'a1', 'migration_id' => 'a1'},{'title' => 'a2', 'migration_id' => 'a2', 'assignment_group_migration_id' => 'a1'}], + 'assignment_groups' => [{'title' => 'a1', 'migration_id' => 'a1'}], + 'calendar_events' => [] + }.to_json) + @formatter = Canvas::Migration::Helpers::SelectiveContentFormatter.new(@migration) + end + + it "should list top-level items" do + @formatter.get_content_list.should == [{:type=>"context_modules", :property=>"copy[all_context_modules]", :title=>"Modules", :count=>1}, + {:type=>"assignments", :property=>"copy[all_assignments]", :title=>"Assignments", :count=>2}, + {:type=>"quizzes", :property=>"copy[all_quizzes]", :title=>"Quizzes", :count=>1}, + {:type=>"wiki_pages", :property=>"copy[all_wiki_pages]", :title=>"Wiki Pages", :count=>1}, + {:type=>"context_external_tools", :property=>"copy[all_context_external_tools]", :title=>"External Tools", :count=>1}, + {:type=>"learning_outcomes", :property=>"copy[all_learning_outcomes]", :title=>"Learning Outcomes", :count=>1}, + {:type=>"attachments", :property=>"copy[all_attachments]", :title=>"Files", :count=>1}] + end + + it "should rename deprecated hash keys" do + @formatter.get_content_list('quizzes').length.should == 1 + @formatter.get_content_list('context_modules').length.should == 1 + @formatter.get_content_list('wiki_pages').length.should == 1 + @formatter.get_content_list('context_external_tools').length.should == 1 + @formatter.get_content_list('learning_outcomes').length.should == 1 + @formatter.get_content_list('attachments').length.should == 1 + end + + it "should group assignments into assignment groups" do + @formatter.get_content_list('assignments').should == [ + {:type => "assignment_groups", :property => "copy[assignment_groups][a1]", :title => "a1", :migration_id => "a1", + "sub_items" => [{:type => "assignments", :property => "copy[assignments][a2]", :title => "a2", :migration_id => "a2"}] + }, + {:type => "assignments", :property => "copy[assignments][a1]", :title => "a1", :migration_id => "a1"} + ] + end + + it "should group attachments by folder" do + @migration.stubs(:read).returns({ + 'file_map' => { + 'a1' => {'path_name' => 'a/a1.html', 'file_name' => 'a1.html', 'migration_id' => 'a1'}, + 'a2' => {'path_name' => 'a/a2.html', 'file_name' => 'a2.html', 'migration_id' => 'a2'}, + 'a3' => {'path_name' => 'a/b/a3.html', 'file_name' => 'a3.html', 'migration_id' => 'a3'}, + 'a4' => {'path_name' => 'a/b/c/a4.html', 'file_name' => 'a4.html', 'migration_id' => 'a4'}, + 'a5' => {'path_name' => 'a5.html', 'file_name' => 'a5.html', 'migration_id' => 'a5'}, + }}.to_json) + @formatter.get_content_list('attachments').should == [{:type => "folders", + :property => "copy[folders][0cc175b9c0f1b6a831c399e269772661]", + :title => "a", + :migration_id => "0cc175b9c0f1b6a831c399e269772661", + :sub_items => + [{:type => "attachments", + :property => "copy[attachments][a1]", + :title => "a1.html", + :migration_id => "a1", + :path => "a"}, + {:type => "attachments", + :property => "copy[attachments][a2]", + :title => "a2.html", + :migration_id => "a2", + :path => "a"}]}, + {:type => "folders", + :property => "copy[folders][a7e86136543b019d72468ceebf71fb8e]", + :title => "a/b", + :migration_id => "a7e86136543b019d72468ceebf71fb8e", + :sub_items => + [{:type => "attachments", + :property => "copy[attachments][a3]", + :title => "a3.html", + :migration_id => "a3", + :path => "a/b"}]}, + {:type => "folders", + :property => "copy[folders][cff49f359f080f71548fcee824af6ad3]", + :title => "a/b/c", + :migration_id => "cff49f359f080f71548fcee824af6ad3", + :sub_items => + [{:type => "attachments", + :property => "copy[attachments][a4]", + :title => "a4.html", + :migration_id => "a4", + :path => "a/b/c"}]}, + {:type => "attachments", + :property => "copy[attachments][a5]", + :title => "a5.html", + :migration_id => "a5", + :path => "a5.html"}] + + end + + end + + context "course copy" do + before do + course_model + @dt1 = @course.discussion_topics.create!(:message => "hi", :title => "discussion title") + @cm = @course.context_modules.create!(:name => "some module") + attachment_model(:context => @course, :filename => 'a5.html') + @wiki = @course.wiki.wiki_pages.create!(:title => "wiki", :body => "ohai") + @migration = mock() + @migration.stubs(:migration_type).returns('course_copy_importer') + @migration.stubs(:source_course).returns(@course) + @formatter = Canvas::Migration::Helpers::SelectiveContentFormatter.new(@migration) + end + + it "should list top-level items" do + @formatter.get_content_list.should == [{:type=>"course_settings", :property=>"copy[all_course_settings]", :title=>"Course Settings"}, + {:type=>"syllabus_body", :property=>"copy[all_syllabus_body]", :title=>"Syllabus Body"}, + {:type=>"context_modules", :property=>"copy[all_context_modules]", :title=>"Modules", :count=>1}, + {:type=>"discussion_topics", :property=>"copy[all_discussion_topics]", :title=>"Discussion Topics", :count=>1}, + {:type=>"wiki_pages", :property=>"copy[all_wiki_pages]", :title=>"Wiki Pages", :count=>1}, + {:type=>"attachments", :property=>"copy[all_attachments]", :title=>"Files", :count=>1}] + end + + it "should individual types" do + @formatter.get_content_list('wiki_pages').length.should == 1 + @formatter.get_content_list('context_modules').length.should == 1 + @formatter.get_content_list('attachments').length.should == 1 + @formatter.get_content_list('discussion_topics').length.should == 1 + end + + it "should group files by folders" do + root = Folder.root_folders(@course).first + a = Folder.create!(:name => 'a', :parent_folder => root, :context => @course) + ab = Folder.create!(:name => 'b', :parent_folder => a, :context => @course) + abc = Folder.create!(:name => 'c', :parent_folder => ab, :context => @course) + + attachment_model(:context => @course, :filename => 'a1.html', :folder => a) + attachment_model(:context => @course, :filename => 'a2.html', :folder => a) + attachment_model(:context => @course, :filename => 'a3.html', :folder => ab) + attachment_model(:context => @course, :filename => 'a4.html', :folder => abc) + @course.reload + + res = @formatter.get_content_list('attachments') + res.length.should == 4 + res[0][:title].should == 'course files' + res[0][:sub_items][0][:title].should == 'a5.html' + res[1][:title].should == 'course files/a' + res[1][:sub_items][0][:title].should == 'a1.html' + res[1][:sub_items][1][:title].should == 'a2.html' + res[2][:title].should == 'course files/a/b' + res[2][:sub_items][0][:title].should == 'a3.html' + res[3][:title].should == 'course files/a/b/c' + res[3][:sub_items][0][:title].should == 'a4.html' + end + + end +end \ No newline at end of file