start canvas cartridge importing

started the framework for importing the canvas flavored
common cartridge packages. There is also some minor
refactoring of the export step to allow for easy testing
of both exporting and importing

course settings, assignment groups, and external tools
are imported and tested with this commit

refs #3396

Change-Id: I4e8a2ebfc9df7e31f8859b189fea23f9584d7ef4
Reviewed-on: https://gerrit.instructure.com/3042
Tested-by: Hudson <hudson@instructure.com>
Reviewed-by: Brian Palmer <brianp@instructure.com>

app/models/assignment.rb

@@ -1373,9 +1373,17 @@ class Assignment < ActiveRecord::Base
     end
     timestamp = hash[:due_date].to_i rescue 0
     item.due_at = Time.at(timestamp / 1000) if timestamp > 0
+    item.assignment_group ||= context.assignment_groups.find_by_migration_id(hash[:assignment_group_migration_id]) if hash[:assignment_group_migration_id]
     item.assignment_group ||= context.assignment_groups.find_or_create_by_name("Imported Assignments")
     context.imported_migration_items << item if context.imported_migration_items && item.new_record?
     item.save_without_broadcasting!
+    
+    if context.respond_to?(:assignment_group_no_drop_assignments) && context.assignment_group_no_drop_assignments
+      if group = context.assignment_group_no_drop_assignments[item.migration_id]
+        AssignmentGroup.add_never_drop_assignment(group, item)
+      end
+    end
+    
     item
   end
 

app/models/assignment_group.rb

@@ -194,10 +194,35 @@ class AssignmentGroup < ActiveRecord::Base
     context.imported_migration_items << item if context.imported_migration_items && item.new_record?
     item.migration_id = hash[:migration_id]
     item.name = hash[:title]
+    item.position = hash[:position].to_i if hash[:position] && hash[:position].to_i > 0
+    item.group_weight = hash[:group_weight] if hash[:group_weight]
+    
+    if hash[:rules] && hash[:rules].length > 0
+      rules = ""
+      hash[:rules].each do |rule|
+        if rule[:drop_type] == "drop_lowest" || rule[:drop_type] == "drop_highest"
+          rules += "#{rule[:drop_type]}:#{rule[:drop_count]}\n"
+        elsif rule[:drop_type] == "never_drop"
+          if context.respond_to?(:assignment_group_no_drop_assignments)
+            context.assignment_group_no_drop_assignments[rule[:assignment_migration_id]] = item
+          end
+        end
+      end
+      item.rules = rules unless rules == ''
+    end
     
     item.save!
-    context.imported_migration_items << item
     item
   end
+  
+  def self.add_never_drop_assignment(group, assignment)
+    rule = "never_drop:#{assignment.id}\n"
+    if group.rules
+      group.rules += rule
+    else
+      group.rules = rule
+    end
+    group.save
+  end
 
 end

app/models/content_export.rb

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License along
 # with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-require 'cc/cc'
+require 'cc'
 class ContentExport < ActiveRecord::Base
   include Workflow
   belongs_to :course

app/models/context_external_tool.rb

@@ -168,4 +168,34 @@ class ContextExternalTool < ActiveRecord::Base
   named_scope :active, :conditions => ['context_external_tools.workflow_state != ?', 'deleted']
   
   def self.serialization_excludes; [:shared_secret,:settings]; end
+  
+  def self.process_migration(data, migration)
+    tools = data['external_tools'] ? data['external_tools']: []
+    to_import = migration.to_import 'external_tools'
+    tools.each do |tool|
+      if tool['migration_id'] && (!to_import || to_import[tool['migration_id']])
+        import_from_migration(tool, migration.context)
+      end
+    end
+  end
+  
+  def self.import_from_migration(hash, context, item=nil)
+    hash = hash.with_indifferent_access
+    return nil if hash[:migration_id] && hash[:external_tools_to_import] && !hash[:external_tools_to_import][hash[:migration_id]]
+    item ||= find_by_context_id_and_context_type_and_migration_id(context.id, context.class.to_s, hash[:migration_id]) if hash[:migration_id]
+    item ||= context.context_external_tools.new
+    item.migration_id = hash[:migration_id]
+    item.name = hash[:title]
+    item.description = hash[:description]
+    item.url = hash[:url] unless hash[:url].blank?
+    item.domain = hash[:domain] unless hash[:domain].blank?
+    item.privacy_level = hash[:privacy_level]
+    item.consumer_key = 'fake'
+    item.shared_secret = 'fake'
+    
+    item.save!
+    context.imported_migration_items << item if context.imported_migration_items && item.new_record?
+    item
+  end
+  
 end

app/models/course.rb

@@ -1176,6 +1176,8 @@ class Course < ActiveRecord::Base
     @imported_migration_items = []
 
     # These only need to be processed once
+    import_settings_from_migration(data)
+    migration.fast_update_progress(0.5)
     process_migration_files(data, migration)
     migration.fast_update_progress(20)
     Attachment.process_migration(data, migration)
@@ -1185,10 +1187,13 @@ class Course < ActiveRecord::Base
     Group.process_migration(data, migration)
     LearningOutcome.process_migration(data, migration)
     Rubric.process_migration(data, migration)
+    @assignment_group_no_drop_assignments = {}
     AssignmentGroup.process_migration(data, migration)
     migration.fast_update_progress(40)
     Quiz.process_migration(data, migration, question_data)
     migration.fast_update_progress(50)
+    #todo - Import external tools when there are post-migration messages to tell the user to add shared secret/password
+    #ContextExternalTool.process_migration(data, migration)
 
     2.times do |i|
       DiscussionTopic.process_migration(data, migration)
@@ -1236,6 +1241,25 @@ class Course < ActiveRecord::Base
   attr_accessor :full_migration_hash
   attr_accessor :external_url_hash
   attr_accessor :folder_name_lookups
+  attr_accessor :assignment_group_no_drop_assignments
+  
+  def import_settings_from_migration(data)
+    return unless data[:course_settings]
+    settings = data[:course_settings]
+    self.name = settings['title']
+    dates = ['conclude_at', 'start_at']
+    settings.slice(*Course.clonable_attributes.map(&:to_s)).each do |key, val|
+      if dates.member? key
+        timestamp = val.to_i/ 1000 rescue 0
+        if timestamp > 0
+          t = Time.at(timestamp)
+          self.send("#{key}=", Time.utc(t.year, t.month, t.day, t.hour, t.min, t.sec))
+        end
+      else
+        self.send("#{key}=", val)
+      end
+    end
+  end
   
   def backup_to_json
     backup.to_json

app/views/content_imports/_cc_config.html.erb

@@ -0,0 +1,11 @@
+<% js_block do %>
+  <script>
+    $(function() {
+
+      $("#plugin_common_cartridge_importer").bind("pluginShown", function(event, enableFileUpload) {
+        enableFileUpload();
+      });
+
+    });
+  </script>
+<% end %>

config/initializers/cc_importer.rb

@@ -0,0 +1,3 @@
+
+# Register the common cartridge importer as a plugin
+require 'cc/importer/init.rb'

db/migrate/20110411214502_add_migration_ids_for_cc_importing.rb

@@ -0,0 +1,15 @@
+class AddMigrationIdsForCcImporting < ActiveRecord::Migration
+  def self.up
+    add_column :context_external_tools, :migration_id, :string
+    add_column :external_feeds, :migration_id, :string
+    add_column :grading_standards, :migration_id, :string
+    add_column :learning_outcome_groups, :migration_id, :string
+  end
+
+  def self.down
+    remove_column :context_external_tools, :migration_id
+    remove_column :external_feeds, :migration_id
+    remove_column :grading_standards, :migration_id
+    remove_column :learning_outcome_groups, :migration_id
+  end
+end

lib/canvas_migration/migrator.rb

@@ -26,18 +26,21 @@ class Canvas::Migrator
   def initialize(settings, migration_type)
     @settings = settings
     @settings[:migration_type] = migration_type
-    download_archive
+    @manifest = nil
+    @error_count = 0
+    @errors = []
     @course = {:file_map=>{}, :wikis=>[]}
+    @course[:name] = @settings[:course_name]
+    
+    return if settings[:testing]
+    
+    download_archive
     @archive_file_path = @settings[:export_archive_path]
     raise "The #{migration_type} archive zip wasn't at the specified location: #{@archive_file_path}." if @archive_file_path.nil? or !File.exists?(@archive_file_path)
     @unzipped_file_path = File.join(File.dirname(@archive_file_path), "#{migration_type}_#{File.basename(@archive_file_path, '.zip')}")
     @base_export_dir = @settings[:base_download_dir] || find_export_dir
     @course[:export_folder_path] = File.expand_path(@base_export_dir)
-    @course[:name] = @settings[:course_name]
     make_export_dir
-    @manifest = nil
-    @error_count = 0
-    @errors = []
   end
 
   def export

lib/canvas_migration/xml_helper.rb

@@ -22,6 +22,13 @@ module Canvas::XMLHelper
     Time.parse(string).to_i * 1000 rescue nil
   end
 
+  def get_node_att(node, selector, attribute, default=nil)
+    if node = node.at_css(selector)
+      return node[attribute]
+    end
+    default
+  end
+
   def get_node_val(node, selector, default=nil)
     node.at_css(selector) ? node.at_css(selector).text : default
   end
@@ -68,10 +75,10 @@ module Canvas::XMLHelper
   end
 
   def open_file(path)
-    ::Nokogiri::HTML(open(path))
+    File.exists?(path) ? ::Nokogiri::HTML(open(path)) : nil
   end
 
   def open_file_xml(path)
-    ::Nokogiri::XML(open(path))
+    File.exists?(path) ? ::Nokogiri::XML(open(path)) : nil
   end
 end

lib/cc.rb

@@ -38,3 +38,4 @@ require "cc/web_links"
 require 'cc/resource'
 require 'cc/organization'
 require 'cc/qti/qti'
+require 'cc/importer'

lib/cc/assignment_groups.rb

@@ -17,12 +17,18 @@
 #
 module CC
   module AssignmentGroups
-    def create_assignment_groups
+    def create_assignment_groups(document=nil)
       return nil unless @course.assignment_groups.active.count > 0
       
-      group_file = File.new(File.join(@canvas_resource_dir, CCHelper::ASSIGNMENT_GROUPS), 'w')
-      rel_path = File.join(CCHelper::COURSE_SETTINGS_DIR, CCHelper::ASSIGNMENT_GROUPS)
-      document = Builder::XmlMarkup.new(:target=>group_file, :indent=>2)
+      if document
+        group_file = nil
+        rel_path = nil
+      else
+        group_file = File.new(File.join(@canvas_resource_dir, CCHelper::ASSIGNMENT_GROUPS), 'w')
+        rel_path = File.join(CCHelper::COURSE_SETTINGS_DIR, CCHelper::ASSIGNMENT_GROUPS)
+        document = Builder::XmlMarkup.new(:target=>group_file, :indent=>2)
+      end
+      
       document.instruct!
       document.assignmentGroups(
               "xmlns" => CCHelper::CANVAS_NAMESPACE,
@@ -60,7 +66,7 @@ module CC
         end
       end
       
-      group_file.close
+      group_file.close if group_file
       rel_path
     end
   end

lib/cc/basic_lti_links.rb

@@ -63,12 +63,18 @@ module CC
 
     end
 
-    def create_external_tools
+    def create_external_tools(document=nil)
       return nil unless @course.context_external_tools.count > 0
       
-      lti_file = File.new(File.join(@canvas_resource_dir, CCHelper::EXTERNAL_TOOLS), 'w')
-      rel_path = File.join(CCHelper::COURSE_SETTINGS_DIR, CCHelper::EXTERNAL_TOOLS)
-      document = Builder::XmlMarkup.new(:target=>lti_file, :indent=>2)
+      if document
+        lti_file = nil
+        rel_path = nil
+      else
+        lti_file = File.new(File.join(@canvas_resource_dir, CCHelper::EXTERNAL_TOOLS), 'w')
+        rel_path = File.join(CCHelper::COURSE_SETTINGS_DIR, CCHelper::EXTERNAL_TOOLS)
+        document = Builder::XmlMarkup.new(:target=>lti_file, :indent=>2)
+      end
+      
       document.instruct!
       document.externalTools(
           "xmlns" => CCHelper::CANVAS_NAMESPACE,
@@ -88,7 +94,7 @@ module CC
         end
       end
 
-      lti_file.close
+      lti_file.close if lti_file
       rel_path
     end
   end

lib/cc/canvas_resource.rb

@@ -59,10 +59,15 @@ module CC
       end
     end
     
-    def create_course_settings(migration_id)
-      course_file = File.new(File.join(@canvas_resource_dir, CCHelper::COURSE_SETTINGS), 'w')
-      rel_path = File.join(CCHelper::COURSE_SETTINGS_DIR, CCHelper::COURSE_SETTINGS)
-      document = Builder::XmlMarkup.new(:target=>course_file, :indent=>2)
+    def create_course_settings(migration_id, document=nil)
+      if document
+        course_file = nil
+        rel_path = nil
+      else
+        course_file = File.new(File.join(@canvas_resource_dir, CCHelper::COURSE_SETTINGS), 'w')
+        rel_path = File.join(CCHelper::COURSE_SETTINGS_DIR, CCHelper::COURSE_SETTINGS)
+        document = Builder::XmlMarkup.new(:target=>course_file, :indent=>2)
+      end
       document.instruct!
       document.course("identifier" => migration_id,
                       "xmlns" => CCHelper::CANVAS_NAMESPACE,
@@ -75,10 +80,10 @@ module CC
         atts = Course.clonable_attributes
         atts -= [:name, :start_at, :conclude_at, :grading_standard_id, :hidden_tabs, :tab_configuration, :syllabus_body]
         atts.each do |att|
-          c.tag!(att, @course.send(att)) unless @course.send(att).blank?
+          c.tag!(att, @course.send(att)) unless @course.send(att).nil? || @course.send(att) == ''
         end
       end
-      course_file.close
+      course_file.close if course_file
       rel_path
     end
   end

lib/cc/cc_helper.rb

@@ -80,12 +80,12 @@ module CCHelper
   
   def self.ims_date(date=nil)
     date ||= Time.now
-    date.strftime(IMS_DATE)
+    date.utc.strftime(IMS_DATE)
   end
   
   def self.ims_datetime(date=nil)
     date ||= Time.now
-    date.strftime(IMS_DATETIME)
+    date.utc.strftime(IMS_DATETIME)
   end
   
   def self.html_page(html, title, course, user)

lib/cc/importer.rb

@@ -0,0 +1,27 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+module CC
+  module Importer
+    include CC::CCHelper
+    include Canvas::XMLHelper
+  end
+end
+
+require 'cc/importer/course_settings'
+require 'cc/importer/cc_converter'

lib/cc/importer/cc_converter.rb

@@ -0,0 +1,46 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+module CC::Importer
+  class CCConverter < Canvas::Migrator
+    include CC::Importer
+    include CourseSettings
+
+    MANIFEST_FILE = "imsmanifest.xml"
+
+    # settings will use these keys: :course_name, :base_download_dir
+    def initialize(settings)
+      super(settings, "cc")
+    end
+
+    # exports the package into the intermediary json
+    def export(to_export = SCRAPE_ALL_HASH)
+      to_export = SCRAPE_ALL_HASH.merge to_export if to_export
+      unzip_archive
+      
+      @manifest = open_file(File.join(@unzipped_file_path, MANIFEST_FILE))
+      convert_non_dependant_course_settings
+      
+      
+      #close up shop
+      save_to_file
+      delete_unzipped_archive
+      @course
+    end
+
+  end
+end

lib/cc/importer/cc_worker.rb

@@ -0,0 +1,54 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+module Canvas
+  module MigrationWorker
+    class CCWorker < Struct.new(:migration_id)
+
+      def perform
+        cm = ContentMigration.find migration_id
+        settings = cm.migration_settings.clone
+        settings[:content_migration_id] = migration_id
+        settings[:user_id] = cm.user_id
+        settings[:attachment_id] = cm.attachment.id rescue nil
+
+        converter = CC::Importer.CCConverter.new(settings)
+        course = converter.export
+        export_folder_path = course[:export_folder_path]
+        overview_file_path = course[:overview_file_path]
+
+        if overview_file_path
+          file = File.new(overview_file_path)
+          Canvas::MigrationWorker::upload_overview_file(file, cm)
+        end
+        if export_folder_path
+          Canvas::MigrationWorker::upload_exported_data(export_folder_path, cm)
+          Canvas::MigrationWorker::clear_exported_data(export_folder_path)
+        end
+
+        cm.migration_settings[:migration_ids_to_import] = {:copy=>{:assessment_questions=>true}}
+        cm.workflow_state = :exported
+        cm.progress = 0
+        cm.save
+      end
+
+      def self.enqueue(content_migration)
+        Delayed::Job.enqueue(new(content_migration.id), :priority => Delayed::LOW_PRIORITY)
+      end
+    end
+  end
+end

lib/cc/importer/course_settings.rb

@@ -0,0 +1,104 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+module CC::Importer
+  module CourseSettings
+    include CC::Importer
+    
+    def course_settings_doc(file)
+      open_file_xml File.join(@unzipped_file_path, COURSE_SETTINGS_DIR, file)
+    end
+    
+    def convert_non_dependant_course_settings
+      @course[:course] = convert_course_settings(course_settings_doc(COURSE_SETTINGS))
+      @course[:assignment_groups] = convert_assignment_groups(course_settings_doc(ASSIGNMENT_GROUPS))
+      @course[:external_tools] = convert_external_tools(course_settings_doc(EXTERNAL_TOOLS))
+    end
+
+    def convert_course_settings(doc)
+      course = {}
+      return course unless doc
+      course[:migration_id] = get_node_att(doc, 'course',  'identifier')
+
+      ['title', 'course_code', 'hashtag', 'default_wiki_editing_roles',
+       'turnitin_comments', 'default_view', 'license',
+       'group_weighting_scheme'].each do |string_type|
+        val = get_node_val(doc, string_type)
+        course[string_type] = val unless val.nil?
+      end
+      ['is_public', 'indexed', 'publish_grades_immediately', 'allow_student_wiki_edits',
+       'allow_student_assignment_edits', 'show_public_context_messages',
+       'allow_student_forum_attachments', 'allow_student_organized_groups',
+       'show_all_discussion_entries', 'open_enrollment', 'allow_wiki_comments',
+       'self_enrollment'].each do |bool_val|
+        val = get_bool_val(doc, bool_val)
+        course[bool_val] = val unless val.nil?
+      end
+      ['start_at', 'conclude_at'].each do |date_type|
+        val = get_time_val(doc, date_type)
+        course[date_type] = val unless val.nil?
+      end
+
+      course['storage_quota'] = get_int_val(doc, 'storage_quota')
+      
+      course
+    end
+
+    def convert_assignment_groups(doc = nil)
+      groups = []
+      return groups unless doc
+      doc.css('assignmentGroup').each do |node|
+        group = {}
+        group['migration_id'] = node['identifier']
+        group['title'] = get_node_val(node, 'title')
+        group['position'] = get_int_val(node, 'position')
+        group['group_weight'] = get_float_val(node, 'group_weight')
+        group['rules'] = []
+        node.css('rules rule').each do |r_node|
+          rule = {}
+          rule['drop_type'] = get_node_val(r_node, 'drop_type')
+          rule['drop_count'] = get_int_val(r_node, 'drop_count')
+          rule['assignment_migration_id'] = get_node_val(r_node, 'identifierref')
+          group['rules'] << rule
+        end
+        
+        groups << group
+      end
+      
+      groups
+    end
+
+    def convert_external_tools(doc)
+      tools = []
+      return tools unless doc
+      doc.css('externalTool').each do |node|
+        tool = {}
+        tool['migration_id'] = node['identifier']
+        tool['title'] = get_node_val(node, 'title')
+        tool['description'] = get_node_val(node, 'description')
+        tool['domain'] = get_node_val(node, 'domain')
+        tool['url'] = get_node_val(node, 'url')
+        tool['privacy_level'] = get_node_val(node, 'privacy_level')
+        
+        tools << tool
+      end
+
+      tools
+    end
+
+  end
+end

lib/cc/importer/init.rb

@@ -0,0 +1,33 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'canvas/plugin'
+require 'cc/importer/cc_worker'
+Rails.configuration.to_prepare do
+  Canvas::Plugin.register :common_cartridge_importer, :export_system, {
+          :name => 'Common Cartridge Importer',
+          :author => 'Instructurecon',
+          :description => 'This enables converting a canvas CC export to the intermediary json format to be imported',
+          :version => '1.0.0',
+          :settings => {
+                  :worker=>'CCWorker',
+                  :migration_partial => 'cc_config',
+                  :select_text => "Canvas Course Export (.imscc)"
+          }
+  }
+end

lib/imported_html_converter.rb

@@ -20,27 +20,49 @@ class ImportedHtmlConverter
   include TextHelper
   def self.convert(html, context)
     doc = Nokogiri::HTML(html || "")
-    root_folder_name = nil
     attrs = ['rel', 'href', 'src', 'data', 'value']
+    course_path = "/#{context.class.to_s.underscore.pluralize}/#{context.id}"
+    root_folder_name = Folder.root_folders(context).first.name
     doc.search("*").each do |node|
       attrs.each do |attr|
         if node[attr]
           if node[attr] =~ /wiki_page_migration_id=([^'"]*)/
-            # This would be from a BB9 migration.
+            # This would be from a BB9 migration. 
+            #todo: refactor migration systems to use new $CANVAS...$ flags
+            #todo: FLAG UNFOUND REFERENCES TO re-attempt in second loop?
             if wiki_migration_id = $1
               if linked_wiki = context.wiki.wiki_pages.find_by_migration_id(wiki_migration_id)
-                node[attr] = URI::escape("/#{context.class.to_s.underscore.pluralize}/#{context.id}/wiki/#{linked_wiki.url}")
+                node[attr] = URI::escape("#{course_path}/wiki/#{linked_wiki.url}")
               end
             end
           elsif node[attr] =~ /discussion_topic_migration_id=([^'"]*)/
             if topic_migration_id = $1
               if linked_topic = context.discussion_topics.find_by_migration_id(topic_migration_id)
-                node[attr] = URI::escape("/#{context.class.to_s.underscore.pluralize}/#{context.id}/discussion_topics/#{linked_topic.id}")
+                node[attr] = URI::escape("#{course_path}/discussion_topics/#{linked_topic.id}")
               end
             end
+          elsif node[attr] =~ %r{(?:\$CANVAS_OBJECT_REFERENCE\$|\$WIKI_REFERENCE\$)/([^/]*)/([^'"]*)}
+            type = $1
+            migration_id = $2
+            if type == 'wiki'
+              if page = context.wiki.wiki_pages.find_by_url(migration_id)
+                node[attr] = URI::escape("#{course_path}/wiki/#{page.url}")
+              end
+            elsif context.respond_to?(type) && context.send(type).respond_to?(:find_by_migration_id)
+              if object = context.send(type).find_by_migration_id(migration_id)
+                node[attr] = URI::escape("#{course_path}/#{type}/#{object.id}")
+              end
+            end
+          elsif node[attr] =~ %r{\$CANVAS_COURSE_REFERENCE\$/([^'"]*)}
+            section = $1
+            node[attr] = URI::escape("#{course_path}/#{section}")
+          elsif node[attr] =~ %r{\$IMS_CC_FILEBASE\$/([^'"]*)}
+            rel_path = $1
+            # the rel_path should already be escaped
+            node[attr] = URI::escape("#{course_path}/file_contents/#{root_folder_name}/") + rel_path
           elsif relative_url?(node[attr])
-            root_folder_name ||= Folder.root_folders(context).first.name
-            node[attr] = URI::escape("/#{context.class.to_s.underscore.pluralize}/#{context.id}/file_contents/#{root_folder_name}/") + node[attr]
+            # the rel_path should already be escaped
+            node[attr] = URI::escape("#{course_path}/file_contents/#{root_folder_name}/") + node[attr]
           end
         end
       end

spec/lib/cc/cc_spec_helper.rb

@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper.rb')
+
+CC_XML_EXPORT_DIR = File.dirname(__FILE__) + '/../../fixtures/cc/cc_export'
+
+def get_cc_converter
+  CC::Importer::CCConverter.new({:testing=>true})
+end
+
+def get_cc_export_file(rel_path)
+  File.join(CC_XML_EXPORT_DIR, rel_path)
+end

spec/lib/cc/importer/cc_converter_spec.rb

@@ -0,0 +1,167 @@
+require File.dirname(__FILE__) + '/../cc_spec_helper'
+
+describe "Common Cartridge importing" do
+  before(:each) do
+    @converter = get_cc_converter
+    @copy_from = course_model
+    @from_teacher = @user
+    @copy_to = course_model
+
+    exporter = CC::CCExporter.new(nil, :course=>@copy_from, :user=>@from_teacher)
+    manifest = CC::Manifest.new(exporter)
+    @resource = CC::Resource.new(manifest, nil)
+    @migration = Object.new
+    @migration.stub!(:to_import).and_return(nil)
+    @migration.stub!(:context).and_return(@copy_to)
+  end
+
+  it "should import course settings" do
+    #set all the possible values to non-default values
+    @copy_from.start_at = 5.minutes.ago
+    @copy_from.conclude_at = 1.month.from_now
+    @copy_from.is_public = false
+    @copy_from.name = "haha copy from test &"
+    @copy_from.course_code = 'something funny'
+    @copy_from.publish_grades_immediately = false
+    @copy_from.allow_student_wiki_edits = true
+    @copy_from.allow_student_assignment_edits = true
+    @copy_from.hashtag = 'oi'
+    @copy_from.show_public_context_messages = false
+    @copy_from.allow_student_forum_attachments = false
+    @copy_from.default_wiki_editing_roles = 'teachers'
+    @copy_from.allow_student_organized_groups = false
+    @copy_from.default_view = 'modules'
+    @copy_from.show_all_discussion_entries = false
+    @copy_from.open_enrollment = true
+    @copy_from.storage_quota = 444
+    @copy_from.allow_wiki_comments = true
+    @copy_from.turnitin_comments = "Don't plagiarize"
+    @copy_from.self_enrollment = true
+    @copy_from.license = "cc_by_nc_nd"
+
+    @copy_from.save!
+
+    #export to xml
+    builder = Builder::XmlMarkup.new(:indent=>2)
+    @resource.create_course_settings("1", builder)
+    #convert to json
+    doc = Nokogiri::XML(builder.target!)
+    hash = @converter.convert_course_settings(doc)
+    #import json into new course
+    hash = hash.with_indifferent_access
+    @copy_to.import_settings_from_migration({:course_settings=>hash})
+    @copy_to.save!
+
+    #compare settings
+    @copy_to.conclude_at.to_i.should == @copy_from.conclude_at.to_i
+    @copy_to.start_at.to_i.should == @copy_from.start_at.to_i
+    atts = Course.clonable_attributes
+    atts -= [:start_at, :conclude_at, :grading_standard_id, :hidden_tabs, :tab_configuration, :syllabus_body]
+    atts.each do |att|
+      @copy_to.send(att).should == @copy_from.send(att)
+    end
+  end
+
+  it "should convert assignment groups" do
+    ag1 = @copy_from.assignment_groups.new
+    ag1.name = "Boring assignments"
+    ag1.position = 1
+    ag1.group_weight = 77.7
+    ag1.save!
+    ag2 = @copy_from.assignment_groups.new
+    ag2.name = "Super not boring assignments"
+    ag2.position = 2
+    ag2.group_weight = 20
+    ag2.save!
+    a = ag2.assignments.new
+    a.title = "Can't drop me"
+    a.context = @copy_from
+    a.save!
+    ag2.rules = "drop_lowest:2\ndrop_highest:5\nnever_drop:%s\n" % a.id
+    ag2.save!
+
+    #export to xml
+    builder = Builder::XmlMarkup.new(:indent=>2)
+    @resource.create_assignment_groups(builder)
+    #convert to json
+    doc = Nokogiri::XML(builder.target!)
+    hash = @converter.convert_assignment_groups(doc)
+    #import json into new course
+    @copy_to.assignment_group_no_drop_assignments = {}
+    AssignmentGroup.process_migration({'assignment_groups'=>hash}, @migration)
+    @copy_to.save!
+
+    #compare settings
+    ag1_2 = @copy_to.assignment_groups.find_by_migration_id(CC::CCHelper.create_key(ag1))
+    ag1_2.name.should == ag1.name
+    ag1_2.position.should == ag1.position
+    ag1_2.group_weight.should == ag1.group_weight
+    ag1_2.rules.should == ag1.rules
+
+    ag2_2 = @copy_to.assignment_groups.find_by_migration_id(CC::CCHelper.create_key(ag2))
+    ag2_2.name.should == ag2.name
+    ag2_2.position.should == ag2.position
+    ag2_2.group_weight.should == ag2.group_weight
+    ag2_2.rules.should == "drop_lowest:2\ndrop_highest:5\n"
+
+    #import assignment
+    hash = {:migration_id=>CC::CCHelper.create_key(a),
+            :title=>a.title,
+            :assignment_group_migration_id=>CC::CCHelper.create_key(ag2)}
+    Assignment.import_from_migration(hash, @copy_to)
+    
+    ag2_2.reload
+    ag2_2.assignments.count.should == 1
+    a_2 = ag2_2.assignments.first 
+    ag2_2.rules.should == "drop_lowest:2\ndrop_highest:5\nnever_drop:%s\n" % a_2.id
+  end
+
+  it "should convert external tools" do
+    tool1 = @copy_from.context_external_tools.new
+    tool1.url = 'http://instructure.com'
+    tool1.name = 'instructure'
+    tool1.description = "description of boring"
+    tool1.privacy_level = 'name_only'
+    tool1.consumer_key = 'haha'
+    tool1.shared_secret = "don't share me"
+    tool1.save!
+    tool2 = @copy_from.context_external_tools.new
+    tool2.domain = 'example.com'
+    tool2.name = 'example'
+    tool2.description = "example.com? That's the best you could come up with?"
+    tool2.privacy_level = 'anonymous'
+    tool2.consumer_key = 'haha'
+    tool2.shared_secret = "don't share me"
+    tool2.save!
+
+    #export to xml
+    builder = Builder::XmlMarkup.new(:indent=>2)
+    @resource.create_external_tools(builder)
+    #convert to json
+    doc = Nokogiri::XML(builder.target!)
+    hash = @converter.convert_external_tools(doc)
+    #import json into new course
+    ContextExternalTool.process_migration({'external_tools'=>hash}, @migration)
+    @copy_to.save!
+
+    #compare settings
+    t1 = @copy_to.context_external_tools.find_by_migration_id(CC::CCHelper.create_key(tool1))
+    t1.url.should == tool1.url
+    t1.name.should == tool1.name
+    t1.description.should == tool1.description
+    t1.workflow_state.should == tool1.workflow_state
+    t1.domain.should == nil
+    t1.consumer_key.should == 'fake'
+    t1.shared_secret.should == 'fake'
+
+    t2 = @copy_to.context_external_tools.find_by_migration_id(CC::CCHelper.create_key(tool2))
+    t2.domain.should == tool2.domain
+    t2.url.should == nil
+    t2.name.should == tool2.name
+    t2.description.should == tool2.description
+    t2.workflow_state.should == tool2.workflow_state
+    t2.consumer_key.should == 'fake'
+    t2.shared_secret.should == 'fake'
+  end
+
+end