strip invalid utf8 data when importing content from other systems

fixes CNVS-4642

test plan: import a course data package that contains invalid utf-8
data, it should import without error and the invalid data stripped out.

Change-Id: I790059a7c3ee7026d47407f12467a3774016fbad
Reviewed-on: https://gerrit.instructure.com/18632
Tested-by: Jenkins <jenkins@instructure.com>
QA-Review: Clare Hetherington <clare@instructure.com>
Reviewed-by: Jeremy Stanley <jeremy@instructure.com>
Product-Review: Brian Palmer <brianp@instructure.com>

app/models/content_migration.rb

@@ -313,8 +313,7 @@ class ContentMigration < ActiveRecord::Base
       @zip_file = Zip::ZipFile.open(@exported_data_zip.path)
       @exported_data_zip.close
       data = JSON.parse(@zip_file.read('course_export.json'), :max_nesting => 50)
-      data = data.with_indifferent_access if data.is_a? Hash
-      data['all_files_export'] ||= {}
+      data = prepare_data(data)
 
       if @zip_file.find_entry('all_files.zip')
         # the file importer needs an actual file to process
@@ -342,6 +341,13 @@ class ContentMigration < ActiveRecord::Base
   end
   handle_asynchronously :import_content, :priority => Delayed::LOW_PRIORITY, :max_attempts => 1
 
+  def prepare_data(data)
+    data = data.with_indifferent_access if data.is_a? Hash
+    TextHelper.recursively_strip_invalid_utf8!(data, true) if RUBY_VERSION >= "1.9"
+    data['all_files_export'] ||= {}
+    data
+  end
+
   def copy_options
     self.migration_settings[:copy_options]
   end

spec/models/content_migration_spec.rb

@@ -1616,6 +1616,18 @@ equation: <img class="equation_image" title="Log_216" src="/equation_images/Log_
 
   end
 
+  context "#prepare_data" do
+    it "should strip invalid utf8" do
+      pending("Ruby 1.9 only") if RUBY_VERSION < "1.9"
+      data = {
+        'assessment_questions' => [{
+          'question_name' => "hai\xfbabcd"
+        }]
+      }
+      ContentMigration.new.prepare_data(data)[:assessment_questions][0][:question_name].should == "haiabcd"
+    end
+  end
+
   context "import_object?" do
     before do
       @cm = ContentMigration.new