strip invalid utf8 data when importing content from other systems
fixes CNVS-4642 test plan: import a course data package that contains invalid utf-8 data, it should import without error and the invalid data stripped out. Change-Id: I790059a7c3ee7026d47407f12467a3774016fbad Reviewed-on: https://gerrit.instructure.com/18632 Tested-by: Jenkins <jenkins@instructure.com> QA-Review: Clare Hetherington <clare@instructure.com> Reviewed-by: Jeremy Stanley <jeremy@instructure.com> Product-Review: Brian Palmer <brianp@instructure.com>
This commit is contained in:
parent
a33b39fbe5
commit
04baec6a43
|
@ -313,8 +313,7 @@ class ContentMigration < ActiveRecord::Base
|
|||
@zip_file = Zip::ZipFile.open(@exported_data_zip.path)
|
||||
@exported_data_zip.close
|
||||
data = JSON.parse(@zip_file.read('course_export.json'), :max_nesting => 50)
|
||||
data = data.with_indifferent_access if data.is_a? Hash
|
||||
data['all_files_export'] ||= {}
|
||||
data = prepare_data(data)
|
||||
|
||||
if @zip_file.find_entry('all_files.zip')
|
||||
# the file importer needs an actual file to process
|
||||
|
@ -342,6 +341,13 @@ class ContentMigration < ActiveRecord::Base
|
|||
end
|
||||
handle_asynchronously :import_content, :priority => Delayed::LOW_PRIORITY, :max_attempts => 1
|
||||
|
||||
def prepare_data(data)
|
||||
data = data.with_indifferent_access if data.is_a? Hash
|
||||
TextHelper.recursively_strip_invalid_utf8!(data, true) if RUBY_VERSION >= "1.9"
|
||||
data['all_files_export'] ||= {}
|
||||
data
|
||||
end
|
||||
|
||||
def copy_options
|
||||
self.migration_settings[:copy_options]
|
||||
end
|
||||
|
|
|
@ -1616,6 +1616,18 @@ equation: <img class="equation_image" title="Log_216" src="/equation_images/Log_
|
|||
|
||||
end
|
||||
|
||||
context "#prepare_data" do
|
||||
it "should strip invalid utf8" do
|
||||
pending("Ruby 1.9 only") if RUBY_VERSION < "1.9"
|
||||
data = {
|
||||
'assessment_questions' => [{
|
||||
'question_name' => "hai\xfbabcd"
|
||||
}]
|
||||
}
|
||||
ContentMigration.new.prepare_data(data)[:assessment_questions][0][:question_name].should == "haiabcd"
|
||||
end
|
||||
end
|
||||
|
||||
context "import_object?" do
|
||||
before do
|
||||
@cm = ContentMigration.new
|
||||
|
|
Loading…
Reference in New Issue