rails4: fix invalid utf8 removal on old serialized attributes

refs #CNVS-21596

Change-Id: I7888abfd6b36a7b685b6d89859d64bc1af72725e
Reviewed-on: https://gerrit.instructure.com/58387
Tested-by: Jenkins
Reviewed-by: Cody Cutrer <cody@instructure.com>
Product-Review: James Williams  <jamesw@instructure.com>
QA-Review: James Williams  <jamesw@instructure.com>

app/models/assessment_question.rb

@@ -45,7 +45,7 @@ class AssessmentQuestion < ActiveRecord::Base
                         "multiple_dropdowns_question", "calculated_question",
                         "essay_question", "true_false_question", "file_upload_question"]
 
-  serialize :question_data
+  serialize_utf8_safe :question_data
 
   set_policy do
     given{|user, session| self.context.grants_right?(user, session, :manage_assignments) }

app/models/context_external_tool.rb

@@ -22,7 +22,7 @@ class ContextExternalTool < ActiveRecord::Base
   validates_presence_of :config_xml, :if => lambda { |t| t.config_type == "by_xml" }
   validates_length_of :domain, :maximum => 253, :allow_blank => true
   validate :url_or_domain_is_set
-  serialize :settings
+  serialize_utf8_safe :settings
   attr_accessor :config_type, :config_url, :config_xml
 
   before_save :infer_defaults, :validate_vendor_help_link

app/models/eportfolio_entry.rb

@@ -39,26 +39,26 @@ class EportfolioEntry < ActiveRecord::Base
   validates_length_of :name, :maximum => maximum_string_length, :allow_nil => false, :allow_blank => true
   validates_length_of :slug, :maximum => maximum_string_length, :allow_nil => false, :allow_blank => true
   has_many :page_comments, :as => :page, :include => :user, :order => 'page_comments.created_at DESC'
-  
 
-  serialize :content
+
+  serialize_utf8_safe :content
 
   set_policy do
     given {|user| user && self.allow_comments }
     can :comment
   end
-  
+
   def infer_comment_visibility
     self.show_comments = false if !self.allow_comments
     true
   end
   protected :infer_comment_visibility
-  
+
   def update_portfolio
     self.eportfolio.save!
   end
   protected :update_portfolio
-  
+
   def content_sections
     (self.content.is_a?(String) && Array(self.content) || self.content || []).map do |section|
       if section.is_a?(Hash)
@@ -68,7 +68,7 @@ class EportfolioEntry < ActiveRecord::Base
       end
     end
   end
-  
+
   def submission_ids
     res = []
     content_sections.each do |section|
@@ -76,11 +76,11 @@ class EportfolioEntry < ActiveRecord::Base
     end
     res
   end
-  
+
   def full_slug
     (self.eportfolio_category.slug rescue "") + "_" + self.slug
   end
-  
+
   def attachments
     res = []
     content_sections.each do |section|
@@ -90,7 +90,7 @@ class EportfolioEntry < ActiveRecord::Base
     end
     res.compact
   end
-  
+
   def submissions
     res = []
     content_sections.each do |section|
@@ -128,7 +128,7 @@ class EportfolioEntry < ActiveRecord::Base
       else
         new_obj = nil
       end
-      
+
       if new_obj
           self.content << new_obj
       end
@@ -136,11 +136,11 @@ class EportfolioEntry < ActiveRecord::Base
     self.content << t(:default_content, "No Content Added Yet") if self.content.empty?
   end
 
-  
+
   def category_slug
     self.eportfolio_category.slug rescue self.eportfolio_category_id
   end
-  
+
   def infer_unique_slug
     pages = self.eportfolio_category.eportfolio_entries rescue []
     self.name ||= t(:default_name, "Page Name")

app/models/error_report.rb

@@ -21,9 +21,9 @@ require 'iconv'
 class ErrorReport < ActiveRecord::Base
   belongs_to :user
   belongs_to :account
-  serialize :http_env
+  serialize_utf8_safe :http_env
   # misc key/value pairs with more details on the error
-  serialize :data, Hash
+  serialize_utf8_safe :data, Hash
 
   before_save :guess_email
 

app/models/learning_outcome.rb

@@ -28,7 +28,7 @@ class LearningOutcome < ActiveRecord::Base
 
   EXPORTABLE_ATTRIBUTES = [:id, :context_id, :context_type, :short_description, :context_code, :description, :data, :workflow_state, :created_at, :updated_at, :vendor_guid, :low_grade, :high_grade]
   EXPORTABLE_ASSOCIATIONS = [:context, :learning_outcome_results, :alignments]
-  serialize :data
+  serialize_utf8_safe :data
 
   before_validation :infer_default_calculation_method, :adjust_calculation_int
   before_save :infer_defaults

app/models/profile.rb

@@ -4,7 +4,7 @@ class Profile < ActiveRecord::Base
 
   attr_accessible :context, :root_account, :title, :path, :description, :visibility, :position
 
-  serialize :data
+  serialize_utf8_safe :data
 
   validates_presence_of :root_account
   validates_presence_of :context

app/models/quizzes/quiz.rb

@@ -96,7 +96,7 @@ class Quizzes::Quiz < ActiveRecord::Base
   after_save :touch_context
   after_save :regrade_if_published
 
-  serialize :quiz_data
+  serialize_utf8_safe :quiz_data
 
   simply_versioned
 

app/models/quizzes/quiz_question.rb

@@ -55,7 +55,7 @@ class Quizzes::QuizQuestion < ActiveRecord::Base
   before_destroy :delete_assessment_question, unless: :generated?
   before_destroy :update_quiz
   validates_presence_of :quiz_id
-  serialize :question_data
+  serialize_utf8_safe :question_data
   after_save :update_quiz
 
   workflow do

app/models/quizzes/quiz_submission.rb

@@ -79,8 +79,8 @@ class Quizzes::QuizSubmission < ActiveRecord::Base
     submission.update_attribute(:workflow_state, "graded")
   end
 
-  serialize :quiz_data
-  serialize :submission_data
+  serialize_utf8_safe :quiz_data
+  serialize_utf8_safe :submission_data
 
   simply_versioned :automatic => false
 

app/models/quizzes/quiz_submission_snapshot.rb

@@ -18,7 +18,7 @@
 
 class Quizzes::QuizSubmissionSnapshot < ActiveRecord::Base
   belongs_to :quiz_submission, class_name: 'Quizzes::QuizSubmission'
-  serialize :data
+  serialize_utf8_safe :data
 
   attr_accessible :quiz_submission, :attempt, :data
 end

app/models/rubric.rb

@@ -42,7 +42,7 @@ class Rubric < ActiveRecord::Base
   after_save :update_alignments
   after_save :touch_associations
 
-  serialize :data
+  serialize_utf8_safe :data
   simply_versioned
 
   scope :publicly_reusable, -> { where(:reusable => true).order(best_unicode_collation_key('title')) }

app/models/rubric_assessment.rb

@@ -31,7 +31,7 @@ class RubricAssessment < ActiveRecord::Base
   belongs_to :artifact, :polymorphic => true, :touch => true
   validates_inclusion_of :artifact_type, :allow_nil => true, :in => ['Submission', 'Assignment']
   has_many :assessment_requests, :dependent => :destroy
-  serialize :data
+  serialize_utf8_safe :data
 
   simply_versioned
 

app/models/sis_batch.rb

@@ -21,8 +21,8 @@ class SisBatch < ActiveRecord::Base
   belongs_to :account
   serialize :data
   serialize :options
-  serialize :processing_errors, Array
-  serialize :processing_warnings, Array
+  serialize_utf8_safe :processing_errors, Array
+  serialize_utf8_safe :processing_warnings, Array
   belongs_to :attachment
   belongs_to :generated_diff, class_name: 'Attachment'
   belongs_to :batch_mode_term, class_name: 'EnrollmentTerm'

app/models/stream_item.rb

@@ -20,7 +20,7 @@ require 'open_object'
 require 'set'
 
 class StreamItem < ActiveRecord::Base
-  serialize :data
+  serialize_utf8_safe :data
 
   has_many :stream_item_instances
   has_many :users, :through => :stream_item_instances

config/initializers/ruby_version_compat.rb

@@ -101,19 +101,24 @@ if CANVAS_RAILS3
 
   class ActiveRecord::Base
     extend HandleInvalidUtf8
-  end
-else
-  # For Rails 4 use Custom Types
-  # https://github.com/rails/rails/blob/08754f12e65a9ec79633a605e986d0f1ffa4b251/activerecord/lib/active_record/attributes.rb#L53?line=53
-end
-
-unless CANVAS_RAILS3
-  module AttributeReadWithUtf8Check
-    def read_attribute(attr_name, &block)
-      self.class.strip_invalid_utf8_from_attribute(attr_name, super)
+    def self.serialize_utf8_safe(*args)
+      serialize(*args)
+    end
+  end
+else
+  module ActiveRecord::Coders
+    class Utf8SafeYAMLColumn < YAMLColumn
+      def load(*args)
+        Utf8Cleaner.recursively_strip_invalid_utf8!(super, true)
+      end
+    end
+  end
+
+  class ActiveRecord::Base
+    def self.serialize_utf8_safe(attr_name, class_name = Object)
+      serialize(attr_name, ::ActiveRecord::Coders::Utf8SafeYAMLColumn.new(class_name))
     end
   end
-  ActiveRecord::AttributeMethods::Read.send(:prepend, AttributeReadWithUtf8Check)
 end
 
 # Fix for https://bugs.ruby-lang.org/issues/7278 , which was filling up our logs with these warnings

gems/plugins/simply_versioned/lib/simply_versioned/version.rb

@@ -26,9 +26,14 @@ class Version < ActiveRecord::Base #:nodoc:
   def model
     obj = versionable_type.constantize.new
     YAML::load( self.yaml ).each do |var_name,var_value|
+      unless CANVAS_RAILS3
+        if (coder = obj.class.serialized_attributes[var_name]) && coder.is_a?(ActiveRecord::Coders::Utf8SafeYAMLColumn)
+          Utf8Cleaner.recursively_strip_invalid_utf8!(var_value, true)
+        end
+      end
 
       # INSTRUCTURE:  added if... so that if a column is removed in a migration after this was versioned it doesen't die with NoMethodError: undefined method `some_column_name=' for ...
-       obj.write_attribute(var_name, var_value) if obj.class.columns_hash[var_name]
+      obj.write_attribute(var_name, var_value) if obj.class.columns_hash[var_name]
     end
     obj.instance_variable_set(:@new_record, false)
     obj.simply_versioned_options[:on_load].try(:call, obj, self)