Encryption: support `support_unencrypted_data` being set at a per-attribute level

activerecord/CHANGELOG.md

@@ -1,3 +1,17 @@
+*   Encryption now supports `support_unencrypted_data` being set per-attribute.
+
+    You can now opt out of `support_unencrypted_data` on a specific encrypted attribute.
+    This only has an effect if `ActiveRecord::Encryption.config.support_unencrypted_data == true`.
+
+    ```ruby
+    class User < ActiveRecord::Base
+      encrypts :name, deterministic: true, support_unencrypted_data: false
+      encrypts :email, deterministic: true
+    end
+    ```
+
+    *Alex Ghiculescu*
+
 *   Add instrumentation for Active Record transactions
 
     Allows subscribing to transaction events for tracking/instrumentation. The event payload contains the connection, as well as timing details.

activerecord/lib/active_record/encryption/encryptable_record.rb

@@ -30,6 +30,10 @@ module ActiveRecord
         #   will use the oldest encryption scheme to encrypt new data by default. You can change this by setting
         #   <tt>deterministic: { fixed: false }</tt>. That will make it use the newest encryption scheme for encrypting new
         #   data.
+        # * <tt>:support_unencrypted_data</tt> - If `config.active_record.encryption.support_unencrypted_data` is +true+,
+        #   you can set this to +false+ to opt out of unencrypted data support for this attribute. This is useful for
+        #   scenarios where you encrypt one column, and want to disable support for unencrypted data without having to tweak
+        #   the global setting.
         # * <tt>:downcase</tt> - When true, it converts the encrypted content to downcase automatically. This allows to
         #   effectively ignore case when querying data. Notice that the case is lost. Use +:ignore_case+ if you are interested
         #   in preserving it.
@@ -42,11 +46,11 @@ module ActiveRecord
         # * <tt>:previous</tt> - List of previous encryption schemes. When provided, they will be used in order when trying to read
         #   the attribute. Each entry of the list can contain the properties supported by #encrypts. Also, when deterministic
         #   encryption is used, they will be used to generate additional ciphertexts to check in the queries.
-        def encrypts(*names, key_provider: nil, key: nil, deterministic: false, downcase: false, ignore_case: false, previous: [], **context_properties)
+        def encrypts(*names, key_provider: nil, key: nil, deterministic: false, support_unencrypted_data: nil, downcase: false, ignore_case: false, previous: [], **context_properties)
           self.encrypted_attributes ||= Set.new # not using :default because the instance would be shared across classes
 
           names.each do |name|
-            encrypt_attribute name, key_provider: key_provider, key: key, deterministic: deterministic, downcase: downcase, ignore_case: ignore_case, previous: previous, **context_properties
+            encrypt_attribute name, key_provider: key_provider, key: key, deterministic: deterministic, support_unencrypted_data: support_unencrypted_data, downcase: downcase, ignore_case: ignore_case, previous: previous, **context_properties
           end
         end
 
@@ -63,9 +67,9 @@ module ActiveRecord
         end
 
         private
-          def scheme_for(key_provider: nil, key: nil, deterministic: false, downcase: false, ignore_case: false, previous: [], **context_properties)
+          def scheme_for(key_provider: nil, key: nil, deterministic: false, support_unencrypted_data: nil, downcase: false, ignore_case: false, previous: [], **context_properties)
             ActiveRecord::Encryption::Scheme.new(key_provider: key_provider, key: key, deterministic: deterministic,
-              downcase: downcase, ignore_case: ignore_case, **context_properties).tap do |scheme|
+              support_unencrypted_data: support_unencrypted_data, downcase: downcase, ignore_case: ignore_case, **context_properties).tap do |scheme|
               scheme.previous_schemes = global_previous_schemes_for(scheme) +
               Array.wrap(previous).collect { |scheme_config| ActiveRecord::Encryption::Scheme.new(**scheme_config) }
             end
@@ -77,14 +81,14 @@ module ActiveRecord
             end
           end
 
-          def encrypt_attribute(name, key_provider: nil, key: nil, deterministic: false, downcase: false, ignore_case: false, previous: [], **context_properties)
+          def encrypt_attribute(name, key_provider: nil, key: nil, deterministic: false, support_unencrypted_data: nil, downcase: false, ignore_case: false, previous: [], **context_properties)
             encrypted_attributes << name.to_sym
 
             attribute name do |cast_type|
-              scheme = scheme_for key_provider: key_provider, key: key, deterministic: deterministic, downcase: downcase, \
-                ignore_case: ignore_case, previous: previous, **context_properties
-              ActiveRecord::Encryption::EncryptedAttributeType.new(scheme: scheme, cast_type: cast_type,
-                default: columns_hash[name.to_s]&.default)
+              scheme = scheme_for key_provider: key_provider, key: key, deterministic: deterministic, support_unencrypted_data: support_unencrypted_data, \
+                downcase: downcase, ignore_case: ignore_case, previous: previous, **context_properties
+
+              ActiveRecord::Encryption::EncryptedAttributeType.new(scheme: scheme, cast_type: cast_type, default: columns_hash[name.to_s]&.default)
             end
 
             preserve_original_encrypted(name) if ignore_case

activerecord/lib/active_record/encryption/encrypted_attribute_type.rb

@@ -54,6 +54,10 @@ module ActiveRecord
         @previous_types[support_unencrypted_data?] ||= build_previous_types_for(previous_schemes_including_clean_text)
       end
 
+      def support_unencrypted_data?
+        ActiveRecord::Encryption.config.support_unencrypted_data && scheme.support_unencrypted_data? && !previous_type?
+      end
+
       private
         def previous_schemes_including_clean_text
           previous_schemes.including((clean_text_scheme if support_unencrypted_data?)).compact
@@ -131,10 +135,6 @@ module ActiveRecord
           ActiveRecord::Encryption.encryptor
         end
 
-        def support_unencrypted_data?
-          ActiveRecord::Encryption.config.support_unencrypted_data && !previous_type?
-        end
-
         def encryption_options
           @encryption_options ||= { key_provider: key_provider, cipher_options: { deterministic: deterministic? } }.compact
         end

activerecord/lib/active_record/encryption/extended_deterministic_queries.rb

@@ -19,25 +19,26 @@ module ActiveRecord
     # * ActiveRecord::Base - Used in <tt>Contact.find_by_email_address(...)</tt>
     # * ActiveRecord::Relation - Used in <tt>Contact.internal.find_by_email_address(...)</tt>
     #
-    # ActiveRecord::Base relies on ActiveRecord::Relation (ActiveRecord::QueryMethods) but it does
-    # some prepared statements caching. That's why we need to intercept +ActiveRecord::Base+ as soon
-    # as it's invoked (so that the proper prepared statement is cached).
-    #
-    # When modifying this file run performance tests in +test/performance/extended_deterministic_queries_performance_test.rb+ to
-    #   make sure performance overhead is acceptable.
-    #
-    # We will extend this to support previous "encryption context" versions in future iterations
-    #
-    # @TODO Experimental. Support for every kind of query is pending
-    # @TODO It should not patch anything if not needed (no previous schemes or no support for previous encryption schemes)
+    # This module is included if `config.active_record.encryption.extend_queries` is `true`.
     module ExtendedDeterministicQueries
       def self.install_support
+        # ActiveRecord::Base relies on ActiveRecord::Relation (ActiveRecord::QueryMethods) but it does
+        # some prepared statements caching. That's why we need to intercept +ActiveRecord::Base+ as soon
+        # as it's invoked (so that the proper prepared statement is cached).
         ActiveRecord::Relation.prepend(RelationQueries)
         ActiveRecord::Base.include(CoreQueries)
         ActiveRecord::Encryption::EncryptedAttributeType.prepend(ExtendedEncryptableType)
         Arel::Nodes::HomogeneousIn.prepend(InWithAdditionalValues)
       end
 
+      # When modifying this file run performance tests in
+      # +activerecord/test/cases/encryption/performance/extended_deterministic_queries_performance_test.rb+
+      # to make sure performance overhead is acceptable.
+      #
+      # @TODO We will extend this to support previous "encryption context" versions in future iterations
+      # @TODO Experimental. Support for every kind of query is pending
+      # @TODO It should not patch anything if not needed (no previous schemes or no support for previous encryption schemes)
+
       module EncryptedQuery # :nodoc:
         class << self
           def process_arguments(owner, args, check_for_additional_values)

activerecord/lib/active_record/encryption/scheme.rb

@@ -10,7 +10,7 @@ module ActiveRecord
     class Scheme
       attr_accessor :previous_schemes
 
-      def initialize(key_provider: nil, key: nil, deterministic: nil, downcase: nil, ignore_case: nil,
+      def initialize(key_provider: nil, key: nil, deterministic: nil, support_unencrypted_data: nil, downcase: nil, ignore_case: nil,
                      previous_schemes: nil, **context_properties)
         # Initializing all attributes to +nil+ as we want to allow a "not set" semantics so that we
         # can merge schemes without overriding values with defaults. See +#merge+
@@ -18,6 +18,7 @@ module ActiveRecord
         @key_provider_param = key_provider
         @key = key
         @deterministic = deterministic
+        @support_unencrypted_data = support_unencrypted_data
         @downcase = downcase || ignore_case
         @ignore_case = ignore_case
         @previous_schemes_param = previous_schemes
@@ -39,6 +40,10 @@ module ActiveRecord
         !!@deterministic
       end
 
+      def support_unencrypted_data?
+        @support_unencrypted_data.nil? ? ActiveRecord::Encryption.config.support_unencrypted_data : @support_unencrypted_data
+      end
+
       def fixed?
         # by default deterministic encryption is fixed
         @fixed ||= @deterministic && (!@deterministic.is_a?(Hash) || @deterministic[:fixed])

activerecord/test/cases/encryption/extended_deterministic_queries_test.rb

@@ -9,19 +9,19 @@ class ActiveRecord::Encryption::ExtendedDeterministicQueriesTest < ActiveRecord:
   end
 
   test "Finds records when data is unencrypted" do
-    ActiveRecord::Encryption.without_encryption { UnencryptedBook.create! name: "Dune" }
+    UnencryptedBook.create!(name: "Dune")
     assert EncryptedBook.find_by(name: "Dune") # core
     assert EncryptedBook.where("id > 0").find_by(name: "Dune") # relation
   end
 
   test "Finds records when data is encrypted" do
-    UnencryptedBook.create! name: "Dune"
+    EncryptedBook.create!(name: "Dune")
     assert EncryptedBook.find_by(name: "Dune") # core
     assert EncryptedBook.where("id > 0").find_by(name: "Dune") # relation
   end
 
   test "Works well with downcased attributes" do
-    ActiveRecord::Encryption.without_encryption { EncryptedBookWithDowncaseName.create! name: "Dune" }
+    EncryptedBookWithDowncaseName.create! name: "Dune"
     assert EncryptedBookWithDowncaseName.find_by(name: "DUNE")
   end
 
@@ -44,7 +44,31 @@ class ActiveRecord::Encryption::ExtendedDeterministicQueriesTest < ActiveRecord:
   end
 
   test "exists?(...) works" do
-    ActiveRecord::Encryption.without_encryption { EncryptedBook.create! name: "Dune" }
+    EncryptedBook.create! name: "Dune"
     assert EncryptedBook.exists?(name: "Dune")
   end
+
+  test "If support_unencrypted_data is opted out at the attribute level, cannot find unencrypted data" do
+    UnencryptedBook.create! name: "Dune"
+    assert_nil EncryptedBookWithUnencryptedDataOptedOut.find_by(name: "Dune") # core
+    assert_nil EncryptedBookWithUnencryptedDataOptedOut.where("id > 0").find_by(name: "Dune") # relation
+  end
+
+  test "If support_unencrypted_data is opted out at the attribute level, can find encrypted data" do
+    EncryptedBook.create! name: "Dune"
+    assert EncryptedBookWithUnencryptedDataOptedOut.find_by(name: "Dune") # core
+    assert EncryptedBookWithUnencryptedDataOptedOut.where("id > 0").find_by(name: "Dune") # relation
+  end
+
+  test "If support_unencrypted_data is opted in at the attribute level, can find unencrypted data" do
+    UnencryptedBook.create! name: "Dune"
+    assert EncryptedBookWithUnencryptedDataOptedIn.find_by(name: "Dune") # core
+    assert EncryptedBookWithUnencryptedDataOptedIn.where("id > 0").find_by(name: "Dune") # relation
+  end
+
+  test "If support_unencrypted_data is opted in at the attribute level, can find encrypted data" do
+    EncryptedBook.create! name: "Dune"
+    assert EncryptedBookWithUnencryptedDataOptedIn.find_by(name: "Dune") # core
+    assert EncryptedBookWithUnencryptedDataOptedIn.where("id > 0").find_by(name: "Dune") # relation
+  end
 end

activerecord/test/cases/encryption/uniqueness_validations_test.rb

@@ -15,10 +15,27 @@ class ActiveRecord::Encryption::UniquenessValidationsTest < ActiveRecord::Encryp
   test "uniqueness validations work when mixing encrypted an unencrypted data" do
     ActiveRecord::Encryption.config.support_unencrypted_data = true
 
-    ActiveRecord::Encryption.without_encryption { EncryptedBookWithDowncaseName.create! name: "dune" }
-
+    UnencryptedBook.create! name: "dune"
     assert_raises ActiveRecord::RecordInvalid do
-      EncryptedBookWithDowncaseName.create!(name: "dune")
+      EncryptedBookWithDowncaseName.create!(name: "DUNE")
+    end
+  end
+
+  test "uniqueness validations do not work when mixing encrypted an unencrypted data and unencrypted data is opted out per-attribute" do
+    ActiveRecord::Encryption.config.support_unencrypted_data = true
+
+    UnencryptedBook.create! name: "dune"
+    assert_nothing_raised do
+      EncryptedBookWithUnencryptedDataOptedOut.create!(name: "dune")
+    end
+  end
+
+  test "uniqueness validations work when mixing encrypted an unencrypted data and unencrypted data is opted in per-attribute" do
+    ActiveRecord::Encryption.config.support_unencrypted_data = true
+
+    UnencryptedBook.create! name: "dune"
+    assert_raises ActiveRecord::RecordInvalid do
+      EncryptedBookWithUnencryptedDataOptedIn.create!(name: "dune")
     end
   end
 

activerecord/test/cases/helper.rb

@@ -70,5 +70,7 @@ ActiveRecord::Encryption.configure \
   deterministic_key: "test deterministic key",
   key_derivation_salt: "testing key derivation salt"
 
+# Simulate https://github.com/rails/rails/blob/735cba5bed7a54c7397dfeec1bed16033ae286f8/activerecord/lib/active_record/railtie.rb#L392
+ActiveRecord::Encryption.config.extend_queries = true
 ActiveRecord::Encryption::ExtendedDeterministicQueries.install_support
 ActiveRecord::Encryption::ExtendedDeterministicUniquenessValidator.install_support

activerecord/test/models/book_encrypted.rb

@@ -22,3 +22,17 @@ class EncryptedBookThatIgnoresCase < ActiveRecord::Base
 
   encrypts :name, deterministic: true, ignore_case: true
 end
+
+class EncryptedBookWithUnencryptedDataOptedOut < ActiveRecord::Base
+  self.table_name = "encrypted_books"
+
+  validates :name, uniqueness: true
+  encrypts :name, deterministic: true, support_unencrypted_data: false
+end
+
+class EncryptedBookWithUnencryptedDataOptedIn < ActiveRecord::Base
+  self.table_name = "encrypted_books"
+
+  validates :name, uniqueness: true
+  encrypts :name, deterministic: true, support_unencrypted_data: true
+end